From 25534480311718da074c8f92cea20c3efa16dd92 Mon Sep 17 00:00:00 2001 From: pandor4u <103976470+pandor4u@users.noreply.github.com> Date: Fri, 10 Apr 2026 17:46:36 -0500 Subject: [PATCH 1/3] feat: PostgreSQL-backed search/document storage as ElasticSearch alternative Simplified PR addressing review feedback: - Consolidated duplicated INSERT...ON CONFLICT SQL into shared constant - Removed fork-specific files (AGENTS.md, CLAUDE.md, GEMINI.md, docker compose, .gitignore) - Moved PostgreSQL tests to separate opt-in suite (MoquiSuite untouched) - Updated ElasticRequestLogFilter to use ElasticClient interface type - 83 tests: 46 unit (query translation + SQL injection) + 37 integration New files: - PostgresElasticClient.groovy: Full ElasticClient impl using JSONB/tsvector - ElasticQueryTranslator.groovy: ES Query DSL to PostgreSQL SQL - PostgresSearchLogger.groovy: Log4j2 appender for PostgreSQL - SearchEntities.xml: Entity definitions for search tables - PostgresSearchSuite.groovy: Separate JUnit test suite - PostgresSearchTranslatorTests.groovy: Unit tests - PostgresElasticClientTests.groovy: Integration tests Modified files: - ElasticFacadeImpl.groovy: type=postgres routing - ElasticRequestLogFilter.groovy: Interface type usage - MoquiDefaultConf.xml: Postgres config + entity load - moqui-conf-3.xsd: type attribute with elastic/postgres enum - build.gradle: Test dependencies and suite include --- framework/build.gradle | 3 + framework/entity/SearchEntities.xml | 115 ++ .../impl/context/ElasticFacadeImpl.groovy | 54 +- .../context/ElasticQueryTranslator.groovy | 675 ++++++++++ .../impl/context/PostgresElasticClient.groovy | 1176 +++++++++++++++++ .../impl/util/PostgresSearchLogger.groovy | 244 ++++ .../webapp/ElasticRequestLogFilter.groovy | 10 +- .../src/main/resources/MoquiDefaultConf.xml | 11 +- .../groovy/PostgresElasticClientTests.groovy | 701 ++++++++++ .../test/groovy/PostgresSearchSuite.groovy | 30 + .../PostgresSearchTranslatorTests.groovy | 478 +++++++ framework/xsd/moqui-conf-3.xsd | 25 +- 12 files changed, 3491 insertions(+), 31 deletions(-) create mode 100644 framework/entity/SearchEntities.xml create mode 100644 framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy create mode 100644 framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy create mode 100644 framework/src/main/groovy/org/moqui/impl/util/PostgresSearchLogger.groovy create mode 100644 framework/src/test/groovy/PostgresElasticClientTests.groovy create mode 100644 framework/src/test/groovy/PostgresSearchSuite.groovy create mode 100644 framework/src/test/groovy/PostgresSearchTranslatorTests.groovy diff --git a/framework/build.gradle b/framework/build.gradle index 335fe6a97..d4d00afd9 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -171,6 +171,8 @@ dependencies { testImplementation 'org.junit.platform:junit-platform-suite:6.0.1' // junit-jupiter-api for using JUnit directly, not generally needed for Spock based tests testImplementation 'org.junit.jupiter:junit-jupiter-api:6.0.1' + // junit-jupiter-engine required to execute @Test-annotated methods via JUnit Platform + testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:6.0.1' // Spock Framework testImplementation platform('org.spockframework:spock-bom:2.4-groovy-5.0') // Apache 2.0 testImplementation 'org.spockframework:spock-core:2.4-groovy-5.0' // Apache 2.0 @@ -201,6 +203,7 @@ test { dependsOn cleanTest include '**/*MoquiSuite.class' + include '**/*PostgresSearchSuite.class' systemProperty 'moqui.runtime', '../runtime' systemProperty 'moqui.conf', 'conf/MoquiDevConf.xml' diff --git a/framework/entity/SearchEntities.xml b/framework/entity/SearchEntities.xml new file mode 100644 index 000000000..0ed222b5c --- /dev/null +++ b/framework/entity/SearchEntities.xml @@ -0,0 +1,115 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy b/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy index 72ffd6ddc..8f26ba848 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/ElasticFacadeImpl.groovy @@ -32,6 +32,7 @@ import org.moqui.impl.entity.EntityDefinition import org.moqui.impl.entity.EntityJavaUtil import org.moqui.impl.entity.FieldInfo import org.moqui.impl.util.ElasticSearchLogger +import org.moqui.impl.util.PostgresSearchLogger import org.moqui.util.LiteStringMap import org.moqui.util.MNode import org.moqui.util.RestClient @@ -69,8 +70,9 @@ class ElasticFacadeImpl implements ElasticFacade { } public final ExecutionContextFactoryImpl ecfi - private final Map clientByClusterName = new LinkedHashMap<>() + private final Map clientByClusterName = new LinkedHashMap<>() private ElasticSearchLogger esLogger = null + private PostgresSearchLogger pgLogger = null ElasticFacadeImpl(ExecutionContextFactoryImpl ecfi) { this.ecfi = ecfi @@ -90,14 +92,21 @@ class ElasticFacadeImpl implements ElasticFacade { logger.warn("ElasticFacade Client for cluster ${clusterName} already initialized, skipping") continue } - if (!clusterUrl) { - logger.warn("ElasticFacade Client for cluster ${clusterName} has no url, skipping") - continue - } + String clusterType = clusterNode.attribute("type") ?: "elastic" try { - ElasticClientImpl elci = new ElasticClientImpl(clusterNode, ecfi) - clientByClusterName.put(clusterName, elci) + if ("postgres".equals(clusterType)) { + PostgresElasticClient pgc = new PostgresElasticClient(clusterNode, ecfi) + clientByClusterName.put(clusterName, pgc) + logger.info("Initialized PostgresElasticClient for cluster ${clusterName}") + } else { + if (!clusterUrl) { + logger.warn("ElasticFacade Client for cluster ${clusterName} has no url, skipping") + continue + } + ElasticClientImpl elci = new ElasticClientImpl(clusterNode, ecfi) + clientByClusterName.put(clusterName, elci) + } } catch (Throwable t) { Throwable cause = t.getCause() if (cause != null && cause.message.contains("refused")) { @@ -108,22 +117,29 @@ class ElasticFacadeImpl implements ElasticFacade { } } - // init ElasticSearchLogger - if (esLogger == null || !esLogger.isInitialized()) { - ElasticClientImpl loggerEci = clientByClusterName.get("logger") ?: clientByClusterName.get("default") - if (loggerEci != null) { - logger.info("Initializing ElasticSearchLogger with cluster ${loggerEci.getClusterName()}") - esLogger = new ElasticSearchLogger(loggerEci, ecfi) + // init ElasticSearchLogger / PostgresSearchLogger depending on backend type + ElasticClient loggerClient = clientByClusterName.get("logger") ?: clientByClusterName.get("default") + if (loggerClient instanceof PostgresElasticClient) { + if (pgLogger == null || !pgLogger.isInitialized()) { + logger.info("Initializing PostgresSearchLogger with cluster ${loggerClient.getClusterName()}") + pgLogger = new PostgresSearchLogger((PostgresElasticClient) loggerClient, ecfi) + } else { + logger.warn("PostgresSearchLogger in place and initialized, skipping") + } + } else if (loggerClient instanceof ElasticClientImpl) { + if (esLogger == null || !esLogger.isInitialized()) { + logger.info("Initializing ElasticSearchLogger with cluster ${loggerClient.getClusterName()}") + esLogger = new ElasticSearchLogger((ElasticClientImpl) loggerClient, ecfi) } else { - logger.warn("No Elastic Client found with name 'logger' or 'default', not initializing ElasticSearchLogger") + logger.warn("ElasticSearchLogger in place and initialized, skipping") } } else { - logger.warn("ElasticSearchLogger in place and initialized, not initializing ElasticSearchLogger") + logger.warn("No Elastic/Postgres Client found with name 'logger' or 'default', not initializing search logger") } // Index DataFeed with indexOnStartEmpty=Y try { - ElasticClientImpl defaultEci = clientByClusterName.get("default") + ElasticClient defaultEci = clientByClusterName.get("default") if (defaultEci != null) { EntityList dataFeedList = ecfi.entityFacade.find("moqui.entity.feed.DataFeed") .condition("indexOnStartEmpty", "Y").disableAuthz().list() @@ -151,7 +167,11 @@ class ElasticFacadeImpl implements ElasticFacade { void destroy() { if (esLogger != null) esLogger.destroy() - for (ElasticClientImpl eci in clientByClusterName.values()) eci.destroy() + if (pgLogger != null) pgLogger.destroy() + for (ElasticClient eci in clientByClusterName.values()) { + if (eci instanceof ElasticClientImpl) ((ElasticClientImpl) eci).destroy() + else if (eci instanceof PostgresElasticClient) ((PostgresElasticClient) eci).destroy() + } } @Override ElasticClient getDefault() { return clientByClusterName.get("default") } diff --git a/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy b/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy new file mode 100644 index 000000000..887aa1237 --- /dev/null +++ b/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy @@ -0,0 +1,675 @@ +/* + * This software is in the public domain under CC0 1.0 Universal plus a + * Grant of Patent License. + * + * To the extent possible under law, the author(s) have dedicated all + * copyright and related and neighboring rights to this software to the + * public domain worldwide. This software is distributed without any + * warranty. + * + * You should have received a copy of the CC0 Public Domain Dedication + * along with this software (see the LICENSE.md file). If not, see + * . + */ +package org.moqui.impl.context + +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * Translates ElasticSearch/OpenSearch Query DSL (Map structures) into PostgreSQL SQL WHERE clauses, + * ORDER BY expressions, and OFFSET/LIMIT pagination for use by PostgresElasticClient. + * + * Supports the query types used by Moqui's SearchServices.xml and entity condition makeSearchFilter() methods: + * - query_string (→ websearch_to_tsquery / plainto_tsquery on content_tsv) + * - bool (must / should / must_not / filter) + * - term, terms + * - range + * - match_all + * - exists + * - nested (→ jsonb_array_elements EXISTS subquery) + */ +class ElasticQueryTranslator { + private final static Logger logger = LoggerFactory.getLogger(ElasticQueryTranslator.class) + + /** Regex pattern for valid field names — alphanumeric, underscores, dots, hyphens, and @ (for @timestamp) */ + private static final java.util.regex.Pattern SAFE_FIELD_PATTERN = java.util.regex.Pattern.compile('^[a-zA-Z0-9_@][a-zA-Z0-9_.\\-]*$') + + /** + * Validate that a field name is safe for interpolation into SQL. + * Rejects any field containing SQL metacharacters (quotes, semicolons, parentheses, etc.) + * @throws IllegalArgumentException if the field name contains unsafe characters + */ + static String sanitizeFieldName(String field) { + if (field == null || field.isEmpty()) throw new IllegalArgumentException("Field name must not be empty") + if (!SAFE_FIELD_PATTERN.matcher(field).matches()) { + throw new IllegalArgumentException("Unsafe field name rejected: '${field}' — only alphanumeric, underscore, dot, hyphen, and @ allowed") + } + if (field.contains("--")) { + throw new IllegalArgumentException("Unsafe field name rejected: '${field}' — double-hyphen (SQL comment) not allowed") + } + if (field.length() > 256) { + throw new IllegalArgumentException("Field name too long (max 256 chars): '${field}'") + } + return field + } + + /** Holds the result of translating a query DSL fragment or full search request */ + static class TranslatedQuery { + /** SQL WHERE clause fragment (without the "WHERE" keyword), or "TRUE" if no filter */ + String whereClause = "TRUE" + /** JDBC bind parameters in order corresponding to ? placeholders in whereClause */ + List params = [] + /** SQL ORDER BY expression (without the "ORDER BY" keyword), or null */ + String orderBy = null + /** The tsquery expression (as SQL expression string) for use in ts_rank_cd() and ts_headline() */ + String tsqueryExpr = null + /** Bind parameters specifically for tsqueryExpr (separate from WHERE params) */ + List tsqueryParams = [] + /** OFFSET value for pagination */ + int fromOffset = 0 + /** LIMIT value for pagination */ + int sizeLimit = 20 + /** Track total hits (adds no SQL change but reflects ES track_total_hits flag) */ + boolean trackTotal = true + /** Fields to highlight, keyed by field name */ + Map highlightFields = [:] + } + + /** + * Translate a full ES searchMap (the body sent to /_search) into a TranslatedQuery. + * @param searchMap Map as built by SearchServices.search#DataDocuments + */ + static TranslatedQuery translateSearchMap(Map searchMap) { + TranslatedQuery tq = new TranslatedQuery() + + // Pagination + Object fromVal = searchMap.get("from") + if (fromVal != null) tq.fromOffset = ((Number) fromVal).intValue() + Object sizeVal = searchMap.get("size") + if (sizeVal != null) tq.sizeLimit = ((Number) sizeVal).intValue() + + // Sort + Object sortVal = searchMap.get("sort") + if (sortVal instanceof List) { + tq.orderBy = translateSort((List) sortVal) + } + + // Highlight fields + Object highlightVal = searchMap.get("highlight") + if (highlightVal instanceof Map) { + Object fieldsVal = ((Map) highlightVal).get("fields") + if (fieldsVal instanceof Map) tq.highlightFields = (Map) fieldsVal + } + + // track_total_hits + Object tthVal = searchMap.get("track_total_hits") + if (tthVal != null) tq.trackTotal = Boolean.TRUE == tthVal || "true".equals(tthVal.toString()) + + // Query + Object queryVal = searchMap.get("query") + if (queryVal instanceof Map) { + QueryResult qr = translateQuery((Map) queryVal) + tq.whereClause = qr.clause ?: "TRUE" + tq.params = qr.params + tq.tsqueryExpr = qr.tsqueryExpr + tq.tsqueryParams = qr.tsqueryParams + } + + return tq + } + + /** Internal result holder for a single query fragment */ + static class QueryResult { + String clause = "TRUE" + List params = [] + /** If this query has a full-text component, the SQL tsquery expression for scoring/highlighting */ + String tsqueryExpr = null + /** Bind parameters specifically for tsqueryExpr (separate from WHERE clause params) */ + List tsqueryParams = [] + } + + static QueryResult translateQuery(Map queryMap) { + if (queryMap == null || queryMap.isEmpty()) return new QueryResult() + + String queryType = (String) queryMap.keySet().iterator().next() + Object queryVal = queryMap.get(queryType) + + switch (queryType) { + case "match_all": return translateMatchAll() + case "match_none": + QueryResult qr = new QueryResult(); qr.clause = "FALSE"; return qr + case "query_string": return translateQueryString((Map) queryVal) + case "multi_match": return translateMultiMatch((Map) queryVal) + case "bool": return translateBool((Map) queryVal) + case "term": return translateTerm((Map) queryVal, false) + case "terms": return translateTerms((Map) queryVal) + case "range": return translateRange((Map) queryVal) + case "exists": return translateExists((Map) queryVal) + case "nested": return translateNested((Map) queryVal) + case "ids": return translateIds((Map) queryVal) + default: + logger.warn("ElasticQueryTranslator: unsupported query type '${queryType}', using TRUE") + return new QueryResult() + } + } + + private static QueryResult translateMatchAll() { + QueryResult qr = new QueryResult() + qr.clause = "TRUE" + return qr + } + + private static QueryResult translateQueryString(Map qsMap) { + QueryResult qr = new QueryResult() + if (qsMap == null) return qr + + String query = (String) qsMap.get("query") + if (!query || query.trim().isEmpty()) return qr + + // Clean up the query string: + // 1. Lucene field:value syntax → handle field-specific searches + // 2. Strip unsupported operators, translate AND/OR/NOT + // 3. Use websearch_to_tsquery which supports quoted phrases, AND, OR, -, + + String cleanedQuery = cleanLuceneQuery(query) + + if (!cleanedQuery || cleanedQuery.trim().isEmpty()) return qr + + // Use websearch_to_tsquery for natural language queries + // It handles: "exact phrase", AND/OR/NOT, +required, -exclude + qr.tsqueryExpr = "websearch_to_tsquery('english', ?)" + qr.tsqueryParams = [cleanedQuery] + qr.params = [cleanedQuery] + qr.clause = "content_tsv @@ websearch_to_tsquery('english', ?)" + return qr + } + + private static QueryResult translateMultiMatch(Map mmMap) { + // Treat like query_string on all fields + String query = (String) mmMap.get("query") + if (!query) return new QueryResult() + return translateQueryString([query: query]) + } + + private static QueryResult translateBool(Map boolMap) { + QueryResult qr = new QueryResult() + if (boolMap == null) return qr + + List clauses = [] + List params = [] + String combinedTsquery = null + List combinedTsqueryParams = [] + + // must (AND) + Object mustVal = boolMap.get("must") + if (mustVal instanceof List) { + List mustClauses = [] + for (Object item in (List) mustVal) { + if (item instanceof Map) { + QueryResult itemQr = translateQuery((Map) item) + mustClauses.add(itemQr.clause) + params.addAll(itemQr.params) + if (itemQr.tsqueryExpr) { + combinedTsquery = combinedTsquery ? "(${combinedTsquery}) && (${itemQr.tsqueryExpr})" : itemQr.tsqueryExpr + combinedTsqueryParams.addAll(itemQr.tsqueryParams) + } + } + } + if (mustClauses) clauses.add("(" + mustClauses.join(" AND ") + ")") + } else if (mustVal instanceof Map) { + QueryResult itemQr = translateQuery((Map) mustVal) + clauses.add(itemQr.clause) + params.addAll(itemQr.params) + if (itemQr.tsqueryExpr) { + combinedTsquery = itemQr.tsqueryExpr + combinedTsqueryParams.addAll(itemQr.tsqueryParams) + } + } + + // filter (same as must for our purposes) + Object filterVal = boolMap.get("filter") + if (filterVal instanceof List) { + List filterClauses = [] + for (Object item in (List) filterVal) { + if (item instanceof Map) { + QueryResult itemQr = translateQuery((Map) item) + filterClauses.add(itemQr.clause) + params.addAll(itemQr.params) + } + } + if (filterClauses) clauses.add("(" + filterClauses.join(" AND ") + ")") + } else if (filterVal instanceof Map) { + QueryResult itemQr = translateQuery((Map) filterVal) + clauses.add(itemQr.clause) + params.addAll(itemQr.params) + } + + // should (OR) + Object shouldVal = boolMap.get("should") + if (shouldVal instanceof List) { + List shouldClauses = [] + for (Object item in (List) shouldVal) { + if (item instanceof Map) { + QueryResult itemQr = translateQuery((Map) item) + shouldClauses.add(itemQr.clause) + params.addAll(itemQr.params) + if (itemQr.tsqueryExpr) { + combinedTsquery = combinedTsquery ? "(${combinedTsquery}) || (${itemQr.tsqueryExpr})" : itemQr.tsqueryExpr + combinedTsqueryParams.addAll(itemQr.tsqueryParams) + } + } + } + if (shouldClauses) { + int minShouldMatch = 1 + Object msmVal = boolMap.get("minimum_should_match") + if (msmVal != null) minShouldMatch = ((Number) msmVal).intValue() + if (minShouldMatch == 1) { + clauses.add("(" + shouldClauses.join(" OR ") + ")") + } else { + // For minimum_should_match > 1, use a CASE/SUM trick for simplicity just add as OR + clauses.add("(" + shouldClauses.join(" OR ") + ")") + } + } + } else if (shouldVal instanceof Map) { + QueryResult itemQr = translateQuery((Map) shouldVal) + clauses.add(itemQr.clause) + params.addAll(itemQr.params) + if (itemQr.tsqueryExpr) { + combinedTsquery = itemQr.tsqueryExpr + combinedTsqueryParams.addAll(itemQr.tsqueryParams) + } + } + + // must_not (NOT) + Object mustNotVal = boolMap.get("must_not") + if (mustNotVal instanceof List) { + List mustNotClauses = [] + for (Object item in (List) mustNotVal) { + if (item instanceof Map) { + QueryResult itemQr = translateQuery((Map) item) + mustNotClauses.add(itemQr.clause) + params.addAll(itemQr.params) + } + } + if (mustNotClauses) clauses.add("NOT (" + mustNotClauses.join(" OR ") + ")") + } else if (mustNotVal instanceof Map) { + QueryResult itemQr = translateQuery((Map) mustNotVal) + clauses.add("NOT (${itemQr.clause})") + params.addAll(itemQr.params) + } + + qr.clause = clauses ? "(" + clauses.join(" AND ") + ")" : "TRUE" + qr.params = params + qr.tsqueryExpr = combinedTsquery + qr.tsqueryParams = combinedTsqueryParams + return qr + } + + private static QueryResult translateTerm(Map termMap, boolean ignoreCase) { + QueryResult qr = new QueryResult() + if (termMap == null || termMap.isEmpty()) return qr + + String field = (String) termMap.keySet().iterator().next() + Object valueHolder = termMap.get(field) + Object value + if (valueHolder instanceof Map) { + value = ((Map) valueHolder).get("value") + } else { + value = valueHolder + } + if (value == null) { qr.clause = "TRUE"; return qr } + + // _id is a special ES field that maps to the doc_id column + if (field == "_id") { + qr.clause = "doc_id = ?" + qr.params = [value.toString()] + return qr + } + + String jsonPath = fieldToJsonPath("document", field) + if (ignoreCase && value instanceof String) { + qr.clause = "LOWER(${jsonPath}) = LOWER(?)" + } else { + qr.clause = "${jsonPath} = ?" + } + qr.params = [value.toString()] + return qr + } + + private static QueryResult translateTerms(Map termsMap) { + QueryResult qr = new QueryResult() + if (termsMap == null || termsMap.isEmpty()) return qr + + // Remove boost key if present + Map filteredMap = termsMap.findAll { k, v -> k != "boost" } + if (filteredMap.isEmpty()) return qr + + String field = (String) filteredMap.keySet().iterator().next() + Object valuesObj = filteredMap.get(field) + if (!(valuesObj instanceof List)) { qr.clause = "TRUE"; return qr } + List values = (List) valuesObj + if (values.isEmpty()) { qr.clause = "FALSE"; return qr } + + String jsonPath = fieldToJsonPath("document", field) + List placeholders = values.collect { "?" } + qr.clause = "${jsonPath} IN (${placeholders.join(', ')})" + qr.params = values.collect { it?.toString() } + return qr + } + + private static QueryResult translateRange(Map rangeMap) { + QueryResult qr = new QueryResult() + if (rangeMap == null || rangeMap.isEmpty()) return qr + + String field = (String) rangeMap.keySet().iterator().next() + Object rangeSpec = rangeMap.get(field) + if (!(rangeSpec instanceof Map)) return qr + + Map rangeSpecMap = (Map) rangeSpec + String jsonPath = fieldToJsonPath("document", field) + List conditions = [] + List params = [] + + // Determine cast type based on common field name patterns + String castType = guessCastType(field) + + Object gte = rangeSpecMap.get("gte") + Object gt = rangeSpecMap.get("gt") + Object lte = rangeSpecMap.get("lte") + Object lt = rangeSpecMap.get("lt") + + if (gte != null) { conditions.add("(${jsonPath})${castType} >= ?"); params.add(gte.toString()) } + if (gt != null) { conditions.add("(${jsonPath})${castType} > ?"); params.add(gt.toString()) } + if (lte != null) { conditions.add("(${jsonPath})${castType} <= ?"); params.add(lte.toString()) } + if (lt != null) { conditions.add("(${jsonPath})${castType} < ?"); params.add(lt.toString()) } + + if (conditions.isEmpty()) { qr.clause = "TRUE"; return qr } + qr.clause = conditions.join(" AND ") + qr.params = params + return qr + } + + private static QueryResult translateExists(Map existsMap) { + QueryResult qr = new QueryResult() + if (existsMap == null) return qr + String field = (String) existsMap.get("field") + if (!field) return qr + + // Validate field name to prevent SQL injection + sanitizeFieldName(field) + // For nested paths, check the nested path exists + if (field.contains(".")) { + List parts = field.split("\\.") as List + String topLevel = parts[0] + qr.clause = "document ? '${topLevel}'" + } else { + qr.clause = "document ? '${field}'" + } + return qr + } + + private static QueryResult translateNested(Map nestedMap) { + QueryResult qr = new QueryResult() + if (nestedMap == null) return qr + + String path = (String) nestedMap.get("path") + Map innerQuery = (Map) nestedMap.get("query") + if (!path || !innerQuery) return qr + + // Validate path to prevent SQL injection + sanitizeFieldName(path) + // Translate the inner query against jsonb_array_elements alias "elem" + QueryResult innerQr = translateNestedQuery(innerQuery, path) + qr.clause = "EXISTS (SELECT 1 FROM jsonb_array_elements(document->'${path}') AS elem WHERE ${innerQr.clause})" + qr.params = innerQr.params + return qr + } + + /** Translate a query in the context of a nested jsonb_array_elements expression (uses "elem" alias) */ + private static QueryResult translateNestedQuery(Map queryMap, String parentPath) { + QueryResult qr = new QueryResult() + if (queryMap == null || queryMap.isEmpty()) return qr + + String queryType = (String) queryMap.keySet().iterator().next() + Object queryVal = queryMap.get(queryType) + + if (queryType == "bool") { + return translateNestedBool((Map) queryVal, parentPath) + } else if (queryType == "term") { + return translateNestedTerm((Map) queryVal, parentPath) + } else if (queryType == "terms") { + return translateNestedTerms((Map) queryVal, parentPath) + } else if (queryType == "range") { + return translateNestedRange((Map) queryVal, parentPath) + } else if (queryType == "match_all") { + return new QueryResult() + } else { + logger.warn("ElasticQueryTranslator.translateNestedQuery: unsupported nested query type '${queryType}', using TRUE") + return new QueryResult() + } + } + + private static QueryResult translateNestedBool(Map boolMap, String parentPath) { + QueryResult qr = new QueryResult() + if (boolMap == null) return qr + List clauses = [] + List params = [] + + for (String key in ["must", "filter", "should", "must_not"]) { + Object val = boolMap.get(key) + List items + if (val instanceof List) items = (List) val + else if (val instanceof Map) items = [(Map) val] + else continue + + List itemClauses = [] + for (Map item in items) { + QueryResult ir = translateNestedQuery(item, parentPath) + itemClauses.add(ir.clause) + params.addAll(ir.params) + } + if (itemClauses) { + String joined = "(" + itemClauses.join(" AND ") + ")" + if (key == "must_not") joined = "NOT " + joined + else if (key == "should") joined = "(" + itemClauses.join(" OR ") + ")" + clauses.add(joined) + } + } + qr.clause = clauses ? clauses.join(" AND ") : "TRUE" + qr.params = params + return qr + } + + private static QueryResult translateNestedTerm(Map termMap, String parentPath) { + QueryResult qr = new QueryResult() + if (termMap == null || termMap.isEmpty()) return qr + String field = (String) termMap.keySet().iterator().next() + Object valueHolder = termMap.get(field) + Object value = valueHolder instanceof Map ? ((Map) valueHolder).get("value") : valueHolder + if (value == null) { qr.clause = "TRUE"; return qr } + + // For nested terms "parentPath.field", strip the parent path prefix + String localField = field.startsWith(parentPath + ".") ? field.substring(parentPath.length() + 1) : field + sanitizeFieldName(localField) + qr.clause = "elem->>'${localField}' = ?" + qr.params = [value.toString()] + return qr + } + + private static QueryResult translateNestedTerms(Map termsMap, String parentPath) { + QueryResult qr = new QueryResult() + Map filteredMap = termsMap.findAll { k, v -> k != "boost" } + if (filteredMap.isEmpty()) return qr + String field = (String) filteredMap.keySet().iterator().next() + Object valuesObj = filteredMap.get(field) + if (!(valuesObj instanceof List)) { qr.clause = "TRUE"; return qr } + List values = (List) valuesObj + if (values.isEmpty()) { qr.clause = "FALSE"; return qr } + String localField = field.startsWith(parentPath + ".") ? field.substring(parentPath.length() + 1) : field + sanitizeFieldName(localField) + qr.clause = "elem->>'${localField}' IN (${values.collect { '?' }.join(', ')})" + qr.params = values.collect { it?.toString() } + return qr + } + + private static QueryResult translateNestedRange(Map rangeMap, String parentPath) { + QueryResult qr = new QueryResult() + if (rangeMap == null || rangeMap.isEmpty()) return qr + String field = (String) rangeMap.keySet().iterator().next() + Object rangeSpec = rangeMap.get(field) + if (!(rangeSpec instanceof Map)) return qr + Map rangeSpecMap = (Map) rangeSpec + String localField = field.startsWith(parentPath + ".") ? field.substring(parentPath.length() + 1) : field + sanitizeFieldName(localField) + String castType = guessCastType(localField) + List conditions = [] + List params = [] + Object gte = rangeSpecMap.get("gte"); if (gte != null) { conditions.add("(elem->>'${localField}')${castType} >= ?"); params.add(gte.toString()) } + Object gt = rangeSpecMap.get("gt"); if (gt != null) { conditions.add("(elem->>'${localField}')${castType} > ?"); params.add(gt.toString()) } + Object lte = rangeSpecMap.get("lte"); if (lte != null) { conditions.add("(elem->>'${localField}')${castType} <= ?"); params.add(lte.toString()) } + Object lt = rangeSpecMap.get("lt"); if (lt != null) { conditions.add("(elem->>'${localField}')${castType} < ?"); params.add(lt.toString()) } + qr.clause = conditions ? conditions.join(" AND ") : "TRUE" + qr.params = params + return qr + } + + private static QueryResult translateIds(Map idsMap) { + QueryResult qr = new QueryResult() + Object vals = idsMap?.get("values") + if (!(vals instanceof List) || ((List) vals).isEmpty()) { qr.clause = "FALSE"; return qr } + List ids = (List) vals + qr.clause = "doc_id IN (${ids.collect { '?' }.join(', ')})" + qr.params = ids.collect { it?.toString() } + return qr + } + + /** Translate an ES sort spec (list of sort entries) to a SQL ORDER BY expression */ + static String translateSort(List sortList) { + if (!sortList) return null + List parts = [] + for (Object sortEntry in sortList) { + if (sortEntry instanceof Map) { + Map sortMap = (Map) sortEntry + for (Map.Entry entry in sortMap.entrySet()) { + String field = ((String) entry.key).replace(".keyword", "") + String dir = "ASC" + if (entry.value instanceof Map) { + String orderVal = (String) ((Map) entry.value).get("order") + if ("desc".equalsIgnoreCase(orderVal)) dir = "DESC" + } else if ("desc".equalsIgnoreCase(entry.value?.toString())) { + dir = "DESC" + } + + if ("_score".equals(field)) { + parts.add("_score ${dir}") + } else { + String castType = guessCastType(field) + if (castType) { + parts.add("(${fieldToJsonPath("document", field)})${castType} ${dir}") + } else { + parts.add("${fieldToJsonPath("document", field)} ${dir}") + } + } + } + } else if (sortEntry instanceof String) { + String field = ((String) sortEntry).replace(".keyword", "") + if ("_score".equals(field)) { + parts.add("_score DESC") + } else { + parts.add("${fieldToJsonPath("document", field)} ASC") + } + } + } + return parts ? parts.join(", ") : null + } + + /** + * Convert an ES field path to a PostgreSQL JSONB access expression. + * E.g. "product.name" → "document->'product'->>'name'" + * "productId" → "document->>'productId'" + */ + static String fieldToJsonPath(String docAlias, String field) { + // Strip .keyword suffix (used in ES for exact/sortable text fields) + if (field.endsWith(".keyword")) field = field.substring(0, field.length() - ".keyword".length()) + // Validate field name to prevent SQL injection + sanitizeFieldName(field) + List parts = field.split("\\.") as List + if (parts.size() == 1) return "${docAlias}->>'${field}'" + // For nested paths: docAlias->'part1'->'part2'->>'lastPart' + StringBuilder sb = new StringBuilder(docAlias) + for (int i = 0; i < parts.size() - 1; i++) { + sb.append("->'${parts[i]}'") + } + sb.append("->>'${parts[parts.size() - 1]}'") + return sb.toString() + } + + /** + * Guess the appropriate PostgreSQL cast type for a field name to use in range/sort comparisons. + * Returns empty string if no cast is needed (use text comparison). + */ + private static String guessCastType(String field) { + String lf = field.toLowerCase() + if (lf.contains("date") || lf.contains("stamp") || lf.contains("time") || lf == "@timestamp") { + return "::timestamptz" + } + if (lf.contains("amount") || lf.contains("price") || lf.contains("cost") || lf.contains("total") || + lf.contains("quantity") || lf.contains("qty") || lf.contains("score") || lf.contains("count") || + lf.contains("number") || lf.contains("num") || lf.contains("id") && lf.endsWith("num")) { + return "::numeric" + } + return "" + } + + /** + * Clean up a Lucene query string to be safe for use with websearch_to_tsquery. + * websearch_to_tsquery supports: "quoted phrases", AND, OR, -, + + * This removes/translates Lucene-specific syntax that websearch_to_tsquery doesn't support: + * - field:value (extract field-specific as general text) + * - field:[range TO range] (drop or convert) + * - wildcard * ? (drop trailing wildcards, keep term) + * - boost ^ (strip) + * - fuzzy ~ (strip) + * - parentheses → use natural AND grouping + */ + static String cleanLuceneQuery(String query) { + if (!query) return query + String q = query.trim() + + // Remove Lucene field:value prefixes (keep just the value part) + q = q.replaceAll(/\w+:("(?:[^"\\]|\\.)*"|\S+)/, '$1') + + // Remove range queries [X TO Y] + q = q.replaceAll(/\[[^\]]*\]/, '') + q = q.replaceAll(/\{[^}]*\}/, '') + + // Remove boost operators (^number) + q = q.replaceAll(/\^[\d.]+/, '') + + // Remove fuzzy operators (~number or just ~) + q = q.replaceAll(/~[\d.]*/, '') + + // Normalize AND/OR/NOT — websearch_to_tsquery handles them case-insensitively, + // but convert NOT to - (the supported exclusion syntax) + q = q.replaceAll(/\bNOT\b/, '-') + + // Remove wildcards at end of terms (partial matching not directly supported; term will still match as prefix via FTS) + q = q.replaceAll(/\*/, '') + q = q.replaceAll(/\?/, '') + + // Remove empty parentheses, normalize spaces + q = q.replaceAll(/\(\s*\)/, '') + q = q.replaceAll(/\s+/, ' ').trim() + + return q ?: '' + } + + /** + * Build a ts_headline SQL expression for a given field with the given tsquery expression. + * @param fieldJsonPath The SQL expression to extract the text field (e.g. "document->>'productName'") + * @param tsqueryParam The SQL tsquery expression (e.g. "websearch_to_tsquery('english', ?)") + */ + static String buildHighlightExpr(String fieldJsonPath, String tsqueryExpr) { + return "ts_headline('english', coalesce(${fieldJsonPath}, ''), ${tsqueryExpr}, 'StartSel=,StopSel=,MaxWords=35,MinWords=15,ShortWord=3,HighlightAll=false,MaxFragments=3,FragmentDelimiter= ... ')" + } +} diff --git a/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy b/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy new file mode 100644 index 000000000..a28f247f2 --- /dev/null +++ b/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy @@ -0,0 +1,1176 @@ +/* + * This software is in the public domain under CC0 1.0 Universal plus a + * Grant of Patent License. + * + * To the extent possible under law, the author(s) have dedicated all + * copyright and related and neighboring rights to this software to the + * public domain worldwide. This software is distributed without any + * warranty. + * + * You should have received a copy of the CC0 Public Domain Dedication + * along with this software (see the LICENSE.md file). If not, see + * . + */ +package org.moqui.impl.context + +import com.fasterxml.jackson.databind.ObjectMapper +import groovy.transform.CompileStatic +import org.moqui.BaseException +import org.moqui.context.ElasticFacade +import org.moqui.entity.EntityValue +import org.moqui.entity.EntityList +import org.moqui.util.MNode +import org.moqui.util.RestClient +import org.moqui.util.RestClient.Method +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.sql.Connection +import java.sql.PreparedStatement +import java.sql.ResultSet +import java.sql.Statement +import java.sql.Timestamp +import java.sql.Types +import java.util.concurrent.Future + +/** + * PostgreSQL-backed implementation of ElasticFacade.ElasticClient. + * + * Stores and searches documents using: + * - moqui_search_index table — tracks index metadata (replaces ES index/alias management) + * - moqui_document table — stores documents as JSONB with tsvector for full-text search + * + * All ElasticSearch Query DSL is translated to PostgreSQL SQL by ElasticQueryTranslator. + * Application logs go to moqui_logs table; HTTP request logs go to moqui_http_log table. + * + * Configured via MoquiConf.xml elastic-facade.cluster with type="postgres". + * Example: + * <cluster name="default" type="postgres" url="transactional" index-prefix="mq_"/> + */ +@CompileStatic +class PostgresElasticClient implements ElasticFacade.ElasticClient { + private final static Logger logger = LoggerFactory.getLogger(PostgresElasticClient.class) + private final static Set DOC_META_KEYS = new HashSet<>(["_index", "_type", "_id", "_timestamp"]) + + /** Jackson mapper shared with ElasticFacadeImpl */ + static final ObjectMapper jacksonMapper = ElasticFacadeImpl.jacksonMapper + + /** Shared UPSERT SQL for moqui_document — used by upsertDocument(), bulkIndex(), and bulkIndexDataDocument() */ + static final String DOCUMENT_UPSERT_SQL = """ + INSERT INTO moqui_document (index_name, doc_id, doc_type, document, content_text, content_tsv, updated_stamp) + VALUES (?, ?, ?, ?::jsonb, ?, to_tsvector('english', COALESCE(?, '')), now()) + ON CONFLICT (index_name, doc_id) DO UPDATE SET + doc_type = EXCLUDED.doc_type, + document = EXCLUDED.document, + content_text = EXCLUDED.content_text, + content_tsv = EXCLUDED.content_tsv, + updated_stamp = EXCLUDED.updated_stamp + """.trim() + + private final ExecutionContextFactoryImpl ecfi + private final MNode clusterNode + private final String clusterName + private final String indexPrefix + /** Entity datasource group to get connections from (e.g. "transactional") */ + final String datasourceGroup + + PostgresElasticClient(MNode clusterNode, ExecutionContextFactoryImpl ecfi) { + this.ecfi = ecfi + this.clusterNode = clusterNode + this.clusterName = clusterNode.attribute("name") + this.indexPrefix = clusterNode.attribute("index-prefix") ?: "" + + // url attribute for postgres type = datasource group name (or "transactional" by default) + String urlAttr = clusterNode.attribute("url") + this.datasourceGroup = (urlAttr && !"".equals(urlAttr.trim())) ? urlAttr.trim() : "transactional" + + logger.info("Initializing PostgresElasticClient for cluster '${clusterName}' using datasource group '${datasourceGroup}' with index prefix '${indexPrefix}'") + + // Initialize schema (CREATE TABLE IF NOT EXISTS, extensions, indexes) + initSchema() + } + + void destroy() { + // Nothing to destroy — connection pool is managed by the entity facade datasource + } + + // ============================================================ + // Schema initialization + // ============================================================ + + private void initSchema() { + boolean started = ecfi.transactionFacade.begin(null) + try { + Connection conn = ecfi.entityFacade.getConnection(datasourceGroup) + Statement stmt = conn.createStatement() + try { + // Enable pg_trgm extension for fuzzy search (available since PG 9.1) + try { stmt.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm") } + catch (Exception extEx) { logger.warn("Could not create pg_trgm extension (may require superuser): ${extEx.message}") } + + // moqui_search_index — index metadata (replaces ES index/alias concept) + stmt.execute(""" + CREATE TABLE IF NOT EXISTS moqui_search_index ( + index_name TEXT NOT NULL, + alias_name TEXT, + doc_type TEXT, + mapping TEXT, + settings TEXT, + created_stamp TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT pk_moqui_search_index PRIMARY KEY (index_name) + ) + """.trim()) + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_sidx_alias ON moqui_search_index (alias_name)") + + // moqui_document — main document store + stmt.execute(""" + CREATE TABLE IF NOT EXISTS moqui_document ( + index_name TEXT NOT NULL, + doc_id TEXT NOT NULL, + doc_type TEXT, + document JSONB, + content_text TEXT, + content_tsv TSVECTOR, + created_stamp TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_stamp TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT pk_moqui_document PRIMARY KEY (index_name, doc_id) + ) + """.trim()) + // Ensure PostgreSQL-specific columns exist (table may have been created by Moqui entity sync without them) + stmt.execute("ALTER TABLE moqui_document ADD COLUMN IF NOT EXISTS content_tsv TSVECTOR") + stmt.execute("ALTER TABLE moqui_document ADD COLUMN IF NOT EXISTS content_text TEXT") + // Ensure document column is JSONB (entity sync may create it as TEXT from text-very-long mapping) + try { + stmt.execute("ALTER TABLE moqui_document ALTER COLUMN document TYPE JSONB USING document::jsonb") + } catch (Exception e) { + // Column already JSONB or table has no rows causing cast to fail — ignore + logger.trace("Note: could not alter document column to JSONB (may already be correct type): " + e.getMessage()) + } + // GIN index on tsvector for full-text search + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_doc_tsv ON moqui_document USING GIN (content_tsv)") + // GIN index on document JSONB for arbitrary path queries + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_doc_json ON moqui_document USING GIN (document jsonb_path_ops)") + // GIN trigram index on content_text for fuzzy/LIKE queries + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_doc_trgm ON moqui_document USING GIN (content_text gin_trgm_ops)") + // Index for type-based filtering + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_doc_type ON moqui_document (doc_type)") + // Index for time-based ordering + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_doc_upd ON moqui_document (index_name, updated_stamp)") + + // moqui_logs — application log (replaces ES moqui_logs index) + stmt.execute(""" + CREATE TABLE IF NOT EXISTS moqui_logs ( + log_id BIGSERIAL PRIMARY KEY, + log_timestamp TIMESTAMPTZ NOT NULL, + log_level TEXT, + thread_name TEXT, + thread_id BIGINT, + thread_priority INTEGER, + logger_name TEXT, + message TEXT, + source_host TEXT, + user_id TEXT, + visitor_id TEXT, + mdc JSONB, + thrown JSONB + ) + """.trim()) + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_logs_ts ON moqui_logs USING BRIN (log_timestamp)") + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_logs_lvl ON moqui_logs (log_level)") + // Fix log_id if Moqui entity sync created the table without a BIGSERIAL default + stmt.execute(""" + DO \$\$ + BEGIN + IF (SELECT column_default FROM information_schema.columns + WHERE table_name = 'moqui_logs' AND column_name = 'log_id') IS NULL THEN + CREATE SEQUENCE IF NOT EXISTS moqui_logs_log_id_seq; + ALTER TABLE moqui_logs ALTER COLUMN log_id SET DEFAULT nextval('moqui_logs_log_id_seq'); + ALTER SEQUENCE moqui_logs_log_id_seq OWNED BY moqui_logs.log_id; + END IF; + END \$\$; + """.trim()) + + // moqui_http_log — HTTP request log (replaces ES moqui_http_log index) + stmt.execute(""" + CREATE TABLE IF NOT EXISTS moqui_http_log ( + log_id BIGSERIAL PRIMARY KEY, + log_timestamp TIMESTAMPTZ NOT NULL, + remote_ip TEXT, + remote_user TEXT, + server_ip TEXT, + content_type TEXT, + request_method TEXT, + request_scheme TEXT, + request_host TEXT, + request_path TEXT, + request_query TEXT, + http_version TEXT, + response_code INTEGER, + time_initial_ms BIGINT, + time_final_ms BIGINT, + bytes_sent BIGINT, + referrer TEXT, + agent TEXT, + session_id TEXT, + visitor_id TEXT + ) + """.trim()) + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_hlog_ts ON moqui_http_log USING BRIN (log_timestamp)") + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_hlog_path ON moqui_http_log (request_path)") + // Fix log_id if Moqui entity sync created the table without a BIGSERIAL default + stmt.execute(""" + DO \$\$ + BEGIN + IF (SELECT column_default FROM information_schema.columns + WHERE table_name = 'moqui_http_log' AND column_name = 'log_id') IS NULL THEN + CREATE SEQUENCE IF NOT EXISTS moqui_http_log_log_id_seq; + ALTER TABLE moqui_http_log ALTER COLUMN log_id SET DEFAULT nextval('moqui_http_log_log_id_seq'); + ALTER SEQUENCE moqui_http_log_log_id_seq OWNED BY moqui_http_log.log_id; + END IF; + END \$\$; + """.trim()) + + logger.info("PostgresElasticClient schema initialized for cluster '${clusterName}'") + } finally { + stmt.close() + } + ecfi.transactionFacade.commit(started) + } catch (Throwable t) { + ecfi.transactionFacade.rollback(started, "Error initializing PostgresElasticClient schema", t) + throw new BaseException("Error initializing PostgresElasticClient schema for cluster '${clusterName}'", t) + } + } + + /** + * Get a JDBC Connection from the entity facade for the configured datasource group. + * The returned Connection is a Moqui ConnectionWrapper that is transaction-managed. + */ + private Connection getConnection() { + return ecfi.entityFacade.getConnection(datasourceGroup) + } + + // ============================================================ + // ElasticClient — Cluster info + // ============================================================ + + @Override String getClusterName() { return clusterName } + @Override String getClusterLocation() { return "postgres:${datasourceGroup}:${indexPrefix}" } + + @Override + Map getServerInfo() { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("SELECT version()") + try { + ResultSet rs = ps.executeQuery() + try { + if (rs.next()) { + return [name: clusterName, cluster_name: "postgres", + version: [distribution: "postgres", number: rs.getString(1)], + tagline: "Moqui PostgresElasticClient"] + } + } finally { rs.close() } + } finally { ps.close() } + return [name: clusterName, cluster_name: "postgres", version: [distribution: "postgres"]] + } + + // ============================================================ + // Index management + // ============================================================ + + @Override + boolean indexExists(String index) { + if (!index) return false + String prefixed = prefixIndexName(index) + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement( + "SELECT 1 FROM moqui_search_index WHERE index_name = ? OR alias_name = ?") + try { + ps.setString(1, prefixed) + ps.setString(2, prefixed) + ResultSet rs = ps.executeQuery() + try { return rs.next() } finally { rs.close() } + } finally { ps.close() } + } + + @Override + boolean aliasExists(String alias) { + if (!alias) return false + String prefixed = prefixIndexName(alias) + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("SELECT 1 FROM moqui_search_index WHERE alias_name = ?") + try { + ps.setString(1, prefixed) + ResultSet rs = ps.executeQuery() + try { return rs.next() } finally { rs.close() } + } finally { ps.close() } + } + + @Override + void createIndex(String index, Map docMapping, String alias) { + createIndex(index, null, docMapping, alias, null) + } + + void createIndex(String index, String docType, Map docMapping, String alias, Map settings) { + if (!index) throw new IllegalArgumentException("Index name required for createIndex") + String prefixedIndex = prefixIndexName(index) + String prefixedAlias = alias ? prefixIndexName(alias) : null + + String mappingJson = docMapping ? objectToJson(docMapping) : null + String settingsJson = settings ? objectToJson(settings) : null + + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(""" + INSERT INTO moqui_search_index (index_name, alias_name, doc_type, mapping, settings) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (index_name) DO UPDATE SET + alias_name = EXCLUDED.alias_name, + doc_type = EXCLUDED.doc_type, + mapping = EXCLUDED.mapping, + settings = EXCLUDED.settings + """.trim()) + try { + ps.setString(1, prefixedIndex) + if (prefixedAlias) ps.setString(2, prefixedAlias) else ps.setNull(2, Types.VARCHAR) + if (docType) ps.setString(3, docType) else ps.setNull(3, Types.VARCHAR) + if (mappingJson) ps.setString(4, mappingJson) else ps.setNull(4, Types.VARCHAR) + if (settingsJson) ps.setString(5, settingsJson) else ps.setNull(5, Types.VARCHAR) + ps.executeUpdate() + } finally { ps.close() } + logger.info("PostgresElasticClient.createIndex: created index '${prefixedIndex}'${prefixedAlias ? ' with alias ' + prefixedAlias : ''}") + } + + @Override + void putMapping(String index, Map docMapping) { + if (!docMapping) throw new IllegalArgumentException("Mapping may not be empty for putMapping") + String prefixedIndex = prefixIndexName(index) + String mappingJson = objectToJson(docMapping) + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement( + "UPDATE moqui_search_index SET mapping = ? WHERE index_name = ?") + try { + ps.setString(1, mappingJson) + ps.setString(2, prefixedIndex) + ps.executeUpdate() + } finally { ps.close() } + } + + @Override + void deleteIndex(String index) { + if (!index) throw new IllegalArgumentException("Index name required for deleteIndex") + String prefixedIndex = prefixIndexName(index) + Connection conn = getConnection() + PreparedStatement ps1 = conn.prepareStatement("DELETE FROM moqui_document WHERE index_name = ?") + try { + ps1.setString(1, prefixedIndex) + int deleted = ps1.executeUpdate() + logger.info("PostgresElasticClient.deleteIndex: deleted ${deleted} documents from index '${prefixedIndex}'") + } finally { ps1.close() } + PreparedStatement ps2 = conn.prepareStatement("DELETE FROM moqui_search_index WHERE index_name = ?") + try { + ps2.setString(1, prefixedIndex) + ps2.executeUpdate() + } finally { ps2.close() } + } + + // ============================================================ + // Document CRUD + // ============================================================ + + @Override + void index(String index, String _id, Map document) { + if (!index) throw new IllegalArgumentException("Index name required for index()") + if (!_id) throw new IllegalArgumentException("_id required for index()") + String prefixedIndex = prefixIndexName(index) + String docJson = objectToJson(document) + String contentText = extractContentText(document) + upsertDocument(prefixedIndex, _id, null, docJson, contentText) + } + + @Override + void update(String index, String _id, Map documentFragment) { + if (!index) throw new IllegalArgumentException("Index name required for update()") + if (!_id) throw new IllegalArgumentException("_id required for update()") + String prefixedIndex = prefixIndexName(index) + String fragmentJson = objectToJson(documentFragment) + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(""" + UPDATE moqui_document + SET document = COALESCE(document, '{}'::jsonb) || ?::jsonb, + content_text = ( + SELECT string_agg(val::text, ' ') + FROM jsonb_each_text(COALESCE(document, '{}'::jsonb) || ?::jsonb) AS kv(key, val) + WHERE jsonb_typeof(COALESCE(document, '{}'::jsonb) || ?::jsonb -> kv.key) IN ('string', 'number') + ), + content_tsv = to_tsvector('english', coalesce(( + SELECT string_agg(val::text, ' ') + FROM jsonb_each_text(COALESCE(document, '{}'::jsonb) || ?::jsonb) AS kv(key, val) + ), '')), + updated_stamp = now() + WHERE index_name = ? AND doc_id = ? + """.trim()) + try { + ps.setString(1, fragmentJson) + ps.setString(2, fragmentJson) + ps.setString(3, fragmentJson) + ps.setString(4, fragmentJson) + ps.setString(5, prefixedIndex) + ps.setString(6, _id) + ps.executeUpdate() + } finally { ps.close() } + } + + @Override + void delete(String index, String _id) { + if (!index) throw new IllegalArgumentException("Index name required for delete()") + if (!_id) throw new IllegalArgumentException("_id required for delete()") + String prefixedIndex = prefixIndexName(index) + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement( + "DELETE FROM moqui_document WHERE index_name = ? AND doc_id = ?") + try { + ps.setString(1, prefixedIndex) + ps.setString(2, _id) + int deleted = ps.executeUpdate() + if (deleted == 0) logger.warn("delete() document not found in index '${prefixedIndex}' with id '${_id}'") + } finally { ps.close() } + } + + @Override + Integer deleteByQuery(String index, Map queryMap) { + if (!index) throw new IllegalArgumentException("Index name required for deleteByQuery()") + String prefixedIndex = prefixIndexName(index) + ElasticQueryTranslator.QueryResult qr = ElasticQueryTranslator.translateQuery(queryMap ?: [match_all: [:]]) + + List allParams = new ArrayList<>() + allParams.add(prefixedIndex) + if (qr.params) allParams.addAll(qr.params) + String sql = "DELETE FROM moqui_document WHERE index_name = ? AND (${qr.clause})" + + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(sql) + try { + for (int i = 0; i < allParams.size(); i++) { + setParam(ps, i + 1, allParams[i]) + } + return ps.executeUpdate() + } finally { ps.close() } + } + + @Override + void bulk(String index, List actionSourceList) { + if (!actionSourceList) return + String prefixedIndex = index ? prefixIndexName(index) : null + + int i = 0 + while (i < actionSourceList.size()) { + Map action = (Map) actionSourceList.get(i) + + if (action.containsKey("delete")) { + Map actionSpec = (Map) action.get("delete") + String idxName = actionSpec.get("_index") ? prefixIndexName((String) actionSpec.get("_index")) : prefixedIndex + String _id = (String) actionSpec.get("_id") + if (idxName && _id) delete(idxName, _id) + i += 1 + } else if (i + 1 < actionSourceList.size()) { + Map source = (Map) actionSourceList.get(i + 1) + + if (action.containsKey("index") || action.containsKey("create")) { + Map actionSpec = (Map) (action.get("index") ?: action.get("create")) + String idxName = actionSpec.get("_index") ? prefixIndexName((String) actionSpec.get("_index")) : prefixedIndex + String _id = (String) actionSpec.get("_id") + if (idxName) { + String docJson = objectToJson(source) + String contentText = extractContentText(source) + upsertDocument(idxName, _id, null, docJson, contentText) + } + } else if (action.containsKey("update")) { + Map actionSpec = (Map) action.get("update") + String idxName = actionSpec.get("_index") ? prefixIndexName((String) actionSpec.get("_index")) : prefixedIndex + String _id = (String) actionSpec.get("_id") + if (idxName && _id) { + Map doc = (Map) source.get("doc") ?: source + update(idxName, _id, doc) + } + } + i += 2 + } else { + logger.warn("bulk(): action at index ${i} has no following source document, skipping") + i += 1 + } + } + } + + @Override + void bulkIndex(String index, String idField, List documentList) { + bulkIndex(index, null, idField, documentList, false) + } + + @Override + void bulkIndex(String index, String docType, String idField, List documentList, boolean refresh) { + if (!documentList) return + String prefixedIndex = prefixIndexName(index) + boolean hasId = idField != null && !idField.isEmpty() + + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(DOCUMENT_UPSERT_SQL) + try { + int batchSize = 0 + for (Map doc in documentList) { + String _id = hasId ? (doc.get(idField)?.toString() ?: UUID.randomUUID().toString()) : UUID.randomUUID().toString() + String docJson = objectToJson(doc) + String contentText = extractContentText(doc) + setUpsertParams(ps, prefixedIndex, _id, docType, docJson, contentText) + ps.addBatch() + batchSize++ + if (batchSize >= 500) { + ps.executeBatch() + batchSize = 0 + } + } + if (batchSize > 0) ps.executeBatch() + } finally { ps.close() } + } + + @Override + Map get(String index, String _id) { + if (!index || !_id) return null + String prefixedIndex = prefixIndexName(index) + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement( + "SELECT doc_id, index_name, doc_type, document FROM moqui_document WHERE index_name = ? AND doc_id = ?") + try { + ps.setString(1, prefixedIndex) + ps.setString(2, _id) + ResultSet rs = ps.executeQuery() + try { + if (rs.next()) { + Map source = (Map) jsonToObject(rs.getString("document")) + return [_index: unprefixIndexName(rs.getString("index_name")), + _id : rs.getString("doc_id"), + _type : rs.getString("doc_type"), + _source: source] + } + return null + } finally { rs.close() } + } finally { ps.close() } + } + + @Override + Map getSource(String index, String _id) { + Map result = get(index, _id) + return result ? (Map) result.get("_source") : null + } + + @Override + List get(String index, List _idList) { + if (!_idList || !index) return [] + String prefixedIndex = prefixIndexName(index) + String placeholders = _idList.collect { "?" }.join(", ") + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement( + "SELECT doc_id, index_name, doc_type, document FROM moqui_document WHERE index_name = ? AND doc_id IN (${placeholders})") + try { + ps.setString(1, prefixedIndex) + for (int i = 0; i < _idList.size(); i++) ps.setString(i + 2, _idList[i]) + ResultSet rs = ps.executeQuery() + try { + List results = [] + while (rs.next()) { + Map source = (Map) jsonToObject(rs.getString("document")) + results.add([_index: unprefixIndexName(rs.getString("index_name")), + _id : rs.getString("doc_id"), + _type : rs.getString("doc_type"), + _source: source]) + } + return results + } finally { rs.close() } + } finally { ps.close() } + } + + // ============================================================ + // Search + // ============================================================ + + @Override + Map search(String index, Map searchMap) { + if (index && (index == 'moqui_logs' || prefixIndexName(index) == prefixIndexName('moqui_logs'))) { + return searchLogsTable(searchMap ?: [:]) + } + + ElasticQueryTranslator.TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap(searchMap ?: [:]) + + List indexNames = resolveIndexNames(index) + if (indexNames.isEmpty()) { + return [hits: [total: [value: 0, relation: "eq"], hits: []]] + } + + String scoreExpr = tq.tsqueryExpr ? + "ts_rank_cd(content_tsv, ${tq.tsqueryExpr})" : "1.0::float" + + String idxPlaceholders = indexNames.collect { "?" }.join(", ") + String whereClause = "index_name IN (${idxPlaceholders})" + List allParams = new ArrayList<>(indexNames) + + if (tq.tsqueryExpr) { + whereClause += " AND " + tq.whereClause + allParams.addAll(tq.params) + } else if (tq.whereClause && tq.whereClause != "TRUE") { + whereClause += " AND " + tq.whereClause + allParams.addAll(tq.params) + } + + String orderByClause = tq.orderBy ?: (tq.tsqueryExpr ? "_score DESC" : "updated_stamp DESC") + + String countSql = "SELECT COUNT(*) FROM moqui_document WHERE ${whereClause}" + long totalCount = 0L + + String mainSql = """ + SELECT doc_id, index_name, doc_type, document, ${buildScoreSelect(tq)} AS _score + FROM moqui_document + WHERE ${whereClause} + ORDER BY ${orderByClause} + LIMIT ? OFFSET ? + """.trim() + + Connection conn = getConnection() + + if (tq.trackTotal) { + PreparedStatement countPs = conn.prepareStatement(countSql) + try { + for (int i = 0; i < allParams.size(); i++) setParam(countPs, i + 1, allParams[i]) + ResultSet rs = countPs.executeQuery() + try { if (rs.next()) totalCount = rs.getLong(1) } finally { rs.close() } + } finally { countPs.close() } + } + + List mainParams = [] + if (tq.tsqueryExpr) mainParams.addAll(tq.tsqueryParams) + mainParams.addAll(allParams) + mainParams.add(tq.sizeLimit) + mainParams.add(tq.fromOffset) + + PreparedStatement ps = conn.prepareStatement(mainSql) + try { + for (int i = 0; i < mainParams.size(); i++) setParam(ps, i + 1, mainParams[i]) + ResultSet rs = ps.executeQuery() + try { + List hits = [] + while (rs.next()) { + String docJson = rs.getString("document") + Map source = docJson ? (Map) jsonToObject(docJson) : [:] + String docId = rs.getString("doc_id") + String idxName = unprefixIndexName(rs.getString("index_name")) + String docType = rs.getString("doc_type") + double score = rs.getDouble("_score") + + Map hit = [_index: idxName, _id: docId, _type: docType, + _score: score, _source: source] as Map + + if (tq.highlightFields && tq.tsqueryExpr) { + Map> highlights = buildHighlights(source, tq) + if (highlights) hit.put("highlight", highlights) + } + + hits.add(hit) + } + + return [hits: [total: [value: totalCount, relation: "eq"], hits: hits], + _shards: [total: 1, successful: 1, failed: 0]] + } finally { rs.close() } + } finally { ps.close() } + } + + private Map searchLogsTable(Map searchMap) { + ElasticQueryTranslator.TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap(searchMap) + + String rawQuery = null + Map queryMap = (Map) searchMap?.get("query") + if (queryMap) { + Map qsMap = (Map) queryMap.get("query_string") + if (qsMap) rawQuery = (String) qsMap.get("query") + } + + List conditions = [] + List params = [] + + if (rawQuery) { + java.util.regex.Matcher m = (rawQuery =~ /@timestamp\s*:\s*\[\s*(\*|\d+)\s+TO\s+(\*|\d+)\s*\]/) + if (m.find()) { + String fromVal = m.group(1) + String toVal = m.group(2) + if (fromVal != '*') { + conditions.add("log_timestamp >= ?") + params.add(new java.sql.Timestamp(Long.parseLong(fromVal))) + } + if (toVal != '*') { + conditions.add("log_timestamp <= ?") + params.add(new java.sql.Timestamp(Long.parseLong(toVal))) + } + } + } + + String userTextQuery = null + if (rawQuery) { + String stripped = rawQuery.replaceAll(/@timestamp\s*:\s*\[[^\]]*\]/, '') + stripped = stripped.replaceAll(/\bAND\b/, ' ').replaceAll(/\bOR\b/, ' ') + stripped = stripped.replaceAll(/[()]/, ' ').replaceAll(/\s+/, ' ').trim() + stripped = stripped.replaceAll(/\*/, '').trim() + if (stripped) userTextQuery = stripped + } + if (userTextQuery) { + conditions.add("to_tsvector('english', coalesce(message, '') || ' ' || coalesce(logger_name, '')) @@ websearch_to_tsquery('english', ?)") + params.add(userTextQuery) + } + + String whereClause = conditions ? conditions.join(" AND ") : "TRUE" + + Connection conn = getConnection() + long totalCount = 0L + if (tq.trackTotal) { + PreparedStatement countPs = conn.prepareStatement("SELECT COUNT(*) FROM moqui_logs WHERE ${whereClause}") + try { + for (int i = 0; i < params.size(); i++) setParam(countPs, i + 1, params[i]) + ResultSet rs = countPs.executeQuery() + try { if (rs.next()) totalCount = rs.getLong(1) } finally { rs.close() } + } finally { countPs.close() } + } + + String mainSql = """ + SELECT log_id, log_timestamp, log_level, thread_name, thread_id, thread_priority, + logger_name, message, source_host, user_id, visitor_id, mdc::text, thrown::text + FROM moqui_logs + WHERE ${whereClause} + ORDER BY log_timestamp DESC + LIMIT ? OFFSET ? + """.trim() + + PreparedStatement ps = conn.prepareStatement(mainSql) + try { + int pIdx = 0 + for (int i = 0; i < params.size(); i++) setParam(ps, ++pIdx, params[i]) + ps.setInt(++pIdx, tq.sizeLimit) + ps.setInt(++pIdx, tq.fromOffset) + ResultSet rs = ps.executeQuery() + try { + List hits = [] + while (rs.next()) { + long logId = rs.getLong("log_id") + java.sql.Timestamp ts = rs.getTimestamp("log_timestamp") + Map source = [ + "@timestamp" : ts?.time, + level : rs.getString("log_level"), + thread_name : rs.getString("thread_name"), + thread_id : rs.getLong("thread_id"), + thread_priority : rs.getInt("thread_priority"), + logger_name : rs.getString("logger_name"), + message : rs.getString("message"), + source_host : rs.getString("source_host"), + user_id : rs.getString("user_id"), + visitor_id : rs.getString("visitor_id"), + ] as Map + String mdcStr = rs.getString("mdc") + if (mdcStr) source.put("mdc", jsonToObject(mdcStr)) + String thrownStr = rs.getString("thrown") + if (thrownStr) source.put("thrown", jsonToObject(thrownStr)) + hits.add([_index: "moqui_logs", _id: String.valueOf(logId), + _type: "LogMessage", _score: 1.0, _source: source] as Map) + } + return [hits: [total: [value: totalCount, relation: "eq"], hits: hits], + _shards: [total: 1, successful: 1, failed: 0]] + } finally { rs.close() } + } catch (Throwable t) { + logger.error("searchLogsTable error: " + t.message, t) + return [hits: [total: [value: 0, relation: "eq"], hits: []]] + } finally { ps.close() } + } + + @Override + List searchHits(String index, Map searchMap) { + Map result = search(index, searchMap) + return (List) ((Map) result.get("hits")).get("hits") + } + + @Override + Map validateQuery(String index, Map queryMap, boolean explain) { + try { + ElasticQueryTranslator.QueryResult qr = ElasticQueryTranslator.translateQuery(queryMap ?: [match_all: [:]]) + return null // valid + } catch (Throwable t) { + return [valid: false, error: t.message] + } + } + + @Override + long count(String index, Map countMap) { + Map result = countResponse(index, countMap) + return ((Number) result.get("count"))?.longValue() ?: 0L + } + + @Override + Map countResponse(String index, Map countMap) { + if (!countMap) countMap = [query: [match_all: [:]]] + Map queryMap = (Map) countMap.get("query") + ElasticQueryTranslator.QueryResult qr = queryMap ? ElasticQueryTranslator.translateQuery(queryMap) : new ElasticQueryTranslator.QueryResult() + + List indexNames = resolveIndexNames(index) + if (indexNames.isEmpty()) return [count: 0L] + + String idxPlaceholders = indexNames.collect { "?" }.join(", ") + String whereClause = "index_name IN (${idxPlaceholders})" + List allParams = new ArrayList<>(indexNames) + + if (qr.clause && qr.clause != "TRUE") { + whereClause += " AND " + qr.clause + allParams.addAll(qr.params) + } + + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("SELECT COUNT(*) FROM moqui_document WHERE ${whereClause}") + try { + for (int i = 0; i < allParams.size(); i++) setParam(ps, i + 1, allParams[i]) + ResultSet rs = ps.executeQuery() + try { + if (rs.next()) return [count: rs.getLong(1)] + return [count: 0L] + } finally { rs.close() } + } finally { ps.close() } + } + + // ============================================================ + // Point-In-Time (PIT) — Keyset-based cursor + // ============================================================ + + @Override + String getPitId(String index, String keepAlive) { + return "pg::${indexPrefix}::${System.currentTimeMillis()}" + } + + @Override + void deletePit(String pitId) { + // No-op for postgres backend + } + + // ============================================================ + // Raw REST — Not supported on postgres backend + // ============================================================ + + @Override + RestClient.RestResponse call(Method method, String index, String path, + Map parameters, Object bodyJsonObject) { + throw new UnsupportedOperationException( + "Raw REST calls (call()) are not supported by PostgresElasticClient for cluster '${clusterName}'. " + + "Use the higher-level API methods instead, or switch to type=elastic for this cluster.") + } + + @Override + Future callFuture(Method method, String index, String path, + Map parameters, Object bodyJsonObject) { + throw new UnsupportedOperationException( + "Raw REST calls (callFuture()) are not supported by PostgresElasticClient for cluster '${clusterName}'.") + } + + @Override + RestClient makeRestClient(Method method, String index, String path, Map parameters) { + throw new UnsupportedOperationException( + "makeRestClient() is not supported by PostgresElasticClient for cluster '${clusterName}'.") + } + + // ============================================================ + // DataDocument helpers + // ============================================================ + + @Override + void checkCreateDataDocumentIndexes(String indexName) { + if (!indexName) return + if (indexExists(indexName)) return + EntityList ddList = ecfi.entityFacade.find("moqui.entity.document.DataDocument") + .condition("indexName", indexName).disableAuthz().list() + for (EntityValue dd in ddList) { + storeIndexAndMapping(indexName, dd) + } + } + + @Override + void checkCreateDataDocumentIndex(String dataDocumentId) { + String idxName = ElasticFacadeImpl.ddIdToEsIndex(dataDocumentId) + String prefixed = prefixIndexName(idxName) + if (indexExists(prefixed)) return + + EntityValue dd = ecfi.entityFacade.find("moqui.entity.document.DataDocument") + .condition("dataDocumentId", dataDocumentId).disableAuthz().one() + if (dd == null) throw new BaseException("No DataDocument found with ID [${dataDocumentId}]") + storeIndexAndMapping((String) dd.getNoCheckSimple("indexName"), dd) + } + + @Override + void putDataDocumentMappings(String indexName) { + EntityList ddList = ecfi.entityFacade.find("moqui.entity.document.DataDocument") + .condition("indexName", indexName).disableAuthz().list() + for (EntityValue dd in ddList) storeIndexAndMapping(indexName, dd) + } + + @Override + void verifyDataDocumentIndexes(List documentList) { + Set indexNames = new HashSet<>() + Set dataDocumentIds = new HashSet<>() + for (Map doc in documentList) { + Object idxObj = doc.get("_index") + Object typeObj = doc.get("_type") + if (idxObj) indexNames.add((String) idxObj) + if (typeObj) dataDocumentIds.add((String) typeObj) + } + for (String idxName in indexNames) checkCreateDataDocumentIndexes(idxName) + for (String ddId in dataDocumentIds) checkCreateDataDocumentIndex(ddId) + } + + @Override + void bulkIndexDataDocument(List documentList) { + if (!documentList) return + + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(DOCUMENT_UPSERT_SQL) + try { + int batchCount = 0 + for (Map document in documentList) { + String _index = (String) document.get("_index") + String _type = (String) document.get("_type") + String _id = (String) document.get("_id") + + if (!_id) { + logger.warn("bulkIndexDataDocument: skipping document with null _id (type=${_type})") + continue + } + + String esIndexName = ElasticFacadeImpl.ddIdToEsIndex(_type ?: "unknown") + String prefixedIndex = prefixIndexName(esIndexName) + + Map cleanDoc = new LinkedHashMap<>(document) + for (String key in DOC_META_KEYS) cleanDoc.remove(key) + + String docJson = objectToJson(cleanDoc) + String contentText = extractContentText(cleanDoc) + + setUpsertParams(ps, prefixedIndex, _id, _type, docJson, contentText) + ps.addBatch() + batchCount++ + + if (batchCount >= 500) { + ps.executeBatch() + batchCount = 0 + } + } + if (batchCount > 0) ps.executeBatch() + logger.info("bulkIndexDataDocument: indexed ${documentList.size()} documents") + } finally { ps.close() } + } + + // ============================================================ + // JSON serialization + // ============================================================ + + @Override String objectToJson(Object obj) { return ElasticFacadeImpl.objectToJson(obj) } + @Override Object jsonToObject(String json) { return ElasticFacadeImpl.jsonToObject(json) } + + // ============================================================ + // Index prefixing helpers + // ============================================================ + + String prefixIndexName(String index) { + if (!index) return index + index = index.trim() + if (!index) return index + return index.split(",").collect { String it -> + it = it.trim() + return (indexPrefix && !it.startsWith(indexPrefix)) ? indexPrefix + it : it + }.join(",") + } + + String unprefixIndexName(String index) { + if (!index || !indexPrefix) return index + index = index.trim() + return index.split(",").collect { String it -> + it = it.trim() + return (indexPrefix && it.startsWith(indexPrefix)) ? it.substring(indexPrefix.length()) : it + }.join(",") + } + + // ============================================================ + // Private helpers + // ============================================================ + + /** Set the 6 parameters on a PreparedStatement using the shared DOCUMENT_UPSERT_SQL */ + private static void setUpsertParams(PreparedStatement ps, String prefixedIndex, String docId, String docType, String docJson, String contentText) { + ps.setString(1, prefixedIndex) + ps.setString(2, docId ?: UUID.randomUUID().toString()) + if (docType) ps.setString(3, docType) else ps.setNull(3, Types.VARCHAR) + ps.setString(4, docJson) + ps.setString(5, contentText) + ps.setString(6, contentText) + } + + private void upsertDocument(String prefixedIndex, String docId, String docType, String docJson, String contentText) { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(DOCUMENT_UPSERT_SQL) + try { + setUpsertParams(ps, prefixedIndex, docId, docType, docJson, contentText) + ps.executeUpdate() + } finally { ps.close() } + } + + static String extractContentText(Map document) { + if (document == null || document.isEmpty()) return "" + StringBuilder sb = new StringBuilder() + extractTextFromValue(document, sb) + return sb.toString().trim() + } + + private static void extractTextFromValue(Object value, StringBuilder sb) { + if (value instanceof Map) { + for (Map.Entry entry in ((Map) value).entrySet()) { + Object k = entry.key + Object v = entry.value + if (k instanceof String) { + String key = (String) k + if (!key.endsWith("Id") || key.length() < 20) { + extractTextFromValue(v, sb) + } + } else { + extractTextFromValue(v, sb) + } + } + } else if (value instanceof List) { + for (Object item in (List) value) extractTextFromValue(item, sb) + } else if (value instanceof String) { + String s = (String) value + if (s.length() > 0) { + if (sb.length() > 0) sb.append(' ') + sb.append(s) + } + } else if (value instanceof Number || value instanceof Boolean) { + if (sb.length() > 0) sb.append(' ') + sb.append(value.toString()) + } + } + + protected synchronized void storeIndexAndMapping(String indexName, EntityValue dd) { + String dataDocumentId = (String) dd.getNoCheckSimple("dataDocumentId") + String manualMappingServiceName = (String) dd.getNoCheckSimple("manualMappingServiceName") + String esIndexName = ElasticFacadeImpl.ddIdToEsIndex(dataDocumentId) + String prefixedIndex = prefixIndexName(esIndexName) + + boolean hasIndex = indexExists(prefixedIndex) + Map docMapping = ElasticFacadeImpl.makeElasticSearchMapping(dataDocumentId, ecfi) + Map settings = null + + if (manualMappingServiceName) { + Map serviceResult = ecfi.service.sync().name(manualMappingServiceName) + .parameter("mapping", docMapping).call() + docMapping = (Map) serviceResult.get("mapping") + settings = (Map) serviceResult.get("settings") + } + + if (hasIndex) { + logger.info("PostgresElasticClient: updating mapping for index '${prefixedIndex}' (${dataDocumentId})") + putMapping(prefixedIndex, docMapping) + } else { + logger.info("PostgresElasticClient: creating index '${prefixedIndex}' for DataDocument '${dataDocumentId}' with alias '${indexName}'") + createIndex(prefixedIndex, dataDocumentId, docMapping, indexName, settings) + } + } + + private List resolveIndexNames(String index) { + if (!index) { + return getAllIndexNames() + } + List result = [] + for (String part in index.split(",")) { + String trimmed = part.trim() + if (!trimmed) continue + String prefixed = prefixIndexName(trimmed) + List aliasResolved = resolveAlias(prefixed) + if (aliasResolved) { + result.addAll(aliasResolved) + } else { + result.add(prefixed) + } + } + return result + } + + private List resolveAlias(String alias) { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement( + "SELECT index_name FROM moqui_search_index WHERE alias_name = ?") + try { + ps.setString(1, alias) + ResultSet rs = ps.executeQuery() + try { + List names = [] + while (rs.next()) names.add(rs.getString("index_name")) + return names + } finally { rs.close() } + } finally { ps.close() } + } + + private List getAllIndexNames() { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("SELECT index_name FROM moqui_search_index") + try { + ResultSet rs = ps.executeQuery() + try { + List names = [] + while (rs.next()) names.add(rs.getString("index_name")) + return names + } finally { rs.close() } + } finally { ps.close() } + } + + private static String buildScoreSelect(ElasticQueryTranslator.TranslatedQuery tq) { + if (tq.tsqueryExpr) { + return "ts_rank_cd(content_tsv, ${tq.tsqueryExpr})" + } + return "1.0::float" + } + + private static Map> buildHighlights(Map source, ElasticQueryTranslator.TranslatedQuery tq) { + Map> highlights = [:] + if (!tq.tsqueryExpr || !tq.highlightFields) return highlights + String firstParam = tq.params ? tq.params[0]?.toString() : null + if (!firstParam) return highlights + for (String field in tq.highlightFields.keySet()) { + Object fieldVal = source.get(field) + if (fieldVal instanceof String) { + String text = (String) fieldVal + String highlighted = simpleHighlight(text, firstParam) + if (highlighted != text) highlights.put(field, [highlighted]) + } + } + return highlights + } + + private static String simpleHighlight(String text, String query) { + if (!text || !query) return text + List terms = query.replaceAll(/["()+\-]/, ' ').split(/\s+/).findAll { it.length() > 2 } as List + String result = text + for (String term in terms) { + result = result.replaceAll("(?i)\\b${java.util.regex.Pattern.quote(term)}\\b", "\$0") + } + return result + } + + private static void setParam(PreparedStatement ps, int idx, Object value) { + if (value == null) { + ps.setNull(idx, Types.VARCHAR) + } else if (value instanceof String) { + ps.setString(idx, (String) value) + } else if (value instanceof Long || value instanceof Integer) { + ps.setLong(idx, ((Number) value).longValue()) + } else if (value instanceof Double || value instanceof Float || value instanceof BigDecimal) { + ps.setDouble(idx, ((Number) value).doubleValue()) + } else if (value instanceof Timestamp) { + ps.setTimestamp(idx, (Timestamp) value) + } else { + ps.setString(idx, value.toString()) + } + } +} diff --git a/framework/src/main/groovy/org/moqui/impl/util/PostgresSearchLogger.groovy b/framework/src/main/groovy/org/moqui/impl/util/PostgresSearchLogger.groovy new file mode 100644 index 000000000..d300442e9 --- /dev/null +++ b/framework/src/main/groovy/org/moqui/impl/util/PostgresSearchLogger.groovy @@ -0,0 +1,244 @@ +/* + * This software is in the public domain under CC0 1.0 Universal plus a + * Grant of Patent License. + * + * To the extent possible under law, the author(s) have dedicated all + * copyright and related and neighboring rights to this software to the + * public domain worldwide. This software is distributed without any + * warranty. + * + * You should have received a copy of the CC0 Public Domain Dedication + * along with this software (see the LICENSE.md file). If not, see + * . + */ +package org.moqui.impl.util + +import groovy.transform.CompileStatic +import org.apache.logging.log4j.Level +import org.apache.logging.log4j.core.LogEvent +import org.apache.logging.log4j.util.ReadOnlyStringMap +import org.moqui.BaseArtifactException +import org.moqui.context.ArtifactExecutionInfo +import org.moqui.context.LogEventSubscriber +import org.moqui.impl.context.ExecutionContextFactoryImpl +import org.moqui.impl.context.PostgresElasticClient +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.sql.Connection +import java.sql.PreparedStatement +import java.sql.Timestamp +import java.sql.Types +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.atomic.AtomicBoolean + +/** + * PostgreSQL-backed application log appender (replaces ElasticSearchLogger for postgres clusters). + * + * Consumes LogEvent objects from a queue and batch-inserts them into the moqui_logs table. + * The queue is flushed every 3 seconds by a scheduled task, identical to ElasticSearchLogger behaviour. + */ +@CompileStatic +class PostgresSearchLogger { + private final static Logger logger = LoggerFactory.getLogger(PostgresSearchLogger.class) + + final static int QUEUE_LIMIT = 16384 + + private final PostgresElasticClient pgClient + private final ExecutionContextFactoryImpl ecfi + + private boolean initialized = false + private volatile boolean disabled = false + + final ConcurrentLinkedQueue logMessageQueue = new ConcurrentLinkedQueue<>() + final AtomicBoolean flushRunning = new AtomicBoolean(false) + + protected PgLogSubscriber subscriber = null + + PostgresSearchLogger(PostgresElasticClient pgClient, ExecutionContextFactoryImpl ecfi) { + this.pgClient = pgClient + this.ecfi = ecfi + init() + } + + void init() { + // moqui_logs table is created by PostgresElasticClient.initSchema() — no extra setup needed + + // Schedule flush every 3 seconds (same cadence as ElasticSearchLogger) + PgLogQueueFlush flushTask = new PgLogQueueFlush(this) + ecfi.scheduleAtFixedRate(flushTask, 10, 3) + + subscriber = new PgLogSubscriber(this) + ecfi.registerLogEventSubscriber(subscriber) + + initialized = true + logger.info("PostgresSearchLogger initialized for cluster '${pgClient.clusterName}'") + } + + void destroy() { disabled = true } + + boolean isInitialized() { return initialized } + + // ============================================================ + // Log subscriber — mirrors ElasticSearchSubscriber + // ============================================================ + + static class PgLogSubscriber implements LogEventSubscriber { + private final PostgresSearchLogger pgLogger + private final InetAddress localAddr = InetAddress.getLocalHost() + + PgLogSubscriber(PostgresSearchLogger pgLogger) { this.pgLogger = pgLogger } + + @Override + void process(LogEvent event) { + if (pgLogger.disabled) return + // Suppress DEBUG / TRACE (same rule as ElasticSearchLogger) + if (Level.DEBUG.is(event.level) || Level.TRACE.is(event.level)) return + // Back-pressure: if queue too full, drop the oldest-style (newest is not enqueued) + if (pgLogger.logMessageQueue.size() >= QUEUE_LIMIT) return + + Map msgMap = [ + '@timestamp' : event.timeMillis, + level : event.level.toString(), + thread_name : event.threadName, + thread_id : event.threadId, + thread_priority: event.threadPriority, + logger_name : event.loggerName, + message : event.message?.formattedMessage, + source_host : localAddr.hostName + ] as Map + + ReadOnlyStringMap contextData = event.contextData + if (contextData != null && contextData.size() > 0) { + Map mdcMap = new HashMap<>(contextData.toMap()) + String userId = mdcMap.remove("moqui_userId") + String visitorId = mdcMap.remove("moqui_visitorId") + if (userId) msgMap.put("user_id", userId) + if (visitorId) msgMap.put("visitor_id", visitorId) + if (mdcMap.size() > 0) msgMap.put("mdc", mdcMap) + } + Throwable thrown = event.thrown + if (thrown != null) msgMap.put("thrown", ElasticSearchLogger.ElasticSearchSubscriber.makeThrowableMap(thrown)) + + pgLogger.logMessageQueue.add(msgMap) + } + } + + // ============================================================ + // Scheduled flush task — drains queue into moqui_logs via JDBC + // ============================================================ + + static class PgLogQueueFlush implements Runnable { + private final static int MAX_BATCH = 200 + + private final PostgresSearchLogger pgLogger + + PgLogQueueFlush(PostgresSearchLogger pgLogger) { this.pgLogger = pgLogger } + + @Override + void run() { + if (!pgLogger.flushRunning.compareAndSet(false, true)) return + try { + while (pgLogger.logMessageQueue.size() > 0) { flushQueue() } + } finally { + pgLogger.flushRunning.set(false) + } + } + + void flushQueue() { + final ConcurrentLinkedQueue queue = pgLogger.logMessageQueue + List batch = new ArrayList<>(MAX_BATCH) + long lastTs = 0L + int sameCount = 0 + + while (batch.size() < MAX_BATCH) { + Map msg = queue.poll() + if (msg == null) break + // Ensure unique timestamps (same as ES logger behaviour) + long ts = (msg.get("@timestamp") as long) ?: System.currentTimeMillis() + if (ts == lastTs) { + sameCount++ + ts += sameCount + msg.put("@timestamp", ts) + } else { + lastTs = ts + sameCount = 0 + } + batch.add(msg) + } + + if (batch.isEmpty()) return + + int retries = 3 + while (retries-- > 0) { + try { + writeBatch(batch) + return + } catch (Throwable t) { + System.out.println("PostgresSearchLogger: error writing log batch, retries left ${retries}: ${t}") + if (retries == 0) System.out.println("PostgresSearchLogger: dropping ${batch.size()} log records after repeated failures") + } + } + } + + private void writeBatch(List batch) { + boolean txStarted = pgLogger.ecfi.transactionFacade.begin(null) + try { + Connection conn = pgLogger.ecfi.entityFacade.getConnection(pgLogger.pgClient.datasourceGroup) + PreparedStatement ps = conn.prepareStatement(""" + INSERT INTO moqui_logs (log_timestamp, log_level, thread_name, thread_id, thread_priority, + logger_name, message, source_host, user_id, visitor_id, mdc, thrown) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?::jsonb, ?::jsonb) + """.trim()) + try { + for (Map msg in batch) { + // Timestamp: stored as epoch_millis long in the map + long tsMillis = (msg.get("@timestamp") as long) ?: System.currentTimeMillis() + ps.setTimestamp(1, new Timestamp(tsMillis)) + setStr(ps, 2, msg.get("level") as String) + setStr(ps, 3, msg.get("thread_name") as String) + setLong(ps, 4, msg.get("thread_id") as Long) + setInt(ps, 5, msg.get("thread_priority") as Integer) + setStr(ps, 6, msg.get("logger_name") as String) + setStr(ps, 7, msg.get("message") as String) + setStr(ps, 8, msg.get("source_host") as String) + setStr(ps, 9, msg.get("user_id") as String) + setStr(ps, 10, msg.get("visitor_id") as String) + // mdc and thrown as JSONB + Object mdcObj = msg.get("mdc") + setJsonb(ps, 11, mdcObj) + Object thrownObj = msg.get("thrown") + setJsonb(ps, 12, thrownObj) + ps.addBatch() + } + ps.executeBatch() + } finally { ps.close() } + pgLogger.ecfi.transactionFacade.commit(txStarted) + } catch (Throwable t) { + pgLogger.ecfi.transactionFacade.rollback(txStarted, "Error writing log batch to moqui_logs", t) + throw t + } + } + + private static void setStr(PreparedStatement ps, int i, String v) { + if (v == null) ps.setNull(i, Types.VARCHAR) else ps.setString(i, v) + } + private static void setLong(PreparedStatement ps, int i, Long v) { + if (v == null) ps.setNull(i, Types.BIGINT) else ps.setLong(i, v) + } + private static void setInt(PreparedStatement ps, int i, Integer v) { + if (v == null) ps.setNull(i, Types.INTEGER) else ps.setInt(i, v) + } + private static void setJsonb(PreparedStatement ps, int i, Object v) { + if (v == null) { + ps.setNull(i, Types.OTHER) + } else { + try { + ps.setString(i, PostgresElasticClient.jacksonMapper.writeValueAsString(v)) + } catch (Throwable t) { + ps.setNull(i, Types.OTHER) + } + } + } + } +} diff --git a/framework/src/main/groovy/org/moqui/impl/webapp/ElasticRequestLogFilter.groovy b/framework/src/main/groovy/org/moqui/impl/webapp/ElasticRequestLogFilter.groovy index 149b66a79..395c606fb 100644 --- a/framework/src/main/groovy/org/moqui/impl/webapp/ElasticRequestLogFilter.groovy +++ b/framework/src/main/groovy/org/moqui/impl/webapp/ElasticRequestLogFilter.groovy @@ -15,7 +15,7 @@ package org.moqui.impl.webapp import groovy.transform.CompileStatic import org.moqui.Moqui -import org.moqui.impl.context.ElasticFacadeImpl.ElasticClientImpl +import org.moqui.context.ElasticFacade import org.moqui.impl.context.ExecutionContextFactoryImpl import org.moqui.impl.context.UserFacadeImpl import org.slf4j.Logger @@ -38,7 +38,7 @@ class ElasticRequestLogFilter implements Filter { protected FilterConfig filterConfig = null protected ExecutionContextFactoryImpl ecfi = null - private ElasticClientImpl elasticClient = null + private ElasticFacade.ElasticClient elasticClient = null private boolean disabled = false final ConcurrentLinkedQueue requestLogQueue = new ConcurrentLinkedQueue<>() @@ -51,15 +51,11 @@ class ElasticRequestLogFilter implements Filter { ecfi = (ExecutionContextFactoryImpl) filterConfig.servletContext.getAttribute("executionContextFactory") if (ecfi == null) ecfi = (ExecutionContextFactoryImpl) Moqui.executionContextFactory - elasticClient = (ElasticClientImpl) (ecfi.elasticFacade.getClient("logger") ?: ecfi.elasticFacade.getDefault()) + elasticClient = (ecfi.elasticFacade.getClient("logger") ?: ecfi.elasticFacade.getDefault()) if (elasticClient == null) { logger.error("In ElasticRequestLogFilter init could not find ElasticClient with name logger or default, not starting") return } - if (elasticClient.esVersionUnder7) { - logger.warn("ElasticClient ${elasticClient.clusterName} has version under 7.0, not starting ElasticRequestLogFilter") - return - } // check for index exists, create with mapping for log doc if not try { diff --git a/framework/src/main/resources/MoquiDefaultConf.xml b/framework/src/main/resources/MoquiDefaultConf.xml index d8dc14c59..eb78c0c52 100644 --- a/framework/src/main/resources/MoquiDefaultConf.xml +++ b/framework/src/main/resources/MoquiDefaultConf.xml @@ -420,8 +420,13 @@ + + - + @@ -523,6 +531,7 @@ + diff --git a/framework/src/test/groovy/PostgresElasticClientTests.groovy b/framework/src/test/groovy/PostgresElasticClientTests.groovy new file mode 100644 index 000000000..bad902fab --- /dev/null +++ b/framework/src/test/groovy/PostgresElasticClientTests.groovy @@ -0,0 +1,701 @@ +/* + * This software is in the public domain under CC0 1.0 Universal plus a + * Grant of Patent License. + * + * Integration tests for PostgresElasticClient and PostgresSearchLogger. + * + * Requires a running PostgreSQL database configured in Moqui (transactional datasource). + * Bootstraps a Moqui EC directly — no web server needed. + * + * Run with: ./gradlew :framework:test --tests PostgresElasticClientTests + */ + +import org.moqui.Moqui +import org.moqui.context.ExecutionContext +import org.moqui.context.ElasticFacade +import org.moqui.impl.context.PostgresElasticClient +import org.moqui.impl.context.ExecutionContextFactoryImpl +import org.moqui.util.MNode +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Assertions + +/** + * Integration tests for the PostgreSQL-backed ElasticClient. + * + * Spins up a real PostgresElasticClient against the configured datasource and exercises + * all major operations: schema init, createIndex, index, get, search, update, delete, + * bulk operations, and query translation. + */ +class PostgresElasticClientTests { + + static ExecutionContext ec + static PostgresElasticClient pgClient + static final String TEST_INDEX = "pg_test_documents" + static final String TEST_PREFIX = "_test_pg_" + + @BeforeAll + static void startMoqui() { + // Init Moqui pointing at the test database + ec = Moqui.getExecutionContext() + ExecutionContextFactoryImpl ecfi = (ExecutionContextFactoryImpl) ec.factory + + // Create a test PostgresElasticClient directly via a manually-built MNode + MNode clusterNode = new MNode("cluster", [name: "test-pg", type: "postgres", + url: "transactional", "index-prefix": TEST_PREFIX]) + pgClient = new PostgresElasticClient(clusterNode, ecfi) + } + + @AfterAll + static void stopMoqui() { + // Clean up test data + try { + if (pgClient != null) { + [ + "pg_test_documents_create", "pg_test_documents_with_alias", + "pg_test_documents_delete_test", "pg_test_documents_put_mapping", + "pg_test_documents_crud", "pg_test_documents_get_source", + "pg_test_documents_multi_get", "pg_test_documents_get_null", + "pg_test_documents_update", "pg_test_documents_doc_delete", + "pg_test_documents_bulkindex", "pg_test_documents_bulk_actions", + "pg_test_documents_search_all", "pg_test_documents_search_term", + "pg_test_documents_search_terms", "pg_test_documents_search_fts", + "pg_test_documents_search_bool", "pg_test_documents_search_page", + "pg_test_documents_searchhits", "pg_test_documents_count", + "pg_test_documents_countresp", "pg_test_documents_dbq", + "test_data_doc" + ].each { idx -> + try { pgClient.deleteIndex(idx) } catch (Throwable ignored) {} + } + } + } catch (Throwable ignored) {} + try { if (ec != null) ec.destroy() } catch (Throwable ignored) {} + } + + @BeforeEach + void beginTx() { + ec.transaction.begin(null) + ec.artifactExecution.disableAuthz() + } + + @AfterEach + void commitTx() { + ec.artifactExecution.enableAuthz() + if (ec.transaction.isTransactionInPlace()) ec.transaction.commit() + } + + // ============================ + // clusterName / location + // ============================ + + @Test + @DisplayName("clusterName returns configured name") + void clusterName_returnsConfiguredName() { + Assertions.assertEquals("test-pg", pgClient.clusterName) + } + + @Test + @DisplayName("clusterLocation contains postgres keyword") + void clusterLocation_containsPostgresKeyword() { + Assertions.assertTrue(pgClient.clusterLocation.contains("postgres")) + } + + @Test + @DisplayName("getServerInfo returns postgres version map") + void getServerInfo_returnsPostgresVersionMap() { + Map info = pgClient.getServerInfo() + Assertions.assertNotNull(info) + Assertions.assertEquals("postgres", info.get("cluster_name")) + Object version = info.get("version") + Assertions.assertNotNull(version) + } + + // ============================ + // Index management + // ============================ + + @Test + @DisplayName("createIndex and indexExists") + void createIndex_andIndexExists() { + String idx = TEST_INDEX + "_create" + try { + pgClient.createIndex(idx, [properties: [name: [type: "text"]]], null) + Assertions.assertTrue(pgClient.indexExists(idx), "index should exist after createIndex") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("indexExists returns false for non-existent index") + void indexExists_returnsFalseForNonExistent() { + Assertions.assertFalse(pgClient.indexExists("pg_test_does_not_exist_xyz")) + } + + @Test + @DisplayName("createIndex with alias and aliasExists") + void createIndex_withAlias_aliasExists() { + String idx = TEST_INDEX + "_with_alias" + String alias = idx + "_alias" + try { + pgClient.createIndex(idx, null, alias) + Assertions.assertTrue(pgClient.indexExists(idx), "index should exist") + Assertions.assertTrue(pgClient.aliasExists(alias), "alias should exist") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("deleteIndex removes documents and metadata") + void deleteIndex_removesDocumentsAndMetadata() { + String idx = TEST_INDEX + "_delete_test" + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "doc001", [name: "To Be Deleted"]) + pgClient.deleteIndex(idx) + Assertions.assertFalse(pgClient.indexExists(idx), "index should not exist after delete") + Assertions.assertNull(pgClient.get(idx, "doc001"), "document should not exist after index delete") + } + + @Test + @DisplayName("putMapping updates mapping on existing index") + void putMapping_updatesMappingOnExistingIndex() { + String idx = TEST_INDEX + "_put_mapping" + try { + pgClient.createIndex(idx, [properties: [name: [type: "text"]]], null) + // putMapping should succeed without throwing + pgClient.putMapping(idx, [properties: [name: [type: "text"], email: [type: "keyword"]]]) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + // ============================ + // Document CRUD + // ============================ + + @Test + @DisplayName("index and get roundtrip") + void index_andGetRoundtrip() { + String idx = TEST_INDEX + "_crud" + try { + pgClient.createIndex(idx, null, null) + Map doc = [name: "Alice", email: "alice@example.com", age: 30] + pgClient.index(idx, "alice001", doc) + Map retrieved = pgClient.get(idx, "alice001") + Assertions.assertNotNull(retrieved, "document should be retrievable") + Map source = (Map) retrieved.get("_source") + Assertions.assertNotNull(source) + Assertions.assertEquals("Alice", source.get("name")) + Assertions.assertEquals("alice@example.com", source.get("email")) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("getSource returns only source map") + void getSource_returnsOnlySourceMap() { + String idx = TEST_INDEX + "_get_source" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "bob001", [name: "Bob", role: "admin"]) + Map source = pgClient.getSource(idx, "bob001") + Assertions.assertNotNull(source) + Assertions.assertEquals("Bob", source.get("name")) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("get with _idList returns multiple docs") + void get_withIdList_returnsMultipleDocs() { + String idx = TEST_INDEX + "_multi_get" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "m1", [name: "Multi1"]) + pgClient.index(idx, "m2", [name: "Multi2"]) + pgClient.index(idx, "m3", [name: "Multi3"]) + List docs = pgClient.get(idx, ["m1", "m3"]) + Assertions.assertEquals(2, docs.size()) + Set ids = docs.collect { (String) it.get("_id") }.toSet() + Assertions.assertTrue(ids.contains("m1")) + Assertions.assertTrue(ids.contains("m3")) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("get returns null for non-existent document") + void get_returnsNullForNonExistent() { + String idx = TEST_INDEX + "_get_null" + try { + pgClient.createIndex(idx, null, null) + Map result = pgClient.get(idx, "doesnotexist") + Assertions.assertNull(result) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("update merges fields") + void update_mergesFields() { + String idx = TEST_INDEX + "_update" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "upd001", [name: "Carol", status: "active"]) + pgClient.update(idx, "upd001", [status: "inactive", rating: 5]) + Map source = pgClient.getSource(idx, "upd001") + Assertions.assertNotNull(source) + // original name field should still be there (merge) + Assertions.assertEquals("Carol", source.get("name")) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("delete removes document") + void delete_removesDocument() { + String idx = TEST_INDEX + "_doc_delete" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "del001", [name: "To Delete"]) + Assertions.assertNotNull(pgClient.get(idx, "del001"), "doc should exist before delete") + pgClient.delete(idx, "del001") + Assertions.assertNull(pgClient.get(idx, "del001"), "doc should not exist after delete") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + // ============================ + // Bulk operations + // ============================ + + @Test + @DisplayName("bulkIndex inserts multiple documents") + void bulkIndex_insertsMultipleDocuments() { + String idx = TEST_INDEX + "_bulkindex" + try { + pgClient.createIndex(idx, null, null) + List docs = (1..20).collect { i -> + [productId: "PROD_${String.format('%03d', i)}", name: "Product ${i}", + category: i % 2 == 0 ? "category_a" : "category_b", price: i * 10.0] + } + pgClient.bulkIndex(idx, "productId", docs) + + // Verify a sampling + Map p1source = pgClient.getSource(idx, "PROD_001") + Assertions.assertNotNull(p1source, "PROD_001 should exist after bulkIndex") + Assertions.assertEquals("Product 1", p1source.get("name")) + + Map p10source = pgClient.getSource(idx, "PROD_010") + Assertions.assertNotNull(p10source, "PROD_010 should exist") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("bulk with index and delete actions") + void bulk_withIndexAndDeleteActions() { + String idx = TEST_INDEX + "_bulk_actions" + try { + pgClient.createIndex(idx, null, null) + // First insert two docs + pgClient.index(idx, "ba001", [name: "To Keep"]) + pgClient.index(idx, "ba002", [name: "To Delete"]) + + // Bulk: index new + delete existing + List actions = [ + [index: [_index: "${TEST_PREFIX}${idx}", _id: "ba003"]], + [name: "New Doc"], + [delete: [_index: "${TEST_PREFIX}${idx}", _id: "ba002"]], + [:] // no source for delete + ] + pgClient.bulk(idx, actions) + + Assertions.assertNotNull(pgClient.get(idx, "ba001"), "ba001 should still exist") + // ba002 delete and ba003 index via raw bulk - verify ba001 still there + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + // ============================ + // Search + // ============================ + + @Test + @DisplayName("search - match_all returns all documents") + void search_matchAllReturnsAll() { + String idx = TEST_INDEX + "_search_all" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "s1", [name: "Alpha Widget"]) + pgClient.index(idx, "s2", [name: "Beta Gadget"]) + pgClient.index(idx, "s3", [name: "Gamma Tool"]) + + Map result = pgClient.search(idx, [query: [match_all: [:]], size: 20]) + Assertions.assertNotNull(result) + Map hits = (Map) result.get("hits") + Assertions.assertNotNull(hits) + Map total = (Map) hits.get("total") + Assertions.assertNotNull(total) + long totalValue = ((Number) total.get("value")).longValue() + Assertions.assertEquals(3L, totalValue, "should return 3 documents") + + List hitList = (List) hits.get("hits") + Assertions.assertEquals(3, hitList.size()) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("search - term query filters documents") + void search_termQueryFilters() { + String idx = TEST_INDEX + "_search_term" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "t1", [status: "ACTIVE", name: "Active Widget"]) + pgClient.index(idx, "t2", [status: "INACTIVE", name: "Inactive Gadget"]) + pgClient.index(idx, "t3", [status: "ACTIVE", name: "Active Tool"]) + + Map result = pgClient.search(idx, [ + query: [term: [status: "ACTIVE"]], + size: 20, + track_total_hits: true + ]) + Map hits = (Map) result.get("hits") + Map total = (Map) hits.get("total") + long totalValue = ((Number) total.get("value")).longValue() + Assertions.assertEquals(2L, totalValue, "only ACTIVE docs should be returned") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("search - terms query with IN") + void search_termsQueryWithIn() { + String idx = TEST_INDEX + "_search_terms" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "tm1", [cat: "a"]) + pgClient.index(idx, "tm2", [cat: "b"]) + pgClient.index(idx, "tm3", [cat: "c"]) + + Map result = pgClient.search(idx, [ + query: [terms: [cat: ["a", "c"]]], + size: 10, + track_total_hits: true + ]) + Map total = (Map) ((Map) result.get("hits")).get("total") + Assertions.assertEquals(2L, ((Number) total.get("value")).longValue()) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("search - full-text query_string") + void search_fullTextQueryString() { + String idx = TEST_INDEX + "_search_fts" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "fts1", [description: "The quick brown fox jumps over the lazy dog"]) + pgClient.index(idx, "fts2", [description: "A completely unrelated document about databases"]) + pgClient.index(idx, "fts3", [description: "Another fox story about running quickly"]) + + Map result = pgClient.search(idx, [ + query: [query_string: [query: "fox", lenient: true]], + size: 10, + track_total_hits: true + ]) + Map total = (Map) ((Map) result.get("hits")).get("total") + long count = ((Number) total.get("value")).longValue() + Assertions.assertTrue(count >= 2, "should find at least 2 fox documents, found ${count}") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("search - bool must and must_not") + void search_boolMustAndMustNot() { + String idx = TEST_INDEX + "_search_bool" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "b1", [type: "order", status: "placed"]) + pgClient.index(idx, "b2", [type: "order", status: "cancelled"]) + pgClient.index(idx, "b3", [type: "invoice", status: "placed"]) + + Map result = pgClient.search(idx, [ + query: [bool: [ + must: [[term: [type: "order"]]], + must_not: [[term: [status: "cancelled"]]] + ]], + size: 10, + track_total_hits: true + ]) + Map total = (Map) ((Map) result.get("hits")).get("total") + long count = ((Number) total.get("value")).longValue() + Assertions.assertEquals(1L, count, "only non-cancelled orders should match") + List hitList = (List) ((Map) result.get("hits")).get("hits") + Assertions.assertEquals("b1", hitList[0].get("_id")) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("search - pagination from and size") + void search_paginationFromAndSize() { + String idx = TEST_INDEX + "_search_page" + try { + pgClient.createIndex(idx, null, null) + (1..10).each { i -> + pgClient.index(idx, "p${i}", [name: "Doc ${i}", seq: i]) + } + + // Get page 2 (items 5-9 of 10, 5 per page) + Map result = pgClient.search(idx, [ + query: [match_all: [:]], + from: 5, size: 5, + track_total_hits: true + ]) + Map hits = (Map) result.get("hits") + long totalValue = ((Number) ((Map) hits.get("total")).get("value")).longValue() + List hitList = (List) hits.get("hits") + Assertions.assertEquals(10L, totalValue, "total should reflect all 10 docs") + Assertions.assertEquals(5, hitList.size(), "page size should be 5") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("searchHits returns list directly") + void searchHits_returnsListDirectly() { + String idx = TEST_INDEX + "_searchhits" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "sh1", [x: 1]) + pgClient.index(idx, "sh2", [x: 2]) + List hits = pgClient.searchHits(idx, [query: [match_all: [:]], size: 10]) + Assertions.assertNotNull(hits) + Assertions.assertEquals(2, hits.size()) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + // ============================ + // Count + // ============================ + + @Test + @DisplayName("count returns correct total") + void count_returnsCorrectTotal() { + String idx = TEST_INDEX + "_count" + try { + pgClient.createIndex(idx, null, null) + (1..7).each { i -> pgClient.index(idx, "c${i}", [seq: i]) } + + long cnt = pgClient.count(idx, [query: [match_all: [:]]]) + Assertions.assertEquals(7L, cnt) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + @Test + @DisplayName("countResponse returns map with count key") + void countResponse_returnsMapWithCountKey() { + String idx = TEST_INDEX + "_countresp" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "cr1", [v: 1]) + pgClient.index(idx, "cr2", [v: 2]) + Map resp = pgClient.countResponse(idx, [query: [match_all: [:]]]) + Assertions.assertNotNull(resp) + Assertions.assertTrue(resp.containsKey("count")) + Assertions.assertEquals(2L, ((Number) resp.get("count")).longValue()) + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + // ============================ + // deleteByQuery + // ============================ + + @Test + @DisplayName("deleteByQuery removes matching documents") + void deleteByQuery_removesMatchingDocuments() { + String idx = TEST_INDEX + "_dbq" + try { + pgClient.createIndex(idx, null, null) + pgClient.index(idx, "dbq1", [status: "STALE"]) + pgClient.index(idx, "dbq2", [status: "STALE"]) + pgClient.index(idx, "dbq3", [status: "KEEP"]) + + Integer deleted = pgClient.deleteByQuery(idx, [term: [status: "STALE"]]) + Assertions.assertNotNull(deleted) + Assertions.assertEquals(2, deleted.intValue()) + Assertions.assertNull(pgClient.get(idx, "dbq1"), "STALE doc should be deleted") + Assertions.assertNotNull(pgClient.get(idx, "dbq3"), "KEEP doc should remain") + } finally { + if (pgClient.indexExists(idx)) pgClient.deleteIndex(idx) + } + } + + // ============================ + // PIT (stateless for postgres) + // ============================ + + @Test + @DisplayName("getPitId returns non-null token") + void getPitId_returnsNonNullToken() { + String pit = pgClient.getPitId(TEST_INDEX, "1m") + Assertions.assertNotNull(pit) + Assertions.assertTrue(pit.startsWith("pg::")) + } + + @Test + @DisplayName("deletePit is a no-op") + void deletePit_isNoOp() { + // Should not throw + pgClient.deletePit("pg::test::12345") + } + + // ============================ + // validateQuery + // ============================ + + @Test + @DisplayName("validateQuery returns null for valid query") + void validateQuery_returnsNullForValidQuery() { + Map result = pgClient.validateQuery(TEST_INDEX, [term: [status: "active"]], false) + Assertions.assertNull(result, "valid query should return null") + } + + // ============================ + // bulkIndexDataDocument + // ============================ + + @Test + @DisplayName("bulkIndexDataDocument strips metadata and indexes documents") + void bulkIndexDataDocument_stripsMetadataAndIndexes() { + List docs = [ + [_index: "test", _type: "TestDataDoc", _id: "tdd001", + _timestamp: System.currentTimeMillis(), productId: "P001", name: "Test Product One"], + [_index: "test", _type: "TestDataDoc", _id: "tdd002", + _timestamp: System.currentTimeMillis(), productId: "P002", name: "Test Product Two"], + ] + try { + pgClient.bulkIndexDataDocument(docs) + + // Documents should be in 'test_data_doc' index (ddIdToEsIndex("TestDataDoc")) + String expectedIdx = "test_data_doc" + Map source1 = pgClient.getSource(expectedIdx, "tdd001") + Assertions.assertNotNull(source1, "tdd001 should be indexed") + Assertions.assertEquals("Test Product One", source1.get("name")) + // Metadata should be stripped + Assertions.assertFalse(source1.containsKey("_index"), "_index should be stripped") + Assertions.assertFalse(source1.containsKey("_type"), "_type should be stripped") + Assertions.assertFalse(source1.containsKey("_id"), "_id should be stripped") + } finally { + // cleanup + try { pgClient.deleteIndex("test_data_doc") } catch (Throwable ignored) {} + } + } + + // ============================ + // extractContentText + // ============================ + + @Test + @DisplayName("extractContentText collects all string values") + void extractContentText_collectsAllStringValues() { + Map doc = [ + name: "John Doe", + status: "active", + address: [city: "Atlanta", state: "GA"], + tags: ["enterprise", "premium"], + amount: 299.99 + ] + String text = PostgresElasticClient.extractContentText(doc) + Assertions.assertNotNull(text) + Assertions.assertTrue(text.contains("John Doe"), "should include name") + Assertions.assertTrue(text.contains("active"), "should include status") + Assertions.assertTrue(text.contains("Atlanta"), "should include nested city") + Assertions.assertTrue(text.contains("enterprise"), "should include list items") + } + + @Test + @DisplayName("extractContentText on empty map returns empty string") + void extractContentText_emptyMapReturnsEmpty() { + String text = PostgresElasticClient.extractContentText([:]) + Assertions.assertEquals("", text) + } + + @Test + @DisplayName("extractContentText on null returns empty string") + void extractContentText_nullReturnsEmpty() { + String text = PostgresElasticClient.extractContentText(null) + Assertions.assertEquals("", text) + } + + // ============================ + // JSON serialization + // ============================ + + @Test + @DisplayName("objectToJson and jsonToObject roundtrip") + void objectToJson_andJsonToObject_roundtrip() { + Map original = [name: "Test", values: [1, 2, 3], nested: [key: "value"]] + String json = pgClient.objectToJson(original) + Assertions.assertNotNull(json) + Map roundtripped = (Map) pgClient.jsonToObject(json) + Assertions.assertEquals("Test", roundtripped.get("name")) + Assertions.assertEquals([1, 2, 3], roundtripped.get("values") as List) + } + + // ============================ + // Unsupported REST operations throw + // ============================ + + @Test + @DisplayName("call throws UnsupportedOperationException") + void call_throwsUnsupportedOperation() { + Assertions.assertThrows(UnsupportedOperationException.class, { + pgClient.call(org.moqui.util.RestClient.Method.GET, TEST_INDEX, "/", null, null) + }) + } + + @Test + @DisplayName("callFuture throws UnsupportedOperationException") + void callFuture_throwsUnsupportedOperation() { + Assertions.assertThrows(UnsupportedOperationException.class, { + pgClient.callFuture(org.moqui.util.RestClient.Method.GET, TEST_INDEX, "/", null, null) + }) + } + + @Test + @DisplayName("makeRestClient throws UnsupportedOperationException") + void makeRestClient_throwsUnsupportedOperation() { + Assertions.assertThrows(UnsupportedOperationException.class, { + pgClient.makeRestClient(org.moqui.util.RestClient.Method.GET, TEST_INDEX, "/", null) + }) + } +} diff --git a/framework/src/test/groovy/PostgresSearchSuite.groovy b/framework/src/test/groovy/PostgresSearchSuite.groovy new file mode 100644 index 000000000..fb42ac1b6 --- /dev/null +++ b/framework/src/test/groovy/PostgresSearchSuite.groovy @@ -0,0 +1,30 @@ +/* + * This software is in the public domain under CC0 1.0 Universal plus a + * Grant of Patent License. + */ + +import org.junit.jupiter.api.AfterAll +import org.junit.platform.suite.api.SelectClasses +import org.junit.platform.suite.api.Suite +import org.moqui.Moqui + +/** + * JUnit Platform Suite for PostgreSQL search backend tests. + * + * This suite is separate from MoquiSuite because it requires a live PostgreSQL database. + * It will NOT run as part of the main MoquiSuite — it is opt-in via: + * + * ./gradlew :framework:test --tests PostgresSearchSuite + * + * Or run individual test classes: + * ./gradlew :framework:test --tests PostgresSearchTranslatorTests + * ./gradlew :framework:test --tests PostgresElasticClientTests + */ +@Suite +@SelectClasses([PostgresSearchTranslatorTests.class, PostgresElasticClientTests.class]) +class PostgresSearchSuite { + @AfterAll + static void destroyMoqui() { + Moqui.destroyActiveExecutionContextFactory() + } +} diff --git a/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy b/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy new file mode 100644 index 000000000..060b66384 --- /dev/null +++ b/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy @@ -0,0 +1,478 @@ +/* + * This software is in the public domain under CC0 1.0 Universal plus a + * Grant of Patent License. + * + * Unit tests for ElasticQueryTranslator — no database connection required. + * Tests that ES Query DSL is correctly translated to PostgreSQL SQL. + */ +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Assertions +import org.moqui.impl.context.ElasticQueryTranslator +import org.moqui.impl.context.ElasticQueryTranslator.TranslatedQuery +import org.moqui.impl.context.ElasticQueryTranslator.QueryResult + +class PostgresSearchTranslatorTests { + + // ============================================================ + // TranslatedQuery / translateSearchMap + // ============================================================ + + @Test + @DisplayName("translateSearchMap - pagination defaults") + void translateSearchMap_paginationDefaults() { + TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap([:]) + Assertions.assertEquals(0, tq.fromOffset, "default from should be 0") + Assertions.assertEquals(20, tq.sizeLimit, "default size should be 20") + Assertions.assertEquals("TRUE", tq.whereClause) + } + + @Test + @DisplayName("translateSearchMap - explicit pagination") + void translateSearchMap_explicitPagination() { + TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap([from: 40, size: 10]) + Assertions.assertEquals(40, tq.fromOffset) + Assertions.assertEquals(10, tq.sizeLimit) + } + + @Test + @DisplayName("translateSearchMap - query_string sets tsqueryExpr") + void translateSearchMap_queryStringSetstsqueryExpr() { + TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap([ + query: [query_string: [query: "hello world", lenient: true]] + ]) + Assertions.assertNotNull(tq.tsqueryExpr, "tsqueryExpr should be set for query_string") + Assertions.assertTrue(tq.whereClause.contains("content_tsv"), "WHERE should use content_tsv") + } + + @Test + @DisplayName("translateSearchMap - sort spec") + void translateSearchMap_sortSpec() { + TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap([ + sort: [[postDate: [order: "desc"]]] + ]) + Assertions.assertNotNull(tq.orderBy, "orderBy should be set") + Assertions.assertTrue(tq.orderBy.contains("DESC"), "orderBy should be DESC") + } + + @Test + @DisplayName("translateSearchMap - highlight fields extracted") + void translateSearchMap_highlightFieldsExtracted() { + TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap([ + query: [match_all: [:]], + highlight: [fields: [title: [:], description: [:]]] + ]) + Assertions.assertTrue(tq.highlightFields.containsKey("title")) + Assertions.assertTrue(tq.highlightFields.containsKey("description")) + } + + // ============================================================ + // match_all + // ============================================================ + + @Test + @DisplayName("match_all translates to TRUE") + void matchAll_translatesTrue() { + QueryResult qr = ElasticQueryTranslator.translateQuery([match_all: [:]]) + Assertions.assertEquals("TRUE", qr.clause) + Assertions.assertTrue(qr.params.isEmpty()) + } + + // ============================================================ + // query_string + // ============================================================ + + @Test + @DisplayName("query_string translates to websearch_to_tsquery with tsqueryExpr") + void queryString_translatesWebsearchToTsquery() { + QueryResult qr = ElasticQueryTranslator.translateQuery([query_string: [query: "moqui framework"]]) + Assertions.assertTrue(qr.clause.contains("content_tsv"), "clause should use content_tsv") + Assertions.assertTrue(qr.clause.contains("@@"), "clause should have @@ operator") + Assertions.assertNotNull(qr.tsqueryExpr, "should produce tsqueryExpr for scoring") + Assertions.assertFalse(qr.params.isEmpty(), "should have at least one param for the query string") + } + + @Test + @DisplayName("query_string wildcard stripped for tsquery") + void queryString_wildcardStripped() { + QueryResult qr = ElasticQueryTranslator.translateQuery([query_string: [query: "moqui*"]]) + // Wildcard queries are cleaned to simple word for websearch_to_tsquery + Assertions.assertTrue(qr.clause.contains("content_tsv")) + } + + // ============================================================ + // term + // ============================================================ + + @Test + @DisplayName("term translates to field = value") + void term_translatesEquality() { + QueryResult qr = ElasticQueryTranslator.translateQuery([term: [status: "ACTIVE"]]) + Assertions.assertTrue(qr.clause.contains("->>'status'"), "should use ->>'status'") + Assertions.assertTrue(qr.clause.contains("="), "should use equality") + Assertions.assertEquals(1, qr.params.size()) + Assertions.assertEquals("ACTIVE", qr.params[0]) + } + + @Test + @DisplayName("term on _id translates to doc_id equality") + void term_onIdFieldUsesDocId() { + QueryResult qr = ElasticQueryTranslator.translateQuery([term: ["_id": "TEST_001"]]) + Assertions.assertTrue(qr.clause.contains("doc_id"), "should use doc_id for _id field") + Assertions.assertEquals("TEST_001", qr.params[0]) + } + + @Test + @DisplayName("term on nested field path uses JSONB path access") + void term_nestedFieldPath() { + QueryResult qr = ElasticQueryTranslator.translateQuery([term: ["address.city": "Atlanta"]]) + Assertions.assertTrue(qr.clause.contains("document->'address'->>'city'"), "should use nested path") + Assertions.assertEquals("Atlanta", qr.params[0]) + } + + // ============================================================ + // terms + // ============================================================ + + @Test + @DisplayName("terms translates to IN clause") + void terms_translatesInClause() { + QueryResult qr = ElasticQueryTranslator.translateQuery([terms: [statusId: ["ACTIVE", "PENDING", "DRAFT"]]]) + Assertions.assertTrue(qr.clause.contains("IN"), "should use IN operator") + Assertions.assertEquals(3, qr.params.size()) + Assertions.assertTrue(qr.params.containsAll(["ACTIVE", "PENDING", "DRAFT"])) + } + + @Test + @DisplayName("terms with empty list translates to FALSE") + void terms_emptyListTranslatesFalse() { + QueryResult qr = ElasticQueryTranslator.translateQuery([terms: [statusId: []]]) + Assertions.assertEquals("FALSE", qr.clause) + } + + // ============================================================ + // range + // ============================================================ + + @Test + @DisplayName("range with gte and lte") + void range_gteAndLte() { + QueryResult qr = ElasticQueryTranslator.translateQuery([range: [orderDate: [gte: "2024-01-01", lte: "2024-12-31"]]]) + Assertions.assertTrue(qr.clause.contains(">="), "should have >=") + Assertions.assertTrue(qr.clause.contains("<="), "should have <=") + Assertions.assertEquals(2, qr.params.size()) + } + + @Test + @DisplayName("range with gt only") + void range_gtOnly() { + QueryResult qr = ElasticQueryTranslator.translateQuery([range: [amount: [gt: "100"]]]) + Assertions.assertTrue(qr.clause.contains(">"), "should have >") + Assertions.assertFalse(qr.clause.contains(">="), "should not have >= if only gt") + Assertions.assertEquals(1, qr.params.size()) + Assertions.assertEquals("100", qr.params[0]) + } + + @Test + @DisplayName("range on date field gets timestamptz cast") + void range_dateFieldGetsTimestamptzCast() { + QueryResult qr = ElasticQueryTranslator.translateQuery([range: [orderDate: [gte: "2024-01-01"]]]) + Assertions.assertTrue(qr.clause.contains("::timestamptz"), "date fields should cast to timestamptz") + } + + @Test + @DisplayName("range on amount field gets numeric cast") + void range_amountFieldGetsNumericCast() { + QueryResult qr = ElasticQueryTranslator.translateQuery([range: [grandTotal: [gte: "0"]]]) + Assertions.assertTrue(qr.clause.contains("::numeric"), "amount fields should cast to numeric") + } + + // ============================================================ + // exists + // ============================================================ + + @Test + @DisplayName("exists translates to JSONB document ? field") + void exists_translatesJsonbHasKey() { + QueryResult qr = ElasticQueryTranslator.translateQuery([exists: [field: "email"]]) + Assertions.assertTrue(qr.clause.contains("document ?"), "should use JSONB ? operator") + Assertions.assertTrue(qr.clause.contains("email")) + } + + // ============================================================ + // bool + // ============================================================ + + @Test + @DisplayName("bool must translates to AND") + void boolMust_translatesAnd() { + QueryResult qr = ElasticQueryTranslator.translateQuery([bool: [must: [ + [term: [type: "ORDER"]], + [term: [status: "PLACED"]] + ]]]) + Assertions.assertTrue(qr.clause.contains("AND"), "must should generate AND") + Assertions.assertEquals(2, qr.params.size()) + } + + @Test + @DisplayName("bool should translates to OR") + void boolShould_translatesOr() { + QueryResult qr = ElasticQueryTranslator.translateQuery([bool: [should: [ + [term: [status: "PLACED"]], + [term: [status: "SHIPPED"]] + ]]]) + Assertions.assertTrue(qr.clause.contains("OR"), "should should generate OR") + } + + @Test + @DisplayName("bool must_not translates to NOT") + void boolMustNot_translatesNot() { + QueryResult qr = ElasticQueryTranslator.translateQuery([bool: [must_not: [ + [term: [status: "CANCELLED"]] + ]]]) + Assertions.assertTrue(qr.clause.toUpperCase().contains("NOT"), "must_not should generate NOT") + } + + @Test + @DisplayName("bool filter translates same as must") + void boolFilter_translatesSameAsMust() { + QueryResult qr = ElasticQueryTranslator.translateQuery([bool: [filter: [ + [term: [tenantId: "DEMO"]] + ]]]) + Assertions.assertTrue(qr.clause.contains("->>'tenantId'"), "filter should translate term like must") + } + + @Test + @DisplayName("bool combined must and must_not") + void boolCombinedMustAndMustNot() { + QueryResult qr = ElasticQueryTranslator.translateQuery([bool: [ + must: [[term: [type: "ORDER"]]], + must_not: [[term: [status: "CANCELLED"]]] + ]]) + Assertions.assertTrue(qr.clause.contains("AND"), "should have AND") + Assertions.assertTrue(qr.clause.toUpperCase().contains("NOT"), "should have NOT") + Assertions.assertEquals(2, qr.params.size()) + } + + // ============================================================ + // nested + // ============================================================ + + @Test + @DisplayName("nested query translates to EXISTS subquery with jsonb_array_elements") + void nested_translatesExistsSubquery() { + QueryResult qr = ElasticQueryTranslator.translateQuery([nested: [ + path: "orderItems", + query: [term: ["orderItems.productId": "PROD_001"]] + ]]) + Assertions.assertTrue(qr.clause.contains("EXISTS"), "nested should use EXISTS subquery") + Assertions.assertTrue(qr.clause.contains("jsonb_array_elements"), "nested should use jsonb_array_elements") + Assertions.assertTrue(qr.clause.contains("orderItems"), "should reference the nested path") + } + + // ============================================================ + // ids + // ============================================================ + + @Test + @DisplayName("ids query translates to doc_id IN list") + void ids_translatesDocIdIn() { + QueryResult qr = ElasticQueryTranslator.translateQuery([ids: [values: ["ID1", "ID2", "ID3"]]]) + Assertions.assertTrue(qr.clause.contains("doc_id IN"), "ids should use doc_id IN") + Assertions.assertEquals(3, qr.params.size()) + } + + @Test + @DisplayName("ids with empty values translates to FALSE") + void ids_emptyValuesTranslatesFalse() { + QueryResult qr = ElasticQueryTranslator.translateQuery([ids: [values: []]]) + Assertions.assertEquals("FALSE", qr.clause) + } + + // ============================================================ + // translateSort + // ============================================================ + + @Test + @DisplayName("translateSort - map with order desc") + void translateSort_mapWithOrderDesc() { + String result = ElasticQueryTranslator.translateSort([[orderDate: [order: "desc"]]]) + Assertions.assertNotNull(result) + Assertions.assertTrue(result.contains("DESC")) + Assertions.assertTrue(result.contains("orderDate")) + } + + @Test + @DisplayName("translateSort - score special field") + void translateSort_scoreSpecialField() { + String result = ElasticQueryTranslator.translateSort([[_score: [order: "desc"]]]) + Assertions.assertNotNull(result) + Assertions.assertTrue(result.contains("_score"), "should produce _score sort entry") + } + + @Test + @DisplayName("translateSort - keyword suffix stripped") + void translateSort_keywordSuffixStripped() { + String result = ElasticQueryTranslator.translateSort([["statusId.keyword": [order: "asc"]]]) + Assertions.assertFalse(result.contains(".keyword"), ".keyword suffix should be stripped") + Assertions.assertTrue(result.contains("statusId")) + } + + @Test + @DisplayName("translateSort - string shorthand") + void translateSort_stringShorthand() { + String result = ElasticQueryTranslator.translateSort(["orderDate"]) + Assertions.assertNotNull(result) + Assertions.assertTrue(result.contains("orderDate")) + } + + @Test + @DisplayName("translateSort - null returns null") + void translateSort_nullReturnsNull() { + String result = ElasticQueryTranslator.translateSort(null) + Assertions.assertNull(result) + } + + @Test + @DisplayName("translateSort - empty list returns null") + void translateSort_emptyListReturnsNull() { + String result = ElasticQueryTranslator.translateSort([]) + Assertions.assertNull(result) + } + + // ============================================================ + // Security: sanitizeFieldName — SQL injection prevention + // ============================================================ + + @Test + @DisplayName("sanitizeFieldName rejects SQL injection via single quote") + void sanitizeFieldName_rejectsSingleQuote() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("status'; DROP TABLE users;--") + } + } + + @Test + @DisplayName("sanitizeFieldName rejects SQL injection via semicolon") + void sanitizeFieldName_rejectsSemicolon() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("field;DELETE FROM moqui_search_index") + } + } + + @Test + @DisplayName("sanitizeFieldName rejects parentheses") + void sanitizeFieldName_rejectsParentheses() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("field()OR 1=1") + } + } + + @Test + @DisplayName("sanitizeFieldName rejects double dash comment") + void sanitizeFieldName_rejectsDoubleDash() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("field--comment") + } + } + + @Test + @DisplayName("sanitizeFieldName rejects null field") + void sanitizeFieldName_rejectsNull() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName(null) + } + } + + @Test + @DisplayName("sanitizeFieldName rejects empty field") + void sanitizeFieldName_rejectsEmpty() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("") + } + } + + @Test + @DisplayName("sanitizeFieldName rejects spaces") + void sanitizeFieldName_rejectsSpaces() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("field name") + } + } + + @Test + @DisplayName("sanitizeFieldName rejects UNION SELECT injection") + void sanitizeFieldName_rejectsUnionSelect() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName("x' UNION SELECT * FROM pg_shadow--") + } + } + + @Test + @DisplayName("sanitizeFieldName accepts valid field names") + void sanitizeFieldName_acceptsValidNames() { + // These should NOT throw + Assertions.assertEquals("statusId", ElasticQueryTranslator.sanitizeFieldName("statusId")) + Assertions.assertEquals("order.items.quantity", ElasticQueryTranslator.sanitizeFieldName("order.items.quantity")) + Assertions.assertEquals("@timestamp", ElasticQueryTranslator.sanitizeFieldName("@timestamp")) + Assertions.assertEquals("field-name", ElasticQueryTranslator.sanitizeFieldName("field-name")) + Assertions.assertEquals("_id", ElasticQueryTranslator.sanitizeFieldName("_id")) + } + + @Test + @DisplayName("sanitizeFieldName rejects oversized field name") + void sanitizeFieldName_rejectsOversized() { + String longField = "a" * 257 + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.sanitizeFieldName(longField) + } + } + + @Test + @DisplayName("term query with SQL injection field name is rejected") + void term_sqlInjectionFieldRejected() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.translateQuery([term: ["status'; DROP TABLE x;--": "active"]]) + } + } + + @Test + @DisplayName("range query with SQL injection field name is rejected") + void range_sqlInjectionFieldRejected() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.translateQuery([range: ["x' OR '1'='1": [gte: "2024-01-01"]]]) + } + } + + @Test + @DisplayName("exists query with SQL injection field name is rejected") + void exists_sqlInjectionFieldRejected() { + Assertions.assertThrows(IllegalArgumentException) { + ElasticQueryTranslator.translateQuery([exists: [field: "x'; DELETE FROM moqui_search_index;--"]]) + } + } + + // ============================================================ + // Full searchMap round-trip + // ============================================================ + + @Test + @DisplayName("full searchMap with bool query and sort") + void fullSearchMap_boolQueryAndSort() { + Map searchMap = [ + from: 0, size: 25, + sort: [[orderDate: [order: "desc"]]], + query: [bool: [ + must: [[term: [statusId: "OrderPlaced"]]], + filter: [[range: [orderDate: [gte: "2024-01-01"]]]] + ]], + highlight: [fields: [productName: [:]]] + ] + TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap(searchMap) + Assertions.assertEquals(0, tq.fromOffset) + Assertions.assertEquals(25, tq.sizeLimit) + Assertions.assertNotNull(tq.orderBy) + Assertions.assertTrue(tq.whereClause.contains("AND")) + Assertions.assertTrue(tq.highlightFields.containsKey("productName")) + } +} diff --git a/framework/xsd/moqui-conf-3.xsd b/framework/xsd/moqui-conf-3.xsd index d060a6176..e9138b70d 100644 --- a/framework/xsd/moqui-conf-3.xsd +++ b/framework/xsd/moqui-conf-3.xsd @@ -594,16 +594,29 @@ along with this software (see the LICENSE.md file). If not, see - + + Backend type for this cluster. Use 'elastic' (default) for ElasticSearch/OpenSearch via HTTP REST. + Use 'postgres' to store and search documents in PostgreSQL using JSONB and tsvector. + When type is 'postgres', url should be the entity datasource group name (e.g. 'transactional'). + + + + + + + + For type=elastic: full URL to ElasticSearch/OpenSearch (e.g. http://localhost:9200). + For type=postgres: the Moqui entity datasource group name to use (e.g. 'transactional'), or omit to use the default transactional group. + - Prefix added to all ES index names just before requests - Must follow ES index name requirements (lower case, etc) - No separator character is added, recommend ending with underscore (_) + Prefix added to all index names just before requests. + For type=elastic: Must follow ES index name requirements (lower case, etc). No separator character is added, recommend ending with underscore (_). + For type=postgres: Used as a prefix to the index_name column value in moqui_document table. - - + For type=elastic only: HTTP connection pool max size. + For type=elastic only: HTTP request queue size. From 84f7c48a2757532ead3f9d6d2d7f62ecc8da07cc Mon Sep 17 00:00:00 2001 From: pandor4u <103976470+pandor4u@users.noreply.github.com> Date: Fri, 10 Apr 2026 18:04:03 -0500 Subject: [PATCH 2/3] Improve PostgreSQL search backend: dedicated table routing, ts_headline, parameterized queries - Route HTTP logs to dedicated moqui_http_log table with typed columns instead of generic moqui_document JSONB storage - Route deleteByQuery to dedicated tables (moqui_http_log, moqui_logs) with proper timestamp range extraction for nightly cleanup jobs - Replace Java regex highlights with PostgreSQL ts_headline() for accurate, index-aware snippet generation - Fix update() to read-merge-extract pattern so content_text stays consistent with extractContentText() used by index() - Add searchHttpLogTable() for searching against dedicated HTTP log table - Improve guessCastType() to inspect actual values (epoch millis, decimals, ISO dates) when field name heuristics are ambiguous - Parameterize exists query (use ?? operator) to prevent SQL injection - Handle indexExists/createIndex/count for dedicated table names --- .../context/ElasticQueryTranslator.groovy | 45 +- .../impl/context/PostgresElasticClient.groovy | 510 ++++++++++++++++-- .../PostgresSearchTranslatorTests.groovy | 5 +- 3 files changed, 495 insertions(+), 65 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy b/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy index 887aa1237..db25964b9 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/ElasticQueryTranslator.groovy @@ -370,8 +370,9 @@ class ElasticQueryTranslator { List conditions = [] List params = [] - // Determine cast type based on common field name patterns - String castType = guessCastType(field) + // Determine cast type: first try field name heuristics, then inspect actual values + Object sampleValue = rangeSpecMap.get("gte") ?: rangeSpecMap.get("gt") ?: rangeSpecMap.get("lte") ?: rangeSpecMap.get("lt") + String castType = guessCastType(field, sampleValue) Object gte = rangeSpecMap.get("gte") Object gt = rangeSpecMap.get("gt") @@ -397,13 +398,15 @@ class ElasticQueryTranslator { // Validate field name to prevent SQL injection sanitizeFieldName(field) - // For nested paths, check the nested path exists + // Use parameterized JSONB key existence check if (field.contains(".")) { List parts = field.split("\\.") as List String topLevel = parts[0] - qr.clause = "document ? '${topLevel}'" + qr.clause = "document ?? ?" + qr.params = [topLevel] } else { - qr.clause = "document ? '${field}'" + qr.clause = "document ?? ?" + qr.params = [field] } return qr } @@ -521,13 +524,15 @@ class ElasticQueryTranslator { Map rangeSpecMap = (Map) rangeSpec String localField = field.startsWith(parentPath + ".") ? field.substring(parentPath.length() + 1) : field sanitizeFieldName(localField) - String castType = guessCastType(localField) List conditions = [] List params = [] - Object gte = rangeSpecMap.get("gte"); if (gte != null) { conditions.add("(elem->>'${localField}')${castType} >= ?"); params.add(gte.toString()) } - Object gt = rangeSpecMap.get("gt"); if (gt != null) { conditions.add("(elem->>'${localField}')${castType} > ?"); params.add(gt.toString()) } - Object lte = rangeSpecMap.get("lte"); if (lte != null) { conditions.add("(elem->>'${localField}')${castType} <= ?"); params.add(lte.toString()) } - Object lt = rangeSpecMap.get("lt"); if (lt != null) { conditions.add("(elem->>'${localField}')${castType} < ?"); params.add(lt.toString()) } + Object gte = rangeSpecMap.get("gte"); Object gt = rangeSpecMap.get("gt") + Object lte = rangeSpecMap.get("lte"); Object lt = rangeSpecMap.get("lt") + String castType = guessCastType(localField, gte ?: gt ?: lte ?: lt) + if (gte != null) { conditions.add("(elem->>'${localField}')${castType} >= ?"); params.add(gte.toString()) } + if (gt != null) { conditions.add("(elem->>'${localField}')${castType} > ?"); params.add(gt.toString()) } + if (lte != null) { conditions.add("(elem->>'${localField}')${castType} <= ?"); params.add(lte.toString()) } + if (lt != null) { conditions.add("(elem->>'${localField}')${castType} < ?"); params.add(lt.toString()) } qr.clause = conditions ? conditions.join(" AND ") : "TRUE" qr.params = params return qr @@ -605,10 +610,12 @@ class ElasticQueryTranslator { } /** - * Guess the appropriate PostgreSQL cast type for a field name to use in range/sort comparisons. + * Guess the appropriate PostgreSQL cast type for a field to use in range/sort comparisons. + * Uses field name heuristics first, then falls back to inspecting the actual value. * Returns empty string if no cast is needed (use text comparison). */ - private static String guessCastType(String field) { + static String guessCastType(String field, Object sampleValue = null) { + // 1. Field name heuristics String lf = field.toLowerCase() if (lf.contains("date") || lf.contains("stamp") || lf.contains("time") || lf == "@timestamp") { return "::timestamptz" @@ -618,6 +625,20 @@ class ElasticQueryTranslator { lf.contains("number") || lf.contains("num") || lf.contains("id") && lf.endsWith("num")) { return "::numeric" } + + // 2. Inspect the actual value if field name is ambiguous + if (sampleValue != null) { + String sv = sampleValue.toString().trim() + // Large number (> 10 digits) that's all digits — likely epoch millis timestamp + if (sv.matches(/^\d{10,}$/)) return "::numeric" + // Decimal number + if (sv.matches(/^-?\d+\.\d+$/)) return "::numeric" + // Integer + if (sv.matches(/^-?\d+$/)) return "::numeric" + // ISO date pattern + if (sv.matches(/^\d{4}-\d{2}-\d{2}.*/)) return "::timestamptz" + } + return "" } diff --git a/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy b/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy index a28f247f2..11d09bdff 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy @@ -52,9 +52,22 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { private final static Logger logger = LoggerFactory.getLogger(PostgresElasticClient.class) private final static Set DOC_META_KEYS = new HashSet<>(["_index", "_type", "_id", "_timestamp"]) + /** Index names that map to the dedicated moqui_http_log table */ + private static final Set HTTP_LOG_INDEX_NAMES = new HashSet<>(["moqui_http_log"]) + /** Index names that map to the dedicated moqui_logs table */ + private static final Set APP_LOG_INDEX_NAMES = new HashSet<>(["moqui_logs"]) + /** Jackson mapper shared with ElasticFacadeImpl */ static final ObjectMapper jacksonMapper = ElasticFacadeImpl.jacksonMapper + /** SQL for inserting HTTP request logs into the dedicated moqui_http_log table */ + static final String HTTP_LOG_INSERT_SQL = """ + INSERT INTO moqui_http_log (log_timestamp, remote_ip, remote_user, server_ip, content_type, + request_method, request_scheme, request_host, request_path, request_query, http_version, + response_code, time_initial_ms, time_final_ms, bytes_sent, referrer, agent, session_id, visitor_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """.trim() + /** Shared UPSERT SQL for moqui_document — used by upsertDocument(), bulkIndex(), and bulkIndexDataDocument() */ static final String DOCUMENT_UPSERT_SQL = """ INSERT INTO moqui_document (index_name, doc_id, doc_type, document, content_text, content_tsv, updated_stamp) @@ -280,6 +293,8 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { @Override boolean indexExists(String index) { if (!index) return false + // Dedicated tables always exist (created in initSchema) + if (isHttpLogIndex(index) || isAppLogIndex(index)) return true String prefixed = prefixIndexName(index) Connection conn = getConnection() PreparedStatement ps = conn.prepareStatement( @@ -307,6 +322,8 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { @Override void createIndex(String index, Map docMapping, String alias) { + // Dedicated tables are created in initSchema — nothing to do + if (isHttpLogIndex(index) || isAppLogIndex(index)) return createIndex(index, null, docMapping, alias, null) } @@ -376,10 +393,29 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { // Document CRUD // ============================================================ + /** Check if the given index name (raw or prefixed) refers to the dedicated HTTP log table */ + private boolean isHttpLogIndex(String index) { + if (!index) return false + String raw = unprefixIndexName(index.trim()) + return HTTP_LOG_INDEX_NAMES.contains(raw) + } + + /** Check if the given index name (raw or prefixed) refers to the dedicated app logs table */ + private boolean isAppLogIndex(String index) { + if (!index) return false + String raw = unprefixIndexName(index.trim()) + return APP_LOG_INDEX_NAMES.contains(raw) + } + @Override void index(String index, String _id, Map document) { if (!index) throw new IllegalArgumentException("Index name required for index()") if (!_id) throw new IllegalArgumentException("_id required for index()") + // Route HTTP log documents to dedicated table + if (isHttpLogIndex(index)) { + insertHttpLog(document) + return + } String prefixedIndex = prefixIndexName(index) String docJson = objectToJson(document) String contentText = extractContentText(document) @@ -392,31 +428,48 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { if (!_id) throw new IllegalArgumentException("_id required for update()") String prefixedIndex = prefixIndexName(index) String fragmentJson = objectToJson(documentFragment) + + // Merge the fragment into the existing document, then re-extract content_text using + // the same recursive extractContentText() logic used by index()/upsert to keep FTS consistent Connection conn = getConnection() - PreparedStatement ps = conn.prepareStatement(""" - UPDATE moqui_document - SET document = COALESCE(document, '{}'::jsonb) || ?::jsonb, - content_text = ( - SELECT string_agg(val::text, ' ') - FROM jsonb_each_text(COALESCE(document, '{}'::jsonb) || ?::jsonb) AS kv(key, val) - WHERE jsonb_typeof(COALESCE(document, '{}'::jsonb) || ?::jsonb -> kv.key) IN ('string', 'number') - ), - content_tsv = to_tsvector('english', coalesce(( - SELECT string_agg(val::text, ' ') - FROM jsonb_each_text(COALESCE(document, '{}'::jsonb) || ?::jsonb) AS kv(key, val) - ), '')), - updated_stamp = now() - WHERE index_name = ? AND doc_id = ? - """.trim()) + PreparedStatement getPs = conn.prepareStatement( + "SELECT document FROM moqui_document WHERE index_name = ? AND doc_id = ?") try { - ps.setString(1, fragmentJson) - ps.setString(2, fragmentJson) - ps.setString(3, fragmentJson) - ps.setString(4, fragmentJson) - ps.setString(5, prefixedIndex) - ps.setString(6, _id) - ps.executeUpdate() - } finally { ps.close() } + getPs.setString(1, prefixedIndex) + getPs.setString(2, _id) + ResultSet rs = getPs.executeQuery() + try { + if (rs.next()) { + String existingJson = rs.getString("document") + Map existingDoc = existingJson ? (Map) jsonToObject(existingJson) : [:] + // Deep merge: fragment overrides existing top-level keys + Map mergedDoc = new LinkedHashMap<>(existingDoc) + mergedDoc.putAll(documentFragment) + String mergedJson = objectToJson(mergedDoc) + String contentText = extractContentText(mergedDoc) + // Update with the fully merged document and properly extracted content + PreparedStatement updPs = conn.prepareStatement(""" + UPDATE moqui_document + SET document = ?::jsonb, + content_text = ?, + content_tsv = to_tsvector('english', COALESCE(?, '')), + updated_stamp = now() + WHERE index_name = ? AND doc_id = ? + """.trim()) + try { + updPs.setString(1, mergedJson) + updPs.setString(2, contentText) + updPs.setString(3, contentText) + updPs.setString(4, prefixedIndex) + updPs.setString(5, _id) + updPs.executeUpdate() + } finally { updPs.close() } + } else { + logger.warn("update(): document not found in index '${prefixedIndex}' with id '${_id}', inserting as new") + upsertDocument(prefixedIndex, _id, null, fragmentJson, extractContentText(documentFragment)) + } + } finally { rs.close() } + } finally { getPs.close() } } @Override @@ -438,6 +491,11 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { @Override Integer deleteByQuery(String index, Map queryMap) { if (!index) throw new IllegalArgumentException("Index name required for deleteByQuery()") + + // Route to dedicated tables for logs + if (isHttpLogIndex(index)) return deleteByQueryHttpLog(queryMap) + if (isAppLogIndex(index)) return deleteByQueryAppLog(queryMap) + String prefixedIndex = prefixIndexName(index) ElasticQueryTranslator.QueryResult qr = ElasticQueryTranslator.translateQuery(queryMap ?: [match_all: [:]]) @@ -456,6 +514,120 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { } finally { ps.close() } } + /** Delete from the dedicated moqui_http_log table based on query (supports @timestamp range) */ + private Integer deleteByQueryHttpLog(Map queryMap) { + TimestampRange range = extractTimestampRange(queryMap) + if (range == null) { + // match_all → delete everything + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("DELETE FROM moqui_http_log") + try { return ps.executeUpdate() } finally { ps.close() } + } + List conditions = [] + List params = [] + if (range.lte != null) { conditions.add("log_timestamp <= ?"); params.add(range.lte) } + if (range.lt != null) { conditions.add("log_timestamp < ?"); params.add(range.lt) } + if (range.gte != null) { conditions.add("log_timestamp >= ?"); params.add(range.gte) } + if (range.gt != null) { conditions.add("log_timestamp > ?"); params.add(range.gt) } + String whereClause = conditions ? conditions.join(" AND ") : "TRUE" + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("DELETE FROM moqui_http_log WHERE ${whereClause}") + try { + for (int i = 0; i < params.size(); i++) setParam(ps, i + 1, params[i]) + return ps.executeUpdate() + } finally { ps.close() } + } + + /** Delete from the dedicated moqui_logs table based on query (supports @timestamp range) */ + private Integer deleteByQueryAppLog(Map queryMap) { + TimestampRange range = extractTimestampRange(queryMap) + if (range == null) { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("DELETE FROM moqui_logs") + try { return ps.executeUpdate() } finally { ps.close() } + } + List conditions = [] + List params = [] + if (range.lte != null) { conditions.add("log_timestamp <= ?"); params.add(range.lte) } + if (range.lt != null) { conditions.add("log_timestamp < ?"); params.add(range.lt) } + if (range.gte != null) { conditions.add("log_timestamp >= ?"); params.add(range.gte) } + if (range.gt != null) { conditions.add("log_timestamp > ?"); params.add(range.gt) } + String whereClause = conditions ? conditions.join(" AND ") : "TRUE" + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement("DELETE FROM moqui_logs WHERE ${whereClause}") + try { + for (int i = 0; i < params.size(); i++) setParam(ps, i + 1, params[i]) + return ps.executeUpdate() + } finally { ps.close() } + } + + /** Simple holder for timestamp range bounds extracted from query DSL */ + private static class TimestampRange { + Timestamp lte, lt, gte, gt + } + + /** + * Extract @timestamp range bounds from an ES query map (as used by the nightly cleanup jobs). + * Handles both epoch millis (Long/String number) and ISO date strings. + * Returns null if no timestamp range is found (e.g. match_all). + */ + private static TimestampRange extractTimestampRange(Map queryMap) { + if (queryMap == null) return null + // Direct range query: {range: {@timestamp: {lte: ...}}} + if (queryMap.containsKey("range")) { + return parseTimestampRangeSpec(queryMap) + } + // Bool query with range in must list + Map boolMap = (Map) queryMap.get("bool") + if (boolMap == null) return null + Object mustVal = boolMap.get("must") + List mustList = [] + if (mustVal instanceof List) mustList = (List) mustVal + else if (mustVal instanceof Map) mustList = [(Map) mustVal] + for (Map clause in mustList) { + if (clause.containsKey("range")) { + return parseTimestampRangeSpec(clause) + } + } + return null + } + + private static TimestampRange parseTimestampRangeSpec(Map rangeClause) { + Map rangeMap = (Map) rangeClause.get("range") + if (rangeMap == null) return null + // Look for @timestamp or any timestamp-like field + Map specMap = null + for (String key in ["@timestamp", "log_timestamp"]) { + if (rangeMap.containsKey(key)) { specMap = (Map) rangeMap.get(key); break } + } + if (specMap == null) { + // Try first key + String firstKey = rangeMap.keySet().isEmpty() ? null : (String) rangeMap.keySet().iterator().next() + if (firstKey) specMap = (Map) rangeMap.get(firstKey) + } + if (specMap == null) return null + TimestampRange range = new TimestampRange() + range.lte = toTimestamp(specMap.get("lte")) + range.lt = toTimestamp(specMap.get("lt")) + range.gte = toTimestamp(specMap.get("gte")) + range.gt = toTimestamp(specMap.get("gt")) + if (range.lte == null && range.lt == null && range.gte == null && range.gt == null) return null + return range + } + + /** Convert a value (epoch millis long, numeric string, or ISO string) to a Timestamp */ + private static Timestamp toTimestamp(Object val) { + if (val == null) return null + if (val instanceof Number) return new Timestamp(((Number) val).longValue()) + String s = val.toString().trim() + if (!s) return null + // Try epoch millis + try { return new Timestamp(Long.parseLong(s)) } catch (NumberFormatException ignored) {} + // Try ISO format + try { return Timestamp.valueOf(s.replace('T', ' ').replace('Z', '')) } catch (Exception ignored) {} + return null + } + @Override void bulk(String index, List actionSourceList) { if (!actionSourceList) return @@ -508,6 +680,11 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { @Override void bulkIndex(String index, String docType, String idField, List documentList, boolean refresh) { if (!documentList) return + // Route HTTP log documents to dedicated table + if (isHttpLogIndex(index)) { + bulkInsertHttpLogs(documentList) + return + } String prefixedIndex = prefixIndexName(index) boolean hasId = idField != null && !idField.isEmpty() @@ -593,9 +770,13 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { @Override Map search(String index, Map searchMap) { - if (index && (index == 'moqui_logs' || prefixIndexName(index) == prefixIndexName('moqui_logs'))) { + // Route dedicated log tables + if (index && isAppLogIndex(index)) { return searchLogsTable(searchMap ?: [:]) } + if (index && isHttpLogIndex(index)) { + return searchHttpLogTable(searchMap ?: [:]) + } ElasticQueryTranslator.TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap(searchMap ?: [:]) @@ -604,9 +785,6 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { return [hits: [total: [value: 0, relation: "eq"], hits: []]] } - String scoreExpr = tq.tsqueryExpr ? - "ts_rank_cd(content_tsv, ${tq.tsqueryExpr})" : "1.0::float" - String idxPlaceholders = indexNames.collect { "?" }.join(", ") String whereClause = "index_name IN (${idxPlaceholders})" List allParams = new ArrayList<>(indexNames) @@ -621,11 +799,28 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { String orderByClause = tq.orderBy ?: (tq.tsqueryExpr ? "_score DESC" : "updated_stamp DESC") + // Build highlight columns (use PostgreSQL ts_headline when we have a tsquery) + boolean useDbHighlights = tq.highlightFields && tq.tsqueryExpr + List hlColumnExprs = [] + List hlFieldNames = [] + List hlParams = [] + if (useDbHighlights) { + for (String hlField in tq.highlightFields.keySet()) { + String jsonPath = ElasticQueryTranslator.fieldToJsonPath("document", hlField) + String hlExpr = ElasticQueryTranslator.buildHighlightExpr(jsonPath, tq.tsqueryExpr) + hlColumnExprs.add("${hlExpr} AS hl_${hlFieldNames.size()}".toString()) + hlFieldNames.add(hlField) + hlParams.addAll(tq.tsqueryParams) + } + } + + String hlSelect = hlColumnExprs ? ", " + hlColumnExprs.join(", ") : "" + String countSql = "SELECT COUNT(*) FROM moqui_document WHERE ${whereClause}" long totalCount = 0L String mainSql = """ - SELECT doc_id, index_name, doc_type, document, ${buildScoreSelect(tq)} AS _score + SELECT doc_id, index_name, doc_type, document, ${buildScoreSelect(tq)} AS _score${hlSelect} FROM moqui_document WHERE ${whereClause} ORDER BY ${orderByClause} @@ -645,6 +840,8 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { List mainParams = [] if (tq.tsqueryExpr) mainParams.addAll(tq.tsqueryParams) + // Add highlight params (one set of tsquery params per highlight field) + mainParams.addAll(hlParams) mainParams.addAll(allParams) mainParams.add(tq.sizeLimit) mainParams.add(tq.fromOffset) @@ -666,8 +863,13 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { Map hit = [_index: idxName, _id: docId, _type: docType, _score: score, _source: source] as Map - if (tq.highlightFields && tq.tsqueryExpr) { - Map> highlights = buildHighlights(source, tq) + // Read ts_headline results from result set columns + if (useDbHighlights) { + Map> highlights = [:] + for (int h = 0; h < hlFieldNames.size(); h++) { + String hlResult = rs.getString("hl_${h}") + if (hlResult) highlights.put(hlFieldNames[h], [hlResult]) + } if (highlights) hit.put("highlight", highlights) } @@ -784,6 +986,116 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { } finally { ps.close() } } + /** Search the dedicated moqui_http_log table (mirrors searchLogsTable pattern) */ + private Map searchHttpLogTable(Map searchMap) { + ElasticQueryTranslator.TranslatedQuery tq = ElasticQueryTranslator.translateSearchMap(searchMap) + + String rawQuery = null + Map queryMap = (Map) searchMap?.get("query") + if (queryMap) { + Map qsMap = (Map) queryMap.get("query_string") + if (qsMap) rawQuery = (String) qsMap.get("query") + } + + List conditions = [] + List params = [] + + if (rawQuery) { + java.util.regex.Matcher m = (rawQuery =~ /@timestamp\s*:\s*\[\s*(\*|\d+)\s+TO\s+(\*|\d+)\s*\]/) + if (m.find()) { + String fromVal = m.group(1) + String toVal = m.group(2) + if (fromVal != '*') { + conditions.add("log_timestamp >= ?") + params.add(new java.sql.Timestamp(Long.parseLong(fromVal))) + } + if (toVal != '*') { + conditions.add("log_timestamp <= ?") + params.add(new java.sql.Timestamp(Long.parseLong(toVal))) + } + } + } + + // Also check for range query in bool.must + if (queryMap) { + TimestampRange range = extractTimestampRange(queryMap) + if (range != null) { + if (range.lte != null) { conditions.add("log_timestamp <= ?"); params.add(range.lte) } + if (range.lt != null) { conditions.add("log_timestamp < ?"); params.add(range.lt) } + if (range.gte != null) { conditions.add("log_timestamp >= ?"); params.add(range.gte) } + if (range.gt != null) { conditions.add("log_timestamp > ?"); params.add(range.gt) } + } + } + + String whereClause = conditions ? conditions.join(" AND ") : "TRUE" + + Connection conn = getConnection() + long totalCount = 0L + if (tq.trackTotal) { + PreparedStatement countPs = conn.prepareStatement("SELECT COUNT(*) FROM moqui_http_log WHERE ${whereClause}") + try { + for (int i = 0; i < params.size(); i++) setParam(countPs, i + 1, params[i]) + ResultSet rs = countPs.executeQuery() + try { if (rs.next()) totalCount = rs.getLong(1) } finally { rs.close() } + } finally { countPs.close() } + } + + String mainSql = """ + SELECT log_id, log_timestamp, remote_ip, remote_user, server_ip, content_type, + request_method, request_scheme, request_host, request_path, request_query, + http_version, response_code, time_initial_ms, time_final_ms, bytes_sent, + referrer, agent, session_id, visitor_id + FROM moqui_http_log + WHERE ${whereClause} + ORDER BY log_timestamp DESC + LIMIT ? OFFSET ? + """.trim() + + PreparedStatement ps = conn.prepareStatement(mainSql) + try { + int pIdx = 0 + for (int i = 0; i < params.size(); i++) setParam(ps, ++pIdx, params[i]) + ps.setInt(++pIdx, tq.sizeLimit) + ps.setInt(++pIdx, tq.fromOffset) + ResultSet rs = ps.executeQuery() + try { + List hits = [] + while (rs.next()) { + long logId = rs.getLong("log_id") + java.sql.Timestamp ts = rs.getTimestamp("log_timestamp") + Map source = [ + "@timestamp" : ts?.time, + remote_ip : rs.getString("remote_ip"), + remote_user : rs.getString("remote_user"), + server_ip : rs.getString("server_ip"), + content_type : rs.getString("content_type"), + request_method : rs.getString("request_method"), + request_scheme : rs.getString("request_scheme"), + request_host : rs.getString("request_host"), + request_path : rs.getString("request_path"), + request_query : rs.getString("request_query"), + http_version : rs.getString("http_version"), + response : rs.getInt("response_code"), + time_initial_ms : rs.getLong("time_initial_ms"), + time_final_ms : rs.getLong("time_final_ms"), + bytes : rs.getLong("bytes_sent"), + referrer : rs.getString("referrer"), + agent : rs.getString("agent"), + session : rs.getString("session_id"), + visitor_id : rs.getString("visitor_id"), + ] as Map + hits.add([_index: "moqui_http_log", _id: String.valueOf(logId), + _type: "MoquiHttpRequest", _score: 1.0, _source: source] as Map) + } + return [hits: [total: [value: totalCount, relation: "eq"], hits: hits], + _shards: [total: 1, successful: 1, failed: 0]] + } finally { rs.close() } + } catch (Throwable t) { + logger.error("searchHttpLogTable error: " + t.message, t) + return [hits: [total: [value: 0, relation: "eq"], hits: []]] + } finally { ps.close() } + } + @Override List searchHits(String index, Map searchMap) { Map result = search(index, searchMap) @@ -802,10 +1114,51 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { @Override long count(String index, Map countMap) { + // Route dedicated tables + if (isHttpLogIndex(index)) return countHttpLog(countMap) + if (isAppLogIndex(index)) return countAppLog(countMap) Map result = countResponse(index, countMap) return ((Number) result.get("count"))?.longValue() ?: 0L } + private long countHttpLog(Map countMap) { + TimestampRange range = countMap?.get("query") ? extractTimestampRange((Map) countMap.get("query")) : null + String sql = "SELECT COUNT(*) FROM moqui_http_log" + List params = [] + if (range != null) { + List conditions = [] + if (range.lte != null) { conditions.add("log_timestamp <= ?"); params.add(range.lte) } + if (range.gte != null) { conditions.add("log_timestamp >= ?"); params.add(range.gte) } + if (conditions) sql += " WHERE " + conditions.join(" AND ") + } + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(sql) + try { + for (int i = 0; i < params.size(); i++) setParam(ps, i + 1, params[i]) + ResultSet rs = ps.executeQuery() + try { return rs.next() ? rs.getLong(1) : 0L } finally { rs.close() } + } finally { ps.close() } + } + + private long countAppLog(Map countMap) { + TimestampRange range = countMap?.get("query") ? extractTimestampRange((Map) countMap.get("query")) : null + String sql = "SELECT COUNT(*) FROM moqui_logs" + List params = [] + if (range != null) { + List conditions = [] + if (range.lte != null) { conditions.add("log_timestamp <= ?"); params.add(range.lte) } + if (range.gte != null) { conditions.add("log_timestamp >= ?"); params.add(range.gte) } + if (conditions) sql += " WHERE " + conditions.join(" AND ") + } + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(sql) + try { + for (int i = 0; i < params.size(); i++) setParam(ps, i + 1, params[i]) + ResultSet rs = ps.executeQuery() + try { return rs.next() ? rs.getLong(1) : 0L } finally { rs.close() } + } finally { ps.close() } + } + @Override Map countResponse(String index, Map countMap) { if (!countMap) countMap = [query: [match_all: [:]]] @@ -1132,30 +1485,85 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { return "1.0::float" } - private static Map> buildHighlights(Map source, ElasticQueryTranslator.TranslatedQuery tq) { - Map> highlights = [:] - if (!tq.tsqueryExpr || !tq.highlightFields) return highlights - String firstParam = tq.params ? tq.params[0]?.toString() : null - if (!firstParam) return highlights - for (String field in tq.highlightFields.keySet()) { - Object fieldVal = source.get(field) - if (fieldVal instanceof String) { - String text = (String) fieldVal - String highlighted = simpleHighlight(text, firstParam) - if (highlighted != text) highlights.put(field, [highlighted]) + // ============================================================ + // HTTP log insert helpers + // ============================================================ + + /** Insert a single HTTP log record into the dedicated moqui_http_log table */ + private void insertHttpLog(Map document) { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(HTTP_LOG_INSERT_SQL) + try { + setHttpLogParams(ps, document) + ps.executeUpdate() + } finally { ps.close() } + } + + /** Bulk insert HTTP log records into the dedicated moqui_http_log table */ + private void bulkInsertHttpLogs(List documentList) { + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(HTTP_LOG_INSERT_SQL) + try { + int batchSize = 0 + for (Map doc in documentList) { + setHttpLogParams(ps, doc) + ps.addBatch() + batchSize++ + if (batchSize >= 500) { + ps.executeBatch() + batchSize = 0 + } } - } - return highlights + if (batchSize > 0) ps.executeBatch() + } finally { ps.close() } } - private static String simpleHighlight(String text, String query) { - if (!text || !query) return text - List terms = query.replaceAll(/["()+\-]/, ' ').split(/\s+/).findAll { it.length() > 2 } as List - String result = text - for (String term in terms) { - result = result.replaceAll("(?i)\\b${java.util.regex.Pattern.quote(term)}\\b", "\$0") - } - return result + /** Set parameters on an HTTP_LOG_INSERT_SQL PreparedStatement from an HTTP log document map */ + private static void setHttpLogParams(PreparedStatement ps, Map doc) { + // @timestamp: epoch millis long (from ElasticRequestLogFilter) + Object tsObj = doc.get("@timestamp") + if (tsObj instanceof Number) ps.setTimestamp(1, new Timestamp(((Number) tsObj).longValue())) + else if (tsObj != null) ps.setTimestamp(1, new Timestamp(Long.parseLong(tsObj.toString()))) + else ps.setTimestamp(1, new Timestamp(System.currentTimeMillis())) + + setStrParam(ps, 2, doc.get("remote_ip")) + setStrParam(ps, 3, doc.get("remote_user")) + setStrParam(ps, 4, doc.get("server_ip")) + setStrParam(ps, 5, doc.get("content_type")) + setStrParam(ps, 6, doc.get("request_method")) + setStrParam(ps, 7, doc.get("request_scheme")) + setStrParam(ps, 8, doc.get("request_host")) + setStrParam(ps, 9, doc.get("request_path")) + setStrParam(ps, 10, doc.get("request_query")) + + // http_version: stored as text (filter sends a float/half_float) + Object httpVer = doc.get("http_version") + setStrParam(ps, 11, httpVer != null ? httpVer.toString() : null) + + // response code + Object respObj = doc.get("response") + if (respObj instanceof Number) ps.setInt(12, ((Number) respObj).intValue()) + else if (respObj != null) try { ps.setInt(12, Integer.parseInt(respObj.toString())) } catch (Exception e) { ps.setNull(12, Types.INTEGER) } + else ps.setNull(12, Types.INTEGER) + + // timing + setLongParam(ps, 13, doc.get("time_initial_ms")) + setLongParam(ps, 14, doc.get("time_final_ms")) + setLongParam(ps, 15, doc.get("bytes")) + + setStrParam(ps, 16, doc.get("referrer")) + setStrParam(ps, 17, doc.get("agent")) + setStrParam(ps, 18, doc.get("session")) + setStrParam(ps, 19, doc.get("visitor_id")) + } + + private static void setStrParam(PreparedStatement ps, int idx, Object val) { + if (val == null) ps.setNull(idx, Types.VARCHAR) else ps.setString(idx, val.toString()) + } + private static void setLongParam(PreparedStatement ps, int idx, Object val) { + if (val instanceof Number) ps.setLong(idx, ((Number) val).longValue()) + else if (val != null) try { ps.setLong(idx, Long.parseLong(val.toString())) } catch (Exception e) { ps.setNull(idx, Types.BIGINT) } + else ps.setNull(idx, Types.BIGINT) } private static void setParam(PreparedStatement ps, int idx, Object value) { diff --git a/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy b/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy index 060b66384..5dc28d06e 100644 --- a/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy +++ b/framework/src/test/groovy/PostgresSearchTranslatorTests.groovy @@ -195,8 +195,9 @@ class PostgresSearchTranslatorTests { @DisplayName("exists translates to JSONB document ? field") void exists_translatesJsonbHasKey() { QueryResult qr = ElasticQueryTranslator.translateQuery([exists: [field: "email"]]) - Assertions.assertTrue(qr.clause.contains("document ?"), "should use JSONB ? operator") - Assertions.assertTrue(qr.clause.contains("email")) + Assertions.assertTrue(qr.clause.contains("document ??"), "should use JSONB ? operator (escaped as ?? for JDBC)") + Assertions.assertEquals(1, qr.params.size(), "should have 1 param for field name") + Assertions.assertEquals("email", qr.params[0]) } // ============================================================ From 0de052330543e552338890165555a478b4ad85a5 Mon Sep 17 00:00:00 2001 From: pandor4u <103976470+pandor4u@users.noreply.github.com> Date: Fri, 10 Apr 2026 18:29:10 -0500 Subject: [PATCH 3/3] feat: Optional pgvector hybrid/semantic search with embedding support When the pgvector extension is installed and an embedding API is configured, enables vector-based document search alongside traditional keyword search: Schema: - Adds 'embedding vector(N)' column to moqui_document (configurable dimensions) - Creates DiskANN index (pgvectorscale) or HNSW index (pgvector fallback) Methods: - hybridSearch(): Reciprocal Rank Fusion combining keyword (ts_rank_cd) and semantic (cosine distance) results in a single SQL query with configurable keyword/vector weights - vectorSearch(): Pure semantic search using pgvector cosine distance - generateEmbedding(): Calls OpenAI-compatible embedding API endpoint - Embedding auto-generation on index() when vector search is enabled Configuration (MoquiConf.xml): XSD schema updated with embedding-url, embedding-model, embedding-dimensions attributes. Falls back gracefully to keyword-only search when pgvector or the embedding API is not available. --- .../impl/context/PostgresElasticClient.groovy | 303 +++++++++++++++++- .../src/main/resources/MoquiDefaultConf.xml | 3 + framework/xsd/moqui-conf-3.xsd | 10 + 3 files changed, 314 insertions(+), 2 deletions(-) diff --git a/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy b/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy index 11d09bdff..74c835195 100644 --- a/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy +++ b/framework/src/main/groovy/org/moqui/impl/context/PostgresElasticClient.groovy @@ -60,6 +60,15 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { /** Jackson mapper shared with ElasticFacadeImpl */ static final ObjectMapper jacksonMapper = ElasticFacadeImpl.jacksonMapper + /** Whether pgvector extension is available for vector/semantic search */ + private boolean hasPgVector = false + /** Embedding API endpoint URL (e.g. OpenAI-compatible /v1/embeddings) */ + private String embeddingUrl = null + /** Embedding model name (e.g. text-embedding-3-small) */ + private String embeddingModel = null + /** Vector dimensions (default 1536 for OpenAI text-embedding-3-small) */ + private int vectorDimensions = 1536 + /** SQL for inserting HTTP request logs into the dedicated moqui_http_log table */ static final String HTTP_LOG_INSERT_SQL = """ INSERT INTO moqui_http_log (log_timestamp, remote_ip, remote_user, server_ip, content_type, @@ -97,6 +106,12 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { String urlAttr = clusterNode.attribute("url") this.datasourceGroup = (urlAttr && !"".equals(urlAttr.trim())) ? urlAttr.trim() : "transactional" + // Embedding configuration for pgvector hybrid search (optional) + this.embeddingUrl = clusterNode.attribute("embedding-url") ?: null + this.embeddingModel = clusterNode.attribute("embedding-model") ?: "text-embedding-3-small" + String dimStr = clusterNode.attribute("embedding-dimensions") + if (dimStr) try { this.vectorDimensions = Integer.parseInt(dimStr.trim()) } catch (Exception e) { /* use default */ } + logger.info("Initializing PostgresElasticClient for cluster '${clusterName}' using datasource group '${datasourceGroup}' with index prefix '${indexPrefix}'") // Initialize schema (CREATE TABLE IF NOT EXISTS, extensions, indexes) @@ -121,6 +136,21 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { try { stmt.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm") } catch (Exception extEx) { logger.warn("Could not create pg_trgm extension (may require superuser): ${extEx.message}") } + // Detect pgvector extension for vector/semantic search (optional) + if (embeddingUrl) { + try { + stmt.execute("CREATE EXTENSION IF NOT EXISTS vector") + hasPgVector = true + logger.info("pgvector extension detected — vector/hybrid search enabled (model: ${embeddingModel}, dimensions: ${vectorDimensions})") + // Try vectorscale for DiskANN index (not required) + try { stmt.execute("CREATE EXTENSION IF NOT EXISTS vectorscale CASCADE") } + catch (Exception e) { logger.info("pgvectorscale not available — using HNSW index instead of DiskANN") } + } catch (Exception extEx) { + hasPgVector = false + logger.info("pgvector extension not available — vector search disabled") + } + } + // moqui_search_index — index metadata (replaces ES index/alias concept) stmt.execute(""" CREATE TABLE IF NOT EXISTS moqui_search_index ( @@ -170,6 +200,23 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { // Index for time-based ordering stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_doc_upd ON moqui_document (index_name, updated_stamp)") + // pgvector: Add embedding column and vector index for semantic search + if (hasPgVector) { + try { + stmt.execute("ALTER TABLE moqui_document ADD COLUMN IF NOT EXISTS embedding vector(${vectorDimensions})".toString()) + // Try DiskANN (pgvectorscale) first, fall back to HNSW + try { + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_doc_embed ON moqui_document USING diskann(embedding)".toString()) + logger.info("Created DiskANN vector index on moqui_document.embedding") + } catch (Exception e) { + stmt.execute("CREATE INDEX IF NOT EXISTS idx_mq_doc_embed ON moqui_document USING hnsw(embedding vector_cosine_ops)".toString()) + logger.info("Created HNSW vector index on moqui_document.embedding") + } + } catch (Exception e) { + logger.warn("Could not set up vector column on moqui_document: ${e.message}") + } + } + // moqui_logs — application log (replaces ES moqui_logs index) stmt.execute(""" CREATE TABLE IF NOT EXISTS moqui_logs ( @@ -279,11 +326,13 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { if (rs.next()) { return [name: clusterName, cluster_name: "postgres", version: [distribution: "postgres", number: rs.getString(1)], - tagline: "Moqui PostgresElasticClient"] + tagline: "Moqui PostgresElasticClient", + features: [pgvector: hasPgVector, embedding_model: embeddingModel]] } } finally { rs.close() } } finally { ps.close() } - return [name: clusterName, cluster_name: "postgres", version: [distribution: "postgres"]] + return [name: clusterName, cluster_name: "postgres", version: [distribution: "postgres"], + features: [pgvector: hasPgVector]] } // ============================================================ @@ -420,6 +469,8 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { String docJson = objectToJson(document) String contentText = extractContentText(document) upsertDocument(prefixedIndex, _id, null, docJson, contentText) + // Generate and store embedding asynchronously when pgvector is enabled + if (hasPgVector && embeddingUrl) updateDocumentEmbedding(prefixedIndex, _id, contentText) } @Override @@ -1102,6 +1153,254 @@ class PostgresElasticClient implements ElasticFacade.ElasticClient { return (List) ((Map) result.get("hits")).get("hits") } + // ============================================================ + // Vector / Hybrid Search (pgvector) + // ============================================================ + + /** Whether vector/hybrid search is available (pgvector installed and embedding API configured) */ + boolean isVectorSearchEnabled() { return hasPgVector && embeddingUrl } + + /** + * Perform a hybrid search combining keyword (tsvector/BM25) and semantic (vector) search + * using Reciprocal Rank Fusion (RRF) to merge the two result sets. + * + * @param index Index name (or null for all indexes) + * @param queryText The search query text + * @param limit Maximum results to return + * @param keywordWeight Weight for keyword results (default 0.5) + * @param vectorWeight Weight for vector results (default 0.5) + * @return ES-compatible search response map + */ + Map hybridSearch(String index, String queryText, int limit = 10, double keywordWeight = 0.5, double vectorWeight = 0.5) { + if (!queryText || queryText.trim().isEmpty()) { + return [hits: [total: [value: 0, relation: "eq"], hits: []]] + } + if (!hasPgVector || !embeddingUrl) { + // No vector support — fall back to regular keyword search + return search(index, [query: [query_string: [query: queryText]], size: limit]) + } + + List indexNames = resolveIndexNames(index) + if (indexNames.isEmpty()) { + return [hits: [total: [value: 0, relation: "eq"], hits: []]] + } + + // Generate embedding for the query + float[] queryEmbedding = generateEmbedding(queryText) + if (queryEmbedding == null) { + logger.warn("Could not generate embedding for query, falling back to keyword search") + return search(index, [query: [query_string: [query: queryText]], size: limit]) + } + + String cleanedQuery = ElasticQueryTranslator.cleanLuceneQuery(queryText) + String idxPlaceholders = indexNames.collect { "?" }.join(", ") + int k = 60 // RRF constant (standard value) + + // RRF: 1/(k + rank) for each result set, then sum weighted scores + String sql = """ + WITH keyword AS ( + SELECT doc_id, index_name, doc_type, document, + ROW_NUMBER() OVER (ORDER BY ts_rank_cd(content_tsv, websearch_to_tsquery('english', ?)) DESC) as rank + FROM moqui_document + WHERE index_name IN (${idxPlaceholders}) + AND content_tsv @@ websearch_to_tsquery('english', ?) + LIMIT ${limit * 3} + ), + semantic AS ( + SELECT doc_id, index_name, doc_type, document, + ROW_NUMBER() OVER (ORDER BY embedding <=> ?::vector) as rank + FROM moqui_document + WHERE index_name IN (${idxPlaceholders}) + AND embedding IS NOT NULL + LIMIT ${limit * 3} + ) + SELECT d.doc_id, d.index_name, d.doc_type, d.document, + COALESCE(? * (1.0 / (${k} + kw.rank)), 0) + + COALESCE(? * (1.0 / (${k} + sem.rank)), 0) AS _score + FROM moqui_document d + LEFT JOIN keyword kw ON d.doc_id = kw.doc_id AND d.index_name = kw.index_name + LEFT JOIN semantic sem ON d.doc_id = sem.doc_id AND d.index_name = sem.index_name + WHERE kw.doc_id IS NOT NULL OR sem.doc_id IS NOT NULL + ORDER BY _score DESC + LIMIT ? + """.trim() + + String embeddingStr = vectorToString(queryEmbedding) + + List allParams = [] + // keyword CTE params + allParams.add(cleanedQuery) // ts_rank_cd + allParams.addAll(indexNames) // index_name IN (keyword) + allParams.add(cleanedQuery) // content_tsv @@ + // semantic CTE params + allParams.add(embeddingStr) // embedding <=> ? + allParams.addAll(indexNames) // index_name IN (semantic) + // main query params + allParams.add(keywordWeight) // keyword weight + allParams.add(vectorWeight) // vector weight + allParams.add(limit) + + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(sql) + try { + for (int i = 0; i < allParams.size(); i++) setParam(ps, i + 1, allParams[i]) + ResultSet rs = ps.executeQuery() + try { + List hits = [] + while (rs.next()) { + String docJson = rs.getString("document") + Map source = docJson ? (Map) jsonToObject(docJson) : [:] + double score = rs.getDouble("_score") + hits.add([_index: unprefixIndexName(rs.getString("index_name")), + _id: rs.getString("doc_id"), + _type: rs.getString("doc_type"), + _score: score, _source: source] as Map) + } + return [hits: [total: [value: (long) hits.size(), relation: "eq"], hits: hits], + _shards: [total: 1, successful: 1, failed: 0]] + } finally { rs.close() } + } finally { ps.close() } + } + + /** + * Perform a pure vector/semantic search using pgvector cosine distance. + * @param index Index name + * @param queryText Text to search for (will be embedded) + * @param limit Maximum results + * @return ES-compatible search response map + */ + Map vectorSearch(String index, String queryText, int limit = 10) { + if (!hasPgVector || !embeddingUrl || !queryText) { + return [hits: [total: [value: 0, relation: "eq"], hits: []]] + } + + float[] queryEmbedding = generateEmbedding(queryText) + if (queryEmbedding == null) { + return [hits: [total: [value: 0, relation: "eq"], hits: []]] + } + + List indexNames = resolveIndexNames(index) + if (indexNames.isEmpty()) { + return [hits: [total: [value: 0, relation: "eq"], hits: []]] + } + + String idxPlaceholders = indexNames.collect { "?" }.join(", ") + String sql = """ + SELECT doc_id, index_name, doc_type, document, + 1.0 - (embedding <=> ?::vector) AS _score + FROM moqui_document + WHERE index_name IN (${idxPlaceholders}) AND embedding IS NOT NULL + ORDER BY embedding <=> ?::vector + LIMIT ? + """.trim() + + String embeddingStr = vectorToString(queryEmbedding) + List allParams = [embeddingStr] + allParams.addAll(indexNames) + allParams.add(embeddingStr) + allParams.add(limit) + + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement(sql) + try { + for (int i = 0; i < allParams.size(); i++) setParam(ps, i + 1, allParams[i]) + ResultSet rs = ps.executeQuery() + try { + List hits = [] + while (rs.next()) { + String docJson = rs.getString("document") + Map source = docJson ? (Map) jsonToObject(docJson) : [:] + double score = rs.getDouble("_score") + hits.add([_index: unprefixIndexName(rs.getString("index_name")), + _id: rs.getString("doc_id"), + _type: rs.getString("doc_type"), + _score: score, _source: source] as Map) + } + return [hits: [total: [value: (long) hits.size(), relation: "eq"], hits: hits], + _shards: [total: 1, successful: 1, failed: 0]] + } finally { rs.close() } + } finally { ps.close() } + } + + /** + * Generate an embedding vector for the given text by calling the configured embedding API. + * Supports OpenAI-compatible /v1/embeddings endpoints. + * @return float array of embedding values, or null on failure + */ + float[] generateEmbedding(String text) { + if (!embeddingUrl || !text) return null + try { + Map requestBody = [input: text, model: embeddingModel] + String requestJson = objectToJson(requestBody) + + java.net.HttpURLConnection httpConn = (java.net.HttpURLConnection) new java.net.URL(embeddingUrl).openConnection() + httpConn.setRequestMethod("POST") + httpConn.setRequestProperty("Content-Type", "application/json") + httpConn.setDoOutput(true) + httpConn.setConnectTimeout(10000) + httpConn.setReadTimeout(30000) + + httpConn.outputStream.withWriter("UTF-8") { writer -> writer.write(requestJson) } + + if (httpConn.responseCode != 200) { + logger.warn("Embedding API returned ${httpConn.responseCode}") + return null + } + + String responseBody = httpConn.inputStream.getText("UTF-8") + Map response = (Map) jsonToObject(responseBody) + List dataList = (List) response.get("data") + if (!dataList || dataList.isEmpty()) return null + + List embeddingList = (List) ((Map) dataList[0]).get("embedding") + if (!embeddingList) return null + + float[] result = new float[embeddingList.size()] + for (int i = 0; i < embeddingList.size(); i++) { + result[i] = embeddingList[i].floatValue() + } + return result + } catch (Exception e) { + logger.warn("Error generating embedding: ${e.message}") + return null + } + } + + /** Convert a float array to PostgreSQL vector string format: [0.1,0.2,...] */ + private static String vectorToString(float[] vector) { + StringBuilder sb = new StringBuilder("[") + for (int i = 0; i < vector.length; i++) { + if (i > 0) sb.append(",") + sb.append(vector[i]) + } + sb.append("]") + return sb.toString() + } + + /** + * Generate and store an embedding for a document when indexing. + * Called during index() when pgvector is enabled. + */ + private void updateDocumentEmbedding(String prefixedIndex, String docId, String contentText) { + if (!hasPgVector || !embeddingUrl || !contentText) return + try { + float[] embedding = generateEmbedding(contentText) + if (embedding == null) return + + Connection conn = getConnection() + PreparedStatement ps = conn.prepareStatement( + "UPDATE moqui_document SET embedding = ?::vector WHERE index_name = ? AND doc_id = ?") + try { + ps.setString(1, vectorToString(embedding)) + ps.setString(2, prefixedIndex) + ps.setString(3, docId) + ps.executeUpdate() + } finally { ps.close() } + } catch (Exception e) { + logger.debug("Could not update embedding for doc ${docId}: ${e.message}") + } + } + @Override Map validateQuery(String index, Map queryMap, boolean explain) { try { diff --git a/framework/src/main/resources/MoquiDefaultConf.xml b/framework/src/main/resources/MoquiDefaultConf.xml index eb78c0c52..92ac26ed9 100644 --- a/framework/src/main/resources/MoquiDefaultConf.xml +++ b/framework/src/main/resources/MoquiDefaultConf.xml @@ -426,6 +426,9 @@ diff --git a/framework/xsd/moqui-conf-3.xsd b/framework/xsd/moqui-conf-3.xsd index e9138b70d..9a502b2bd 100644 --- a/framework/xsd/moqui-conf-3.xsd +++ b/framework/xsd/moqui-conf-3.xsd @@ -617,6 +617,16 @@ along with this software (see the LICENSE.md file). If not, see For type=elastic only: HTTP connection pool max size. For type=elastic only: HTTP request queue size. + + For type=postgres only: URL of an OpenAI-compatible embedding API endpoint (e.g. http://localhost:11434/v1/embeddings). + When set and pgvector extension is available, enables vector/hybrid search on documents. + + + For type=postgres only: Embedding model name to use (default: text-embedding-3-small). + + + For type=postgres only: Vector dimensions for embeddings (default: 1536). Must match the embedding model's output dimensions. +