spice-couchbase 2.0.60 → 2.0.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.auto-optimize.test.js +578 -0
- package/build/index.js +153 -13
- package/build/utility.js +156 -48
- package/build/utility.test.js +53 -5
- package/package.json +2 -1
- package/src/index.auto-optimize.test.js +393 -0
- package/src/index.js +165 -8
- package/src/utility.js +165 -62
- package/src/utility.test.js +73 -4
package/build/index.js
CHANGED
|
@@ -24,6 +24,11 @@ var {
|
|
|
24
24
|
QueryScanConsistency
|
|
25
25
|
} = require("couchbase");
|
|
26
26
|
|
|
27
|
+
var {
|
|
28
|
+
parseResourceList,
|
|
29
|
+
extractStatementResources
|
|
30
|
+
} = require("./utility");
|
|
31
|
+
|
|
27
32
|
if (!Promise.allSettled) {
|
|
28
33
|
Promise.allSettled = promises => Promise.all(promises.map((promise, i) => promise.then(value => ({
|
|
29
34
|
status: "fulfilled",
|
|
@@ -35,13 +40,13 @@ if (!Promise.allSettled) {
|
|
|
35
40
|
} // Couchbase lookupIn has a limit of 16 operations per call
|
|
36
41
|
|
|
37
42
|
|
|
38
|
-
var LOOKUP_IN_MAX_SPECS = 16; // Auto-optimisation guard rails. `query()` fires `optimize()` (an EXPLAIN
|
|
39
|
-
//
|
|
43
|
+
var LOOKUP_IN_MAX_SPECS = 16; // Auto-optimisation guard rails. `query()` fires `optimize()` (an EXPLAIN,
|
|
44
|
+
// index inventory check, and possible CREATE INDEX) above SLOW_QUERY_MS. When
|
|
40
45
|
// the cluster is degraded every statement is slow, so without limits each
|
|
41
46
|
// query spawned two more heavy ones and deepened the outage. These caps make it
|
|
42
47
|
// a bounded, once-per-statement background task that backs off on failure.
|
|
43
48
|
|
|
44
|
-
var SLOW_QUERY_MS = Number(process.env.SPICE_COUCHBASE_SLOW_QUERY_MS ||
|
|
49
|
+
var SLOW_QUERY_MS = Number(process.env.SPICE_COUCHBASE_SLOW_QUERY_MS || 3000);
|
|
45
50
|
var AUTO_OPTIMIZE_ENABLED = !["0", "false", "no", "off"].includes(String((_process$env$SPICE_CO = process.env.SPICE_COUCHBASE_AUTO_OPTIMIZE) !== null && _process$env$SPICE_CO !== void 0 ? _process$env$SPICE_CO : "true").trim().toLowerCase());
|
|
46
51
|
var OPTIMIZE_MAX_INFLIGHT = Number(process.env.SPICE_COUCHBASE_OPTIMIZE_MAX_INFLIGHT || 1);
|
|
47
52
|
var OPTIMIZE_FAILURE_COOLDOWN_MS = Number(process.env.SPICE_COUCHBASE_OPTIMIZE_COOLDOWN_MS || 5 * 60 * 1000); // Statements already optimised in this process. Bounded so it cannot grow
|
|
@@ -52,7 +57,35 @@ var optimizeState = {
|
|
|
52
57
|
inFlight: 0,
|
|
53
58
|
seen: new Set(),
|
|
54
59
|
cooldownUntil: 0
|
|
55
|
-
};
|
|
60
|
+
}; // Auto cover indexing is opt-in per resource. A resource is the collection a
|
|
61
|
+
// Database instance serves (scoped deployments) or the `type = '...'` literal
|
|
62
|
+
// legacy bucket mode adds to every statement; raw statements also count their
|
|
63
|
+
// FROM / JOIN keyspaces. Names are case-insensitive. A slow statement is
|
|
64
|
+
// optimised when, in order of precedence:
|
|
65
|
+
// 1. the call says so: query_options.auto_optimize (true / false)
|
|
66
|
+
// 2. the instance says so: constructor option or connection config
|
|
67
|
+
// `auto_optimize` (true / false)
|
|
68
|
+
// 3. the application policy: Database.setAutoOptimizePolicy(fn), where
|
|
69
|
+
// fn(resource, context) returns true / false, or
|
|
70
|
+
// undefined to defer to the static lists
|
|
71
|
+
// 4. a static list names a resource it touches: comma-separated
|
|
72
|
+
// SPICE_COUCHBASE_OPTIMIZE_RESOURCES (`*` = every resource, which is the
|
|
73
|
+
// pre-2.0.62 behaviour), or connection config / constructor option
|
|
74
|
+
// `auto_optimize_resources`.
|
|
75
|
+
|
|
76
|
+
var OPTIMIZE_ENABLED_RESOURCES = parseResourceList(process.env.SPICE_COUCHBASE_OPTIMIZE_RESOURCES);
|
|
77
|
+
var autoOptimizePolicy = null;
|
|
78
|
+
|
|
79
|
+
function requestedAutoOptimize(query_options) {
|
|
80
|
+
var _query_options$auto_o;
|
|
81
|
+
|
|
82
|
+
if (!query_options || typeof query_options !== "object" || Array.isArray(query_options)) {
|
|
83
|
+
return undefined;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
var value = (_query_options$auto_o = query_options.auto_optimize) !== null && _query_options$auto_o !== void 0 ? _query_options$auto_o : query_options.optimize;
|
|
87
|
+
return typeof value === "boolean" ? value : undefined;
|
|
88
|
+
}
|
|
56
89
|
|
|
57
90
|
function shouldOptimize(query) {
|
|
58
91
|
if (!AUTO_OPTIMIZE_ENABLED) return false;
|
|
@@ -124,7 +157,9 @@ module.exports = class Database {
|
|
|
124
157
|
collection,
|
|
125
158
|
scope,
|
|
126
159
|
query_scope = "cluster",
|
|
127
|
-
auto_create_collection
|
|
160
|
+
auto_create_collection,
|
|
161
|
+
auto_optimize,
|
|
162
|
+
auto_optimize_resources
|
|
128
163
|
} = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
|
|
129
164
|
|
|
130
165
|
if (connection) {
|
|
@@ -163,8 +198,93 @@ module.exports = class Database {
|
|
|
163
198
|
this.upsertCollection();
|
|
164
199
|
}
|
|
165
200
|
|
|
166
|
-
this.use_bucket_name = spice.config.database.connections[this.connection_name].scope == undefined;
|
|
201
|
+
this.use_bucket_name = spice.config.database.connections[this.connection_name].scope == undefined; // Auto cover indexing. See OPTIMIZE_ENABLED_RESOURCES for precedence.
|
|
202
|
+
|
|
203
|
+
var configured_auto_optimize = auto_optimize !== null && auto_optimize !== void 0 ? auto_optimize : this.db_config.auto_optimize;
|
|
204
|
+
this.auto_optimize = typeof configured_auto_optimize === "boolean" ? configured_auto_optimize : undefined;
|
|
205
|
+
this.enabled_resources = parseResourceList([...parseResourceList(this.db_config.auto_optimize_resources), ...parseResourceList(auto_optimize_resources)]);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Lets the application decide per resource whether slow statements may
|
|
210
|
+
* create covering indexes, e.g. from a per-resource setting an operator
|
|
211
|
+
* toggles at runtime. `policy(resource, { query, connection, collection })`
|
|
212
|
+
* returns true or false, or undefined to defer to the static lists. It is
|
|
213
|
+
* synchronous because it runs on the query path. Pass null to clear.
|
|
214
|
+
*/
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
static setAutoOptimizePolicy(policy) {
|
|
218
|
+
if (policy !== null && policy !== undefined && typeof policy !== "function") {
|
|
219
|
+
throw new TypeError("Auto-optimize policy must be a function or null");
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
autoOptimizePolicy = policy || null;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
static getAutoOptimizePolicy() {
|
|
226
|
+
return autoOptimizePolicy;
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Resources a statement touches, plus this instance's collection: that is
|
|
230
|
+
* the keyspace `optimize()` would create the index on.
|
|
231
|
+
*/
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
statementResources(query) {
|
|
235
|
+
var resources = extractStatementResources(query);
|
|
236
|
+
|
|
237
|
+
if (this.collection_name && this.collection_name !== "_default") {
|
|
238
|
+
resources.add(String(this.collection_name).toLowerCase());
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
return resources;
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* True when a slow statement may create a covering index. See
|
|
245
|
+
* OPTIMIZE_ENABLED_RESOURCES for the order of precedence.
|
|
246
|
+
*/
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
shouldAutoOptimize(query) {
|
|
250
|
+
var query_options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
|
|
251
|
+
var requested = requestedAutoOptimize(query_options);
|
|
252
|
+
if (requested !== undefined) return requested;
|
|
253
|
+
if (typeof this.auto_optimize === "boolean") return this.auto_optimize;
|
|
254
|
+
var resources = this.statementResources(query);
|
|
255
|
+
|
|
256
|
+
if (typeof autoOptimizePolicy === "function" && resources.size > 0) {
|
|
257
|
+
var deferred = false;
|
|
258
|
+
|
|
259
|
+
for (var resource of resources) {
|
|
260
|
+
var verdict = void 0;
|
|
261
|
+
|
|
262
|
+
try {
|
|
263
|
+
verdict = autoOptimizePolicy(resource, {
|
|
264
|
+
query,
|
|
265
|
+
connection: this.connection_name,
|
|
266
|
+
collection: this.collection_name
|
|
267
|
+
});
|
|
268
|
+
} catch (e) {
|
|
269
|
+
// A broken policy must not create indexes nobody asked for.
|
|
270
|
+
verdict = false;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
if (verdict === true) return true;
|
|
274
|
+
if (verdict === undefined || verdict === null) deferred = true;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
if (!deferred) return false;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
var enabled = new Set([...OPTIMIZE_ENABLED_RESOURCES, ...(this.enabled_resources || [])]);
|
|
281
|
+
if (enabled.has("*")) return true;
|
|
282
|
+
|
|
283
|
+
for (var _resource of resources) {
|
|
284
|
+
if (enabled.has(_resource)) return true;
|
|
167
285
|
}
|
|
286
|
+
|
|
287
|
+
return false;
|
|
168
288
|
}
|
|
169
289
|
|
|
170
290
|
setGlobalConnectionObject(collection) {
|
|
@@ -904,10 +1024,11 @@ module.exports = class Database {
|
|
|
904
1024
|
optimizeState.inFlight += 1;
|
|
905
1025
|
|
|
906
1026
|
try {
|
|
907
|
-
var _explainResponse$rows;
|
|
1027
|
+
var _explainResponse$rows, _indexResponse$meta, _indexResponse$meta$w;
|
|
908
1028
|
|
|
909
1029
|
var {
|
|
910
|
-
|
|
1030
|
+
buildCoveringIndexDefinition,
|
|
1031
|
+
indexSatisfiesCoveringDefinition
|
|
911
1032
|
} = require("./utility"); // 1. Run EXPLAIN to get the query plan
|
|
912
1033
|
|
|
913
1034
|
|
|
@@ -922,15 +1043,34 @@ module.exports = class Database {
|
|
|
922
1043
|
} // 3. Build covering index from the query
|
|
923
1044
|
|
|
924
1045
|
|
|
925
|
-
var
|
|
1046
|
+
var candidate = buildCoveringIndexDefinition(query, {
|
|
926
1047
|
bucket: _this19.bucket,
|
|
927
1048
|
scope: _this19.scope_name,
|
|
928
1049
|
collection: _this19.collection_name
|
|
929
|
-
});
|
|
1050
|
+
});
|
|
1051
|
+
if (!candidate) return; // EXPLAIN only describes the chosen plan. An equivalent or wider index
|
|
1052
|
+
// may already exist under an older name, or still be deferred/building.
|
|
1053
|
+
// Include both catalog representations of the default collection.
|
|
1054
|
+
|
|
1055
|
+
var indexResponse = yield exec_scope.query("SELECT RAW i FROM system:indexes AS i\n WHERE i.namespace_id = 'default'\n AND ((i.bucket_id = $bucket AND i.scope_id = $scope AND i.keyspace_id = $collection)\n OR ($scope = '_default' AND $collection = '_default'\n AND i.bucket_id IS MISSING AND i.scope_id IS MISSING AND i.keyspace_id = $bucket))", {
|
|
1056
|
+
parameters: {
|
|
1057
|
+
bucket: candidate.bucket,
|
|
1058
|
+
scope: candidate.scope,
|
|
1059
|
+
collection: candidate.collection
|
|
1060
|
+
}
|
|
1061
|
+
}); // A failed or incomplete inventory is not evidence that no index exists.
|
|
930
1062
|
|
|
931
|
-
if (
|
|
932
|
-
|
|
1063
|
+
if (!Array.isArray(indexResponse === null || indexResponse === void 0 ? void 0 : indexResponse.rows) || (indexResponse === null || indexResponse === void 0 ? void 0 : (_indexResponse$meta = indexResponse.meta) === null || _indexResponse$meta === void 0 ? void 0 : (_indexResponse$meta$w = _indexResponse$meta.warnings) === null || _indexResponse$meta$w === void 0 ? void 0 : _indexResponse$meta$w.length)) {
|
|
1064
|
+
throw new Error("Cannot verify existing indexes before auto-optimization");
|
|
933
1065
|
}
|
|
1066
|
+
|
|
1067
|
+
if (indexResponse.rows.some(index => index.name === candidate.name || indexSatisfiesCoveringDefinition(index, candidate))) {
|
|
1068
|
+
return;
|
|
1069
|
+
} // Identical definitions use the same name across processes. Couchbase's
|
|
1070
|
+
// per-keyspace name uniqueness also prevents a concurrent duplicate.
|
|
1071
|
+
|
|
1072
|
+
|
|
1073
|
+
yield exec_scope.query(candidate.statement);
|
|
934
1074
|
} catch (e) {
|
|
935
1075
|
// A failure here (timeout, index already exists, cluster busy) almost
|
|
936
1076
|
// always means the cluster is struggling. Stop adding EXPLAIN and
|
|
@@ -973,7 +1113,7 @@ module.exports = class Database {
|
|
|
973
1113
|
var durationInMilliseconds = (endTime - startTime) / BigInt(1000000); // Optimise slow statements in the background, subject to the guard rails
|
|
974
1114
|
// declared at the top of this file. Never awaited, never allowed to reject.
|
|
975
1115
|
|
|
976
|
-
if (durationInMilliseconds > SLOW_QUERY_MS && shouldOptimize(sanitizedQuery)) {
|
|
1116
|
+
if (durationInMilliseconds > SLOW_QUERY_MS && _this20.shouldAutoOptimize(sanitizedQuery, query_options) && shouldOptimize(sanitizedQuery)) {
|
|
977
1117
|
_this20.optimize(sanitizedQuery, exec_scope).catch(() => {});
|
|
978
1118
|
}
|
|
979
1119
|
|
package/build/utility.js
CHANGED
|
@@ -235,7 +235,7 @@ function buildOrderDirectionMap(orderByIndexFields) {
|
|
|
235
235
|
direction
|
|
236
236
|
} of orderByIndexFields) {
|
|
237
237
|
if (direction === "DESC") {
|
|
238
|
-
directionMap.set(name
|
|
238
|
+
directionMap.set(name, direction);
|
|
239
239
|
}
|
|
240
240
|
}
|
|
241
241
|
|
|
@@ -243,14 +243,9 @@ function buildOrderDirectionMap(orderByIndexFields) {
|
|
|
243
243
|
}
|
|
244
244
|
|
|
245
245
|
function formatIndexField(field, orderDirectionMap) {
|
|
246
|
-
var direction = orderDirectionMap.get(field
|
|
246
|
+
var direction = orderDirectionMap.get(field);
|
|
247
247
|
return "`".concat(field, "`").concat(direction ? " ".concat(direction) : "");
|
|
248
248
|
}
|
|
249
|
-
|
|
250
|
-
function formatIndexKey(field, orderDirectionMap) {
|
|
251
|
-
var direction = orderDirectionMap.get(field.toLowerCase());
|
|
252
|
-
return "".concat(field).concat(direction ? " ".concat(direction) : "");
|
|
253
|
-
}
|
|
254
249
|
/**
|
|
255
250
|
* Splits SELECT clause by commas, respecting parentheses
|
|
256
251
|
* @param {string} clause - The SELECT clause content
|
|
@@ -284,21 +279,8 @@ function splitSelectClause(clause) {
|
|
|
284
279
|
|
|
285
280
|
return parts;
|
|
286
281
|
}
|
|
287
|
-
|
|
288
|
-
function hasQualifiedWildcardProjection(query) {
|
|
289
|
-
var selectMatch = query.match(/\bSELECT\b([\s\S]+?)\bFROM\b/i);
|
|
290
|
-
|
|
291
|
-
if (!selectMatch) {
|
|
292
|
-
return false;
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
return splitSelectClause(selectMatch[1].trim()).some(part => {
|
|
296
|
-
var trimmedPart = part.trim();
|
|
297
|
-
return /^\(\s*`?\w+`?\s*\)\s*\.\s*\*$/.test(trimmedPart) || /^`?\w+`?\s*\.\s*\*$/.test(trimmedPart);
|
|
298
|
-
});
|
|
299
|
-
}
|
|
300
282
|
/**
|
|
301
|
-
* Builds a covering
|
|
283
|
+
* Builds a covering index CREATE statement from a query
|
|
302
284
|
* @param {string} query - The N1QL query to analyze
|
|
303
285
|
* @param {Object} options - Configuration options
|
|
304
286
|
* @param {string} options.bucket - The bucket name
|
|
@@ -309,6 +291,14 @@ function hasQualifiedWildcardProjection(query) {
|
|
|
309
291
|
|
|
310
292
|
|
|
311
293
|
function buildCoveringIndex(query, options) {
|
|
294
|
+
var _ref, _buildCoveringIndexDe;
|
|
295
|
+
|
|
296
|
+
return (_ref = (_buildCoveringIndexDe = buildCoveringIndexDefinition(query, options)) === null || _buildCoveringIndexDe === void 0 ? void 0 : _buildCoveringIndexDe.statement) !== null && _ref !== void 0 ? _ref : null;
|
|
297
|
+
} // Keep the access keys separate from projection-only keys so an older, wider
|
|
298
|
+
// index can satisfy the same request without depending on its generated name.
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
function buildCoveringIndexDefinition(query, options) {
|
|
312
302
|
var {
|
|
313
303
|
bucket,
|
|
314
304
|
scope = "_default",
|
|
@@ -321,9 +311,10 @@ function buildCoveringIndex(query, options) {
|
|
|
321
311
|
|
|
322
312
|
|
|
323
313
|
var selectFields = extractSelectFields(query);
|
|
324
|
-
var isNonCoveringFallback = selectFields === null && hasQualifiedWildcardProjection(query);
|
|
325
314
|
|
|
326
|
-
if (selectFields === null
|
|
315
|
+
if (selectFields === null) {
|
|
316
|
+
// Full-document projections cannot be covered by the extracted fields.
|
|
317
|
+
// Skip them instead of creating a non-covering fallback index.
|
|
327
318
|
return null;
|
|
328
319
|
} // Extract fields from WHERE clause
|
|
329
320
|
|
|
@@ -336,42 +327,36 @@ function buildCoveringIndex(query, options) {
|
|
|
336
327
|
var orderDirectionMap = buildOrderDirectionMap(orderByIndexFields);
|
|
337
328
|
var orderByFields = orderByIndexFields.map(field => field.name); // Remove 'deleted' from whereFields as it will be added first with INCLUDE MISSING
|
|
338
329
|
|
|
339
|
-
var filteredWhereFields = whereFields.filter(f => f
|
|
330
|
+
var filteredWhereFields = whereFields.filter(f => f !== "deleted"); // Track fields already added to avoid duplicates
|
|
340
331
|
|
|
341
|
-
var addedFields = new Set(filteredWhereFields
|
|
332
|
+
var addedFields = new Set(filteredWhereFields); // Filter GROUP BY fields (remove duplicates from WHERE, remove deleted)
|
|
342
333
|
|
|
343
334
|
var filteredGroupByFields = groupByFields.filter(f => {
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
if (addedFields.has(lower) || lower === "deleted") {
|
|
335
|
+
if (addedFields.has(f) || f === "deleted") {
|
|
347
336
|
return false;
|
|
348
337
|
}
|
|
349
338
|
|
|
350
|
-
addedFields.add(
|
|
339
|
+
addedFields.add(f);
|
|
351
340
|
return true;
|
|
352
341
|
}); // Filter ORDER BY fields (remove duplicates from WHERE and GROUP BY, remove deleted)
|
|
353
342
|
|
|
354
343
|
var filteredOrderByFields = orderByFields.filter(f => {
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
if (addedFields.has(lower) || lower === "deleted") {
|
|
344
|
+
if (addedFields.has(f) || f === "deleted") {
|
|
358
345
|
return false;
|
|
359
346
|
}
|
|
360
347
|
|
|
361
|
-
addedFields.add(
|
|
348
|
+
addedFields.add(f);
|
|
362
349
|
return true;
|
|
363
350
|
}); // Filter SELECT fields (remove duplicates from WHERE, GROUP BY, and ORDER BY, remove deleted)
|
|
364
351
|
|
|
365
|
-
var selectFieldsFiltered =
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
if (addedFields.has(lower) || lower === "deleted") {
|
|
352
|
+
var selectFieldsFiltered = selectFields.filter(f => {
|
|
353
|
+
if (addedFields.has(f) || f === "deleted") {
|
|
369
354
|
return false;
|
|
370
355
|
}
|
|
371
356
|
|
|
372
|
-
addedFields.add(
|
|
357
|
+
addedFields.add(f);
|
|
373
358
|
return true;
|
|
374
|
-
}); //
|
|
359
|
+
}).sort(); // Projection order has no effect on coverage. Keep access-key order intact.
|
|
375
360
|
|
|
376
361
|
var allFields = [...filteredWhereFields, ...filteredGroupByFields, ...filteredOrderByFields, ...selectFieldsFiltered];
|
|
377
362
|
|
|
@@ -381,22 +366,145 @@ function buildCoveringIndex(query, options) {
|
|
|
381
366
|
} // Build the field list with deleted INCLUDE MISSING first
|
|
382
367
|
|
|
383
368
|
|
|
384
|
-
var
|
|
385
|
-
//
|
|
369
|
+
var keys = ["`deleted` INCLUDE MISSING", ...allFields.map(f => formatIndexField(f, orderDirectionMap))]; // Hash the actual ordered definition, not query syntax. In particular, an
|
|
370
|
+
// explicit ORDER BY ... ASC must not rename an otherwise identical index.
|
|
386
371
|
|
|
387
|
-
var
|
|
388
|
-
var orderFieldsKey = orderByIndexFields.map(field => formatIndexKey(field.name, orderDirectionMap)).join(",");
|
|
389
|
-
var indexMode = isNonCoveringFallback ? "noncovered" : "covered";
|
|
390
|
-
var indexKey = "".concat(bucket, "|").concat(scope, "|").concat(collection, "|mode:").concat(indexMode, "|fields:").concat(indexFieldsKey, "|order:").concat(orderFieldsKey);
|
|
372
|
+
var indexKey = JSON.stringify([bucket, scope, collection, keys]);
|
|
391
373
|
var hash = generateHash(indexKey);
|
|
392
|
-
var indexName = "
|
|
374
|
+
var indexName = "adv_covered_".concat(hash); // Build the CREATE INDEX statement
|
|
375
|
+
|
|
376
|
+
var quote = identifier => "`".concat(identifier.replace(/`/g, "``"), "`");
|
|
377
|
+
|
|
378
|
+
var indexStatement = "CREATE INDEX `".concat(indexName, "` ON `default`:").concat(quote(bucket), ".").concat(quote(scope), ".").concat(quote(collection), "(").concat(keys.join(", "), ")");
|
|
379
|
+
return {
|
|
380
|
+
name: indexName,
|
|
381
|
+
statement: indexStatement,
|
|
382
|
+
keys,
|
|
383
|
+
accessKeyCount: 1 + filteredWhereFields.length + filteredGroupByFields.length + filteredOrderByFields.length,
|
|
384
|
+
bucket,
|
|
385
|
+
scope,
|
|
386
|
+
collection
|
|
387
|
+
};
|
|
388
|
+
} // Only compare simple scalar keys. Do not treat expressions, array indexes,
|
|
389
|
+
// partial indexes, or case-distinct document fields as equivalent.
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
function normalizeIndexKey(key) {
|
|
393
|
+
if (typeof key !== "string") return null;
|
|
394
|
+
var match = key.trim().match(/^(\(*[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]*(?:`(?:[\0-_a-\uFFFF]|``)+`|[A-Z_a-z][0-9A-Z_a-z]*)[\t-\r \xA0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF]*\)*)([\s\S]*?)$/);
|
|
395
|
+
if (!match) return null;
|
|
396
|
+
var expression = match[1].trim();
|
|
397
|
+
|
|
398
|
+
while (expression.startsWith("(") && expression.endsWith(")")) {
|
|
399
|
+
expression = expression.slice(1, -1).trim();
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
var identifier = expression.match(/^(?:`((?:[^`]|``)+)`|([A-Za-z_]\w*))$/);
|
|
403
|
+
if (!identifier) return null;
|
|
404
|
+
var attributes = match[2].trim().toUpperCase();
|
|
405
|
+
|
|
406
|
+
if (!/^(?:(?:ASC|DESC)(?:\s+INCLUDE\s+MISSING)?|INCLUDE\s+MISSING(?:\s+(?:ASC|DESC))?)?$/.test(attributes)) {
|
|
407
|
+
return null;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
return {
|
|
411
|
+
field: identifier[1] !== undefined ? identifier[1].replace(/``/g, "`") : identifier[2],
|
|
412
|
+
direction: /\bDESC\b/.test(attributes) ? "DESC" : "ASC",
|
|
413
|
+
includeMissing: /\bINCLUDE\s+MISSING\b/.test(attributes)
|
|
414
|
+
};
|
|
415
|
+
} // Call with indexes from the candidate's exact bucket/scope/collection. State
|
|
416
|
+
// is deliberately not filtered: a matching deferred/building index is already
|
|
417
|
+
// committed work, even though EXPLAIN cannot use it yet.
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
function indexSatisfiesCoveringDefinition(index, candidate) {
|
|
421
|
+
if (index.is_primary || index.condition || index.using !== "gsi" || !Array.isArray(index.index_key)) {
|
|
422
|
+
return false;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
var existingKeys = index.index_key.map(normalizeIndexKey);
|
|
426
|
+
var requiredKeys = candidate.keys.map(normalizeIndexKey);
|
|
427
|
+
if (existingKeys.some(key => !key) || requiredKeys.some(key => !key)) return false;
|
|
428
|
+
|
|
429
|
+
for (var i = 0; i < candidate.accessKeyCount; i++) {
|
|
430
|
+
var existing = existingKeys[i];
|
|
431
|
+
var required = requiredKeys[i];
|
|
432
|
+
if (!existing || existing.field !== required.field || existing.direction !== required.direction || existing.includeMissing !== required.includeMissing) return false;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
var existingFields = new Set(existingKeys.map(key => key.field));
|
|
436
|
+
return requiredKeys.slice(candidate.accessKeyCount).every(key => existingFields.has(key.field));
|
|
437
|
+
}
|
|
438
|
+
/**
|
|
439
|
+
* Normalizes an exemption list into a Set of lower-case resource names.
|
|
440
|
+
* Accepts an array, a Set, or a comma-separated string (the env var form).
|
|
441
|
+
* Surrounding whitespace and backticks are ignored; empty entries are dropped.
|
|
442
|
+
* @param {string|string[]|Set<string>} value
|
|
443
|
+
* @returns {Set<string>}
|
|
444
|
+
*/
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
function parseResourceList(value) {
|
|
448
|
+
var names = new Set();
|
|
449
|
+
var items = [];
|
|
450
|
+
|
|
451
|
+
if (Array.isArray(value) || value instanceof Set) {
|
|
452
|
+
items = [...value];
|
|
453
|
+
} else if (typeof value === "string") {
|
|
454
|
+
items = value.split(",");
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
for (var item of items) {
|
|
458
|
+
if (typeof item !== "string") continue;
|
|
459
|
+
var name = item.trim().replace(/^`+|`+$/g, "").trim().toLowerCase();
|
|
460
|
+
if (name) names.add(name);
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
return names;
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* Extracts the resource names a statement touches, lower-cased:
|
|
467
|
+
* - the last path segment of every FROM / JOIN keyspace, unquoted, so
|
|
468
|
+
* `default`:`bucket`.`scope`.`users` and users both yield "users";
|
|
469
|
+
* - every `type = '...'` literal, which is how legacy bucket-mode
|
|
470
|
+
* deployments distinguish resources that share one keyspace.
|
|
471
|
+
* Only simple identifiers are recognised; subqueries and expressions are skipped.
|
|
472
|
+
* @param {string} query
|
|
473
|
+
* @returns {Set<string>}
|
|
474
|
+
*/
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
function extractStatementResources(query) {
|
|
478
|
+
var names = new Set();
|
|
479
|
+
if (typeof query !== "string") return names;
|
|
480
|
+
var identifier = "(?:`(?:[^`]|``)+`|[A-Za-z_][\\w$-]*)";
|
|
481
|
+
var keyspacePattern = new RegExp("\\b(?:FROM|JOIN)\\s+(".concat(identifier, "(?:\\s*[.:]\\s*").concat(identifier, ")*)"), "gi");
|
|
482
|
+
var segmentPattern = new RegExp(identifier, "g");
|
|
483
|
+
|
|
484
|
+
for (var match of query.matchAll(keyspacePattern)) {
|
|
485
|
+
var segments = match[1].match(segmentPattern) || [];
|
|
486
|
+
var last = (segments[segments.length - 1] || "").trim();
|
|
487
|
+
var name = last.startsWith("`") ? last.slice(1, -1).replace(/``/g, "`") : last;
|
|
488
|
+
if (name) names.add(name.toLowerCase());
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
var typePattern = /(?:`type`|\btype)\s*=\s*'((?:[^']|'')*)'/gi;
|
|
492
|
+
|
|
493
|
+
for (var _match of query.matchAll(typePattern)) {
|
|
494
|
+
var _name = _match[1].replace(/''/g, "'").trim();
|
|
495
|
+
|
|
496
|
+
if (_name) names.add(_name.toLowerCase());
|
|
497
|
+
}
|
|
393
498
|
|
|
394
|
-
|
|
395
|
-
return indexStatement;
|
|
499
|
+
return names;
|
|
396
500
|
}
|
|
397
501
|
|
|
398
502
|
module.exports = {
|
|
399
503
|
buildCoveringIndex,
|
|
504
|
+
buildCoveringIndexDefinition,
|
|
505
|
+
indexSatisfiesCoveringDefinition,
|
|
506
|
+
parseResourceList,
|
|
507
|
+
extractStatementResources,
|
|
400
508
|
extractWhereFields,
|
|
401
509
|
extractSelectFields,
|
|
402
510
|
extractGroupByFields,
|
package/build/utility.test.js
CHANGED
|
@@ -8,7 +8,9 @@ var {
|
|
|
8
8
|
extractSelectFields,
|
|
9
9
|
extractGroupByFields,
|
|
10
10
|
extractOrderByFields,
|
|
11
|
-
generateHash
|
|
11
|
+
generateHash,
|
|
12
|
+
parseResourceList,
|
|
13
|
+
extractStatementResources
|
|
12
14
|
} = require("./utility"); // Test utilities
|
|
13
15
|
|
|
14
16
|
|
|
@@ -504,21 +506,21 @@ test("returns null when missing bucket option", () => {
|
|
|
504
506
|
});
|
|
505
507
|
assert.strictEqual(result, null);
|
|
506
508
|
});
|
|
507
|
-
test("
|
|
509
|
+
test("uses the default scope when scope option is missing", () => {
|
|
508
510
|
var query = "SELECT name FROM users WHERE status = 'active'";
|
|
509
511
|
var result = buildCoveringIndex(query, {
|
|
510
512
|
bucket: "test",
|
|
511
513
|
collection: "users"
|
|
512
514
|
});
|
|
513
|
-
assert.
|
|
515
|
+
assert.ok(result.includes("`test`.`_default`.`users`"));
|
|
514
516
|
});
|
|
515
|
-
test("
|
|
517
|
+
test("uses the default collection when collection option is missing", () => {
|
|
516
518
|
var query = "SELECT name FROM users WHERE status = 'active'";
|
|
517
519
|
var result = buildCoveringIndex(query, {
|
|
518
520
|
bucket: "test",
|
|
519
521
|
scope: "scope"
|
|
520
522
|
});
|
|
521
|
-
assert.
|
|
523
|
+
assert.ok(result.includes("`test`.`scope`.`_default`"));
|
|
522
524
|
});
|
|
523
525
|
test("generates unique index name from query hash", () => {
|
|
524
526
|
var query1 = "SELECT name FROM users WHERE status = 'active'";
|
|
@@ -825,6 +827,52 @@ test("field order: WHERE -> GROUP BY -> ORDER BY -> SELECT", () => {
|
|
|
825
827
|
|
|
826
828
|
assert.ok(createdAtIndex < emailIndex); // ORDER BY before SELECT
|
|
827
829
|
}); // ============================================
|
|
830
|
+
// Tests for parseResourceList / extractStatementResources
|
|
831
|
+
// ============================================
|
|
832
|
+
|
|
833
|
+
console.log("\n--- parseResourceList tests ---");
|
|
834
|
+
test("parses a comma-separated string, trimming, unquoting and lower-casing", () => {
|
|
835
|
+
var result = parseResourceList(" Users, `Audit_Log` ,, sessions ");
|
|
836
|
+
assert.deepStrictEqual([...result].sort(), ["audit_log", "sessions", "users"]);
|
|
837
|
+
});
|
|
838
|
+
test("accepts arrays and Sets and ignores non-strings", () => {
|
|
839
|
+
assert.deepStrictEqual([...parseResourceList(["Users", 42, null, " logs "])].sort(), ["logs", "users"]);
|
|
840
|
+
assert.deepStrictEqual([...parseResourceList(new Set(["A", "b"]))].sort(), ["a", "b"]);
|
|
841
|
+
});
|
|
842
|
+
test("returns an empty set for undefined, empty or unsupported input", () => {
|
|
843
|
+
assert.strictEqual(parseResourceList(undefined).size, 0);
|
|
844
|
+
assert.strictEqual(parseResourceList("").size, 0);
|
|
845
|
+
assert.strictEqual(parseResourceList(", ,").size, 0);
|
|
846
|
+
assert.strictEqual(parseResourceList({
|
|
847
|
+
users: true
|
|
848
|
+
}).size, 0);
|
|
849
|
+
});
|
|
850
|
+
console.log("\n--- extractStatementResources tests ---");
|
|
851
|
+
test("extracts a bare FROM keyspace", () => {
|
|
852
|
+
assert.deepStrictEqual([...extractStatementResources("SELECT name FROM users WHERE x = 1")], ["users"]);
|
|
853
|
+
});
|
|
854
|
+
test("extracts the last segment of a qualified or quoted keyspace", () => {
|
|
855
|
+
assert.deepStrictEqual([...extractStatementResources("SELECT name FROM `default`:`saturn`.`app`.`Users` AS u")], ["users"]);
|
|
856
|
+
assert.deepStrictEqual([...extractStatementResources("SELECT name FROM saturn.app.logs")], ["logs"]);
|
|
857
|
+
});
|
|
858
|
+
test("extracts JOIN keyspaces alongside FROM", () => {
|
|
859
|
+
var result = extractStatementResources("SELECT u.name FROM users AS u JOIN `orders` AS o ON o.user_id = META(u).id");
|
|
860
|
+
assert.deepStrictEqual([...result].sort(), ["orders", "users"]);
|
|
861
|
+
});
|
|
862
|
+
test("extracts type literals used by legacy bucket mode", () => {
|
|
863
|
+
var result = extractStatementResources("SELECT actor FROM `saturn` WHERE type = 'Audit_Log' AND deleted = false");
|
|
864
|
+
assert.deepStrictEqual([...result].sort(), ["audit_log", "saturn"]);
|
|
865
|
+
assert.ok(extractStatementResources("SELECT a FROM b WHERE `type`='x'").has("x"));
|
|
866
|
+
assert.ok(extractStatementResources("SELECT a FROM b WHERE type = 'it''s'").has("it's"));
|
|
867
|
+
});
|
|
868
|
+
test("does not treat META() arguments, subqueries or column names as keyspaces", () => {
|
|
869
|
+
var result = extractStatementResources("SELECT META(`users`).id AS id, from_date FROM (SELECT * FROM users) AS sub WHERE status = 'from'");
|
|
870
|
+
assert.deepStrictEqual([...result], ["users"]);
|
|
871
|
+
});
|
|
872
|
+
test("handles non-string input", () => {
|
|
873
|
+
assert.strictEqual(extractStatementResources(undefined).size, 0);
|
|
874
|
+
assert.strictEqual(extractStatementResources(null).size, 0);
|
|
875
|
+
}); // ============================================
|
|
828
876
|
// Summary
|
|
829
877
|
// ============================================
|
|
830
878
|
|
package/package.json
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "spice-couchbase",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.62",
|
|
4
4
|
"description": "",
|
|
5
5
|
"main": "build/index.js",
|
|
6
6
|
"scripts": {
|
|
7
7
|
"test": "node src/utility.test.js",
|
|
8
|
+
"test:auto-optimize": "node --test src/index.auto-optimize.test.js",
|
|
8
9
|
"test:query-consistency": "node src/index.query-consistency.test.js",
|
|
9
10
|
"test:count-query": "node src/index.count-query.test.js"
|
|
10
11
|
},
|