s3db.js 13.6.1 → 14.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -15
- package/dist/s3db.cjs +72446 -39022
- package/dist/s3db.cjs.map +1 -1
- package/dist/s3db.es.js +72172 -38790
- package/dist/s3db.es.js.map +1 -1
- package/mcp/lib/base-handler.js +157 -0
- package/mcp/lib/handlers/connection-handler.js +280 -0
- package/mcp/lib/handlers/query-handler.js +533 -0
- package/mcp/lib/handlers/resource-handler.js +428 -0
- package/mcp/lib/tool-registry.js +336 -0
- package/mcp/lib/tools/connection-tools.js +161 -0
- package/mcp/lib/tools/query-tools.js +267 -0
- package/mcp/lib/tools/resource-tools.js +404 -0
- package/package.json +85 -50
- package/src/clients/memory-client.class.js +346 -191
- package/src/clients/memory-storage.class.js +300 -84
- package/src/clients/s3-client.class.js +7 -6
- package/src/concerns/geo-encoding.js +19 -2
- package/src/concerns/ip.js +59 -9
- package/src/concerns/money.js +8 -1
- package/src/concerns/password-hashing.js +49 -8
- package/src/concerns/plugin-storage.js +186 -18
- package/src/concerns/storage-drivers/filesystem-driver.js +284 -0
- package/src/database.class.js +139 -29
- package/src/errors.js +332 -42
- package/src/plugins/api/auth/oidc-auth.js +66 -17
- package/src/plugins/api/auth/strategies/base-strategy.class.js +74 -0
- package/src/plugins/api/auth/strategies/factory.class.js +63 -0
- package/src/plugins/api/auth/strategies/global-strategy.class.js +44 -0
- package/src/plugins/api/auth/strategies/path-based-strategy.class.js +83 -0
- package/src/plugins/api/auth/strategies/path-rules-strategy.class.js +118 -0
- package/src/plugins/api/concerns/failban-manager.js +106 -57
- package/src/plugins/api/concerns/route-context.js +601 -0
- package/src/plugins/api/index.js +168 -40
- package/src/plugins/api/routes/auth-routes.js +198 -30
- package/src/plugins/api/routes/resource-routes.js +19 -4
- package/src/plugins/api/server/health-manager.class.js +163 -0
- package/src/plugins/api/server/middleware-chain.class.js +310 -0
- package/src/plugins/api/server/router.class.js +472 -0
- package/src/plugins/api/server.js +280 -1303
- package/src/plugins/api/utils/custom-routes.js +17 -5
- package/src/plugins/api/utils/guards.js +76 -17
- package/src/plugins/api/utils/openapi-generator-cached.class.js +133 -0
- package/src/plugins/api/utils/openapi-generator.js +7 -6
- package/src/plugins/audit.plugin.js +30 -8
- package/src/plugins/backup.plugin.js +110 -14
- package/src/plugins/cache/cache.class.js +22 -5
- package/src/plugins/cache/filesystem-cache.class.js +116 -19
- package/src/plugins/cache/memory-cache.class.js +211 -57
- package/src/plugins/cache/multi-tier-cache.class.js +371 -0
- package/src/plugins/cache/partition-aware-filesystem-cache.class.js +168 -47
- package/src/plugins/cache/redis-cache.class.js +552 -0
- package/src/plugins/cache/s3-cache.class.js +17 -8
- package/src/plugins/cache.plugin.js +176 -61
- package/src/plugins/cloud-inventory/drivers/alibaba-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/aws-driver.js +60 -29
- package/src/plugins/cloud-inventory/drivers/azure-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/base-driver.js +16 -2
- package/src/plugins/cloud-inventory/drivers/cloudflare-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/digitalocean-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/hetzner-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/linode-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/mongodb-atlas-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/vultr-driver.js +8 -1
- package/src/plugins/cloud-inventory/index.js +29 -8
- package/src/plugins/cloud-inventory/registry.js +64 -42
- package/src/plugins/cloud-inventory.plugin.js +240 -138
- package/src/plugins/concerns/plugin-dependencies.js +54 -0
- package/src/plugins/concerns/resource-names.js +100 -0
- package/src/plugins/consumers/index.js +10 -2
- package/src/plugins/consumers/sqs-consumer.js +12 -2
- package/src/plugins/cookie-farm-suite.plugin.js +278 -0
- package/src/plugins/cookie-farm.errors.js +73 -0
- package/src/plugins/cookie-farm.plugin.js +869 -0
- package/src/plugins/costs.plugin.js +7 -1
- package/src/plugins/eventual-consistency/analytics.js +94 -19
- package/src/plugins/eventual-consistency/config.js +15 -7
- package/src/plugins/eventual-consistency/consolidation.js +29 -11
- package/src/plugins/eventual-consistency/garbage-collection.js +3 -1
- package/src/plugins/eventual-consistency/helpers.js +39 -14
- package/src/plugins/eventual-consistency/install.js +21 -2
- package/src/plugins/eventual-consistency/utils.js +32 -10
- package/src/plugins/fulltext.plugin.js +38 -11
- package/src/plugins/geo.plugin.js +61 -9
- package/src/plugins/identity/concerns/config.js +61 -0
- package/src/plugins/identity/concerns/mfa-manager.js +15 -2
- package/src/plugins/identity/concerns/rate-limit.js +124 -0
- package/src/plugins/identity/concerns/resource-schemas.js +9 -1
- package/src/plugins/identity/concerns/token-generator.js +29 -4
- package/src/plugins/identity/drivers/auth-driver.interface.js +76 -0
- package/src/plugins/identity/drivers/client-credentials-driver.js +127 -0
- package/src/plugins/identity/drivers/index.js +18 -0
- package/src/plugins/identity/drivers/password-driver.js +122 -0
- package/src/plugins/identity/email-service.js +17 -2
- package/src/plugins/identity/index.js +413 -69
- package/src/plugins/identity/oauth2-server.js +413 -30
- package/src/plugins/identity/oidc-discovery.js +16 -8
- package/src/plugins/identity/rsa-keys.js +115 -35
- package/src/plugins/identity/server.js +166 -45
- package/src/plugins/identity/session-manager.js +53 -7
- package/src/plugins/identity/ui/pages/mfa-verification.js +17 -15
- package/src/plugins/identity/ui/routes.js +363 -255
- package/src/plugins/importer/index.js +153 -20
- package/src/plugins/index.js +9 -2
- package/src/plugins/kubernetes-inventory/index.js +6 -0
- package/src/plugins/kubernetes-inventory/k8s-driver.js +867 -0
- package/src/plugins/kubernetes-inventory/resource-types.js +274 -0
- package/src/plugins/kubernetes-inventory.plugin.js +980 -0
- package/src/plugins/metrics.plugin.js +64 -16
- package/src/plugins/ml/base-model.class.js +25 -15
- package/src/plugins/ml/regression-model.class.js +1 -1
- package/src/plugins/ml.errors.js +57 -25
- package/src/plugins/ml.plugin.js +28 -4
- package/src/plugins/namespace.js +210 -0
- package/src/plugins/plugin.class.js +180 -8
- package/src/plugins/puppeteer/console-monitor.js +729 -0
- package/src/plugins/puppeteer/cookie-manager.js +492 -0
- package/src/plugins/puppeteer/network-monitor.js +816 -0
- package/src/plugins/puppeteer/performance-manager.js +746 -0
- package/src/plugins/puppeteer/proxy-manager.js +478 -0
- package/src/plugins/puppeteer/stealth-manager.js +556 -0
- package/src/plugins/puppeteer.errors.js +81 -0
- package/src/plugins/puppeteer.plugin.js +1327 -0
- package/src/plugins/queue-consumer.plugin.js +69 -14
- package/src/plugins/recon/behaviors/uptime-behavior.js +691 -0
- package/src/plugins/recon/concerns/command-runner.js +148 -0
- package/src/plugins/recon/concerns/diff-detector.js +372 -0
- package/src/plugins/recon/concerns/fingerprint-builder.js +307 -0
- package/src/plugins/recon/concerns/process-manager.js +338 -0
- package/src/plugins/recon/concerns/report-generator.js +478 -0
- package/src/plugins/recon/concerns/security-analyzer.js +571 -0
- package/src/plugins/recon/concerns/target-normalizer.js +68 -0
- package/src/plugins/recon/config/defaults.js +321 -0
- package/src/plugins/recon/config/resources.js +370 -0
- package/src/plugins/recon/index.js +778 -0
- package/src/plugins/recon/managers/dependency-manager.js +174 -0
- package/src/plugins/recon/managers/scheduler-manager.js +179 -0
- package/src/plugins/recon/managers/storage-manager.js +745 -0
- package/src/plugins/recon/managers/target-manager.js +274 -0
- package/src/plugins/recon/stages/asn-stage.js +314 -0
- package/src/plugins/recon/stages/certificate-stage.js +84 -0
- package/src/plugins/recon/stages/dns-stage.js +107 -0
- package/src/plugins/recon/stages/dnsdumpster-stage.js +362 -0
- package/src/plugins/recon/stages/fingerprint-stage.js +71 -0
- package/src/plugins/recon/stages/google-dorks-stage.js +440 -0
- package/src/plugins/recon/stages/http-stage.js +89 -0
- package/src/plugins/recon/stages/latency-stage.js +148 -0
- package/src/plugins/recon/stages/massdns-stage.js +302 -0
- package/src/plugins/recon/stages/osint-stage.js +1373 -0
- package/src/plugins/recon/stages/ports-stage.js +169 -0
- package/src/plugins/recon/stages/screenshot-stage.js +94 -0
- package/src/plugins/recon/stages/secrets-stage.js +514 -0
- package/src/plugins/recon/stages/subdomains-stage.js +295 -0
- package/src/plugins/recon/stages/tls-audit-stage.js +78 -0
- package/src/plugins/recon/stages/vulnerability-stage.js +78 -0
- package/src/plugins/recon/stages/web-discovery-stage.js +113 -0
- package/src/plugins/recon/stages/whois-stage.js +349 -0
- package/src/plugins/recon.plugin.js +75 -0
- package/src/plugins/recon.plugin.js.backup +2635 -0
- package/src/plugins/relation.errors.js +87 -14
- package/src/plugins/replicator.plugin.js +514 -137
- package/src/plugins/replicators/base-replicator.class.js +89 -1
- package/src/plugins/replicators/bigquery-replicator.class.js +66 -22
- package/src/plugins/replicators/dynamodb-replicator.class.js +22 -15
- package/src/plugins/replicators/mongodb-replicator.class.js +22 -15
- package/src/plugins/replicators/mysql-replicator.class.js +52 -17
- package/src/plugins/replicators/planetscale-replicator.class.js +30 -4
- package/src/plugins/replicators/postgres-replicator.class.js +62 -27
- package/src/plugins/replicators/s3db-replicator.class.js +25 -18
- package/src/plugins/replicators/schema-sync.helper.js +3 -3
- package/src/plugins/replicators/sqs-replicator.class.js +8 -2
- package/src/plugins/replicators/turso-replicator.class.js +23 -3
- package/src/plugins/replicators/webhook-replicator.class.js +42 -4
- package/src/plugins/s3-queue.plugin.js +464 -65
- package/src/plugins/scheduler.plugin.js +20 -6
- package/src/plugins/state-machine.plugin.js +40 -9
- package/src/plugins/tfstate/base-driver.js +28 -4
- package/src/plugins/tfstate/errors.js +65 -10
- package/src/plugins/tfstate/filesystem-driver.js +52 -8
- package/src/plugins/tfstate/index.js +163 -90
- package/src/plugins/tfstate/s3-driver.js +64 -6
- package/src/plugins/ttl.plugin.js +72 -17
- package/src/plugins/vector/distances.js +18 -12
- package/src/plugins/vector/kmeans.js +26 -4
- package/src/resource.class.js +115 -19
- package/src/testing/factory.class.js +20 -3
- package/src/testing/seeder.class.js +7 -1
- package/src/clients/memory-client.md +0 -917
- package/src/plugins/cloud-inventory/drivers/mock-drivers.js +0 -449
|
@@ -396,7 +396,13 @@
|
|
|
396
396
|
* const hourlyCost = currentCosts - hourStartCosts;
|
|
397
397
|
*
|
|
398
398
|
* if (hourlyCost > HOURLY_COST_LIMIT) {
|
|
399
|
-
* throw new
|
|
399
|
+
* throw new PluginError('Hourly cost limit exceeded', {
|
|
400
|
+
* pluginName: 'CostsPlugin',
|
|
401
|
+
* operation: 'rateLimiter',
|
|
402
|
+
* statusCode: 429,
|
|
403
|
+
* retriable: true,
|
|
404
|
+
* suggestion: 'Delay the operation until costs fall below the configured threshold.'
|
|
405
|
+
* });
|
|
400
406
|
* }
|
|
401
407
|
*
|
|
402
408
|
* // Proceed with operation
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import tryFn from "../../concerns/try-fn.js";
|
|
7
7
|
import { groupByCohort, ensureCohortHours } from "./utils.js";
|
|
8
|
+
import { PluginError, AnalyticsNotEnabledError } from '../../errors.js';
|
|
8
9
|
|
|
9
10
|
/**
|
|
10
11
|
* Update analytics with consolidated transactions
|
|
@@ -20,13 +21,19 @@ export async function updateAnalytics(transactions, analyticsResource, config) {
|
|
|
20
21
|
// CRITICAL VALIDATION: Ensure field is set in config
|
|
21
22
|
// This can be undefined due to race conditions when multiple handlers share config
|
|
22
23
|
if (!config.field) {
|
|
23
|
-
throw new
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
24
|
+
throw new PluginError('config.field is undefined in updateAnalytics()', {
|
|
25
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
26
|
+
operation: 'updateAnalytics',
|
|
27
|
+
statusCode: 500,
|
|
28
|
+
retriable: false,
|
|
29
|
+
suggestion: 'Ensure each field handler uses its own configuration object when invoking analytics updates.',
|
|
30
|
+
context: {
|
|
31
|
+
resource: config.resource,
|
|
32
|
+
field: config.field,
|
|
33
|
+
transactions: transactions.length,
|
|
34
|
+
analyticsResource: analyticsResource?.name || 'unknown'
|
|
35
|
+
}
|
|
36
|
+
});
|
|
30
37
|
}
|
|
31
38
|
|
|
32
39
|
if (config.verbose) {
|
|
@@ -93,9 +100,16 @@ export async function updateAnalytics(transactions, analyticsResource, config) {
|
|
|
93
100
|
}
|
|
94
101
|
);
|
|
95
102
|
// Re-throw to prevent silent failures
|
|
96
|
-
throw new
|
|
97
|
-
|
|
98
|
-
|
|
103
|
+
throw new PluginError(`Analytics update failed for ${config.resource}.${config.field}: ${error.message}`, {
|
|
104
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
105
|
+
operation: 'updateAnalytics',
|
|
106
|
+
statusCode: 500,
|
|
107
|
+
retriable: true,
|
|
108
|
+
suggestion: 'Check the console logs above for the failing transaction and fix the reducer or analytics configuration.',
|
|
109
|
+
resource: config.resource,
|
|
110
|
+
field: config.field,
|
|
111
|
+
original: error
|
|
112
|
+
});
|
|
99
113
|
}
|
|
100
114
|
}
|
|
101
115
|
|
|
@@ -447,16 +461,31 @@ export async function getAnalytics(resourceName, field, options, fieldHandlers)
|
|
|
447
461
|
// Get handler for this resource/field combination
|
|
448
462
|
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
449
463
|
if (!resourceHandlers) {
|
|
450
|
-
throw new
|
|
464
|
+
throw new PluginError(`No eventual consistency configured for resource: ${resourceName}`, {
|
|
465
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
466
|
+
operation: 'getAnalytics',
|
|
467
|
+
statusCode: 404,
|
|
468
|
+
retriable: false,
|
|
469
|
+
suggestion: 'Ensure the resource is registered under EventualConsistencyPlugin resources.',
|
|
470
|
+
resourceName
|
|
471
|
+
});
|
|
451
472
|
}
|
|
452
473
|
|
|
453
474
|
const handler = resourceHandlers.get(field);
|
|
454
475
|
if (!handler) {
|
|
455
|
-
throw new
|
|
476
|
+
throw new PluginError(`No eventual consistency configured for field: ${resourceName}.${field}`, {
|
|
477
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
478
|
+
operation: 'getAnalytics',
|
|
479
|
+
statusCode: 404,
|
|
480
|
+
retriable: false,
|
|
481
|
+
suggestion: 'Add the field to EventualConsistencyPlugin resources configuration.',
|
|
482
|
+
resourceName,
|
|
483
|
+
field
|
|
484
|
+
});
|
|
456
485
|
}
|
|
457
486
|
|
|
458
487
|
if (!handler.analyticsResource) {
|
|
459
|
-
throw new
|
|
488
|
+
throw new AnalyticsNotEnabledError({ resourceName, field, pluginName: 'EventualConsistencyPlugin' });
|
|
460
489
|
}
|
|
461
490
|
|
|
462
491
|
const { period = 'day', date, startDate, endDate, month, year, breakdown = false, recordId } = options;
|
|
@@ -898,16 +927,39 @@ export async function getTopRecords(resourceName, field, options, fieldHandlers)
|
|
|
898
927
|
// Get handler for this resource/field combination
|
|
899
928
|
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
900
929
|
if (!resourceHandlers) {
|
|
901
|
-
throw new
|
|
930
|
+
throw new PluginError(`No eventual consistency configured for resource: ${resourceName}`, {
|
|
931
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
932
|
+
operation: 'getTopRecords',
|
|
933
|
+
statusCode: 404,
|
|
934
|
+
retriable: false,
|
|
935
|
+
suggestion: 'Add the resource to EventualConsistencyPlugin resources configuration.',
|
|
936
|
+
resourceName
|
|
937
|
+
});
|
|
902
938
|
}
|
|
903
939
|
|
|
904
940
|
const handler = resourceHandlers.get(field);
|
|
905
941
|
if (!handler) {
|
|
906
|
-
throw new
|
|
942
|
+
throw new PluginError(`No eventual consistency configured for field: ${resourceName}.${field}`, {
|
|
943
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
944
|
+
operation: 'getTopRecords',
|
|
945
|
+
statusCode: 404,
|
|
946
|
+
retriable: false,
|
|
947
|
+
suggestion: 'Ensure the field is configured for eventual consistency before querying analytics.',
|
|
948
|
+
resourceName,
|
|
949
|
+
field
|
|
950
|
+
});
|
|
907
951
|
}
|
|
908
952
|
|
|
909
953
|
if (!handler.transactionResource) {
|
|
910
|
-
throw new
|
|
954
|
+
throw new PluginError('Transaction resource not initialized', {
|
|
955
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
956
|
+
operation: 'getTopRecords',
|
|
957
|
+
statusCode: 500,
|
|
958
|
+
retriable: false,
|
|
959
|
+
suggestion: 'Verify plugin installation completed successfully and transaction resources were created.',
|
|
960
|
+
resourceName,
|
|
961
|
+
field
|
|
962
|
+
});
|
|
911
963
|
}
|
|
912
964
|
|
|
913
965
|
const { period = 'day', date, metric = 'transactionCount', limit = 10 } = options;
|
|
@@ -1254,16 +1306,39 @@ export async function getRawEvents(resourceName, field, options, fieldHandlers)
|
|
|
1254
1306
|
// Get handler for this resource/field combination
|
|
1255
1307
|
const resourceHandlers = fieldHandlers.get(resourceName);
|
|
1256
1308
|
if (!resourceHandlers) {
|
|
1257
|
-
throw new
|
|
1309
|
+
throw new PluginError(`No eventual consistency configured for resource: ${resourceName}`, {
|
|
1310
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
1311
|
+
operation: 'getRawEvents',
|
|
1312
|
+
statusCode: 404,
|
|
1313
|
+
retriable: false,
|
|
1314
|
+
suggestion: 'Add the resource under EventualConsistencyPlugin configuration to retrieve raw events.',
|
|
1315
|
+
resourceName
|
|
1316
|
+
});
|
|
1258
1317
|
}
|
|
1259
1318
|
|
|
1260
1319
|
const handler = resourceHandlers.get(field);
|
|
1261
1320
|
if (!handler) {
|
|
1262
|
-
throw new
|
|
1321
|
+
throw new PluginError(`No eventual consistency configured for field: ${resourceName}.${field}`, {
|
|
1322
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
1323
|
+
operation: 'getRawEvents',
|
|
1324
|
+
statusCode: 404,
|
|
1325
|
+
retriable: false,
|
|
1326
|
+
suggestion: 'Ensure the field is included in EventualConsistencyPlugin configuration.',
|
|
1327
|
+
resourceName,
|
|
1328
|
+
field
|
|
1329
|
+
});
|
|
1263
1330
|
}
|
|
1264
1331
|
|
|
1265
1332
|
if (!handler.transactionResource) {
|
|
1266
|
-
throw new
|
|
1333
|
+
throw new PluginError('Transaction resource not initialized', {
|
|
1334
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
1335
|
+
operation: 'getRawEvents',
|
|
1336
|
+
statusCode: 500,
|
|
1337
|
+
retriable: false,
|
|
1338
|
+
suggestion: 'Verify plugin installation completed successfully and transaction resources were created.',
|
|
1339
|
+
resourceName,
|
|
1340
|
+
field
|
|
1341
|
+
});
|
|
1267
1342
|
}
|
|
1268
1343
|
|
|
1269
1344
|
const {
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Configuration handling for EventualConsistencyPlugin
|
|
3
3
|
* @module eventual-consistency/config
|
|
4
4
|
*/
|
|
5
|
+
import { PluginError } from '../../errors.js';
|
|
5
6
|
|
|
6
7
|
/**
|
|
7
8
|
* Create default configuration with options
|
|
@@ -96,17 +97,24 @@ export function createConfig(options, detectedTimezone) {
|
|
|
96
97
|
*/
|
|
97
98
|
export function validateResourcesConfig(resources) {
|
|
98
99
|
if (!resources || typeof resources !== 'object') {
|
|
99
|
-
throw new
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
100
|
+
throw new PluginError("EventualConsistencyPlugin requires a 'resources' option", {
|
|
101
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
102
|
+
operation: 'validateResourcesConfig',
|
|
103
|
+
statusCode: 400,
|
|
104
|
+
retriable: false,
|
|
105
|
+
suggestion: "Provide resources configuration, e.g., { resources: { urls: ['clicks', 'views'] } }"
|
|
106
|
+
});
|
|
103
107
|
}
|
|
104
108
|
|
|
105
109
|
for (const [resourceName, fields] of Object.entries(resources)) {
|
|
106
110
|
if (!Array.isArray(fields)) {
|
|
107
|
-
throw new
|
|
108
|
-
|
|
109
|
-
|
|
111
|
+
throw new PluginError(`EventualConsistencyPlugin resources.${resourceName} must be an array of field names`, {
|
|
112
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
113
|
+
operation: 'validateResourcesConfig',
|
|
114
|
+
statusCode: 400,
|
|
115
|
+
retriable: false,
|
|
116
|
+
suggestion: 'Ensure each resource entry maps to an array of field names (e.g., resources.users = ["logins", "visits"]).'
|
|
117
|
+
});
|
|
110
118
|
}
|
|
111
119
|
}
|
|
112
120
|
}
|
|
@@ -7,6 +7,7 @@ import tryFn from "../../concerns/try-fn.js";
|
|
|
7
7
|
import { PromisePool } from "@supercharge/promise-pool";
|
|
8
8
|
import { idGenerator } from "../../concerns/id.js";
|
|
9
9
|
import { getCohortInfo, createSyntheticSetTransaction, ensureCohortHour } from "./utils.js";
|
|
10
|
+
import { PluginError } from '../../errors.js';
|
|
10
11
|
|
|
11
12
|
/**
|
|
12
13
|
* Start consolidation timer for a handler
|
|
@@ -696,12 +697,17 @@ export async function consolidateRecord(
|
|
|
696
697
|
return consolidatedValue;
|
|
697
698
|
} finally {
|
|
698
699
|
// Always release the lock
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
700
|
+
if (lock) {
|
|
701
|
+
const [lockReleased, lockReleaseErr] = await tryFn(() =>
|
|
702
|
+
storage.releaseLock(lock)
|
|
703
|
+
);
|
|
702
704
|
|
|
703
|
-
|
|
704
|
-
|
|
705
|
+
if (!lockReleased && config.verbose) {
|
|
706
|
+
console.warn(
|
|
707
|
+
`[EventualConsistency] Failed to release lock ${lock?.name || lockKey}:`,
|
|
708
|
+
lockReleaseErr?.message
|
|
709
|
+
);
|
|
710
|
+
}
|
|
705
711
|
}
|
|
706
712
|
}
|
|
707
713
|
}
|
|
@@ -853,7 +859,14 @@ export async function recalculateRecord(
|
|
|
853
859
|
if (config.verbose) {
|
|
854
860
|
console.log(`[EventualConsistency] Recalculate lock for ${originalId} already held, skipping`);
|
|
855
861
|
}
|
|
856
|
-
throw new
|
|
862
|
+
throw new PluginError(`Cannot recalculate ${originalId}: lock already held by another worker`, {
|
|
863
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
864
|
+
operation: 'recalculateRecord',
|
|
865
|
+
statusCode: 409,
|
|
866
|
+
retriable: true,
|
|
867
|
+
suggestion: 'Retry after the other worker releases the lock or increase lock TTL if necessary.',
|
|
868
|
+
recordId: originalId
|
|
869
|
+
});
|
|
857
870
|
}
|
|
858
871
|
|
|
859
872
|
try {
|
|
@@ -997,12 +1010,17 @@ export async function recalculateRecord(
|
|
|
997
1010
|
return consolidatedValue;
|
|
998
1011
|
} finally {
|
|
999
1012
|
// Always release the lock
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1013
|
+
if (lock) {
|
|
1014
|
+
const [lockReleased, lockReleaseErr] = await tryFn(() =>
|
|
1015
|
+
storage.releaseLock(lock)
|
|
1016
|
+
);
|
|
1003
1017
|
|
|
1004
|
-
|
|
1005
|
-
|
|
1018
|
+
if (!lockReleased && config.verbose) {
|
|
1019
|
+
console.warn(
|
|
1020
|
+
`[EventualConsistency] Failed to release recalculate lock ${lock?.name || lockKey}:`,
|
|
1021
|
+
lockReleaseErr?.message
|
|
1022
|
+
);
|
|
1023
|
+
}
|
|
1006
1024
|
}
|
|
1007
1025
|
}
|
|
1008
1026
|
}
|
|
@@ -119,6 +119,8 @@ export async function runGarbageCollection(transactionResource, storage, config,
|
|
|
119
119
|
}
|
|
120
120
|
} finally {
|
|
121
121
|
// Always release GC lock
|
|
122
|
-
|
|
122
|
+
if (lock) {
|
|
123
|
+
await tryFn(() => storage.releaseLock(lock));
|
|
124
|
+
}
|
|
123
125
|
}
|
|
124
126
|
}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { idGenerator } from "../../concerns/id.js";
|
|
7
7
|
import tryFn from "../../concerns/try-fn.js";
|
|
8
8
|
import { getCohortInfo, resolveFieldAndPlugin } from "./utils.js";
|
|
9
|
+
import { PluginError } from '../../errors.js';
|
|
9
10
|
|
|
10
11
|
/**
|
|
11
12
|
* Add helper methods to resources
|
|
@@ -155,17 +156,27 @@ export function addHelperMethods(resource, plugin, config) {
|
|
|
155
156
|
// Signature: consolidate(id, field)
|
|
156
157
|
resource.consolidate = async (id, field) => {
|
|
157
158
|
if (!field) {
|
|
158
|
-
throw new
|
|
159
|
+
throw new PluginError('Field parameter is required: consolidate(id, field)', {
|
|
160
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
161
|
+
operation: 'resource.consolidate',
|
|
162
|
+
statusCode: 400,
|
|
163
|
+
retriable: false,
|
|
164
|
+
suggestion: 'Invoke consolidate with both id and field parameters.'
|
|
165
|
+
});
|
|
159
166
|
}
|
|
160
167
|
|
|
161
168
|
const handler = resource._eventualConsistencyPlugins[field];
|
|
162
169
|
|
|
163
170
|
if (!handler) {
|
|
164
171
|
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
|
|
165
|
-
throw new
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
172
|
+
throw new PluginError(`No eventual consistency plugin found for field "${field}"`, {
|
|
173
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
174
|
+
operation: 'resource.consolidate',
|
|
175
|
+
statusCode: 404,
|
|
176
|
+
retriable: false,
|
|
177
|
+
suggestion: `Available fields: ${availableFields}`,
|
|
178
|
+
field
|
|
179
|
+
});
|
|
169
180
|
}
|
|
170
181
|
|
|
171
182
|
return await plugin._consolidateWithHandler(handler, id);
|
|
@@ -178,10 +189,14 @@ export function addHelperMethods(resource, plugin, config) {
|
|
|
178
189
|
|
|
179
190
|
if (!handler) {
|
|
180
191
|
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
|
|
181
|
-
throw new
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
192
|
+
throw new PluginError(`No eventual consistency plugin found for field "${field}"`, {
|
|
193
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
194
|
+
operation: 'resource.getConsolidatedValue',
|
|
195
|
+
statusCode: 404,
|
|
196
|
+
retriable: false,
|
|
197
|
+
suggestion: `Available fields: ${availableFields}`,
|
|
198
|
+
field
|
|
199
|
+
});
|
|
185
200
|
}
|
|
186
201
|
|
|
187
202
|
return await plugin._getConsolidatedValueWithHandler(handler, id, options);
|
|
@@ -191,17 +206,27 @@ export function addHelperMethods(resource, plugin, config) {
|
|
|
191
206
|
// Signature: recalculate(id, field)
|
|
192
207
|
resource.recalculate = async (id, field) => {
|
|
193
208
|
if (!field) {
|
|
194
|
-
throw new
|
|
209
|
+
throw new PluginError('Field parameter is required: recalculate(id, field)', {
|
|
210
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
211
|
+
operation: 'resource.recalculate',
|
|
212
|
+
statusCode: 400,
|
|
213
|
+
retriable: false,
|
|
214
|
+
suggestion: 'Invoke recalculate with both id and field parameters.'
|
|
215
|
+
});
|
|
195
216
|
}
|
|
196
217
|
|
|
197
218
|
const handler = resource._eventualConsistencyPlugins[field];
|
|
198
219
|
|
|
199
220
|
if (!handler) {
|
|
200
221
|
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
|
|
201
|
-
throw new
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
222
|
+
throw new PluginError(`No eventual consistency plugin found for field "${field}"`, {
|
|
223
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
224
|
+
operation: 'resource.recalculate',
|
|
225
|
+
statusCode: 404,
|
|
226
|
+
retriable: false,
|
|
227
|
+
suggestion: `Available fields: ${availableFields}`,
|
|
228
|
+
field
|
|
229
|
+
});
|
|
205
230
|
}
|
|
206
231
|
|
|
207
232
|
return await plugin._recalculateWithHandler(handler, id);
|
|
@@ -9,6 +9,7 @@ import { addHelperMethods } from "./helpers.js";
|
|
|
9
9
|
import { flushPendingTransactions } from "./transactions.js";
|
|
10
10
|
import { startConsolidationTimer } from "./consolidation.js";
|
|
11
11
|
import { startGarbageCollectionTimer } from "./garbage-collection.js";
|
|
12
|
+
import { PluginError } from '../../errors.js';
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
15
|
* Install plugin for all configured resources
|
|
@@ -115,7 +116,16 @@ export async function completeFieldSetup(handler, database, config, plugin) {
|
|
|
115
116
|
);
|
|
116
117
|
|
|
117
118
|
if (!ok && !database.resources[transactionResourceName]) {
|
|
118
|
-
throw new
|
|
119
|
+
throw new PluginError(`Failed to create transaction resource for ${resourceName}.${fieldName}`, {
|
|
120
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
121
|
+
operation: 'createTransactionResource',
|
|
122
|
+
statusCode: 500,
|
|
123
|
+
retriable: false,
|
|
124
|
+
suggestion: 'Verify database permissions and configuration for creating plugin resources.',
|
|
125
|
+
resourceName,
|
|
126
|
+
fieldName,
|
|
127
|
+
original: err
|
|
128
|
+
});
|
|
119
129
|
}
|
|
120
130
|
|
|
121
131
|
handler.transactionResource = ok ? transactionResource : database.resources[transactionResourceName];
|
|
@@ -200,7 +210,16 @@ async function createAnalyticsResource(handler, database, resourceName, fieldNam
|
|
|
200
210
|
);
|
|
201
211
|
|
|
202
212
|
if (!ok && !database.resources[analyticsResourceName]) {
|
|
203
|
-
throw new
|
|
213
|
+
throw new PluginError(`Failed to create analytics resource for ${resourceName}.${fieldName}`, {
|
|
214
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
215
|
+
operation: 'createAnalyticsResource',
|
|
216
|
+
statusCode: 500,
|
|
217
|
+
retriable: false,
|
|
218
|
+
suggestion: 'Verify database permissions and configuration for creating analytics resources.',
|
|
219
|
+
resourceName,
|
|
220
|
+
fieldName,
|
|
221
|
+
original: err
|
|
222
|
+
});
|
|
204
223
|
}
|
|
205
224
|
|
|
206
225
|
handler.analyticsResource = ok ? analyticsResource : database.resources[analyticsResourceName];
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { PluginError } from '../../errors.js';
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* Utility functions for EventualConsistencyPlugin
|
|
3
5
|
* @module eventual-consistency/utils
|
|
@@ -291,7 +293,13 @@ export function validateNestedPath(resource, fieldPath) {
|
|
|
291
293
|
*/
|
|
292
294
|
export function resolveFieldAndPlugin(resource, field, value) {
|
|
293
295
|
if (!resource._eventualConsistencyPlugins) {
|
|
294
|
-
throw new
|
|
296
|
+
throw new PluginError('No eventual consistency plugins configured for this resource', {
|
|
297
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
298
|
+
operation: 'resolveFieldAndPlugin',
|
|
299
|
+
statusCode: 404,
|
|
300
|
+
retriable: false,
|
|
301
|
+
suggestion: 'Configure EventualConsistencyPlugin resources before using helper methods.'
|
|
302
|
+
});
|
|
295
303
|
}
|
|
296
304
|
|
|
297
305
|
// Check if field contains dot notation (nested path)
|
|
@@ -299,7 +307,13 @@ export function resolveFieldAndPlugin(resource, field, value) {
|
|
|
299
307
|
const validation = validateNestedPath(resource, field);
|
|
300
308
|
|
|
301
309
|
if (!validation.valid) {
|
|
302
|
-
throw new
|
|
310
|
+
throw new PluginError(validation.error, {
|
|
311
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
312
|
+
operation: 'resolveFieldAndPlugin',
|
|
313
|
+
statusCode: 400,
|
|
314
|
+
retriable: false,
|
|
315
|
+
suggestion: 'Ensure nested field paths exist on the resource before using dot notation.'
|
|
316
|
+
});
|
|
303
317
|
}
|
|
304
318
|
|
|
305
319
|
// Get plugin for root field
|
|
@@ -308,10 +322,14 @@ export function resolveFieldAndPlugin(resource, field, value) {
|
|
|
308
322
|
|
|
309
323
|
if (!fieldPlugin) {
|
|
310
324
|
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
|
|
311
|
-
throw new
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
325
|
+
throw new PluginError(`No eventual consistency plugin found for root field "${rootField}"`, {
|
|
326
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
327
|
+
operation: 'resolveFieldAndPlugin',
|
|
328
|
+
statusCode: 404,
|
|
329
|
+
retriable: false,
|
|
330
|
+
suggestion: `Available fields: ${availableFields}`,
|
|
331
|
+
field: rootField
|
|
332
|
+
});
|
|
315
333
|
}
|
|
316
334
|
|
|
317
335
|
return {
|
|
@@ -327,10 +345,14 @@ export function resolveFieldAndPlugin(resource, field, value) {
|
|
|
327
345
|
|
|
328
346
|
if (!fieldPlugin) {
|
|
329
347
|
const availableFields = Object.keys(resource._eventualConsistencyPlugins).join(', ');
|
|
330
|
-
throw new
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
348
|
+
throw new PluginError(`No eventual consistency plugin found for field "${field}"`, {
|
|
349
|
+
pluginName: 'EventualConsistencyPlugin',
|
|
350
|
+
operation: 'resolveFieldAndPlugin',
|
|
351
|
+
statusCode: 404,
|
|
352
|
+
retriable: false,
|
|
353
|
+
suggestion: `Available fields: ${availableFields}`,
|
|
354
|
+
field
|
|
355
|
+
});
|
|
334
356
|
}
|
|
335
357
|
|
|
336
358
|
return { field, fieldPath: field, value, plugin: fieldPlugin };
|
|
@@ -485,11 +485,18 @@
|
|
|
485
485
|
import { Plugin } from "./plugin.class.js";
|
|
486
486
|
import tryFn from "../concerns/try-fn.js";
|
|
487
487
|
import { FulltextError } from "./fulltext.errors.js";
|
|
488
|
+
import { resolveResourceName } from "./concerns/resource-names.js";
|
|
488
489
|
|
|
489
490
|
export class FullTextPlugin extends Plugin {
|
|
490
491
|
constructor(options = {}) {
|
|
491
|
-
super();
|
|
492
|
+
super(options);
|
|
492
493
|
this.indexResource = null;
|
|
494
|
+
const resourceNamesOption = options.resourceNames || {};
|
|
495
|
+
this._indexResourceDescriptor = {
|
|
496
|
+
defaultName: 'plg_fulltext_indexes',
|
|
497
|
+
override: resourceNamesOption.index || options.indexResource
|
|
498
|
+
};
|
|
499
|
+
this.indexResourceName = this._resolveIndexResourceName();
|
|
493
500
|
this.config = {
|
|
494
501
|
minWordLength: options.minWordLength || 3,
|
|
495
502
|
maxResults: options.maxResults || 100,
|
|
@@ -500,11 +507,21 @@ export class FullTextPlugin extends Plugin {
|
|
|
500
507
|
this.deletedIndexes = new Set(); // Track deleted index keys
|
|
501
508
|
}
|
|
502
509
|
|
|
510
|
+
_resolveIndexResourceName() {
|
|
511
|
+
return resolveResourceName('fulltext', this._indexResourceDescriptor, {
|
|
512
|
+
namespace: this.namespace
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
onNamespaceChanged() {
|
|
517
|
+
this.indexResourceName = this._resolveIndexResourceName();
|
|
518
|
+
}
|
|
519
|
+
|
|
503
520
|
async onInstall() {
|
|
504
521
|
|
|
505
522
|
// Create index resource if it doesn't exist
|
|
506
523
|
const [ok, err, indexResource] = await tryFn(() => this.database.createResource({
|
|
507
|
-
name:
|
|
524
|
+
name: this.indexResourceName,
|
|
508
525
|
attributes: {
|
|
509
526
|
id: 'string|required',
|
|
510
527
|
resourceName: 'string|required',
|
|
@@ -519,7 +536,13 @@ export class FullTextPlugin extends Plugin {
|
|
|
519
536
|
},
|
|
520
537
|
behavior: 'body-overflow'
|
|
521
538
|
}));
|
|
522
|
-
|
|
539
|
+
if (ok) {
|
|
540
|
+
this.indexResource = indexResource;
|
|
541
|
+
} else if (this.database.resources[this.indexResourceName]) {
|
|
542
|
+
this.indexResource = this.database.resources[this.indexResourceName];
|
|
543
|
+
} else {
|
|
544
|
+
throw err;
|
|
545
|
+
}
|
|
523
546
|
|
|
524
547
|
// Load existing indexes
|
|
525
548
|
await this.loadIndexes();
|
|
@@ -543,6 +566,10 @@ export class FullTextPlugin extends Plugin {
|
|
|
543
566
|
this.removeDatabaseHooks();
|
|
544
567
|
}
|
|
545
568
|
|
|
569
|
+
isInternalResource(name) {
|
|
570
|
+
return name === this.indexResourceName || name === 'plg_fulltext_indexes';
|
|
571
|
+
}
|
|
572
|
+
|
|
546
573
|
async loadIndexes() {
|
|
547
574
|
if (!this.indexResource) return;
|
|
548
575
|
|
|
@@ -631,7 +658,7 @@ export class FullTextPlugin extends Plugin {
|
|
|
631
658
|
installDatabaseHooks() {
|
|
632
659
|
// Use the new database hooks system for automatic resource discovery
|
|
633
660
|
this.database.addHook('afterCreateResource', (resource) => {
|
|
634
|
-
if (resource.name
|
|
661
|
+
if (!this.isInternalResource(resource.name)) {
|
|
635
662
|
this.installResourceHooks(resource);
|
|
636
663
|
}
|
|
637
664
|
});
|
|
@@ -650,7 +677,7 @@ export class FullTextPlugin extends Plugin {
|
|
|
650
677
|
this.database.plugins.fulltext = this;
|
|
651
678
|
|
|
652
679
|
for (const resource of Object.values(this.database.resources)) {
|
|
653
|
-
if (resource.name
|
|
680
|
+
if (this.isInternalResource(resource.name)) continue;
|
|
654
681
|
|
|
655
682
|
this.installResourceHooks(resource);
|
|
656
683
|
}
|
|
@@ -661,9 +688,9 @@ export class FullTextPlugin extends Plugin {
|
|
|
661
688
|
this.database._previousCreateResourceForFullText = this.database.createResource;
|
|
662
689
|
this.database.createResource = async function (...args) {
|
|
663
690
|
const resource = await this._previousCreateResourceForFullText(...args);
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
691
|
+
if (this.plugins?.fulltext && !this.plugins.fulltext.isInternalResource(resource.name)) {
|
|
692
|
+
this.plugins.fulltext.installResourceHooks(resource);
|
|
693
|
+
}
|
|
667
694
|
return resource;
|
|
668
695
|
};
|
|
669
696
|
this.database._fulltextProxyInstalled = true;
|
|
@@ -671,7 +698,7 @@ export class FullTextPlugin extends Plugin {
|
|
|
671
698
|
|
|
672
699
|
// Ensure all existing resources have hooks (even if created before plugin setup)
|
|
673
700
|
for (const resource of Object.values(this.database.resources)) {
|
|
674
|
-
if (resource.name
|
|
701
|
+
if (!this.isInternalResource(resource.name)) {
|
|
675
702
|
this.installResourceHooks(resource);
|
|
676
703
|
}
|
|
677
704
|
}
|
|
@@ -1009,7 +1036,7 @@ export class FullTextPlugin extends Plugin {
|
|
|
1009
1036
|
}
|
|
1010
1037
|
|
|
1011
1038
|
async _rebuildAllIndexesInternal() {
|
|
1012
|
-
const resourceNames = Object.keys(this.database.resources).filter(name => name
|
|
1039
|
+
const resourceNames = Object.keys(this.database.resources).filter(name => !this.isInternalResource(name));
|
|
1013
1040
|
|
|
1014
1041
|
// Process resources sequentially to avoid overwhelming the system
|
|
1015
1042
|
for (const resourceName of resourceNames) {
|
|
@@ -1038,4 +1065,4 @@ export class FullTextPlugin extends Plugin {
|
|
|
1038
1065
|
// Save changes
|
|
1039
1066
|
await this.saveIndexes();
|
|
1040
1067
|
}
|
|
1041
|
-
}
|
|
1068
|
+
}
|