s3db.js 13.6.1 → 14.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -15
- package/dist/s3db.cjs +72446 -39022
- package/dist/s3db.cjs.map +1 -1
- package/dist/s3db.es.js +72172 -38790
- package/dist/s3db.es.js.map +1 -1
- package/mcp/lib/base-handler.js +157 -0
- package/mcp/lib/handlers/connection-handler.js +280 -0
- package/mcp/lib/handlers/query-handler.js +533 -0
- package/mcp/lib/handlers/resource-handler.js +428 -0
- package/mcp/lib/tool-registry.js +336 -0
- package/mcp/lib/tools/connection-tools.js +161 -0
- package/mcp/lib/tools/query-tools.js +267 -0
- package/mcp/lib/tools/resource-tools.js +404 -0
- package/package.json +85 -50
- package/src/clients/memory-client.class.js +346 -191
- package/src/clients/memory-storage.class.js +300 -84
- package/src/clients/s3-client.class.js +7 -6
- package/src/concerns/geo-encoding.js +19 -2
- package/src/concerns/ip.js +59 -9
- package/src/concerns/money.js +8 -1
- package/src/concerns/password-hashing.js +49 -8
- package/src/concerns/plugin-storage.js +186 -18
- package/src/concerns/storage-drivers/filesystem-driver.js +284 -0
- package/src/database.class.js +139 -29
- package/src/errors.js +332 -42
- package/src/plugins/api/auth/oidc-auth.js +66 -17
- package/src/plugins/api/auth/strategies/base-strategy.class.js +74 -0
- package/src/plugins/api/auth/strategies/factory.class.js +63 -0
- package/src/plugins/api/auth/strategies/global-strategy.class.js +44 -0
- package/src/plugins/api/auth/strategies/path-based-strategy.class.js +83 -0
- package/src/plugins/api/auth/strategies/path-rules-strategy.class.js +118 -0
- package/src/plugins/api/concerns/failban-manager.js +106 -57
- package/src/plugins/api/concerns/route-context.js +601 -0
- package/src/plugins/api/index.js +168 -40
- package/src/plugins/api/routes/auth-routes.js +198 -30
- package/src/plugins/api/routes/resource-routes.js +19 -4
- package/src/plugins/api/server/health-manager.class.js +163 -0
- package/src/plugins/api/server/middleware-chain.class.js +310 -0
- package/src/plugins/api/server/router.class.js +472 -0
- package/src/plugins/api/server.js +280 -1303
- package/src/plugins/api/utils/custom-routes.js +17 -5
- package/src/plugins/api/utils/guards.js +76 -17
- package/src/plugins/api/utils/openapi-generator-cached.class.js +133 -0
- package/src/plugins/api/utils/openapi-generator.js +7 -6
- package/src/plugins/audit.plugin.js +30 -8
- package/src/plugins/backup.plugin.js +110 -14
- package/src/plugins/cache/cache.class.js +22 -5
- package/src/plugins/cache/filesystem-cache.class.js +116 -19
- package/src/plugins/cache/memory-cache.class.js +211 -57
- package/src/plugins/cache/multi-tier-cache.class.js +371 -0
- package/src/plugins/cache/partition-aware-filesystem-cache.class.js +168 -47
- package/src/plugins/cache/redis-cache.class.js +552 -0
- package/src/plugins/cache/s3-cache.class.js +17 -8
- package/src/plugins/cache.plugin.js +176 -61
- package/src/plugins/cloud-inventory/drivers/alibaba-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/aws-driver.js +60 -29
- package/src/plugins/cloud-inventory/drivers/azure-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/base-driver.js +16 -2
- package/src/plugins/cloud-inventory/drivers/cloudflare-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/digitalocean-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/hetzner-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/linode-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/mongodb-atlas-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/vultr-driver.js +8 -1
- package/src/plugins/cloud-inventory/index.js +29 -8
- package/src/plugins/cloud-inventory/registry.js +64 -42
- package/src/plugins/cloud-inventory.plugin.js +240 -138
- package/src/plugins/concerns/plugin-dependencies.js +54 -0
- package/src/plugins/concerns/resource-names.js +100 -0
- package/src/plugins/consumers/index.js +10 -2
- package/src/plugins/consumers/sqs-consumer.js +12 -2
- package/src/plugins/cookie-farm-suite.plugin.js +278 -0
- package/src/plugins/cookie-farm.errors.js +73 -0
- package/src/plugins/cookie-farm.plugin.js +869 -0
- package/src/plugins/costs.plugin.js +7 -1
- package/src/plugins/eventual-consistency/analytics.js +94 -19
- package/src/plugins/eventual-consistency/config.js +15 -7
- package/src/plugins/eventual-consistency/consolidation.js +29 -11
- package/src/plugins/eventual-consistency/garbage-collection.js +3 -1
- package/src/plugins/eventual-consistency/helpers.js +39 -14
- package/src/plugins/eventual-consistency/install.js +21 -2
- package/src/plugins/eventual-consistency/utils.js +32 -10
- package/src/plugins/fulltext.plugin.js +38 -11
- package/src/plugins/geo.plugin.js +61 -9
- package/src/plugins/identity/concerns/config.js +61 -0
- package/src/plugins/identity/concerns/mfa-manager.js +15 -2
- package/src/plugins/identity/concerns/rate-limit.js +124 -0
- package/src/plugins/identity/concerns/resource-schemas.js +9 -1
- package/src/plugins/identity/concerns/token-generator.js +29 -4
- package/src/plugins/identity/drivers/auth-driver.interface.js +76 -0
- package/src/plugins/identity/drivers/client-credentials-driver.js +127 -0
- package/src/plugins/identity/drivers/index.js +18 -0
- package/src/plugins/identity/drivers/password-driver.js +122 -0
- package/src/plugins/identity/email-service.js +17 -2
- package/src/plugins/identity/index.js +413 -69
- package/src/plugins/identity/oauth2-server.js +413 -30
- package/src/plugins/identity/oidc-discovery.js +16 -8
- package/src/plugins/identity/rsa-keys.js +115 -35
- package/src/plugins/identity/server.js +166 -45
- package/src/plugins/identity/session-manager.js +53 -7
- package/src/plugins/identity/ui/pages/mfa-verification.js +17 -15
- package/src/plugins/identity/ui/routes.js +363 -255
- package/src/plugins/importer/index.js +153 -20
- package/src/plugins/index.js +9 -2
- package/src/plugins/kubernetes-inventory/index.js +6 -0
- package/src/plugins/kubernetes-inventory/k8s-driver.js +867 -0
- package/src/plugins/kubernetes-inventory/resource-types.js +274 -0
- package/src/plugins/kubernetes-inventory.plugin.js +980 -0
- package/src/plugins/metrics.plugin.js +64 -16
- package/src/plugins/ml/base-model.class.js +25 -15
- package/src/plugins/ml/regression-model.class.js +1 -1
- package/src/plugins/ml.errors.js +57 -25
- package/src/plugins/ml.plugin.js +28 -4
- package/src/plugins/namespace.js +210 -0
- package/src/plugins/plugin.class.js +180 -8
- package/src/plugins/puppeteer/console-monitor.js +729 -0
- package/src/plugins/puppeteer/cookie-manager.js +492 -0
- package/src/plugins/puppeteer/network-monitor.js +816 -0
- package/src/plugins/puppeteer/performance-manager.js +746 -0
- package/src/plugins/puppeteer/proxy-manager.js +478 -0
- package/src/plugins/puppeteer/stealth-manager.js +556 -0
- package/src/plugins/puppeteer.errors.js +81 -0
- package/src/plugins/puppeteer.plugin.js +1327 -0
- package/src/plugins/queue-consumer.plugin.js +69 -14
- package/src/plugins/recon/behaviors/uptime-behavior.js +691 -0
- package/src/plugins/recon/concerns/command-runner.js +148 -0
- package/src/plugins/recon/concerns/diff-detector.js +372 -0
- package/src/plugins/recon/concerns/fingerprint-builder.js +307 -0
- package/src/plugins/recon/concerns/process-manager.js +338 -0
- package/src/plugins/recon/concerns/report-generator.js +478 -0
- package/src/plugins/recon/concerns/security-analyzer.js +571 -0
- package/src/plugins/recon/concerns/target-normalizer.js +68 -0
- package/src/plugins/recon/config/defaults.js +321 -0
- package/src/plugins/recon/config/resources.js +370 -0
- package/src/plugins/recon/index.js +778 -0
- package/src/plugins/recon/managers/dependency-manager.js +174 -0
- package/src/plugins/recon/managers/scheduler-manager.js +179 -0
- package/src/plugins/recon/managers/storage-manager.js +745 -0
- package/src/plugins/recon/managers/target-manager.js +274 -0
- package/src/plugins/recon/stages/asn-stage.js +314 -0
- package/src/plugins/recon/stages/certificate-stage.js +84 -0
- package/src/plugins/recon/stages/dns-stage.js +107 -0
- package/src/plugins/recon/stages/dnsdumpster-stage.js +362 -0
- package/src/plugins/recon/stages/fingerprint-stage.js +71 -0
- package/src/plugins/recon/stages/google-dorks-stage.js +440 -0
- package/src/plugins/recon/stages/http-stage.js +89 -0
- package/src/plugins/recon/stages/latency-stage.js +148 -0
- package/src/plugins/recon/stages/massdns-stage.js +302 -0
- package/src/plugins/recon/stages/osint-stage.js +1373 -0
- package/src/plugins/recon/stages/ports-stage.js +169 -0
- package/src/plugins/recon/stages/screenshot-stage.js +94 -0
- package/src/plugins/recon/stages/secrets-stage.js +514 -0
- package/src/plugins/recon/stages/subdomains-stage.js +295 -0
- package/src/plugins/recon/stages/tls-audit-stage.js +78 -0
- package/src/plugins/recon/stages/vulnerability-stage.js +78 -0
- package/src/plugins/recon/stages/web-discovery-stage.js +113 -0
- package/src/plugins/recon/stages/whois-stage.js +349 -0
- package/src/plugins/recon.plugin.js +75 -0
- package/src/plugins/recon.plugin.js.backup +2635 -0
- package/src/plugins/relation.errors.js +87 -14
- package/src/plugins/replicator.plugin.js +514 -137
- package/src/plugins/replicators/base-replicator.class.js +89 -1
- package/src/plugins/replicators/bigquery-replicator.class.js +66 -22
- package/src/plugins/replicators/dynamodb-replicator.class.js +22 -15
- package/src/plugins/replicators/mongodb-replicator.class.js +22 -15
- package/src/plugins/replicators/mysql-replicator.class.js +52 -17
- package/src/plugins/replicators/planetscale-replicator.class.js +30 -4
- package/src/plugins/replicators/postgres-replicator.class.js +62 -27
- package/src/plugins/replicators/s3db-replicator.class.js +25 -18
- package/src/plugins/replicators/schema-sync.helper.js +3 -3
- package/src/plugins/replicators/sqs-replicator.class.js +8 -2
- package/src/plugins/replicators/turso-replicator.class.js +23 -3
- package/src/plugins/replicators/webhook-replicator.class.js +42 -4
- package/src/plugins/s3-queue.plugin.js +464 -65
- package/src/plugins/scheduler.plugin.js +20 -6
- package/src/plugins/state-machine.plugin.js +40 -9
- package/src/plugins/tfstate/base-driver.js +28 -4
- package/src/plugins/tfstate/errors.js +65 -10
- package/src/plugins/tfstate/filesystem-driver.js +52 -8
- package/src/plugins/tfstate/index.js +163 -90
- package/src/plugins/tfstate/s3-driver.js +64 -6
- package/src/plugins/ttl.plugin.js +72 -17
- package/src/plugins/vector/distances.js +18 -12
- package/src/plugins/vector/kmeans.js +26 -4
- package/src/resource.class.js +115 -19
- package/src/testing/factory.class.js +20 -3
- package/src/testing/seeder.class.js +7 -1
- package/src/clients/memory-client.md +0 -917
- package/src/plugins/cloud-inventory/drivers/mock-drivers.js +0 -449
|
@@ -249,6 +249,7 @@ import { Plugin } from '../plugin.class.js';
|
|
|
249
249
|
import tryFn from '../../concerns/try-fn.js';
|
|
250
250
|
import requirePluginDependency from '../concerns/plugin-dependencies.js';
|
|
251
251
|
import { idGenerator } from '../../concerns/id.js';
|
|
252
|
+
import { resolveResourceNames } from '../concerns/resource-names.js';
|
|
252
253
|
import {
|
|
253
254
|
TfStateError,
|
|
254
255
|
InvalidStateFileError,
|
|
@@ -271,10 +272,31 @@ export class TfStatePlugin extends Plugin {
|
|
|
271
272
|
this.driverConfig = config.config || {};
|
|
272
273
|
|
|
273
274
|
// Resource names
|
|
274
|
-
const
|
|
275
|
-
|
|
276
|
-
this.
|
|
277
|
-
|
|
275
|
+
const resourcesConfig = config.resources || {};
|
|
276
|
+
const resourceNamesOption = config.resourceNames || {};
|
|
277
|
+
this._resourceDescriptors = {
|
|
278
|
+
resources: {
|
|
279
|
+
defaultName: 'plg_tfstate_resources',
|
|
280
|
+
override: resourceNamesOption.resources || resourcesConfig.resources || config.resourceName
|
|
281
|
+
},
|
|
282
|
+
stateFiles: {
|
|
283
|
+
defaultName: 'plg_tfstate_state_files',
|
|
284
|
+
override: resourceNamesOption.stateFiles || resourcesConfig.stateFiles || config.stateFilesName
|
|
285
|
+
},
|
|
286
|
+
diffs: {
|
|
287
|
+
defaultName: 'plg_tfstate_state_diffs',
|
|
288
|
+
override: resourceNamesOption.diffs || resourcesConfig.diffs || config.diffsName
|
|
289
|
+
},
|
|
290
|
+
lineages: {
|
|
291
|
+
defaultName: 'plg_tfstate_lineages',
|
|
292
|
+
override: resourceNamesOption.lineages || resourcesConfig.lineages
|
|
293
|
+
}
|
|
294
|
+
};
|
|
295
|
+
const resolvedNames = this._resolveResourceNames();
|
|
296
|
+
this.resourceName = resolvedNames.resources;
|
|
297
|
+
this.stateFilesName = resolvedNames.stateFiles;
|
|
298
|
+
this.diffsName = resolvedNames.diffs;
|
|
299
|
+
this.lineagesName = resolvedNames.lineages;
|
|
278
300
|
|
|
279
301
|
// Monitoring configuration
|
|
280
302
|
const monitor = config.monitor || {};
|
|
@@ -323,6 +345,20 @@ export class TfStatePlugin extends Plugin {
|
|
|
323
345
|
};
|
|
324
346
|
}
|
|
325
347
|
|
|
348
|
+
_resolveResourceNames() {
|
|
349
|
+
return resolveResourceNames('tfstate', this._resourceDescriptors, {
|
|
350
|
+
namespace: this.namespace
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
onNamespaceChanged() {
|
|
355
|
+
const names = this._resolveResourceNames();
|
|
356
|
+
this.resourceName = names.resources;
|
|
357
|
+
this.stateFilesName = names.stateFiles;
|
|
358
|
+
this.diffsName = names.diffs;
|
|
359
|
+
this.lineagesName = names.lineages;
|
|
360
|
+
}
|
|
361
|
+
|
|
326
362
|
/**
|
|
327
363
|
* Install the plugin
|
|
328
364
|
* @override
|
|
@@ -355,102 +391,131 @@ export class TfStatePlugin extends Plugin {
|
|
|
355
391
|
|
|
356
392
|
// Resource 0: Terraform Lineages (Master tracking resource)
|
|
357
393
|
// NEW: Tracks unique Tfstate lineages for efficient diff tracking
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
394
|
+
{
|
|
395
|
+
const [created, createErr, resource] = await tryFn(() => this.database.createResource({
|
|
396
|
+
name: this.lineagesName,
|
|
397
|
+
attributes: {
|
|
398
|
+
id: 'string|required',
|
|
399
|
+
latestSerial: 'number',
|
|
400
|
+
latestStateId: 'string',
|
|
401
|
+
totalStates: 'number',
|
|
402
|
+
firstImportedAt: 'number',
|
|
403
|
+
lastImportedAt: 'number',
|
|
404
|
+
metadata: 'json'
|
|
405
|
+
},
|
|
406
|
+
timestamps: true,
|
|
407
|
+
asyncPartitions: this.asyncPartitions,
|
|
408
|
+
partitions: {},
|
|
409
|
+
createdBy: 'TfStatePlugin'
|
|
410
|
+
}));
|
|
411
|
+
|
|
412
|
+
if (created) {
|
|
413
|
+
this.lineagesResource = resource;
|
|
414
|
+
} else {
|
|
415
|
+
this.lineagesResource = this.database.resources?.[this.lineagesName];
|
|
416
|
+
if (!this.lineagesResource) {
|
|
417
|
+
throw createErr;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
}
|
|
375
421
|
|
|
376
422
|
// Resource 1: Tfstate Files Metadata
|
|
377
423
|
// Dedicated to tracking state file metadata with SHA256 hash for deduplication
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
424
|
+
{
|
|
425
|
+
const [created, createErr, resource] = await tryFn(() => this.database.createResource({
|
|
426
|
+
name: this.stateFilesName,
|
|
427
|
+
attributes: {
|
|
428
|
+
id: 'string|required',
|
|
429
|
+
lineageId: 'string|required',
|
|
430
|
+
sourceFile: 'string|required',
|
|
431
|
+
serial: 'number|required',
|
|
432
|
+
lineage: 'string|required',
|
|
433
|
+
terraformVersion: 'string',
|
|
434
|
+
stateVersion: 'number|required',
|
|
435
|
+
resourceCount: 'number',
|
|
436
|
+
sha256Hash: 'string|required',
|
|
437
|
+
importedAt: 'number|required'
|
|
438
|
+
},
|
|
439
|
+
timestamps: true,
|
|
440
|
+
asyncPartitions: this.asyncPartitions,
|
|
441
|
+
partitions: {
|
|
442
|
+
byLineage: { fields: { lineageId: 'string' } },
|
|
443
|
+
byLineageSerial: { fields: { lineageId: 'string', serial: 'number' } },
|
|
444
|
+
bySourceFile: { fields: { sourceFile: 'string' } },
|
|
445
|
+
bySerial: { fields: { serial: 'number' } },
|
|
446
|
+
bySha256: { fields: { sha256Hash: 'string' } }
|
|
447
|
+
},
|
|
448
|
+
createdBy: 'TfStatePlugin'
|
|
449
|
+
}));
|
|
450
|
+
|
|
451
|
+
if (created) {
|
|
452
|
+
this.stateFilesResource = resource;
|
|
453
|
+
} else {
|
|
454
|
+
this.stateFilesResource = this.database.resources?.[this.stateFilesName];
|
|
455
|
+
if (!this.stateFilesResource) {
|
|
456
|
+
throw createErr;
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
}
|
|
403
460
|
|
|
404
461
|
// Resource 2: Terraform Resources
|
|
405
462
|
// Store extracted resources with foreign key to state files
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
463
|
+
{
|
|
464
|
+
const [created, createErr, resource] = await tryFn(() => this.database.createResource({
|
|
465
|
+
name: this.resourceName,
|
|
466
|
+
attributes: {
|
|
467
|
+
id: 'string|required',
|
|
468
|
+
stateFileId: 'string|required',
|
|
469
|
+
lineageId: 'string|required',
|
|
470
|
+
stateSerial: 'number|required',
|
|
471
|
+
sourceFile: 'string|required',
|
|
472
|
+
resourceType: 'string|required',
|
|
473
|
+
resourceName: 'string|required',
|
|
474
|
+
resourceAddress: 'string|required',
|
|
475
|
+
providerName: 'string|required',
|
|
476
|
+
mode: 'string',
|
|
477
|
+
attributes: 'json',
|
|
478
|
+
dependencies: 'array',
|
|
479
|
+
importedAt: 'number|required'
|
|
480
|
+
},
|
|
481
|
+
timestamps: true,
|
|
482
|
+
asyncPartitions: this.asyncPartitions,
|
|
483
|
+
partitions: {
|
|
484
|
+
byLineageSerial: { fields: { lineageId: 'string', stateSerial: 'number' } },
|
|
485
|
+
byLineage: { fields: { lineageId: 'string' } },
|
|
486
|
+
byType: { fields: { resourceType: 'string' } },
|
|
487
|
+
byProvider: { fields: { providerName: 'string' } },
|
|
488
|
+
bySerial: { fields: { stateSerial: 'number' } },
|
|
489
|
+
bySourceFile: { fields: { sourceFile: 'string' } },
|
|
490
|
+
byProviderAndType: { fields: { providerName: 'string', resourceType: 'string' } },
|
|
491
|
+
byLineageType: { fields: { lineageId: 'string', resourceType: 'string' } }
|
|
492
|
+
},
|
|
493
|
+
createdBy: 'TfStatePlugin'
|
|
494
|
+
}));
|
|
495
|
+
|
|
496
|
+
if (created) {
|
|
497
|
+
this.resource = resource;
|
|
498
|
+
} else {
|
|
499
|
+
this.resource = this.database.resources?.[this.resourceName];
|
|
500
|
+
if (!this.resource) {
|
|
501
|
+
throw createErr;
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
}
|
|
439
505
|
|
|
440
506
|
// Resource 3: Tfstate Diffs
|
|
441
507
|
// Track changes between state versions (if diff tracking enabled)
|
|
442
508
|
if (this.trackDiffs) {
|
|
443
|
-
|
|
509
|
+
const [created, createErr, resource] = await tryFn(() => this.database.createResource({
|
|
444
510
|
name: this.diffsName,
|
|
445
511
|
attributes: {
|
|
446
512
|
id: 'string|required',
|
|
447
|
-
lineageId: 'string|required',
|
|
513
|
+
lineageId: 'string|required',
|
|
448
514
|
oldSerial: 'number|required',
|
|
449
515
|
newSerial: 'number|required',
|
|
450
|
-
oldStateId: 'string',
|
|
451
|
-
newStateId: 'string|required',
|
|
516
|
+
oldStateId: 'string',
|
|
517
|
+
newStateId: 'string|required',
|
|
452
518
|
calculatedAt: 'number|required',
|
|
453
|
-
// Summary statistics
|
|
454
519
|
summary: {
|
|
455
520
|
type: 'object',
|
|
456
521
|
props: {
|
|
@@ -459,7 +524,6 @@ export class TfStatePlugin extends Plugin {
|
|
|
459
524
|
deletedCount: 'number'
|
|
460
525
|
}
|
|
461
526
|
},
|
|
462
|
-
// Detailed changes
|
|
463
527
|
changes: {
|
|
464
528
|
type: 'object',
|
|
465
529
|
props: {
|
|
@@ -469,17 +533,26 @@ export class TfStatePlugin extends Plugin {
|
|
|
469
533
|
}
|
|
470
534
|
}
|
|
471
535
|
},
|
|
472
|
-
behavior: 'body-only',
|
|
536
|
+
behavior: 'body-only',
|
|
473
537
|
timestamps: true,
|
|
474
|
-
asyncPartitions: this.asyncPartitions,
|
|
538
|
+
asyncPartitions: this.asyncPartitions,
|
|
475
539
|
partitions: {
|
|
476
|
-
byLineage: { fields: { lineageId: 'string' } },
|
|
477
|
-
byLineageNewSerial: { fields: { lineageId: 'string', newSerial: 'number' } },
|
|
540
|
+
byLineage: { fields: { lineageId: 'string' } },
|
|
541
|
+
byLineageNewSerial: { fields: { lineageId: 'string', newSerial: 'number' } },
|
|
478
542
|
byNewSerial: { fields: { newSerial: 'number' } },
|
|
479
543
|
byOldSerial: { fields: { oldSerial: 'number' } }
|
|
480
544
|
},
|
|
481
545
|
createdBy: 'TfStatePlugin'
|
|
482
|
-
});
|
|
546
|
+
}));
|
|
547
|
+
|
|
548
|
+
if (created) {
|
|
549
|
+
this.diffsResource = resource;
|
|
550
|
+
} else {
|
|
551
|
+
this.diffsResource = this.database.resources?.[this.diffsName];
|
|
552
|
+
if (!this.diffsResource) {
|
|
553
|
+
throw createErr;
|
|
554
|
+
}
|
|
555
|
+
}
|
|
483
556
|
}
|
|
484
557
|
|
|
485
558
|
if (this.verbose) {
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { TfStateDriver } from './base-driver.js';
|
|
7
7
|
import { S3Client } from '../../clients/s3-client.class.js';
|
|
8
8
|
import tryFn from '../../concerns/try-fn.js';
|
|
9
|
+
import { TfStateError, InvalidStateFileError, StateFileNotFoundError } from './errors.js';
|
|
9
10
|
|
|
10
11
|
export class S3TfStateDriver extends TfStateDriver {
|
|
11
12
|
constructor(config = {}) {
|
|
@@ -36,7 +37,13 @@ export class S3TfStateDriver extends TfStateDriver {
|
|
|
36
37
|
const url = new URL(connectionString);
|
|
37
38
|
|
|
38
39
|
if (url.protocol !== 's3:') {
|
|
39
|
-
throw new
|
|
40
|
+
throw new TfStateError('Connection string must use s3:// protocol', {
|
|
41
|
+
operation: 'parseConnectionString',
|
|
42
|
+
statusCode: 400,
|
|
43
|
+
retriable: false,
|
|
44
|
+
suggestion: 'Use format s3://accessKey:secretKey@bucket/prefix?region=us-east-1',
|
|
45
|
+
connectionString
|
|
46
|
+
});
|
|
40
47
|
}
|
|
41
48
|
|
|
42
49
|
const credentials = {};
|
|
@@ -61,7 +68,14 @@ export class S3TfStateDriver extends TfStateDriver {
|
|
|
61
68
|
region
|
|
62
69
|
};
|
|
63
70
|
} catch (error) {
|
|
64
|
-
throw new
|
|
71
|
+
throw new TfStateError('Invalid S3 connection string', {
|
|
72
|
+
operation: 'parseConnectionString',
|
|
73
|
+
statusCode: 400,
|
|
74
|
+
retriable: false,
|
|
75
|
+
suggestion: 'Ensure the connection string follows s3://accessKey:secretKey@bucket/prefix?region=REGION.',
|
|
76
|
+
connectionString,
|
|
77
|
+
original: error
|
|
78
|
+
});
|
|
65
79
|
}
|
|
66
80
|
}
|
|
67
81
|
|
|
@@ -95,7 +109,14 @@ export class S3TfStateDriver extends TfStateDriver {
|
|
|
95
109
|
});
|
|
96
110
|
|
|
97
111
|
if (!ok) {
|
|
98
|
-
throw new
|
|
112
|
+
throw new TfStateError('Failed to list Terraform state objects from S3', {
|
|
113
|
+
operation: 'listStateFiles',
|
|
114
|
+
retriable: false,
|
|
115
|
+
suggestion: 'Validate S3 permissions (s3:ListBucket) and prefix configuration.',
|
|
116
|
+
bucket,
|
|
117
|
+
prefix,
|
|
118
|
+
original: err
|
|
119
|
+
});
|
|
99
120
|
}
|
|
100
121
|
|
|
101
122
|
const objects = data.Contents || [];
|
|
@@ -133,14 +154,35 @@ export class S3TfStateDriver extends TfStateDriver {
|
|
|
133
154
|
});
|
|
134
155
|
|
|
135
156
|
if (!ok) {
|
|
136
|
-
|
|
157
|
+
if (err?.$metadata?.httpStatusCode === 404) {
|
|
158
|
+
throw new StateFileNotFoundError(path, {
|
|
159
|
+
operation: 'readStateFile',
|
|
160
|
+
retriable: false,
|
|
161
|
+
suggestion: 'Ensure the state file exists in S3 and the IAM role can access it.',
|
|
162
|
+
bucket,
|
|
163
|
+
original: err
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
throw new TfStateError(`Failed to read state file ${path}`, {
|
|
167
|
+
operation: 'readStateFile',
|
|
168
|
+
retriable: false,
|
|
169
|
+
suggestion: 'Verify S3 permissions (s3:GetObject) and network connectivity.',
|
|
170
|
+
bucket,
|
|
171
|
+
path,
|
|
172
|
+
original: err
|
|
173
|
+
});
|
|
137
174
|
}
|
|
138
175
|
|
|
139
176
|
try {
|
|
140
177
|
const content = data.Body.toString('utf-8');
|
|
141
178
|
return JSON.parse(content);
|
|
142
179
|
} catch (parseError) {
|
|
143
|
-
throw new
|
|
180
|
+
throw new InvalidStateFileError(path, parseError.message, {
|
|
181
|
+
operation: 'readStateFile',
|
|
182
|
+
retriable: false,
|
|
183
|
+
suggestion: 'Check if the state file contains valid JSON exported by Terraform.',
|
|
184
|
+
original: parseError
|
|
185
|
+
});
|
|
144
186
|
}
|
|
145
187
|
}
|
|
146
188
|
|
|
@@ -158,7 +200,23 @@ export class S3TfStateDriver extends TfStateDriver {
|
|
|
158
200
|
});
|
|
159
201
|
|
|
160
202
|
if (!ok) {
|
|
161
|
-
|
|
203
|
+
if (err?.$metadata?.httpStatusCode === 404) {
|
|
204
|
+
throw new StateFileNotFoundError(path, {
|
|
205
|
+
operation: 'getStateFileMetadata',
|
|
206
|
+
retriable: false,
|
|
207
|
+
suggestion: 'Ensure the state file exists in S3 and the IAM role can access it.',
|
|
208
|
+
bucket,
|
|
209
|
+
original: err
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
throw new TfStateError(`Failed to get metadata for ${path}`, {
|
|
213
|
+
operation: 'getStateFileMetadata',
|
|
214
|
+
retriable: false,
|
|
215
|
+
suggestion: 'Verify S3 permissions (s3:HeadObject) and bucket configuration.',
|
|
216
|
+
bucket,
|
|
217
|
+
path,
|
|
218
|
+
original: err
|
|
219
|
+
});
|
|
162
220
|
}
|
|
163
221
|
|
|
164
222
|
return {
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { Plugin } from "./plugin.class.js";
|
|
2
2
|
import tryFn from "../concerns/try-fn.js";
|
|
3
3
|
import { idGenerator } from "../concerns/id.js";
|
|
4
|
+
import { resolveResourceName } from "./concerns/resource-names.js";
|
|
5
|
+
import { PluginError } from "../errors.js";
|
|
4
6
|
|
|
5
7
|
// Time constants (in seconds)
|
|
6
8
|
const ONE_MINUTE_SEC = 60;
|
|
@@ -171,7 +173,13 @@ export class TTLPlugin extends Plugin {
|
|
|
171
173
|
this.isRunning = false;
|
|
172
174
|
|
|
173
175
|
// Expiration index (plugin storage)
|
|
176
|
+
const resourceNamesOption = config.resourceNames || {};
|
|
174
177
|
this.expirationIndex = null;
|
|
178
|
+
this._indexResourceDescriptor = {
|
|
179
|
+
defaultName: 'plg_ttl_expiration_index',
|
|
180
|
+
override: resourceNamesOption.index || config.indexResourceName
|
|
181
|
+
};
|
|
182
|
+
this.indexResourceName = this._resolveIndexResourceName();
|
|
175
183
|
}
|
|
176
184
|
|
|
177
185
|
/**
|
|
@@ -206,23 +214,44 @@ export class TTLPlugin extends Plugin {
|
|
|
206
214
|
});
|
|
207
215
|
}
|
|
208
216
|
|
|
217
|
+
_resolveIndexResourceName() {
|
|
218
|
+
return resolveResourceName('ttl', this._indexResourceDescriptor, {
|
|
219
|
+
namespace: this.namespace
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
onNamespaceChanged() {
|
|
224
|
+
if (!this._indexResourceDescriptor) return;
|
|
225
|
+
this.indexResourceName = this._resolveIndexResourceName();
|
|
226
|
+
}
|
|
227
|
+
|
|
209
228
|
/**
|
|
210
229
|
* Validate resource configuration
|
|
211
230
|
*/
|
|
212
231
|
_validateResourceConfig(resourceName, config) {
|
|
213
232
|
// Must have either ttl or field
|
|
214
233
|
if (!config.ttl && !config.field) {
|
|
215
|
-
throw new
|
|
216
|
-
|
|
217
|
-
|
|
234
|
+
throw new PluginError('[TTLPlugin] Missing TTL configuration', {
|
|
235
|
+
pluginName: 'TTLPlugin',
|
|
236
|
+
operation: 'validateResourceConfig',
|
|
237
|
+
resourceName,
|
|
238
|
+
statusCode: 400,
|
|
239
|
+
retriable: false,
|
|
240
|
+
suggestion: 'Provide either ttl (in seconds) or field (absolute expiration timestamp) for each resource.'
|
|
241
|
+
});
|
|
218
242
|
}
|
|
219
243
|
|
|
220
244
|
const validStrategies = ['soft-delete', 'hard-delete', 'archive', 'callback'];
|
|
221
245
|
if (!config.onExpire || !validStrategies.includes(config.onExpire)) {
|
|
222
|
-
throw new
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
246
|
+
throw new PluginError('[TTLPlugin] Invalid onExpire strategy', {
|
|
247
|
+
pluginName: 'TTLPlugin',
|
|
248
|
+
operation: 'validateResourceConfig',
|
|
249
|
+
resourceName,
|
|
250
|
+
statusCode: 400,
|
|
251
|
+
retriable: false,
|
|
252
|
+
suggestion: `Set onExpire to one of: ${validStrategies.join(', ')}`,
|
|
253
|
+
onExpire: config.onExpire
|
|
254
|
+
});
|
|
226
255
|
}
|
|
227
256
|
|
|
228
257
|
if (config.onExpire === 'soft-delete' && !config.deleteField) {
|
|
@@ -230,15 +259,27 @@ export class TTLPlugin extends Plugin {
|
|
|
230
259
|
}
|
|
231
260
|
|
|
232
261
|
if (config.onExpire === 'archive' && !config.archiveResource) {
|
|
233
|
-
throw new
|
|
234
|
-
|
|
235
|
-
|
|
262
|
+
throw new PluginError('[TTLPlugin] Archive resource required', {
|
|
263
|
+
pluginName: 'TTLPlugin',
|
|
264
|
+
operation: 'validateResourceConfig',
|
|
265
|
+
resourceName,
|
|
266
|
+
statusCode: 400,
|
|
267
|
+
retriable: false,
|
|
268
|
+
suggestion: 'Provide archiveResource pointing to the resource that stores archived records.',
|
|
269
|
+
onExpire: config.onExpire
|
|
270
|
+
});
|
|
236
271
|
}
|
|
237
272
|
|
|
238
273
|
if (config.onExpire === 'callback' && typeof config.callback !== 'function') {
|
|
239
|
-
throw new
|
|
240
|
-
|
|
241
|
-
|
|
274
|
+
throw new PluginError('[TTLPlugin] Callback handler required', {
|
|
275
|
+
pluginName: 'TTLPlugin',
|
|
276
|
+
operation: 'validateResourceConfig',
|
|
277
|
+
resourceName,
|
|
278
|
+
statusCode: 400,
|
|
279
|
+
retriable: false,
|
|
280
|
+
suggestion: 'Provide a callback function: { onExpire: "callback", callback: async (ctx) => {...} }',
|
|
281
|
+
onExpire: config.onExpire
|
|
282
|
+
});
|
|
242
283
|
}
|
|
243
284
|
|
|
244
285
|
// Set default field if not specified
|
|
@@ -249,7 +290,7 @@ export class TTLPlugin extends Plugin {
|
|
|
249
290
|
// Validate timestamp field availability
|
|
250
291
|
if (config.field === '_createdAt' && this.database) {
|
|
251
292
|
const resource = this.database.resources[resourceName];
|
|
252
|
-
if (resource && resource.
|
|
293
|
+
if (resource && resource.$schema.timestamps === false) {
|
|
253
294
|
console.warn(
|
|
254
295
|
`[TTLPlugin] WARNING: Resource "${resourceName}" uses TTL with field "_createdAt" ` +
|
|
255
296
|
`but timestamps are disabled. TTL will be calculated from indexing time, not creation time.`
|
|
@@ -266,7 +307,7 @@ export class TTLPlugin extends Plugin {
|
|
|
266
307
|
*/
|
|
267
308
|
async _createExpirationIndex() {
|
|
268
309
|
this.expirationIndex = await this.database.createResource({
|
|
269
|
-
name:
|
|
310
|
+
name: this.indexResourceName,
|
|
270
311
|
attributes: {
|
|
271
312
|
resourceName: 'string|required',
|
|
272
313
|
recordId: 'string|required',
|
|
@@ -626,7 +667,14 @@ export class TTLPlugin extends Plugin {
|
|
|
626
667
|
async _archive(resource, record, config) {
|
|
627
668
|
// Check if archive resource exists
|
|
628
669
|
if (!this.database.resources[config.archiveResource]) {
|
|
629
|
-
throw new
|
|
670
|
+
throw new PluginError(`Archive resource "${config.archiveResource}" not found`, {
|
|
671
|
+
pluginName: 'TTLPlugin',
|
|
672
|
+
operation: '_archive',
|
|
673
|
+
resourceName: config.archiveResource,
|
|
674
|
+
statusCode: 404,
|
|
675
|
+
retriable: false,
|
|
676
|
+
suggestion: 'Create the archive resource before using onExpire: "archive" or update archiveResource config.'
|
|
677
|
+
});
|
|
630
678
|
}
|
|
631
679
|
|
|
632
680
|
const archiveResource = this.database.resources[config.archiveResource];
|
|
@@ -666,7 +714,14 @@ export class TTLPlugin extends Plugin {
|
|
|
666
714
|
async cleanupResource(resourceName) {
|
|
667
715
|
const config = this.resources[resourceName];
|
|
668
716
|
if (!config) {
|
|
669
|
-
throw new
|
|
717
|
+
throw new PluginError(`Resource "${resourceName}" not configured in TTLPlugin`, {
|
|
718
|
+
pluginName: 'TTLPlugin',
|
|
719
|
+
operation: 'cleanupResource',
|
|
720
|
+
resourceName,
|
|
721
|
+
statusCode: 404,
|
|
722
|
+
retriable: false,
|
|
723
|
+
suggestion: 'Add the resource under TTLPlugin configuration before invoking cleanupResource.'
|
|
724
|
+
});
|
|
670
725
|
}
|
|
671
726
|
|
|
672
727
|
const granularity = config.granularity;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { ValidationError } from '../../errors.js';
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* Vector Distance Functions
|
|
3
5
|
*
|
|
@@ -5,6 +7,18 @@
|
|
|
5
7
|
* All distance functions return lower values for more similar vectors.
|
|
6
8
|
*/
|
|
7
9
|
|
|
10
|
+
function assertSameDimensions(a, b, operation) {
|
|
11
|
+
if (a.length !== b.length) {
|
|
12
|
+
throw new ValidationError(`Dimension mismatch: ${a.length} vs ${b.length}`, {
|
|
13
|
+
operation,
|
|
14
|
+
pluginName: 'VectorPlugin',
|
|
15
|
+
retriable: false,
|
|
16
|
+
suggestion: 'Ensure both vectors have identical lengths before calling distance utilities.',
|
|
17
|
+
metadata: { vectorALength: a.length, vectorBLength: b.length }
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
8
22
|
/**
|
|
9
23
|
* Calculate cosine distance between two vectors
|
|
10
24
|
*
|
|
@@ -17,9 +31,7 @@
|
|
|
17
31
|
* @throws {Error} If vectors have different dimensions
|
|
18
32
|
*/
|
|
19
33
|
export function cosineDistance(a, b) {
|
|
20
|
-
|
|
21
|
-
throw new Error(`Dimension mismatch: ${a.length} vs ${b.length}`);
|
|
22
|
-
}
|
|
34
|
+
assertSameDimensions(a, b, 'cosineDistance');
|
|
23
35
|
|
|
24
36
|
let dotProduct = 0;
|
|
25
37
|
let normA = 0;
|
|
@@ -56,9 +68,7 @@ export function cosineDistance(a, b) {
|
|
|
56
68
|
* @throws {Error} If vectors have different dimensions
|
|
57
69
|
*/
|
|
58
70
|
export function euclideanDistance(a, b) {
|
|
59
|
-
|
|
60
|
-
throw new Error(`Dimension mismatch: ${a.length} vs ${b.length}`);
|
|
61
|
-
}
|
|
71
|
+
assertSameDimensions(a, b, 'euclideanDistance');
|
|
62
72
|
|
|
63
73
|
let sum = 0;
|
|
64
74
|
for (let i = 0; i < a.length; i++) {
|
|
@@ -81,9 +91,7 @@ export function euclideanDistance(a, b) {
|
|
|
81
91
|
* @throws {Error} If vectors have different dimensions
|
|
82
92
|
*/
|
|
83
93
|
export function manhattanDistance(a, b) {
|
|
84
|
-
|
|
85
|
-
throw new Error(`Dimension mismatch: ${a.length} vs ${b.length}`);
|
|
86
|
-
}
|
|
94
|
+
assertSameDimensions(a, b, 'manhattanDistance');
|
|
87
95
|
|
|
88
96
|
let sum = 0;
|
|
89
97
|
for (let i = 0; i < a.length; i++) {
|
|
@@ -104,9 +112,7 @@ export function manhattanDistance(a, b) {
|
|
|
104
112
|
* @throws {Error} If vectors have different dimensions
|
|
105
113
|
*/
|
|
106
114
|
export function dotProduct(a, b) {
|
|
107
|
-
|
|
108
|
-
throw new Error(`Dimension mismatch: ${a.length} vs ${b.length}`);
|
|
109
|
-
}
|
|
115
|
+
assertSameDimensions(a, b, 'dotProduct');
|
|
110
116
|
|
|
111
117
|
let sum = 0;
|
|
112
118
|
for (let i = 0; i < a.length; i++) {
|