s3db.js 12.1.0 → 12.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +212 -196
- package/dist/s3db.cjs.js +1286 -226
- package/dist/s3db.cjs.js.map +1 -1
- package/dist/s3db.es.js +1284 -226
- package/dist/s3db.es.js.map +1 -1
- package/package.json +6 -1
- package/src/cli/index.js +954 -43
- package/src/cli/migration-manager.js +270 -0
- package/src/concerns/calculator.js +0 -4
- package/src/concerns/metadata-encoding.js +1 -21
- package/src/concerns/plugin-storage.js +17 -4
- package/src/concerns/typescript-generator.d.ts +171 -0
- package/src/concerns/typescript-generator.js +275 -0
- package/src/database.class.js +171 -28
- package/src/index.js +15 -9
- package/src/plugins/api/index.js +0 -1
- package/src/plugins/api/routes/resource-routes.js +86 -1
- package/src/plugins/api/server.js +79 -3
- package/src/plugins/api/utils/openapi-generator.js +195 -5
- package/src/plugins/backup/multi-backup-driver.class.js +0 -1
- package/src/plugins/backup.plugin.js +7 -14
- package/src/plugins/concerns/plugin-dependencies.js +73 -19
- package/src/plugins/eventual-consistency/analytics.js +0 -2
- package/src/plugins/eventual-consistency/consolidation.js +2 -13
- package/src/plugins/eventual-consistency/index.js +0 -1
- package/src/plugins/eventual-consistency/install.js +1 -1
- package/src/plugins/geo.plugin.js +5 -6
- package/src/plugins/importer/index.js +1 -1
- package/src/plugins/relation.plugin.js +11 -11
- package/src/plugins/replicator.plugin.js +12 -21
- package/src/plugins/s3-queue.plugin.js +4 -4
- package/src/plugins/scheduler.plugin.js +10 -12
- package/src/plugins/state-machine.plugin.js +8 -12
- package/src/plugins/tfstate/README.md +1 -1
- package/src/plugins/tfstate/errors.js +3 -3
- package/src/plugins/tfstate/index.js +41 -67
- package/src/plugins/ttl.plugin.js +3 -3
- package/src/resource.class.js +263 -61
- package/src/schema.class.js +0 -2
- package/src/testing/factory.class.js +286 -0
- package/src/testing/index.js +15 -0
- package/src/testing/seeder.class.js +183 -0
|
@@ -20,7 +20,7 @@ export class TfStateError extends Error {
|
|
|
20
20
|
*/
|
|
21
21
|
export class InvalidStateFileError extends TfStateError {
|
|
22
22
|
constructor(filePath, reason, context = {}) {
|
|
23
|
-
super(`Invalid
|
|
23
|
+
super(`Invalid Tfstate file "${filePath}": ${reason}`, context);
|
|
24
24
|
this.name = 'InvalidStateFileError';
|
|
25
25
|
this.filePath = filePath;
|
|
26
26
|
this.reason = reason;
|
|
@@ -33,7 +33,7 @@ export class InvalidStateFileError extends TfStateError {
|
|
|
33
33
|
export class UnsupportedStateVersionError extends TfStateError {
|
|
34
34
|
constructor(version, supportedVersions, context = {}) {
|
|
35
35
|
super(
|
|
36
|
-
`
|
|
36
|
+
`Tfstate version ${version} is not supported. Supported versions: ${supportedVersions.join(', ')}`,
|
|
37
37
|
context
|
|
38
38
|
);
|
|
39
39
|
this.name = 'UnsupportedStateVersionError';
|
|
@@ -47,7 +47,7 @@ export class UnsupportedStateVersionError extends TfStateError {
|
|
|
47
47
|
*/
|
|
48
48
|
export class StateFileNotFoundError extends TfStateError {
|
|
49
49
|
constructor(filePath, context = {}) {
|
|
50
|
-
super(`
|
|
50
|
+
super(`Tfstate file not found: ${filePath}`, context);
|
|
51
51
|
this.name = 'StateFileNotFoundError';
|
|
52
52
|
this.filePath = filePath;
|
|
53
53
|
}
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* OpenTofu maintains backward compatibility with Terraform's state file format, so this plugin works seamlessly with both.
|
|
9
9
|
*
|
|
10
10
|
* === 🚀 Key Features ===
|
|
11
|
-
* ✅ **Multi-version support**:
|
|
11
|
+
* ✅ **Multi-version support**: Tfstate v3 and v4
|
|
12
12
|
* ✅ **Multiple sources**: Local files, S3 buckets, remote backends
|
|
13
13
|
* ✅ **SHA256 deduplication**: Prevent duplicate state imports automatically
|
|
14
14
|
* ✅ **Historical tracking**: Full audit trail of infrastructure changes
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* ✅ **Batch import**: Process multiple state files with controlled parallelism
|
|
17
17
|
* ✅ **Resource filtering**: Include/exclude resources by type or pattern
|
|
18
18
|
* ✅ **Auto-sync**: File watching and cron-based monitoring (optional)
|
|
19
|
-
* ✅ **Export capability**: Convert back to
|
|
19
|
+
* ✅ **Export capability**: Convert back to Tfstate format
|
|
20
20
|
* ✅ **Automatic partition optimization**: 10-100x faster queries with zero configuration
|
|
21
21
|
*
|
|
22
22
|
* === ⚡ Performance Optimizations (Auto-Applied) ===
|
|
@@ -266,59 +266,36 @@ export class TfStatePlugin extends Plugin {
|
|
|
266
266
|
constructor(config = {}) {
|
|
267
267
|
super(config);
|
|
268
268
|
|
|
269
|
-
//
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
this.filters = config.filters || {};
|
|
300
|
-
this.verbose = config.verbose || false;
|
|
301
|
-
} else {
|
|
302
|
-
// Legacy configuration (backward compatible)
|
|
303
|
-
this.driverType = null; // Will use legacy methods
|
|
304
|
-
this.driverConfig = {};
|
|
305
|
-
this.resourceName = config.resourceName || 'plg_tfstate_resources';
|
|
306
|
-
this.stateFilesName = config.stateFilesName || 'plg_tfstate_state_files';
|
|
307
|
-
// Support both 'diffsName' and 'stateHistoryName' (legacy) for backward compatibility
|
|
308
|
-
this.diffsName = config.diffsName || config.stateHistoryName || 'plg_tfstate_state_diffs';
|
|
309
|
-
this.stateHistoryName = this.diffsName; // Alias for backward compatibility
|
|
310
|
-
this.autoSync = config.autoSync || false;
|
|
311
|
-
this.watchPaths = Array.isArray(config.watchPaths) ? config.watchPaths : [];
|
|
312
|
-
this.filters = config.filters || {};
|
|
313
|
-
this.trackDiffs = config.trackDiffs !== undefined ? config.trackDiffs : true;
|
|
314
|
-
this.diffsLookback = 10;
|
|
315
|
-
this.asyncPartitions = config.asyncPartitions !== undefined ? config.asyncPartitions : true;
|
|
316
|
-
this.verbose = config.verbose || false;
|
|
317
|
-
this.monitorEnabled = false;
|
|
318
|
-
this.monitorCron = '*/5 * * * *';
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
// Supported Terraform state versions
|
|
269
|
+
// Driver-based configuration
|
|
270
|
+
this.driverType = config.driver || null;
|
|
271
|
+
this.driverConfig = config.config || {};
|
|
272
|
+
|
|
273
|
+
// Resource names
|
|
274
|
+
const resources = config.resources || {};
|
|
275
|
+
this.resourceName = resources.resources || config.resourceName || 'plg_tfstate_resources';
|
|
276
|
+
this.stateFilesName = resources.stateFiles || config.stateFilesName || 'plg_tfstate_state_files';
|
|
277
|
+
this.diffsName = resources.diffs || config.diffsName || 'plg_tfstate_state_diffs';
|
|
278
|
+
|
|
279
|
+
// Monitoring configuration
|
|
280
|
+
const monitor = config.monitor || {};
|
|
281
|
+
this.monitorEnabled = monitor.enabled || false;
|
|
282
|
+
this.monitorCron = monitor.cron || '*/5 * * * *';
|
|
283
|
+
|
|
284
|
+
// Diff configuration
|
|
285
|
+
const diffs = config.diffs || {};
|
|
286
|
+
this.trackDiffs = diffs.enabled !== undefined ? diffs.enabled : (config.trackDiffs !== undefined ? config.trackDiffs : true);
|
|
287
|
+
this.diffsLookback = diffs.lookback || 10;
|
|
288
|
+
|
|
289
|
+
// Partition configuration
|
|
290
|
+
this.asyncPartitions = config.asyncPartitions !== undefined ? config.asyncPartitions : true;
|
|
291
|
+
|
|
292
|
+
// Other config
|
|
293
|
+
this.autoSync = config.autoSync || false;
|
|
294
|
+
this.watchPaths = config.watchPaths || [];
|
|
295
|
+
this.filters = config.filters || {};
|
|
296
|
+
this.verbose = config.verbose || false;
|
|
297
|
+
|
|
298
|
+
// Supported Tfstate versions
|
|
322
299
|
this.supportedVersions = [3, 4];
|
|
323
300
|
|
|
324
301
|
// Internal state
|
|
@@ -377,12 +354,12 @@ export class TfStatePlugin extends Plugin {
|
|
|
377
354
|
}
|
|
378
355
|
|
|
379
356
|
// Resource 0: Terraform Lineages (Master tracking resource)
|
|
380
|
-
// NEW: Tracks unique
|
|
357
|
+
// NEW: Tracks unique Tfstate lineages for efficient diff tracking
|
|
381
358
|
this.lineagesName = 'plg_tfstate_lineages';
|
|
382
359
|
this.lineagesResource = await this.database.createResource({
|
|
383
360
|
name: this.lineagesName,
|
|
384
361
|
attributes: {
|
|
385
|
-
id: 'string|required', // = lineage UUID from
|
|
362
|
+
id: 'string|required', // = lineage UUID from Tfstate
|
|
386
363
|
latestSerial: 'number', // Track latest for quick access
|
|
387
364
|
latestStateId: 'string', // FK to stateFilesResource
|
|
388
365
|
totalStates: 'number', // Counter
|
|
@@ -396,7 +373,7 @@ export class TfStatePlugin extends Plugin {
|
|
|
396
373
|
createdBy: 'TfStatePlugin'
|
|
397
374
|
});
|
|
398
375
|
|
|
399
|
-
// Resource 1:
|
|
376
|
+
// Resource 1: Tfstate Files Metadata
|
|
400
377
|
// Dedicated to tracking state file metadata with SHA256 hash for deduplication
|
|
401
378
|
this.stateFilesResource = await this.database.createResource({
|
|
402
379
|
name: this.stateFilesName,
|
|
@@ -460,7 +437,7 @@ export class TfStatePlugin extends Plugin {
|
|
|
460
437
|
createdBy: 'TfStatePlugin'
|
|
461
438
|
});
|
|
462
439
|
|
|
463
|
-
// Resource 3:
|
|
440
|
+
// Resource 3: Tfstate Diffs
|
|
464
441
|
// Track changes between state versions (if diff tracking enabled)
|
|
465
442
|
if (this.trackDiffs) {
|
|
466
443
|
this.diffsResource = await this.database.createResource({
|
|
@@ -511,12 +488,10 @@ export class TfStatePlugin extends Plugin {
|
|
|
511
488
|
console.log(`[TfStatePlugin] Created resources: ${resourcesCreated.join(', ')}`);
|
|
512
489
|
}
|
|
513
490
|
|
|
514
|
-
// Setup file watchers if autoSync is enabled (legacy mode)
|
|
515
491
|
if (this.autoSync && this.watchPaths.length > 0) {
|
|
516
492
|
await this._setupFileWatchers();
|
|
517
493
|
}
|
|
518
494
|
|
|
519
|
-
// Setup cron monitoring if enabled (new driver mode)
|
|
520
495
|
if (this.monitorEnabled && this.driver) {
|
|
521
496
|
await this._setupCronMonitoring();
|
|
522
497
|
}
|
|
@@ -556,7 +531,6 @@ export class TfStatePlugin extends Plugin {
|
|
|
556
531
|
}
|
|
557
532
|
}
|
|
558
533
|
|
|
559
|
-
// Stop all file watchers (legacy mode)
|
|
560
534
|
for (const watcher of this.watchers) {
|
|
561
535
|
try {
|
|
562
536
|
// fs.promises.watch returns an AsyncIterator with a return() method
|
|
@@ -1281,7 +1255,7 @@ export class TfStatePlugin extends Plugin {
|
|
|
1281
1255
|
}
|
|
1282
1256
|
|
|
1283
1257
|
/**
|
|
1284
|
-
* Read and parse
|
|
1258
|
+
* Read and parse Tfstate file
|
|
1285
1259
|
* @private
|
|
1286
1260
|
*/
|
|
1287
1261
|
async _readStateFile(filePath) {
|
|
@@ -1327,7 +1301,7 @@ export class TfStatePlugin extends Plugin {
|
|
|
1327
1301
|
}
|
|
1328
1302
|
|
|
1329
1303
|
/**
|
|
1330
|
-
* Validate
|
|
1304
|
+
* Validate Tfstate version
|
|
1331
1305
|
* @private
|
|
1332
1306
|
*/
|
|
1333
1307
|
_validateStateVersion(state) {
|
|
@@ -1343,7 +1317,7 @@ export class TfStatePlugin extends Plugin {
|
|
|
1343
1317
|
}
|
|
1344
1318
|
|
|
1345
1319
|
/**
|
|
1346
|
-
* Extract resources from
|
|
1320
|
+
* Extract resources from Tfstate
|
|
1347
1321
|
* @private
|
|
1348
1322
|
*/
|
|
1349
1323
|
async _extractResources(state, filePath, stateFileId, lineageId) {
|
|
@@ -2065,14 +2039,14 @@ export class TfStatePlugin extends Plugin {
|
|
|
2065
2039
|
}
|
|
2066
2040
|
|
|
2067
2041
|
/**
|
|
2068
|
-
* Export resources to
|
|
2042
|
+
* Export resources to Tfstate format
|
|
2069
2043
|
* @param {Object} options - Export options
|
|
2070
2044
|
* @param {number} options.serial - Specific serial to export (default: latest)
|
|
2071
2045
|
* @param {string[]} options.resourceTypes - Filter by resource types
|
|
2072
2046
|
* @param {string} options.terraformVersion - Terraform version for output (default: '1.5.0')
|
|
2073
2047
|
* @param {string} options.lineage - State lineage (default: auto-generated)
|
|
2074
2048
|
* @param {Object} options.outputs - Terraform outputs to include
|
|
2075
|
-
* @returns {Promise<Object>}
|
|
2049
|
+
* @returns {Promise<Object>} Tfstate object
|
|
2076
2050
|
*
|
|
2077
2051
|
* @example
|
|
2078
2052
|
* // Export latest state
|
|
@@ -284,7 +284,7 @@ class TTLPlugin extends Plugin {
|
|
|
284
284
|
return;
|
|
285
285
|
}
|
|
286
286
|
|
|
287
|
-
const resource = this.database.
|
|
287
|
+
const resource = this.database.resources[resourceName];
|
|
288
288
|
|
|
289
289
|
// Verify methods exist before adding middleware
|
|
290
290
|
if (typeof resource.insert !== 'function' || typeof resource.delete !== 'function') {
|
|
@@ -520,7 +520,7 @@ class TTLPlugin extends Plugin {
|
|
|
520
520
|
return;
|
|
521
521
|
}
|
|
522
522
|
|
|
523
|
-
const resource = this.database.
|
|
523
|
+
const resource = this.database.resources[entry.resourceName];
|
|
524
524
|
|
|
525
525
|
// Get the actual record
|
|
526
526
|
const [ok, err, record] = await tryFn(() => resource.get(entry.recordId));
|
|
@@ -612,7 +612,7 @@ class TTLPlugin extends Plugin {
|
|
|
612
612
|
throw new Error(`Archive resource "${config.archiveResource}" not found`);
|
|
613
613
|
}
|
|
614
614
|
|
|
615
|
-
const archiveResource = this.database.
|
|
615
|
+
const archiveResource = this.database.resources[config.archiveResource];
|
|
616
616
|
|
|
617
617
|
// Copy only user data fields (not system fields like _etag, _lastModified, etc.)
|
|
618
618
|
const archiveData = {};
|