s3db.js 13.6.0 → 14.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +139 -43
- package/dist/s3db.cjs +72425 -38970
- package/dist/s3db.cjs.map +1 -1
- package/dist/s3db.es.js +72177 -38764
- package/dist/s3db.es.js.map +1 -1
- package/mcp/lib/base-handler.js +157 -0
- package/mcp/lib/handlers/connection-handler.js +280 -0
- package/mcp/lib/handlers/query-handler.js +533 -0
- package/mcp/lib/handlers/resource-handler.js +428 -0
- package/mcp/lib/tool-registry.js +336 -0
- package/mcp/lib/tools/connection-tools.js +161 -0
- package/mcp/lib/tools/query-tools.js +267 -0
- package/mcp/lib/tools/resource-tools.js +404 -0
- package/package.json +94 -49
- package/src/clients/memory-client.class.js +346 -191
- package/src/clients/memory-storage.class.js +300 -84
- package/src/clients/s3-client.class.js +7 -6
- package/src/concerns/geo-encoding.js +19 -2
- package/src/concerns/ip.js +59 -9
- package/src/concerns/money.js +8 -1
- package/src/concerns/password-hashing.js +49 -8
- package/src/concerns/plugin-storage.js +186 -18
- package/src/concerns/storage-drivers/filesystem-driver.js +284 -0
- package/src/database.class.js +139 -29
- package/src/errors.js +332 -42
- package/src/plugins/api/auth/oidc-auth.js +66 -17
- package/src/plugins/api/auth/strategies/base-strategy.class.js +74 -0
- package/src/plugins/api/auth/strategies/factory.class.js +63 -0
- package/src/plugins/api/auth/strategies/global-strategy.class.js +44 -0
- package/src/plugins/api/auth/strategies/path-based-strategy.class.js +83 -0
- package/src/plugins/api/auth/strategies/path-rules-strategy.class.js +118 -0
- package/src/plugins/api/concerns/failban-manager.js +106 -57
- package/src/plugins/api/concerns/opengraph-helper.js +116 -0
- package/src/plugins/api/concerns/route-context.js +601 -0
- package/src/plugins/api/concerns/state-machine.js +288 -0
- package/src/plugins/api/index.js +180 -41
- package/src/plugins/api/routes/auth-routes.js +198 -30
- package/src/plugins/api/routes/resource-routes.js +19 -4
- package/src/plugins/api/server/health-manager.class.js +163 -0
- package/src/plugins/api/server/middleware-chain.class.js +310 -0
- package/src/plugins/api/server/router.class.js +472 -0
- package/src/plugins/api/server.js +280 -1303
- package/src/plugins/api/utils/custom-routes.js +17 -5
- package/src/plugins/api/utils/guards.js +76 -17
- package/src/plugins/api/utils/openapi-generator-cached.class.js +133 -0
- package/src/plugins/api/utils/openapi-generator.js +7 -6
- package/src/plugins/api/utils/template-engine.js +77 -3
- package/src/plugins/audit.plugin.js +30 -8
- package/src/plugins/backup.plugin.js +110 -14
- package/src/plugins/cache/cache.class.js +22 -5
- package/src/plugins/cache/filesystem-cache.class.js +116 -19
- package/src/plugins/cache/memory-cache.class.js +211 -57
- package/src/plugins/cache/multi-tier-cache.class.js +371 -0
- package/src/plugins/cache/partition-aware-filesystem-cache.class.js +168 -47
- package/src/plugins/cache/redis-cache.class.js +552 -0
- package/src/plugins/cache/s3-cache.class.js +17 -8
- package/src/plugins/cache.plugin.js +176 -61
- package/src/plugins/cloud-inventory/drivers/alibaba-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/aws-driver.js +60 -29
- package/src/plugins/cloud-inventory/drivers/azure-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/base-driver.js +16 -2
- package/src/plugins/cloud-inventory/drivers/cloudflare-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/digitalocean-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/hetzner-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/linode-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/mongodb-atlas-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/vultr-driver.js +8 -1
- package/src/plugins/cloud-inventory/index.js +29 -8
- package/src/plugins/cloud-inventory/registry.js +64 -42
- package/src/plugins/cloud-inventory.plugin.js +240 -138
- package/src/plugins/concerns/plugin-dependencies.js +54 -0
- package/src/plugins/concerns/resource-names.js +100 -0
- package/src/plugins/consumers/index.js +10 -2
- package/src/plugins/consumers/sqs-consumer.js +12 -2
- package/src/plugins/cookie-farm-suite.plugin.js +278 -0
- package/src/plugins/cookie-farm.errors.js +73 -0
- package/src/plugins/cookie-farm.plugin.js +869 -0
- package/src/plugins/costs.plugin.js +7 -1
- package/src/plugins/eventual-consistency/analytics.js +94 -19
- package/src/plugins/eventual-consistency/config.js +15 -7
- package/src/plugins/eventual-consistency/consolidation.js +29 -11
- package/src/plugins/eventual-consistency/garbage-collection.js +3 -1
- package/src/plugins/eventual-consistency/helpers.js +39 -14
- package/src/plugins/eventual-consistency/install.js +21 -2
- package/src/plugins/eventual-consistency/utils.js +32 -10
- package/src/plugins/fulltext.plugin.js +38 -11
- package/src/plugins/geo.plugin.js +61 -9
- package/src/plugins/identity/concerns/config.js +61 -0
- package/src/plugins/identity/concerns/mfa-manager.js +15 -2
- package/src/plugins/identity/concerns/rate-limit.js +124 -0
- package/src/plugins/identity/concerns/resource-schemas.js +9 -1
- package/src/plugins/identity/concerns/token-generator.js +29 -4
- package/src/plugins/identity/drivers/auth-driver.interface.js +76 -0
- package/src/plugins/identity/drivers/client-credentials-driver.js +127 -0
- package/src/plugins/identity/drivers/index.js +18 -0
- package/src/plugins/identity/drivers/password-driver.js +122 -0
- package/src/plugins/identity/email-service.js +17 -2
- package/src/plugins/identity/index.js +413 -69
- package/src/plugins/identity/oauth2-server.js +413 -30
- package/src/plugins/identity/oidc-discovery.js +16 -8
- package/src/plugins/identity/rsa-keys.js +115 -35
- package/src/plugins/identity/server.js +166 -45
- package/src/plugins/identity/session-manager.js +53 -7
- package/src/plugins/identity/ui/pages/mfa-verification.js +17 -15
- package/src/plugins/identity/ui/routes.js +363 -255
- package/src/plugins/importer/index.js +153 -20
- package/src/plugins/index.js +9 -2
- package/src/plugins/kubernetes-inventory/index.js +6 -0
- package/src/plugins/kubernetes-inventory/k8s-driver.js +867 -0
- package/src/plugins/kubernetes-inventory/resource-types.js +274 -0
- package/src/plugins/kubernetes-inventory.plugin.js +980 -0
- package/src/plugins/metrics.plugin.js +64 -16
- package/src/plugins/ml/base-model.class.js +25 -15
- package/src/plugins/ml/regression-model.class.js +1 -1
- package/src/plugins/ml.errors.js +57 -25
- package/src/plugins/ml.plugin.js +28 -4
- package/src/plugins/namespace.js +210 -0
- package/src/plugins/plugin.class.js +180 -8
- package/src/plugins/puppeteer/console-monitor.js +729 -0
- package/src/plugins/puppeteer/cookie-manager.js +492 -0
- package/src/plugins/puppeteer/network-monitor.js +816 -0
- package/src/plugins/puppeteer/performance-manager.js +746 -0
- package/src/plugins/puppeteer/proxy-manager.js +478 -0
- package/src/plugins/puppeteer/stealth-manager.js +556 -0
- package/src/plugins/puppeteer.errors.js +81 -0
- package/src/plugins/puppeteer.plugin.js +1327 -0
- package/src/plugins/queue-consumer.plugin.js +69 -14
- package/src/plugins/recon/behaviors/uptime-behavior.js +691 -0
- package/src/plugins/recon/concerns/command-runner.js +148 -0
- package/src/plugins/recon/concerns/diff-detector.js +372 -0
- package/src/plugins/recon/concerns/fingerprint-builder.js +307 -0
- package/src/plugins/recon/concerns/process-manager.js +338 -0
- package/src/plugins/recon/concerns/report-generator.js +478 -0
- package/src/plugins/recon/concerns/security-analyzer.js +571 -0
- package/src/plugins/recon/concerns/target-normalizer.js +68 -0
- package/src/plugins/recon/config/defaults.js +321 -0
- package/src/plugins/recon/config/resources.js +370 -0
- package/src/plugins/recon/index.js +778 -0
- package/src/plugins/recon/managers/dependency-manager.js +174 -0
- package/src/plugins/recon/managers/scheduler-manager.js +179 -0
- package/src/plugins/recon/managers/storage-manager.js +745 -0
- package/src/plugins/recon/managers/target-manager.js +274 -0
- package/src/plugins/recon/stages/asn-stage.js +314 -0
- package/src/plugins/recon/stages/certificate-stage.js +84 -0
- package/src/plugins/recon/stages/dns-stage.js +107 -0
- package/src/plugins/recon/stages/dnsdumpster-stage.js +362 -0
- package/src/plugins/recon/stages/fingerprint-stage.js +71 -0
- package/src/plugins/recon/stages/google-dorks-stage.js +440 -0
- package/src/plugins/recon/stages/http-stage.js +89 -0
- package/src/plugins/recon/stages/latency-stage.js +148 -0
- package/src/plugins/recon/stages/massdns-stage.js +302 -0
- package/src/plugins/recon/stages/osint-stage.js +1373 -0
- package/src/plugins/recon/stages/ports-stage.js +169 -0
- package/src/plugins/recon/stages/screenshot-stage.js +94 -0
- package/src/plugins/recon/stages/secrets-stage.js +514 -0
- package/src/plugins/recon/stages/subdomains-stage.js +295 -0
- package/src/plugins/recon/stages/tls-audit-stage.js +78 -0
- package/src/plugins/recon/stages/vulnerability-stage.js +78 -0
- package/src/plugins/recon/stages/web-discovery-stage.js +113 -0
- package/src/plugins/recon/stages/whois-stage.js +349 -0
- package/src/plugins/recon.plugin.js +75 -0
- package/src/plugins/recon.plugin.js.backup +2635 -0
- package/src/plugins/relation.errors.js +87 -14
- package/src/plugins/replicator.plugin.js +514 -137
- package/src/plugins/replicators/base-replicator.class.js +89 -1
- package/src/plugins/replicators/bigquery-replicator.class.js +66 -22
- package/src/plugins/replicators/dynamodb-replicator.class.js +22 -15
- package/src/plugins/replicators/mongodb-replicator.class.js +22 -15
- package/src/plugins/replicators/mysql-replicator.class.js +52 -17
- package/src/plugins/replicators/planetscale-replicator.class.js +30 -4
- package/src/plugins/replicators/postgres-replicator.class.js +62 -27
- package/src/plugins/replicators/s3db-replicator.class.js +25 -18
- package/src/plugins/replicators/schema-sync.helper.js +3 -3
- package/src/plugins/replicators/sqs-replicator.class.js +8 -2
- package/src/plugins/replicators/turso-replicator.class.js +23 -3
- package/src/plugins/replicators/webhook-replicator.class.js +42 -4
- package/src/plugins/s3-queue.plugin.js +464 -65
- package/src/plugins/scheduler.plugin.js +20 -6
- package/src/plugins/state-machine.plugin.js +40 -9
- package/src/plugins/tfstate/README.md +126 -126
- package/src/plugins/tfstate/base-driver.js +28 -4
- package/src/plugins/tfstate/errors.js +65 -10
- package/src/plugins/tfstate/filesystem-driver.js +52 -8
- package/src/plugins/tfstate/index.js +163 -90
- package/src/plugins/tfstate/s3-driver.js +64 -6
- package/src/plugins/ttl.plugin.js +72 -17
- package/src/plugins/vector/distances.js +18 -12
- package/src/plugins/vector/kmeans.js +26 -4
- package/src/resource.class.js +115 -19
- package/src/testing/factory.class.js +20 -3
- package/src/testing/seeder.class.js +7 -1
- package/src/clients/memory-client.md +0 -917
- package/src/plugins/cloud-inventory/drivers/mock-drivers.js +0 -449
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { PluginError } from '../../errors.js';
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* TfStatePlugin Error Classes
|
|
3
5
|
* Custom errors for Terraform/OpenTofu state operations
|
|
@@ -6,12 +8,19 @@
|
|
|
6
8
|
/**
|
|
7
9
|
* Base error for all Terraform/OpenTofu state operations
|
|
8
10
|
*/
|
|
9
|
-
export class TfStateError extends
|
|
11
|
+
export class TfStateError extends PluginError {
|
|
10
12
|
constructor(message, context = {}) {
|
|
11
|
-
|
|
13
|
+
const merged = {
|
|
14
|
+
pluginName: context.pluginName || 'TfStatePlugin',
|
|
15
|
+
operation: context.operation || 'unknown',
|
|
16
|
+
statusCode: context.statusCode ?? 500,
|
|
17
|
+
retriable: context.retriable ?? false,
|
|
18
|
+
suggestion: context.suggestion ?? 'Verify Terraform/OpenTofu configuration and state storage before retrying.',
|
|
19
|
+
...context
|
|
20
|
+
};
|
|
21
|
+
super(message, merged);
|
|
12
22
|
this.name = 'TfStateError';
|
|
13
23
|
this.context = context;
|
|
14
|
-
Error.captureStackTrace(this, this.constructor);
|
|
15
24
|
}
|
|
16
25
|
}
|
|
17
26
|
|
|
@@ -20,7 +29,14 @@ export class TfStateError extends Error {
|
|
|
20
29
|
*/
|
|
21
30
|
export class InvalidStateFileError extends TfStateError {
|
|
22
31
|
constructor(filePath, reason, context = {}) {
|
|
23
|
-
super(`Invalid Tfstate file "${filePath}": ${reason}`,
|
|
32
|
+
super(`Invalid Tfstate file "${filePath}": ${reason}`, {
|
|
33
|
+
statusCode: context.statusCode ?? 422,
|
|
34
|
+
retriable: false,
|
|
35
|
+
suggestion: context.suggestion ?? 'Validate Terraform state integrity or re-run terraform state pull.',
|
|
36
|
+
filePath,
|
|
37
|
+
reason,
|
|
38
|
+
...context
|
|
39
|
+
});
|
|
24
40
|
this.name = 'InvalidStateFileError';
|
|
25
41
|
this.filePath = filePath;
|
|
26
42
|
this.reason = reason;
|
|
@@ -34,7 +50,14 @@ export class UnsupportedStateVersionError extends TfStateError {
|
|
|
34
50
|
constructor(version, supportedVersions, context = {}) {
|
|
35
51
|
super(
|
|
36
52
|
`Tfstate version ${version} is not supported. Supported versions: ${supportedVersions.join(', ')}`,
|
|
37
|
-
|
|
53
|
+
{
|
|
54
|
+
statusCode: context.statusCode ?? 400,
|
|
55
|
+
retriable: false,
|
|
56
|
+
suggestion: context.suggestion ?? `Upgrade/downgrade Terraform state to one of the supported versions: ${supportedVersions.join(', ')}.`,
|
|
57
|
+
version,
|
|
58
|
+
supportedVersions,
|
|
59
|
+
...context
|
|
60
|
+
}
|
|
38
61
|
);
|
|
39
62
|
this.name = 'UnsupportedStateVersionError';
|
|
40
63
|
this.version = version;
|
|
@@ -47,7 +70,13 @@ export class UnsupportedStateVersionError extends TfStateError {
|
|
|
47
70
|
*/
|
|
48
71
|
export class StateFileNotFoundError extends TfStateError {
|
|
49
72
|
constructor(filePath, context = {}) {
|
|
50
|
-
super(`Tfstate file not found: ${filePath}`,
|
|
73
|
+
super(`Tfstate file not found: ${filePath}`, {
|
|
74
|
+
statusCode: context.statusCode ?? 404,
|
|
75
|
+
retriable: false,
|
|
76
|
+
suggestion: context.suggestion ?? 'Ensure the state file exists at the configured path/bucket.',
|
|
77
|
+
filePath,
|
|
78
|
+
...context
|
|
79
|
+
});
|
|
51
80
|
this.name = 'StateFileNotFoundError';
|
|
52
81
|
this.filePath = filePath;
|
|
53
82
|
}
|
|
@@ -60,7 +89,13 @@ export class ResourceExtractionError extends TfStateError {
|
|
|
60
89
|
constructor(resourceAddress, originalError, context = {}) {
|
|
61
90
|
super(
|
|
62
91
|
`Failed to extract resource "${resourceAddress}": ${originalError.message}`,
|
|
63
|
-
|
|
92
|
+
{
|
|
93
|
+
retriable: context.retriable ?? false,
|
|
94
|
+
suggestion: context.suggestion ?? 'Check resource address and state structure; rerun extraction after fixing the state.',
|
|
95
|
+
resourceAddress,
|
|
96
|
+
originalError,
|
|
97
|
+
...context
|
|
98
|
+
}
|
|
64
99
|
);
|
|
65
100
|
this.name = 'ResourceExtractionError';
|
|
66
101
|
this.resourceAddress = resourceAddress;
|
|
@@ -75,7 +110,14 @@ export class StateDiffError extends TfStateError {
|
|
|
75
110
|
constructor(oldSerial, newSerial, originalError, context = {}) {
|
|
76
111
|
super(
|
|
77
112
|
`Failed to calculate diff between state serials ${oldSerial} and ${newSerial}: ${originalError.message}`,
|
|
78
|
-
|
|
113
|
+
{
|
|
114
|
+
retriable: context.retriable ?? true,
|
|
115
|
+
suggestion: context.suggestion ?? 'Refresh the latest state snapshots and retry the diff operation.',
|
|
116
|
+
oldSerial,
|
|
117
|
+
newSerial,
|
|
118
|
+
originalError,
|
|
119
|
+
...context
|
|
120
|
+
}
|
|
79
121
|
);
|
|
80
122
|
this.name = 'StateDiffError';
|
|
81
123
|
this.oldSerial = oldSerial;
|
|
@@ -89,7 +131,13 @@ export class StateDiffError extends TfStateError {
|
|
|
89
131
|
*/
|
|
90
132
|
export class FileWatchError extends TfStateError {
|
|
91
133
|
constructor(path, originalError, context = {}) {
|
|
92
|
-
super(`Failed to watch path "${path}": ${originalError.message}`,
|
|
134
|
+
super(`Failed to watch path "${path}": ${originalError.message}`, {
|
|
135
|
+
retriable: context.retriable ?? true,
|
|
136
|
+
suggestion: context.suggestion ?? 'Verify filesystem permissions and that the watch path exists.',
|
|
137
|
+
path,
|
|
138
|
+
originalError,
|
|
139
|
+
...context
|
|
140
|
+
});
|
|
93
141
|
this.name = 'FileWatchError';
|
|
94
142
|
this.path = path;
|
|
95
143
|
this.originalError = originalError;
|
|
@@ -103,7 +151,14 @@ export class ResourceFilterError extends TfStateError {
|
|
|
103
151
|
constructor(filterExpression, originalError, context = {}) {
|
|
104
152
|
super(
|
|
105
153
|
`Failed to apply resource filter "${filterExpression}": ${originalError.message}`,
|
|
106
|
-
|
|
154
|
+
{
|
|
155
|
+
statusCode: context.statusCode ?? 400,
|
|
156
|
+
retriable: context.retriable ?? false,
|
|
157
|
+
suggestion: context.suggestion ?? 'Validate the filter expression syntax and ensure referenced resources exist.',
|
|
158
|
+
filterExpression,
|
|
159
|
+
originalError,
|
|
160
|
+
...context
|
|
161
|
+
}
|
|
107
162
|
);
|
|
108
163
|
this.name = 'ResourceFilterError';
|
|
109
164
|
this.filterExpression = filterExpression;
|
|
@@ -6,8 +6,9 @@
|
|
|
6
6
|
*/
|
|
7
7
|
import { TfStateDriver } from './base-driver.js';
|
|
8
8
|
import { readFile, stat } from 'fs/promises';
|
|
9
|
-
import { join
|
|
9
|
+
import { join } from 'path';
|
|
10
10
|
import { glob } from 'glob';
|
|
11
|
+
import { TfStateError, StateFileNotFoundError } from './errors.js';
|
|
11
12
|
|
|
12
13
|
export class FilesystemTfStateDriver extends TfStateDriver {
|
|
13
14
|
constructor(config = {}) {
|
|
@@ -23,10 +24,23 @@ export class FilesystemTfStateDriver extends TfStateDriver {
|
|
|
23
24
|
try {
|
|
24
25
|
const stats = await stat(this.basePath);
|
|
25
26
|
if (!stats.isDirectory()) {
|
|
26
|
-
throw new
|
|
27
|
+
throw new TfStateError(`Base path is not a directory: ${this.basePath}`, {
|
|
28
|
+
operation: 'initialize',
|
|
29
|
+
statusCode: 400,
|
|
30
|
+
retriable: false,
|
|
31
|
+
suggestion: 'Update the TfState filesystem driver configuration to point to a directory containing .tfstate files.',
|
|
32
|
+
basePath: this.basePath
|
|
33
|
+
});
|
|
27
34
|
}
|
|
28
35
|
} catch (error) {
|
|
29
|
-
throw new
|
|
36
|
+
throw new TfStateError(`Invalid base path: ${this.basePath}`, {
|
|
37
|
+
operation: 'initialize',
|
|
38
|
+
statusCode: 400,
|
|
39
|
+
retriable: false,
|
|
40
|
+
suggestion: 'Ensure the basePath exists and is readable by the current process.',
|
|
41
|
+
basePath: this.basePath,
|
|
42
|
+
original: error
|
|
43
|
+
});
|
|
30
44
|
}
|
|
31
45
|
}
|
|
32
46
|
|
|
@@ -60,7 +74,15 @@ export class FilesystemTfStateDriver extends TfStateDriver {
|
|
|
60
74
|
|
|
61
75
|
return stateFiles;
|
|
62
76
|
} catch (error) {
|
|
63
|
-
throw new
|
|
77
|
+
throw new TfStateError('Failed to list Terraform state files', {
|
|
78
|
+
operation: 'listStateFiles',
|
|
79
|
+
statusCode: 500,
|
|
80
|
+
retriable: false,
|
|
81
|
+
suggestion: 'Verify filesystem permissions and glob selector pattern.',
|
|
82
|
+
selector: this.selector,
|
|
83
|
+
basePath: this.basePath,
|
|
84
|
+
original: error
|
|
85
|
+
});
|
|
64
86
|
}
|
|
65
87
|
}
|
|
66
88
|
|
|
@@ -77,9 +99,20 @@ export class FilesystemTfStateDriver extends TfStateDriver {
|
|
|
77
99
|
return JSON.parse(content);
|
|
78
100
|
} catch (error) {
|
|
79
101
|
if (error.code === 'ENOENT') {
|
|
80
|
-
throw new
|
|
102
|
+
throw new StateFileNotFoundError(path, {
|
|
103
|
+
operation: 'readStateFile',
|
|
104
|
+
retriable: false,
|
|
105
|
+
suggestion: 'Ensure the Terraform state file exists at the specified path.',
|
|
106
|
+
original: error
|
|
107
|
+
});
|
|
81
108
|
}
|
|
82
|
-
throw new
|
|
109
|
+
throw new TfStateError(`Failed to read state file ${path}`, {
|
|
110
|
+
operation: 'readStateFile',
|
|
111
|
+
retriable: false,
|
|
112
|
+
suggestion: 'Validate file permissions and state file contents (must be valid JSON).',
|
|
113
|
+
path,
|
|
114
|
+
original: error
|
|
115
|
+
});
|
|
83
116
|
}
|
|
84
117
|
}
|
|
85
118
|
|
|
@@ -103,9 +136,20 @@ export class FilesystemTfStateDriver extends TfStateDriver {
|
|
|
103
136
|
};
|
|
104
137
|
} catch (error) {
|
|
105
138
|
if (error.code === 'ENOENT') {
|
|
106
|
-
throw new
|
|
139
|
+
throw new StateFileNotFoundError(path, {
|
|
140
|
+
operation: 'getStateFileMetadata',
|
|
141
|
+
retriable: false,
|
|
142
|
+
suggestion: 'Ensure the Terraform state file exists at the specified path.',
|
|
143
|
+
original: error
|
|
144
|
+
});
|
|
107
145
|
}
|
|
108
|
-
throw new
|
|
146
|
+
throw new TfStateError(`Failed to get metadata for ${path}`, {
|
|
147
|
+
operation: 'getStateFileMetadata',
|
|
148
|
+
retriable: false,
|
|
149
|
+
suggestion: 'Check filesystem permissions and path configuration for TfStatePlugin.',
|
|
150
|
+
path,
|
|
151
|
+
original: error
|
|
152
|
+
});
|
|
109
153
|
}
|
|
110
154
|
}
|
|
111
155
|
|
|
@@ -249,6 +249,7 @@ import { Plugin } from '../plugin.class.js';
|
|
|
249
249
|
import tryFn from '../../concerns/try-fn.js';
|
|
250
250
|
import requirePluginDependency from '../concerns/plugin-dependencies.js';
|
|
251
251
|
import { idGenerator } from '../../concerns/id.js';
|
|
252
|
+
import { resolveResourceNames } from '../concerns/resource-names.js';
|
|
252
253
|
import {
|
|
253
254
|
TfStateError,
|
|
254
255
|
InvalidStateFileError,
|
|
@@ -271,10 +272,31 @@ export class TfStatePlugin extends Plugin {
|
|
|
271
272
|
this.driverConfig = config.config || {};
|
|
272
273
|
|
|
273
274
|
// Resource names
|
|
274
|
-
const
|
|
275
|
-
|
|
276
|
-
this.
|
|
277
|
-
|
|
275
|
+
const resourcesConfig = config.resources || {};
|
|
276
|
+
const resourceNamesOption = config.resourceNames || {};
|
|
277
|
+
this._resourceDescriptors = {
|
|
278
|
+
resources: {
|
|
279
|
+
defaultName: 'plg_tfstate_resources',
|
|
280
|
+
override: resourceNamesOption.resources || resourcesConfig.resources || config.resourceName
|
|
281
|
+
},
|
|
282
|
+
stateFiles: {
|
|
283
|
+
defaultName: 'plg_tfstate_state_files',
|
|
284
|
+
override: resourceNamesOption.stateFiles || resourcesConfig.stateFiles || config.stateFilesName
|
|
285
|
+
},
|
|
286
|
+
diffs: {
|
|
287
|
+
defaultName: 'plg_tfstate_state_diffs',
|
|
288
|
+
override: resourceNamesOption.diffs || resourcesConfig.diffs || config.diffsName
|
|
289
|
+
},
|
|
290
|
+
lineages: {
|
|
291
|
+
defaultName: 'plg_tfstate_lineages',
|
|
292
|
+
override: resourceNamesOption.lineages || resourcesConfig.lineages
|
|
293
|
+
}
|
|
294
|
+
};
|
|
295
|
+
const resolvedNames = this._resolveResourceNames();
|
|
296
|
+
this.resourceName = resolvedNames.resources;
|
|
297
|
+
this.stateFilesName = resolvedNames.stateFiles;
|
|
298
|
+
this.diffsName = resolvedNames.diffs;
|
|
299
|
+
this.lineagesName = resolvedNames.lineages;
|
|
278
300
|
|
|
279
301
|
// Monitoring configuration
|
|
280
302
|
const monitor = config.monitor || {};
|
|
@@ -323,6 +345,20 @@ export class TfStatePlugin extends Plugin {
|
|
|
323
345
|
};
|
|
324
346
|
}
|
|
325
347
|
|
|
348
|
+
_resolveResourceNames() {
|
|
349
|
+
return resolveResourceNames('tfstate', this._resourceDescriptors, {
|
|
350
|
+
namespace: this.namespace
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
onNamespaceChanged() {
|
|
355
|
+
const names = this._resolveResourceNames();
|
|
356
|
+
this.resourceName = names.resources;
|
|
357
|
+
this.stateFilesName = names.stateFiles;
|
|
358
|
+
this.diffsName = names.diffs;
|
|
359
|
+
this.lineagesName = names.lineages;
|
|
360
|
+
}
|
|
361
|
+
|
|
326
362
|
/**
|
|
327
363
|
* Install the plugin
|
|
328
364
|
* @override
|
|
@@ -355,102 +391,131 @@ export class TfStatePlugin extends Plugin {
|
|
|
355
391
|
|
|
356
392
|
// Resource 0: Terraform Lineages (Master tracking resource)
|
|
357
393
|
// NEW: Tracks unique Tfstate lineages for efficient diff tracking
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
394
|
+
{
|
|
395
|
+
const [created, createErr, resource] = await tryFn(() => this.database.createResource({
|
|
396
|
+
name: this.lineagesName,
|
|
397
|
+
attributes: {
|
|
398
|
+
id: 'string|required',
|
|
399
|
+
latestSerial: 'number',
|
|
400
|
+
latestStateId: 'string',
|
|
401
|
+
totalStates: 'number',
|
|
402
|
+
firstImportedAt: 'number',
|
|
403
|
+
lastImportedAt: 'number',
|
|
404
|
+
metadata: 'json'
|
|
405
|
+
},
|
|
406
|
+
timestamps: true,
|
|
407
|
+
asyncPartitions: this.asyncPartitions,
|
|
408
|
+
partitions: {},
|
|
409
|
+
createdBy: 'TfStatePlugin'
|
|
410
|
+
}));
|
|
411
|
+
|
|
412
|
+
if (created) {
|
|
413
|
+
this.lineagesResource = resource;
|
|
414
|
+
} else {
|
|
415
|
+
this.lineagesResource = this.database.resources?.[this.lineagesName];
|
|
416
|
+
if (!this.lineagesResource) {
|
|
417
|
+
throw createErr;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
}
|
|
375
421
|
|
|
376
422
|
// Resource 1: Tfstate Files Metadata
|
|
377
423
|
// Dedicated to tracking state file metadata with SHA256 hash for deduplication
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
424
|
+
{
|
|
425
|
+
const [created, createErr, resource] = await tryFn(() => this.database.createResource({
|
|
426
|
+
name: this.stateFilesName,
|
|
427
|
+
attributes: {
|
|
428
|
+
id: 'string|required',
|
|
429
|
+
lineageId: 'string|required',
|
|
430
|
+
sourceFile: 'string|required',
|
|
431
|
+
serial: 'number|required',
|
|
432
|
+
lineage: 'string|required',
|
|
433
|
+
terraformVersion: 'string',
|
|
434
|
+
stateVersion: 'number|required',
|
|
435
|
+
resourceCount: 'number',
|
|
436
|
+
sha256Hash: 'string|required',
|
|
437
|
+
importedAt: 'number|required'
|
|
438
|
+
},
|
|
439
|
+
timestamps: true,
|
|
440
|
+
asyncPartitions: this.asyncPartitions,
|
|
441
|
+
partitions: {
|
|
442
|
+
byLineage: { fields: { lineageId: 'string' } },
|
|
443
|
+
byLineageSerial: { fields: { lineageId: 'string', serial: 'number' } },
|
|
444
|
+
bySourceFile: { fields: { sourceFile: 'string' } },
|
|
445
|
+
bySerial: { fields: { serial: 'number' } },
|
|
446
|
+
bySha256: { fields: { sha256Hash: 'string' } }
|
|
447
|
+
},
|
|
448
|
+
createdBy: 'TfStatePlugin'
|
|
449
|
+
}));
|
|
450
|
+
|
|
451
|
+
if (created) {
|
|
452
|
+
this.stateFilesResource = resource;
|
|
453
|
+
} else {
|
|
454
|
+
this.stateFilesResource = this.database.resources?.[this.stateFilesName];
|
|
455
|
+
if (!this.stateFilesResource) {
|
|
456
|
+
throw createErr;
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
}
|
|
403
460
|
|
|
404
461
|
// Resource 2: Terraform Resources
|
|
405
462
|
// Store extracted resources with foreign key to state files
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
463
|
+
{
|
|
464
|
+
const [created, createErr, resource] = await tryFn(() => this.database.createResource({
|
|
465
|
+
name: this.resourceName,
|
|
466
|
+
attributes: {
|
|
467
|
+
id: 'string|required',
|
|
468
|
+
stateFileId: 'string|required',
|
|
469
|
+
lineageId: 'string|required',
|
|
470
|
+
stateSerial: 'number|required',
|
|
471
|
+
sourceFile: 'string|required',
|
|
472
|
+
resourceType: 'string|required',
|
|
473
|
+
resourceName: 'string|required',
|
|
474
|
+
resourceAddress: 'string|required',
|
|
475
|
+
providerName: 'string|required',
|
|
476
|
+
mode: 'string',
|
|
477
|
+
attributes: 'json',
|
|
478
|
+
dependencies: 'array',
|
|
479
|
+
importedAt: 'number|required'
|
|
480
|
+
},
|
|
481
|
+
timestamps: true,
|
|
482
|
+
asyncPartitions: this.asyncPartitions,
|
|
483
|
+
partitions: {
|
|
484
|
+
byLineageSerial: { fields: { lineageId: 'string', stateSerial: 'number' } },
|
|
485
|
+
byLineage: { fields: { lineageId: 'string' } },
|
|
486
|
+
byType: { fields: { resourceType: 'string' } },
|
|
487
|
+
byProvider: { fields: { providerName: 'string' } },
|
|
488
|
+
bySerial: { fields: { stateSerial: 'number' } },
|
|
489
|
+
bySourceFile: { fields: { sourceFile: 'string' } },
|
|
490
|
+
byProviderAndType: { fields: { providerName: 'string', resourceType: 'string' } },
|
|
491
|
+
byLineageType: { fields: { lineageId: 'string', resourceType: 'string' } }
|
|
492
|
+
},
|
|
493
|
+
createdBy: 'TfStatePlugin'
|
|
494
|
+
}));
|
|
495
|
+
|
|
496
|
+
if (created) {
|
|
497
|
+
this.resource = resource;
|
|
498
|
+
} else {
|
|
499
|
+
this.resource = this.database.resources?.[this.resourceName];
|
|
500
|
+
if (!this.resource) {
|
|
501
|
+
throw createErr;
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
}
|
|
439
505
|
|
|
440
506
|
// Resource 3: Tfstate Diffs
|
|
441
507
|
// Track changes between state versions (if diff tracking enabled)
|
|
442
508
|
if (this.trackDiffs) {
|
|
443
|
-
|
|
509
|
+
const [created, createErr, resource] = await tryFn(() => this.database.createResource({
|
|
444
510
|
name: this.diffsName,
|
|
445
511
|
attributes: {
|
|
446
512
|
id: 'string|required',
|
|
447
|
-
lineageId: 'string|required',
|
|
513
|
+
lineageId: 'string|required',
|
|
448
514
|
oldSerial: 'number|required',
|
|
449
515
|
newSerial: 'number|required',
|
|
450
|
-
oldStateId: 'string',
|
|
451
|
-
newStateId: 'string|required',
|
|
516
|
+
oldStateId: 'string',
|
|
517
|
+
newStateId: 'string|required',
|
|
452
518
|
calculatedAt: 'number|required',
|
|
453
|
-
// Summary statistics
|
|
454
519
|
summary: {
|
|
455
520
|
type: 'object',
|
|
456
521
|
props: {
|
|
@@ -459,7 +524,6 @@ export class TfStatePlugin extends Plugin {
|
|
|
459
524
|
deletedCount: 'number'
|
|
460
525
|
}
|
|
461
526
|
},
|
|
462
|
-
// Detailed changes
|
|
463
527
|
changes: {
|
|
464
528
|
type: 'object',
|
|
465
529
|
props: {
|
|
@@ -469,17 +533,26 @@ export class TfStatePlugin extends Plugin {
|
|
|
469
533
|
}
|
|
470
534
|
}
|
|
471
535
|
},
|
|
472
|
-
behavior: 'body-only',
|
|
536
|
+
behavior: 'body-only',
|
|
473
537
|
timestamps: true,
|
|
474
|
-
asyncPartitions: this.asyncPartitions,
|
|
538
|
+
asyncPartitions: this.asyncPartitions,
|
|
475
539
|
partitions: {
|
|
476
|
-
byLineage: { fields: { lineageId: 'string' } },
|
|
477
|
-
byLineageNewSerial: { fields: { lineageId: 'string', newSerial: 'number' } },
|
|
540
|
+
byLineage: { fields: { lineageId: 'string' } },
|
|
541
|
+
byLineageNewSerial: { fields: { lineageId: 'string', newSerial: 'number' } },
|
|
478
542
|
byNewSerial: { fields: { newSerial: 'number' } },
|
|
479
543
|
byOldSerial: { fields: { oldSerial: 'number' } }
|
|
480
544
|
},
|
|
481
545
|
createdBy: 'TfStatePlugin'
|
|
482
|
-
});
|
|
546
|
+
}));
|
|
547
|
+
|
|
548
|
+
if (created) {
|
|
549
|
+
this.diffsResource = resource;
|
|
550
|
+
} else {
|
|
551
|
+
this.diffsResource = this.database.resources?.[this.diffsName];
|
|
552
|
+
if (!this.diffsResource) {
|
|
553
|
+
throw createErr;
|
|
554
|
+
}
|
|
555
|
+
}
|
|
483
556
|
}
|
|
484
557
|
|
|
485
558
|
if (this.verbose) {
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { TfStateDriver } from './base-driver.js';
|
|
7
7
|
import { S3Client } from '../../clients/s3-client.class.js';
|
|
8
8
|
import tryFn from '../../concerns/try-fn.js';
|
|
9
|
+
import { TfStateError, InvalidStateFileError, StateFileNotFoundError } from './errors.js';
|
|
9
10
|
|
|
10
11
|
export class S3TfStateDriver extends TfStateDriver {
|
|
11
12
|
constructor(config = {}) {
|
|
@@ -36,7 +37,13 @@ export class S3TfStateDriver extends TfStateDriver {
|
|
|
36
37
|
const url = new URL(connectionString);
|
|
37
38
|
|
|
38
39
|
if (url.protocol !== 's3:') {
|
|
39
|
-
throw new
|
|
40
|
+
throw new TfStateError('Connection string must use s3:// protocol', {
|
|
41
|
+
operation: 'parseConnectionString',
|
|
42
|
+
statusCode: 400,
|
|
43
|
+
retriable: false,
|
|
44
|
+
suggestion: 'Use format s3://accessKey:secretKey@bucket/prefix?region=us-east-1',
|
|
45
|
+
connectionString
|
|
46
|
+
});
|
|
40
47
|
}
|
|
41
48
|
|
|
42
49
|
const credentials = {};
|
|
@@ -61,7 +68,14 @@ export class S3TfStateDriver extends TfStateDriver {
|
|
|
61
68
|
region
|
|
62
69
|
};
|
|
63
70
|
} catch (error) {
|
|
64
|
-
throw new
|
|
71
|
+
throw new TfStateError('Invalid S3 connection string', {
|
|
72
|
+
operation: 'parseConnectionString',
|
|
73
|
+
statusCode: 400,
|
|
74
|
+
retriable: false,
|
|
75
|
+
suggestion: 'Ensure the connection string follows s3://accessKey:secretKey@bucket/prefix?region=REGION.',
|
|
76
|
+
connectionString,
|
|
77
|
+
original: error
|
|
78
|
+
});
|
|
65
79
|
}
|
|
66
80
|
}
|
|
67
81
|
|
|
@@ -95,7 +109,14 @@ export class S3TfStateDriver extends TfStateDriver {
|
|
|
95
109
|
});
|
|
96
110
|
|
|
97
111
|
if (!ok) {
|
|
98
|
-
throw new
|
|
112
|
+
throw new TfStateError('Failed to list Terraform state objects from S3', {
|
|
113
|
+
operation: 'listStateFiles',
|
|
114
|
+
retriable: false,
|
|
115
|
+
suggestion: 'Validate S3 permissions (s3:ListBucket) and prefix configuration.',
|
|
116
|
+
bucket,
|
|
117
|
+
prefix,
|
|
118
|
+
original: err
|
|
119
|
+
});
|
|
99
120
|
}
|
|
100
121
|
|
|
101
122
|
const objects = data.Contents || [];
|
|
@@ -133,14 +154,35 @@ export class S3TfStateDriver extends TfStateDriver {
|
|
|
133
154
|
});
|
|
134
155
|
|
|
135
156
|
if (!ok) {
|
|
136
|
-
|
|
157
|
+
if (err?.$metadata?.httpStatusCode === 404) {
|
|
158
|
+
throw new StateFileNotFoundError(path, {
|
|
159
|
+
operation: 'readStateFile',
|
|
160
|
+
retriable: false,
|
|
161
|
+
suggestion: 'Ensure the state file exists in S3 and the IAM role can access it.',
|
|
162
|
+
bucket,
|
|
163
|
+
original: err
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
throw new TfStateError(`Failed to read state file ${path}`, {
|
|
167
|
+
operation: 'readStateFile',
|
|
168
|
+
retriable: false,
|
|
169
|
+
suggestion: 'Verify S3 permissions (s3:GetObject) and network connectivity.',
|
|
170
|
+
bucket,
|
|
171
|
+
path,
|
|
172
|
+
original: err
|
|
173
|
+
});
|
|
137
174
|
}
|
|
138
175
|
|
|
139
176
|
try {
|
|
140
177
|
const content = data.Body.toString('utf-8');
|
|
141
178
|
return JSON.parse(content);
|
|
142
179
|
} catch (parseError) {
|
|
143
|
-
throw new
|
|
180
|
+
throw new InvalidStateFileError(path, parseError.message, {
|
|
181
|
+
operation: 'readStateFile',
|
|
182
|
+
retriable: false,
|
|
183
|
+
suggestion: 'Check if the state file contains valid JSON exported by Terraform.',
|
|
184
|
+
original: parseError
|
|
185
|
+
});
|
|
144
186
|
}
|
|
145
187
|
}
|
|
146
188
|
|
|
@@ -158,7 +200,23 @@ export class S3TfStateDriver extends TfStateDriver {
|
|
|
158
200
|
});
|
|
159
201
|
|
|
160
202
|
if (!ok) {
|
|
161
|
-
|
|
203
|
+
if (err?.$metadata?.httpStatusCode === 404) {
|
|
204
|
+
throw new StateFileNotFoundError(path, {
|
|
205
|
+
operation: 'getStateFileMetadata',
|
|
206
|
+
retriable: false,
|
|
207
|
+
suggestion: 'Ensure the state file exists in S3 and the IAM role can access it.',
|
|
208
|
+
bucket,
|
|
209
|
+
original: err
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
throw new TfStateError(`Failed to get metadata for ${path}`, {
|
|
213
|
+
operation: 'getStateFileMetadata',
|
|
214
|
+
retriable: false,
|
|
215
|
+
suggestion: 'Verify S3 permissions (s3:HeadObject) and bucket configuration.',
|
|
216
|
+
bucket,
|
|
217
|
+
path,
|
|
218
|
+
original: err
|
|
219
|
+
});
|
|
162
220
|
}
|
|
163
221
|
|
|
164
222
|
return {
|