s3db.js 13.6.0 → 14.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/README.md +139 -43
  2. package/dist/s3db.cjs +72425 -38970
  3. package/dist/s3db.cjs.map +1 -1
  4. package/dist/s3db.es.js +72177 -38764
  5. package/dist/s3db.es.js.map +1 -1
  6. package/mcp/lib/base-handler.js +157 -0
  7. package/mcp/lib/handlers/connection-handler.js +280 -0
  8. package/mcp/lib/handlers/query-handler.js +533 -0
  9. package/mcp/lib/handlers/resource-handler.js +428 -0
  10. package/mcp/lib/tool-registry.js +336 -0
  11. package/mcp/lib/tools/connection-tools.js +161 -0
  12. package/mcp/lib/tools/query-tools.js +267 -0
  13. package/mcp/lib/tools/resource-tools.js +404 -0
  14. package/package.json +94 -49
  15. package/src/clients/memory-client.class.js +346 -191
  16. package/src/clients/memory-storage.class.js +300 -84
  17. package/src/clients/s3-client.class.js +7 -6
  18. package/src/concerns/geo-encoding.js +19 -2
  19. package/src/concerns/ip.js +59 -9
  20. package/src/concerns/money.js +8 -1
  21. package/src/concerns/password-hashing.js +49 -8
  22. package/src/concerns/plugin-storage.js +186 -18
  23. package/src/concerns/storage-drivers/filesystem-driver.js +284 -0
  24. package/src/database.class.js +139 -29
  25. package/src/errors.js +332 -42
  26. package/src/plugins/api/auth/oidc-auth.js +66 -17
  27. package/src/plugins/api/auth/strategies/base-strategy.class.js +74 -0
  28. package/src/plugins/api/auth/strategies/factory.class.js +63 -0
  29. package/src/plugins/api/auth/strategies/global-strategy.class.js +44 -0
  30. package/src/plugins/api/auth/strategies/path-based-strategy.class.js +83 -0
  31. package/src/plugins/api/auth/strategies/path-rules-strategy.class.js +118 -0
  32. package/src/plugins/api/concerns/failban-manager.js +106 -57
  33. package/src/plugins/api/concerns/opengraph-helper.js +116 -0
  34. package/src/plugins/api/concerns/route-context.js +601 -0
  35. package/src/plugins/api/concerns/state-machine.js +288 -0
  36. package/src/plugins/api/index.js +180 -41
  37. package/src/plugins/api/routes/auth-routes.js +198 -30
  38. package/src/plugins/api/routes/resource-routes.js +19 -4
  39. package/src/plugins/api/server/health-manager.class.js +163 -0
  40. package/src/plugins/api/server/middleware-chain.class.js +310 -0
  41. package/src/plugins/api/server/router.class.js +472 -0
  42. package/src/plugins/api/server.js +280 -1303
  43. package/src/plugins/api/utils/custom-routes.js +17 -5
  44. package/src/plugins/api/utils/guards.js +76 -17
  45. package/src/plugins/api/utils/openapi-generator-cached.class.js +133 -0
  46. package/src/plugins/api/utils/openapi-generator.js +7 -6
  47. package/src/plugins/api/utils/template-engine.js +77 -3
  48. package/src/plugins/audit.plugin.js +30 -8
  49. package/src/plugins/backup.plugin.js +110 -14
  50. package/src/plugins/cache/cache.class.js +22 -5
  51. package/src/plugins/cache/filesystem-cache.class.js +116 -19
  52. package/src/plugins/cache/memory-cache.class.js +211 -57
  53. package/src/plugins/cache/multi-tier-cache.class.js +371 -0
  54. package/src/plugins/cache/partition-aware-filesystem-cache.class.js +168 -47
  55. package/src/plugins/cache/redis-cache.class.js +552 -0
  56. package/src/plugins/cache/s3-cache.class.js +17 -8
  57. package/src/plugins/cache.plugin.js +176 -61
  58. package/src/plugins/cloud-inventory/drivers/alibaba-driver.js +8 -1
  59. package/src/plugins/cloud-inventory/drivers/aws-driver.js +60 -29
  60. package/src/plugins/cloud-inventory/drivers/azure-driver.js +8 -1
  61. package/src/plugins/cloud-inventory/drivers/base-driver.js +16 -2
  62. package/src/plugins/cloud-inventory/drivers/cloudflare-driver.js +8 -1
  63. package/src/plugins/cloud-inventory/drivers/digitalocean-driver.js +8 -1
  64. package/src/plugins/cloud-inventory/drivers/hetzner-driver.js +8 -1
  65. package/src/plugins/cloud-inventory/drivers/linode-driver.js +8 -1
  66. package/src/plugins/cloud-inventory/drivers/mongodb-atlas-driver.js +8 -1
  67. package/src/plugins/cloud-inventory/drivers/vultr-driver.js +8 -1
  68. package/src/plugins/cloud-inventory/index.js +29 -8
  69. package/src/plugins/cloud-inventory/registry.js +64 -42
  70. package/src/plugins/cloud-inventory.plugin.js +240 -138
  71. package/src/plugins/concerns/plugin-dependencies.js +54 -0
  72. package/src/plugins/concerns/resource-names.js +100 -0
  73. package/src/plugins/consumers/index.js +10 -2
  74. package/src/plugins/consumers/sqs-consumer.js +12 -2
  75. package/src/plugins/cookie-farm-suite.plugin.js +278 -0
  76. package/src/plugins/cookie-farm.errors.js +73 -0
  77. package/src/plugins/cookie-farm.plugin.js +869 -0
  78. package/src/plugins/costs.plugin.js +7 -1
  79. package/src/plugins/eventual-consistency/analytics.js +94 -19
  80. package/src/plugins/eventual-consistency/config.js +15 -7
  81. package/src/plugins/eventual-consistency/consolidation.js +29 -11
  82. package/src/plugins/eventual-consistency/garbage-collection.js +3 -1
  83. package/src/plugins/eventual-consistency/helpers.js +39 -14
  84. package/src/plugins/eventual-consistency/install.js +21 -2
  85. package/src/plugins/eventual-consistency/utils.js +32 -10
  86. package/src/plugins/fulltext.plugin.js +38 -11
  87. package/src/plugins/geo.plugin.js +61 -9
  88. package/src/plugins/identity/concerns/config.js +61 -0
  89. package/src/plugins/identity/concerns/mfa-manager.js +15 -2
  90. package/src/plugins/identity/concerns/rate-limit.js +124 -0
  91. package/src/plugins/identity/concerns/resource-schemas.js +9 -1
  92. package/src/plugins/identity/concerns/token-generator.js +29 -4
  93. package/src/plugins/identity/drivers/auth-driver.interface.js +76 -0
  94. package/src/plugins/identity/drivers/client-credentials-driver.js +127 -0
  95. package/src/plugins/identity/drivers/index.js +18 -0
  96. package/src/plugins/identity/drivers/password-driver.js +122 -0
  97. package/src/plugins/identity/email-service.js +17 -2
  98. package/src/plugins/identity/index.js +413 -69
  99. package/src/plugins/identity/oauth2-server.js +413 -30
  100. package/src/plugins/identity/oidc-discovery.js +16 -8
  101. package/src/plugins/identity/rsa-keys.js +115 -35
  102. package/src/plugins/identity/server.js +166 -45
  103. package/src/plugins/identity/session-manager.js +53 -7
  104. package/src/plugins/identity/ui/pages/mfa-verification.js +17 -15
  105. package/src/plugins/identity/ui/routes.js +363 -255
  106. package/src/plugins/importer/index.js +153 -20
  107. package/src/plugins/index.js +9 -2
  108. package/src/plugins/kubernetes-inventory/index.js +6 -0
  109. package/src/plugins/kubernetes-inventory/k8s-driver.js +867 -0
  110. package/src/plugins/kubernetes-inventory/resource-types.js +274 -0
  111. package/src/plugins/kubernetes-inventory.plugin.js +980 -0
  112. package/src/plugins/metrics.plugin.js +64 -16
  113. package/src/plugins/ml/base-model.class.js +25 -15
  114. package/src/plugins/ml/regression-model.class.js +1 -1
  115. package/src/plugins/ml.errors.js +57 -25
  116. package/src/plugins/ml.plugin.js +28 -4
  117. package/src/plugins/namespace.js +210 -0
  118. package/src/plugins/plugin.class.js +180 -8
  119. package/src/plugins/puppeteer/console-monitor.js +729 -0
  120. package/src/plugins/puppeteer/cookie-manager.js +492 -0
  121. package/src/plugins/puppeteer/network-monitor.js +816 -0
  122. package/src/plugins/puppeteer/performance-manager.js +746 -0
  123. package/src/plugins/puppeteer/proxy-manager.js +478 -0
  124. package/src/plugins/puppeteer/stealth-manager.js +556 -0
  125. package/src/plugins/puppeteer.errors.js +81 -0
  126. package/src/plugins/puppeteer.plugin.js +1327 -0
  127. package/src/plugins/queue-consumer.plugin.js +69 -14
  128. package/src/plugins/recon/behaviors/uptime-behavior.js +691 -0
  129. package/src/plugins/recon/concerns/command-runner.js +148 -0
  130. package/src/plugins/recon/concerns/diff-detector.js +372 -0
  131. package/src/plugins/recon/concerns/fingerprint-builder.js +307 -0
  132. package/src/plugins/recon/concerns/process-manager.js +338 -0
  133. package/src/plugins/recon/concerns/report-generator.js +478 -0
  134. package/src/plugins/recon/concerns/security-analyzer.js +571 -0
  135. package/src/plugins/recon/concerns/target-normalizer.js +68 -0
  136. package/src/plugins/recon/config/defaults.js +321 -0
  137. package/src/plugins/recon/config/resources.js +370 -0
  138. package/src/plugins/recon/index.js +778 -0
  139. package/src/plugins/recon/managers/dependency-manager.js +174 -0
  140. package/src/plugins/recon/managers/scheduler-manager.js +179 -0
  141. package/src/plugins/recon/managers/storage-manager.js +745 -0
  142. package/src/plugins/recon/managers/target-manager.js +274 -0
  143. package/src/plugins/recon/stages/asn-stage.js +314 -0
  144. package/src/plugins/recon/stages/certificate-stage.js +84 -0
  145. package/src/plugins/recon/stages/dns-stage.js +107 -0
  146. package/src/plugins/recon/stages/dnsdumpster-stage.js +362 -0
  147. package/src/plugins/recon/stages/fingerprint-stage.js +71 -0
  148. package/src/plugins/recon/stages/google-dorks-stage.js +440 -0
  149. package/src/plugins/recon/stages/http-stage.js +89 -0
  150. package/src/plugins/recon/stages/latency-stage.js +148 -0
  151. package/src/plugins/recon/stages/massdns-stage.js +302 -0
  152. package/src/plugins/recon/stages/osint-stage.js +1373 -0
  153. package/src/plugins/recon/stages/ports-stage.js +169 -0
  154. package/src/plugins/recon/stages/screenshot-stage.js +94 -0
  155. package/src/plugins/recon/stages/secrets-stage.js +514 -0
  156. package/src/plugins/recon/stages/subdomains-stage.js +295 -0
  157. package/src/plugins/recon/stages/tls-audit-stage.js +78 -0
  158. package/src/plugins/recon/stages/vulnerability-stage.js +78 -0
  159. package/src/plugins/recon/stages/web-discovery-stage.js +113 -0
  160. package/src/plugins/recon/stages/whois-stage.js +349 -0
  161. package/src/plugins/recon.plugin.js +75 -0
  162. package/src/plugins/recon.plugin.js.backup +2635 -0
  163. package/src/plugins/relation.errors.js +87 -14
  164. package/src/plugins/replicator.plugin.js +514 -137
  165. package/src/plugins/replicators/base-replicator.class.js +89 -1
  166. package/src/plugins/replicators/bigquery-replicator.class.js +66 -22
  167. package/src/plugins/replicators/dynamodb-replicator.class.js +22 -15
  168. package/src/plugins/replicators/mongodb-replicator.class.js +22 -15
  169. package/src/plugins/replicators/mysql-replicator.class.js +52 -17
  170. package/src/plugins/replicators/planetscale-replicator.class.js +30 -4
  171. package/src/plugins/replicators/postgres-replicator.class.js +62 -27
  172. package/src/plugins/replicators/s3db-replicator.class.js +25 -18
  173. package/src/plugins/replicators/schema-sync.helper.js +3 -3
  174. package/src/plugins/replicators/sqs-replicator.class.js +8 -2
  175. package/src/plugins/replicators/turso-replicator.class.js +23 -3
  176. package/src/plugins/replicators/webhook-replicator.class.js +42 -4
  177. package/src/plugins/s3-queue.plugin.js +464 -65
  178. package/src/plugins/scheduler.plugin.js +20 -6
  179. package/src/plugins/state-machine.plugin.js +40 -9
  180. package/src/plugins/tfstate/README.md +126 -126
  181. package/src/plugins/tfstate/base-driver.js +28 -4
  182. package/src/plugins/tfstate/errors.js +65 -10
  183. package/src/plugins/tfstate/filesystem-driver.js +52 -8
  184. package/src/plugins/tfstate/index.js +163 -90
  185. package/src/plugins/tfstate/s3-driver.js +64 -6
  186. package/src/plugins/ttl.plugin.js +72 -17
  187. package/src/plugins/vector/distances.js +18 -12
  188. package/src/plugins/vector/kmeans.js +26 -4
  189. package/src/resource.class.js +115 -19
  190. package/src/testing/factory.class.js +20 -3
  191. package/src/testing/seeder.class.js +7 -1
  192. package/src/clients/memory-client.md +0 -917
  193. package/src/plugins/cloud-inventory/drivers/mock-drivers.js +0 -449
@@ -0,0 +1,980 @@
1
+ /**
2
+ * KubernetesInventoryPlugin
3
+ *
4
+ * Collects and tracks all resources from Kubernetes clusters.
5
+ * For each discovered resource we store:
6
+ * - A snapshot with the latest configuration digest
7
+ * - Immutable configuration versions (history)
8
+ * - Structured diffs between versions
9
+ *
10
+ * Supports:
11
+ * - Multi-cluster inventory
12
+ * - Core resources + Custom Resource Definitions (CRDs)
13
+ * - Select/ignore filtering
14
+ * - Scheduled discovery
15
+ * - Distributed locking
16
+ */
17
+
18
+ import { createHash } from 'crypto';
19
+ import jsonStableStringify from 'json-stable-stringify';
20
+ import isEqual from 'lodash-es/isEqual.js';
21
+
22
+ import { Plugin } from './plugin.class.js';
23
+ import { PluginError } from '../errors.js';
24
+ import tryFn from '../concerns/try-fn.js';
25
+ import { KubernetesDriver } from './kubernetes-inventory/k8s-driver.js';
26
+ import { formatResourceTypeId, parseResourceTypeId } from './kubernetes-inventory/resource-types.js';
27
+ import { resolveResourceNames } from './concerns/resource-names.js';
28
+
29
+ const DEFAULT_DISCOVERY = {
30
+ concurrency: 2,
31
+ select: null, // null = allow all
32
+ ignore: [], // empty = ignore nothing
33
+ runOnInstall: true,
34
+ dryRun: false,
35
+ };
36
+
37
+ const DEFAULT_LOCK = {
38
+ ttl: 600, // 10 minutes (K8s can be slow)
39
+ timeout: 0,
40
+ };
41
+
42
+ const BASE_SCHEDULE = {
43
+ enabled: false,
44
+ cron: null,
45
+ timezone: undefined,
46
+ runOnStart: false,
47
+ };
48
+
49
+ /**
50
+ * Normalize cluster definitions
51
+ */
52
+ function normalizeClusterDefinitions(clusters, logger) {
53
+ return clusters.map((cluster, index) => {
54
+ // Auto-generate ID if not provided
55
+ if (!cluster.id) {
56
+ cluster.id = `k8s-cluster-${index + 1}`;
57
+ logger('debug', `Auto-generated cluster ID: ${cluster.id}`);
58
+ }
59
+
60
+ // Set name to ID if not provided
61
+ if (!cluster.name) {
62
+ cluster.name = cluster.id;
63
+ }
64
+
65
+ // Normalize discovery options
66
+ cluster.discovery = cluster.discovery || {};
67
+
68
+ // Normalize scheduled
69
+ cluster.scheduled = normalizeSchedule(cluster.scheduled);
70
+
71
+ // Tags and metadata
72
+ cluster.tags = cluster.tags || {};
73
+ cluster.metadata = cluster.metadata || {};
74
+
75
+ return cluster;
76
+ });
77
+ }
78
+
79
+ /**
80
+ * Normalize schedule configuration
81
+ */
82
+ function normalizeSchedule(schedule) {
83
+ if (!schedule) return { ...BASE_SCHEDULE };
84
+ if (typeof schedule !== 'object') return { ...BASE_SCHEDULE };
85
+
86
+ return {
87
+ enabled: schedule.enabled === true,
88
+ cron: schedule.cron || null,
89
+ timezone: schedule.timezone,
90
+ runOnStart: schedule.runOnStart === true,
91
+ };
92
+ }
93
+
94
+ /**
95
+ * KubernetesInventoryPlugin Class
96
+ */
97
+ export class KubernetesInventoryPlugin extends Plugin {
98
+ constructor(options = {}) {
99
+ super(options);
100
+
101
+ const pendingLogs = [];
102
+ const normalizedClusters = normalizeClusterDefinitions(
103
+ Array.isArray(options.clusters) ? options.clusters : [],
104
+ (level, message, meta) => pendingLogs.push({ level, message, meta })
105
+ );
106
+
107
+ this._internalResourceOverrides = options.resourceNames || {};
108
+ this._internalResourceDescriptors = {
109
+ snapshots: {
110
+ defaultName: 'plg_k8s_inventory_snapshots',
111
+ override: this._internalResourceOverrides.snapshots,
112
+ },
113
+ versions: {
114
+ defaultName: 'plg_k8s_inventory_versions',
115
+ override: this._internalResourceOverrides.versions,
116
+ },
117
+ changes: {
118
+ defaultName: 'plg_k8s_inventory_changes',
119
+ override: this._internalResourceOverrides.changes,
120
+ },
121
+ clusters: {
122
+ defaultName: 'plg_k8s_inventory_clusters',
123
+ override: this._internalResourceOverrides.clusters,
124
+ },
125
+ };
126
+ this.internalResourceNames = this._resolveInternalResourceNames();
127
+
128
+ this.config = {
129
+ clusters: normalizedClusters,
130
+ discovery: {
131
+ ...DEFAULT_DISCOVERY,
132
+ ...(options.discovery || {}),
133
+ },
134
+ resourceNames: this.internalResourceNames,
135
+ logger: typeof options.logger === 'function' ? options.logger : null,
136
+ verbose: options.verbose === true,
137
+ scheduled: normalizeSchedule(options.scheduled),
138
+ lock: {
139
+ ttl: options.lock?.ttl ?? DEFAULT_LOCK.ttl,
140
+ timeout: options.lock?.timeout ?? DEFAULT_LOCK.timeout,
141
+ },
142
+ };
143
+
144
+ this.clusterDrivers = new Map();
145
+ this._resourceHandles = {};
146
+ this._scheduledJobs = [];
147
+ this._cron = null;
148
+ this.resourceNames = this.internalResourceNames;
149
+
150
+ for (const entry of pendingLogs) {
151
+ this._log(entry.level, entry.message, entry.meta);
152
+ }
153
+ }
154
+
155
+ // ============================================
156
+ // LIFECYCLE HOOKS
157
+ // ============================================
158
+
159
+ async onInstall() {
160
+ this._validateConfiguration();
161
+ await this._ensureResources();
162
+ await this._initializeDrivers();
163
+
164
+ if (this.config.discovery.runOnInstall) {
165
+ await this.syncAll();
166
+ }
167
+ }
168
+
169
+ async onStart() {
170
+ await this._setupSchedules();
171
+ }
172
+
173
+ async onStop() {
174
+ await this._teardownSchedules();
175
+ await this._destroyDrivers();
176
+ }
177
+
178
+ async onUninstall() {
179
+ await this._teardownSchedules();
180
+ await this._destroyDrivers();
181
+ }
182
+
183
+ onNamespaceChanged() {
184
+ this.internalResourceNames = this._resolveInternalResourceNames();
185
+ if (this.config) {
186
+ this.config.resourceNames = this.internalResourceNames;
187
+ }
188
+ this.resourceNames = this.internalResourceNames;
189
+ this._resourceHandles = {};
190
+ }
191
+
192
+ // ============================================
193
+ // PUBLIC API
194
+ // ============================================
195
+
196
+ /**
197
+ * Sync all clusters
198
+ */
199
+ async syncAll(options = {}) {
200
+ const results = [];
201
+ for (const cluster of this.config.clusters) {
202
+ const result = await this.syncCluster(cluster.id, options);
203
+ results.push(result);
204
+ }
205
+ return results;
206
+ }
207
+
208
+ /**
209
+ * Sync specific cluster
210
+ */
211
+ async syncCluster(clusterId, options = {}) {
212
+ const driverEntry = this.clusterDrivers.get(clusterId);
213
+ if (!driverEntry) {
214
+ throw new PluginError(`Cluster "${clusterId}" is not registered`, {
215
+ pluginName: 'KubernetesInventoryPlugin',
216
+ operation: 'syncCluster',
217
+ statusCode: 404,
218
+ retriable: false,
219
+ suggestion: `Register the cluster in KubernetesInventoryPlugin configuration. Available: ${[...this.clusterDrivers.keys()].join(', ') || 'none'}.`,
220
+ clusterId,
221
+ });
222
+ }
223
+
224
+ const { driver, definition } = driverEntry;
225
+ const summaryResource = this._resourceHandles.clusters;
226
+
227
+ const summaryBefore = (await summaryResource.getOrNull(clusterId))
228
+ ?? await this._ensureClusterSummaryRecord(clusterId, definition, definition.scheduled);
229
+
230
+ const storage = this.getStorage();
231
+ const lockKey = `k8s-inventory-sync-${clusterId}`;
232
+ const lock = await storage.acquireLock(lockKey, {
233
+ ttl: this.config.lock.ttl,
234
+ timeout: this.config.lock.timeout,
235
+ });
236
+
237
+ if (!lock) {
238
+ this._log('warn', `Could not acquire lock for cluster sync: ${clusterId}`, { clusterId, lockKey });
239
+ return {
240
+ clusterId,
241
+ skipped: true,
242
+ reason: 'lock-not-acquired',
243
+ lockKey,
244
+ };
245
+ }
246
+
247
+ const startTime = Date.now();
248
+
249
+ try {
250
+ await summaryResource.patch(clusterId, {
251
+ status: 'running',
252
+ lastRunAt: new Date().toISOString(),
253
+ });
254
+
255
+ const checkpoint = summaryBefore.checkpoint || null;
256
+ const state = summaryBefore.state || {};
257
+
258
+ const runtime = {
259
+ emitProgress: (data) => this._emitProgress(clusterId, data),
260
+ emitCheckpoint: async (data) => {
261
+ await summaryResource.patch(clusterId, { checkpoint: data });
262
+ },
263
+ };
264
+
265
+ const counters = {
266
+ total: 0,
267
+ created: 0,
268
+ updated: 0,
269
+ unchanged: 0,
270
+ errors: 0,
271
+ };
272
+
273
+ const discoveryOptions = {
274
+ checkpoint,
275
+ state,
276
+ runtime,
277
+ ...options,
278
+ };
279
+
280
+ // List resources from driver
281
+ const resourceIterator = driver.listResources(discoveryOptions);
282
+
283
+ for await (const resource of resourceIterator) {
284
+ counters.total++;
285
+
286
+ // Apply filtering
287
+ if (!this._shouldIncludeResource(resource)) {
288
+ continue;
289
+ }
290
+
291
+ // Persist snapshot
292
+ if (!this.config.discovery.dryRun) {
293
+ const [ok, err, result] = await tryFn(async () => {
294
+ return await this._persistSnapshot(resource, resource);
295
+ });
296
+
297
+ if (ok) {
298
+ if (result.status === 'created') counters.created++;
299
+ else if (result.status === 'updated') counters.updated++;
300
+ else if (result.status === 'unchanged') counters.unchanged++;
301
+ } else {
302
+ counters.errors++;
303
+ this._log('error', `Failed to persist resource snapshot: ${err.message}`, {
304
+ clusterId,
305
+ resourceType: resource.resourceType,
306
+ resourceId: resource.resourceId,
307
+ error: err.message,
308
+ });
309
+ }
310
+ }
311
+ }
312
+
313
+ const duration = Date.now() - startTime;
314
+
315
+ // Update cluster summary
316
+ await summaryResource.patch(clusterId, {
317
+ status: 'idle',
318
+ lastResult: {
319
+ success: true,
320
+ timestamp: new Date().toISOString(),
321
+ duration,
322
+ counters,
323
+ },
324
+ checkpoint: null, // Clear checkpoint on success
325
+ state: {}, // Clear state on success
326
+ });
327
+
328
+ this._log('info', `Cluster sync completed: ${clusterId}`, { clusterId, duration, counters });
329
+
330
+ return {
331
+ clusterId,
332
+ success: true,
333
+ duration,
334
+ ...counters,
335
+ };
336
+ } catch (error) {
337
+ const duration = Date.now() - startTime;
338
+
339
+ await summaryResource.patch(clusterId, {
340
+ status: 'error',
341
+ lastResult: {
342
+ success: false,
343
+ timestamp: new Date().toISOString(),
344
+ duration,
345
+ error: error.message,
346
+ },
347
+ });
348
+
349
+ this._log('error', `Cluster sync failed: ${clusterId}`, {
350
+ clusterId,
351
+ error: error.message,
352
+ stack: error.stack,
353
+ });
354
+
355
+ throw error;
356
+ } finally {
357
+ await storage.releaseLock(lock);
358
+ }
359
+ }
360
+
361
+ /**
362
+ * Discover available resource types in a cluster
363
+ */
364
+ async discoverResourceTypes(clusterId, options = {}) {
365
+ const driverEntry = this.clusterDrivers.get(clusterId);
366
+ if (!driverEntry) {
367
+ throw new PluginError(`Cluster "${clusterId}" is not registered`, {
368
+ pluginName: 'KubernetesInventoryPlugin',
369
+ operation: 'discoverResourceTypes',
370
+ clusterId,
371
+ });
372
+ }
373
+
374
+ const { driver } = driverEntry;
375
+ return await driver.discoverResourceTypes(options);
376
+ }
377
+
378
+ /**
379
+ * Get snapshots (latest state of resources)
380
+ */
381
+ async getSnapshots(filter = {}) {
382
+ const resource = this._resourceHandles.snapshots;
383
+ const query = {};
384
+
385
+ if (filter.clusterId) query.clusterId = filter.clusterId;
386
+ if (filter.resourceType) query.resourceType = filter.resourceType;
387
+ if (filter.namespace) query.namespace = filter.namespace;
388
+
389
+ return await resource.query(query);
390
+ }
391
+
392
+ /**
393
+ * Get version history for a resource
394
+ */
395
+ async getVersions(filter = {}) {
396
+ const resource = this._resourceHandles.versions;
397
+ const query = {};
398
+
399
+ if (filter.clusterId) query.clusterId = filter.clusterId;
400
+ if (filter.resourceType) query.resourceType = filter.resourceType;
401
+ if (filter.resourceId) query.resourceId = filter.resourceId;
402
+
403
+ return await resource.query(query);
404
+ }
405
+
406
+ /**
407
+ * Get changes (diffs) for resources
408
+ */
409
+ async getChanges(filter = {}) {
410
+ const resource = this._resourceHandles.changes;
411
+ const query = {};
412
+
413
+ if (filter.clusterId) query.clusterId = filter.clusterId;
414
+ if (filter.resourceType) query.resourceType = filter.resourceType;
415
+ if (filter.resourceId) query.resourceId = filter.resourceId;
416
+
417
+ // Filter by time if provided
418
+ if (filter.since) {
419
+ const results = await resource.query(query);
420
+ return results.filter(change => new Date(change.createdAt) >= new Date(filter.since));
421
+ }
422
+
423
+ return await resource.query(query);
424
+ }
425
+
426
+ // ============================================
427
+ // INTERNAL METHODS
428
+ // ============================================
429
+
430
+ /**
431
+ * Validate plugin configuration
432
+ */
433
+ _validateConfiguration() {
434
+ if (!this.config.clusters || this.config.clusters.length === 0) {
435
+ throw new PluginError('At least one cluster must be configured', {
436
+ pluginName: 'KubernetesInventoryPlugin',
437
+ operation: 'validateConfiguration',
438
+ });
439
+ }
440
+
441
+ // Validate cluster IDs are unique
442
+ const clusterIds = this.config.clusters.map(c => c.id);
443
+ const duplicates = clusterIds.filter((id, index) => clusterIds.indexOf(id) !== index);
444
+ if (duplicates.length > 0) {
445
+ throw new PluginError(`Duplicate cluster IDs found: ${duplicates.join(', ')}`, {
446
+ pluginName: 'KubernetesInventoryPlugin',
447
+ operation: 'validateConfiguration',
448
+ });
449
+ }
450
+ }
451
+
452
+ /**
453
+ * Create internal resources for storing inventory data
454
+ */
455
+ async _ensureResources() {
456
+ const { database } = this;
457
+
458
+ // Snapshots resource (latest state of each resource)
459
+ this._resourceHandles.snapshots = await database.createResource({
460
+ name: this.config.resourceNames.snapshots,
461
+ createdBy: 'KubernetesInventoryPlugin',
462
+ attributes: {
463
+ clusterId: 'string|required',
464
+ namespace: 'string',
465
+ resourceType: 'string|required',
466
+ resourceId: 'string|required',
467
+ uid: 'string',
468
+ name: 'string',
469
+ apiVersion: 'string',
470
+ kind: 'string',
471
+ labels: 'object',
472
+ annotations: 'object',
473
+ latestDigest: 'string|required',
474
+ latestVersion: 'number|required|integer|min:1',
475
+ changelogSize: 'number|required|integer|min:0',
476
+ firstSeenAt: 'string|required',
477
+ lastSeenAt: 'string|required',
478
+ createdAt: 'string|required',
479
+ updatedAt: 'string|required',
480
+ },
481
+ partitions: {
482
+ byClusterId: { fields: { clusterId: 'string' } },
483
+ byResourceType: { fields: { resourceType: 'string' } },
484
+ byClusterAndType: { fields: { clusterId: 'string', resourceType: 'string' } },
485
+ byNamespace: { fields: { namespace: 'string' } },
486
+ },
487
+ timestamps: false,
488
+ });
489
+
490
+ // Versions resource (immutable history)
491
+ this._resourceHandles.versions = await database.createResource({
492
+ name: this.config.resourceNames.versions,
493
+ createdBy: 'KubernetesInventoryPlugin',
494
+ attributes: {
495
+ clusterId: 'string|required',
496
+ resourceType: 'string|required',
497
+ resourceId: 'string|required',
498
+ uid: 'string',
499
+ namespace: 'string',
500
+ version: 'number|required|integer|min:1',
501
+ digest: 'string|required',
502
+ capturedAt: 'string|required',
503
+ configuration: 'object|required',
504
+ summary: 'object',
505
+ raw: 'object',
506
+ },
507
+ partitions: {
508
+ byResourceKey: {
509
+ fields: { clusterId: 'string', resourceType: 'string', resourceId: 'string' },
510
+ },
511
+ byClusterId: { fields: { clusterId: 'string' } },
512
+ },
513
+ timestamps: true,
514
+ });
515
+
516
+ // Changes resource (diffs between versions)
517
+ this._resourceHandles.changes = await database.createResource({
518
+ name: this.config.resourceNames.changes,
519
+ createdBy: 'KubernetesInventoryPlugin',
520
+ attributes: {
521
+ clusterId: 'string|required',
522
+ resourceType: 'string|required',
523
+ resourceId: 'string|required',
524
+ fromVersion: 'number|required|integer|min:1',
525
+ toVersion: 'number|required|integer|min:1',
526
+ diff: 'object|required',
527
+ },
528
+ partitions: {
529
+ byResourceKey: {
530
+ fields: { clusterId: 'string', resourceType: 'string', resourceId: 'string' },
531
+ },
532
+ byClusterId: { fields: { clusterId: 'string' } },
533
+ },
534
+ timestamps: true,
535
+ });
536
+
537
+ // Clusters resource (cluster metadata and sync status)
538
+ this._resourceHandles.clusters = await database.createResource({
539
+ name: this.config.resourceNames.clusters,
540
+ createdBy: 'KubernetesInventoryPlugin',
541
+ attributes: {
542
+ id: 'string|required',
543
+ name: 'string|required',
544
+ status: 'string|required|enum:idle,running,error',
545
+ lastRunAt: 'string',
546
+ lastResult: 'object',
547
+ checkpoint: 'object',
548
+ state: 'object',
549
+ schedule: 'object',
550
+ tags: 'object',
551
+ metadata: 'object',
552
+ },
553
+ timestamps: true,
554
+ });
555
+
556
+ this._log('info', 'Internal resources created successfully');
557
+ }
558
+
559
+ /**
560
+ * Initialize Kubernetes drivers for each cluster
561
+ */
562
+ async _initializeDrivers() {
563
+ for (const clusterDef of this.config.clusters) {
564
+ const driver = new KubernetesDriver({
565
+ ...clusterDef,
566
+ logger: this.config.logger,
567
+ verbose: this.config.verbose,
568
+ });
569
+
570
+ await driver.initialize();
571
+
572
+ this.clusterDrivers.set(clusterDef.id, {
573
+ driver,
574
+ definition: clusterDef,
575
+ });
576
+
577
+ this._log('info', `Initialized driver for cluster: ${clusterDef.id}`);
578
+ }
579
+ }
580
+
581
+ /**
582
+ * Destroy all drivers
583
+ */
584
+ async _destroyDrivers() {
585
+ for (const [clusterId, { driver }] of this.clusterDrivers.entries()) {
586
+ await driver.destroy();
587
+ this._log('info', `Destroyed driver for cluster: ${clusterId}`);
588
+ }
589
+ this.clusterDrivers.clear();
590
+ }
591
+
592
+ /**
593
+ * Ensure cluster summary record exists
594
+ */
595
+ async _ensureClusterSummaryRecord(clusterId, definition, schedule) {
596
+ const summaryResource = this._resourceHandles.clusters;
597
+
598
+ return await summaryResource.insert({
599
+ id: clusterId,
600
+ name: definition.name || clusterId,
601
+ status: 'idle',
602
+ lastRunAt: null,
603
+ lastResult: null,
604
+ checkpoint: null,
605
+ state: {},
606
+ schedule,
607
+ tags: definition.tags || {},
608
+ metadata: definition.metadata || {},
609
+ });
610
+ }
611
+
612
+ /**
613
+ * Persist snapshot and track version history
614
+ */
615
+ async _persistSnapshot(normalized, rawItem) {
616
+ const snapshotResource = this._resourceHandles.snapshots;
617
+ const versionResource = this._resourceHandles.versions;
618
+ const changeResource = this._resourceHandles.changes;
619
+
620
+ const resourceKey = this._buildResourceKey(normalized);
621
+ const digest = this._computeDigest(normalized.configuration);
622
+
623
+ // Try to get existing snapshot
624
+ const existingSnapshot = await snapshotResource.getOrNull(resourceKey);
625
+
626
+ const now = new Date().toISOString();
627
+
628
+ if (!existingSnapshot) {
629
+ // NEW RESOURCE
630
+ await versionResource.insert({
631
+ clusterId: normalized.clusterId,
632
+ resourceType: normalized.resourceType,
633
+ resourceId: normalized.resourceId,
634
+ uid: normalized.uid,
635
+ namespace: normalized.namespace,
636
+ version: 1,
637
+ digest,
638
+ capturedAt: now,
639
+ configuration: normalized.configuration,
640
+ summary: this._extractSummary(normalized),
641
+ raw: rawItem,
642
+ });
643
+
644
+ await snapshotResource.insert({
645
+ id: resourceKey,
646
+ clusterId: normalized.clusterId,
647
+ namespace: normalized.namespace,
648
+ resourceType: normalized.resourceType,
649
+ resourceId: normalized.resourceId,
650
+ uid: normalized.uid,
651
+ name: normalized.name,
652
+ apiVersion: normalized.apiVersion,
653
+ kind: normalized.kind,
654
+ labels: normalized.labels,
655
+ annotations: normalized.annotations,
656
+ latestDigest: digest,
657
+ latestVersion: 1,
658
+ changelogSize: 0,
659
+ firstSeenAt: now,
660
+ lastSeenAt: now,
661
+ createdAt: now,
662
+ updatedAt: now,
663
+ });
664
+
665
+ return { status: 'created', resourceKey, version: 1, digest };
666
+ }
667
+
668
+ // EXISTING RESOURCE
669
+ if (existingSnapshot.latestDigest === digest) {
670
+ // UNCHANGED
671
+ await snapshotResource.patch(resourceKey, {
672
+ lastSeenAt: now,
673
+ updatedAt: now,
674
+ });
675
+
676
+ return { status: 'unchanged', resourceKey, version: existingSnapshot.latestVersion, digest };
677
+ }
678
+
679
+ // CHANGED
680
+ const newVersion = existingSnapshot.latestVersion + 1;
681
+
682
+ // Store new version
683
+ await versionResource.insert({
684
+ clusterId: normalized.clusterId,
685
+ resourceType: normalized.resourceType,
686
+ resourceId: normalized.resourceId,
687
+ uid: normalized.uid,
688
+ namespace: normalized.namespace,
689
+ version: newVersion,
690
+ digest,
691
+ capturedAt: now,
692
+ configuration: normalized.configuration,
693
+ summary: this._extractSummary(normalized),
694
+ raw: rawItem,
695
+ });
696
+
697
+ // Compute diff
698
+ const previousVersions = await versionResource.query({
699
+ clusterId: normalized.clusterId,
700
+ resourceType: normalized.resourceType,
701
+ resourceId: normalized.resourceId,
702
+ version: existingSnapshot.latestVersion,
703
+ });
704
+
705
+ if (previousVersions.length > 0) {
706
+ const previousVersion = previousVersions[0];
707
+ const diff = this._computeDiff(previousVersion.configuration, normalized.configuration);
708
+
709
+ await changeResource.insert({
710
+ clusterId: normalized.clusterId,
711
+ resourceType: normalized.resourceType,
712
+ resourceId: normalized.resourceId,
713
+ fromVersion: existingSnapshot.latestVersion,
714
+ toVersion: newVersion,
715
+ diff,
716
+ });
717
+ }
718
+
719
+ // Update snapshot
720
+ await snapshotResource.patch(resourceKey, {
721
+ latestDigest: digest,
722
+ latestVersion: newVersion,
723
+ changelogSize: existingSnapshot.changelogSize + 1,
724
+ lastSeenAt: now,
725
+ updatedAt: now,
726
+ labels: normalized.labels,
727
+ annotations: normalized.annotations,
728
+ });
729
+
730
+ return { status: 'updated', resourceKey, version: newVersion, digest };
731
+ }
732
+
733
+ /**
734
+ * Build unique resource key
735
+ */
736
+ _buildResourceKey(normalized) {
737
+ const parts = [
738
+ normalized.clusterId,
739
+ normalized.resourceType,
740
+ normalized.namespace || 'cluster',
741
+ normalized.resourceId,
742
+ ];
743
+ return parts.join('::');
744
+ }
745
+
746
+ /**
747
+ * Compute digest (SHA256) of configuration
748
+ */
749
+ _computeDigest(configuration) {
750
+ const canonical = jsonStableStringify(configuration);
751
+ return createHash('sha256').update(canonical).digest('hex');
752
+ }
753
+
754
+ /**
755
+ * Extract summary information
756
+ */
757
+ _extractSummary(normalized) {
758
+ return {
759
+ name: normalized.name,
760
+ namespace: normalized.namespace,
761
+ kind: normalized.kind,
762
+ apiVersion: normalized.apiVersion,
763
+ labels: normalized.labels,
764
+ annotations: normalized.annotations,
765
+ };
766
+ }
767
+
768
+ /**
769
+ * Compute diff between two configurations
770
+ */
771
+ _computeDiff(oldConfig, newConfig) {
772
+ const diff = {
773
+ added: {},
774
+ removed: {},
775
+ updated: {},
776
+ };
777
+
778
+ const oldKeys = new Set(Object.keys(oldConfig || {}));
779
+ const newKeys = new Set(Object.keys(newConfig || {}));
780
+
781
+ // Added keys
782
+ for (const key of newKeys) {
783
+ if (!oldKeys.has(key)) {
784
+ diff.added[key] = newConfig[key];
785
+ }
786
+ }
787
+
788
+ // Removed keys
789
+ for (const key of oldKeys) {
790
+ if (!newKeys.has(key)) {
791
+ diff.removed[key] = oldConfig[key];
792
+ }
793
+ }
794
+
795
+ // Updated keys
796
+ for (const key of newKeys) {
797
+ if (oldKeys.has(key) && !isEqual(oldConfig[key], newConfig[key])) {
798
+ diff.updated[key] = {
799
+ old: oldConfig[key],
800
+ new: newConfig[key],
801
+ };
802
+ }
803
+ }
804
+
805
+ return diff;
806
+ }
807
+
808
+ /**
809
+ * Check if resource should be included based on filters
810
+ */
811
+ _shouldIncludeResource(resource) {
812
+ const { select, ignore } = this.config.discovery;
813
+
814
+ // Apply select filter first (whitelist)
815
+ if (select !== null) {
816
+ if (!this._matchesFilter(resource, select)) {
817
+ return false;
818
+ }
819
+ }
820
+
821
+ // Apply ignore filter (blacklist)
822
+ if (ignore && ignore.length > 0) {
823
+ if (this._matchesFilter(resource, ignore)) {
824
+ return false;
825
+ }
826
+ }
827
+
828
+ return true;
829
+ }
830
+
831
+ /**
832
+ * Check if resource matches a filter
833
+ */
834
+ _matchesFilter(resource, filter) {
835
+ // Function filter
836
+ if (typeof filter === 'function') {
837
+ return filter(resource);
838
+ }
839
+
840
+ // Array filter
841
+ if (Array.isArray(filter)) {
842
+ for (const pattern of filter) {
843
+ if (this._matchesPattern(resource, pattern)) {
844
+ return true;
845
+ }
846
+ }
847
+ return false;
848
+ }
849
+
850
+ // Single pattern
851
+ return this._matchesPattern(resource, filter);
852
+ }
853
+
854
+ /**
855
+ * Check if resource matches a pattern
856
+ */
857
+ _matchesPattern(resource, pattern) {
858
+ // Function pattern
859
+ if (typeof pattern === 'function') {
860
+ return pattern(resource);
861
+ }
862
+
863
+ // String pattern (resource type matching)
864
+ if (typeof pattern === 'string') {
865
+ const resourceType = resource.resourceType;
866
+
867
+ // Exact match
868
+ if (resourceType === pattern) {
869
+ return true;
870
+ }
871
+
872
+ // Wildcard matching (e.g., "core.*", "*.Pod")
873
+ const regex = new RegExp('^' + pattern.replace(/\*/g, '.*') + '$');
874
+ return regex.test(resourceType);
875
+ }
876
+
877
+ return false;
878
+ }
879
+
880
+ /**
881
+ * Setup scheduled jobs
882
+ */
883
+ async _setupSchedules() {
884
+ // Require node-cron if scheduling is enabled
885
+ const needsCron = this.config.scheduled.enabled ||
886
+ this.config.clusters.some(c => c.scheduled?.enabled);
887
+
888
+ if (!needsCron) {
889
+ this._log('debug', 'No schedules configured, skipping cron setup');
890
+ return;
891
+ }
892
+
893
+ this._cron = requirePluginDependency('node-cron', 'KubernetesInventoryPlugin (for scheduling)');
894
+
895
+ // Global schedule (applies to all clusters without per-cluster schedule)
896
+ if (this.config.scheduled.enabled && this.config.scheduled.cron) {
897
+ const job = this._scheduleJob(
898
+ this.config.scheduled,
899
+ async () => {
900
+ this._log('info', 'Running global scheduled discovery');
901
+ await this.syncAll();
902
+ }
903
+ );
904
+ this._scheduledJobs.push({ type: 'global', job });
905
+ this._log('info', `Global schedule configured: ${this.config.scheduled.cron}`);
906
+ }
907
+
908
+ // Per-cluster schedules
909
+ for (const cluster of this.config.clusters) {
910
+ if (cluster.scheduled?.enabled && cluster.scheduled?.cron) {
911
+ const job = this._scheduleJob(
912
+ cluster.scheduled,
913
+ async () => {
914
+ this._log('info', `Running scheduled discovery for cluster: ${cluster.id}`, { clusterId: cluster.id });
915
+ await this.syncCluster(cluster.id);
916
+ }
917
+ );
918
+ this._scheduledJobs.push({ type: 'cluster', clusterId: cluster.id, job });
919
+ this._log('info', `Cluster schedule configured: ${cluster.id} -> ${cluster.scheduled.cron}`);
920
+ }
921
+ }
922
+ }
923
+
924
+ /**
925
+ * Schedule a single job
926
+ */
927
+ _scheduleJob(schedule, handler) {
928
+ const { cron, timezone, runOnStart } = schedule;
929
+
930
+ const job = this._cron.schedule(cron, handler, {
931
+ scheduled: true,
932
+ timezone: timezone || 'UTC',
933
+ });
934
+
935
+ if (runOnStart) {
936
+ setImmediate(handler);
937
+ }
938
+
939
+ return job;
940
+ }
941
+
942
+ /**
943
+ * Teardown all scheduled jobs
944
+ */
945
+ async _teardownSchedules() {
946
+ for (const entry of this._scheduledJobs) {
947
+ entry.job.stop();
948
+ this._log('debug', `Stopped scheduled job: ${entry.type}`, { type: entry.type, clusterId: entry.clusterId });
949
+ }
950
+ this._scheduledJobs = [];
951
+ }
952
+
953
+ /**
954
+ * Emit progress event
955
+ */
956
+ _emitProgress(clusterId, data) {
957
+ // Can be extended to emit events to external systems
958
+ this._log('debug', 'Progress update', { clusterId, ...data });
959
+ }
960
+
961
+ /**
962
+ * Resolve internal resource names
963
+ */
964
+ _resolveInternalResourceNames() {
965
+ return resolveResourceNames(this.database, this._internalResourceDescriptors);
966
+ }
967
+
968
+ /**
969
+ * Internal logger
970
+ */
971
+ _log(level, message, meta = {}) {
972
+ if (this.config.logger) {
973
+ this.config.logger(level, message, { plugin: 'KubernetesInventoryPlugin', ...meta });
974
+ }
975
+
976
+ if (this.config.verbose && level !== 'debug') {
977
+ console.log(`[${level.toUpperCase()}] [KubernetesInventoryPlugin] ${message}`);
978
+ }
979
+ }
980
+ }