s3db.js 13.6.0 → 14.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/README.md +139 -43
  2. package/dist/s3db.cjs +72425 -38970
  3. package/dist/s3db.cjs.map +1 -1
  4. package/dist/s3db.es.js +72177 -38764
  5. package/dist/s3db.es.js.map +1 -1
  6. package/mcp/lib/base-handler.js +157 -0
  7. package/mcp/lib/handlers/connection-handler.js +280 -0
  8. package/mcp/lib/handlers/query-handler.js +533 -0
  9. package/mcp/lib/handlers/resource-handler.js +428 -0
  10. package/mcp/lib/tool-registry.js +336 -0
  11. package/mcp/lib/tools/connection-tools.js +161 -0
  12. package/mcp/lib/tools/query-tools.js +267 -0
  13. package/mcp/lib/tools/resource-tools.js +404 -0
  14. package/package.json +94 -49
  15. package/src/clients/memory-client.class.js +346 -191
  16. package/src/clients/memory-storage.class.js +300 -84
  17. package/src/clients/s3-client.class.js +7 -6
  18. package/src/concerns/geo-encoding.js +19 -2
  19. package/src/concerns/ip.js +59 -9
  20. package/src/concerns/money.js +8 -1
  21. package/src/concerns/password-hashing.js +49 -8
  22. package/src/concerns/plugin-storage.js +186 -18
  23. package/src/concerns/storage-drivers/filesystem-driver.js +284 -0
  24. package/src/database.class.js +139 -29
  25. package/src/errors.js +332 -42
  26. package/src/plugins/api/auth/oidc-auth.js +66 -17
  27. package/src/plugins/api/auth/strategies/base-strategy.class.js +74 -0
  28. package/src/plugins/api/auth/strategies/factory.class.js +63 -0
  29. package/src/plugins/api/auth/strategies/global-strategy.class.js +44 -0
  30. package/src/plugins/api/auth/strategies/path-based-strategy.class.js +83 -0
  31. package/src/plugins/api/auth/strategies/path-rules-strategy.class.js +118 -0
  32. package/src/plugins/api/concerns/failban-manager.js +106 -57
  33. package/src/plugins/api/concerns/opengraph-helper.js +116 -0
  34. package/src/plugins/api/concerns/route-context.js +601 -0
  35. package/src/plugins/api/concerns/state-machine.js +288 -0
  36. package/src/plugins/api/index.js +180 -41
  37. package/src/plugins/api/routes/auth-routes.js +198 -30
  38. package/src/plugins/api/routes/resource-routes.js +19 -4
  39. package/src/plugins/api/server/health-manager.class.js +163 -0
  40. package/src/plugins/api/server/middleware-chain.class.js +310 -0
  41. package/src/plugins/api/server/router.class.js +472 -0
  42. package/src/plugins/api/server.js +280 -1303
  43. package/src/plugins/api/utils/custom-routes.js +17 -5
  44. package/src/plugins/api/utils/guards.js +76 -17
  45. package/src/plugins/api/utils/openapi-generator-cached.class.js +133 -0
  46. package/src/plugins/api/utils/openapi-generator.js +7 -6
  47. package/src/plugins/api/utils/template-engine.js +77 -3
  48. package/src/plugins/audit.plugin.js +30 -8
  49. package/src/plugins/backup.plugin.js +110 -14
  50. package/src/plugins/cache/cache.class.js +22 -5
  51. package/src/plugins/cache/filesystem-cache.class.js +116 -19
  52. package/src/plugins/cache/memory-cache.class.js +211 -57
  53. package/src/plugins/cache/multi-tier-cache.class.js +371 -0
  54. package/src/plugins/cache/partition-aware-filesystem-cache.class.js +168 -47
  55. package/src/plugins/cache/redis-cache.class.js +552 -0
  56. package/src/plugins/cache/s3-cache.class.js +17 -8
  57. package/src/plugins/cache.plugin.js +176 -61
  58. package/src/plugins/cloud-inventory/drivers/alibaba-driver.js +8 -1
  59. package/src/plugins/cloud-inventory/drivers/aws-driver.js +60 -29
  60. package/src/plugins/cloud-inventory/drivers/azure-driver.js +8 -1
  61. package/src/plugins/cloud-inventory/drivers/base-driver.js +16 -2
  62. package/src/plugins/cloud-inventory/drivers/cloudflare-driver.js +8 -1
  63. package/src/plugins/cloud-inventory/drivers/digitalocean-driver.js +8 -1
  64. package/src/plugins/cloud-inventory/drivers/hetzner-driver.js +8 -1
  65. package/src/plugins/cloud-inventory/drivers/linode-driver.js +8 -1
  66. package/src/plugins/cloud-inventory/drivers/mongodb-atlas-driver.js +8 -1
  67. package/src/plugins/cloud-inventory/drivers/vultr-driver.js +8 -1
  68. package/src/plugins/cloud-inventory/index.js +29 -8
  69. package/src/plugins/cloud-inventory/registry.js +64 -42
  70. package/src/plugins/cloud-inventory.plugin.js +240 -138
  71. package/src/plugins/concerns/plugin-dependencies.js +54 -0
  72. package/src/plugins/concerns/resource-names.js +100 -0
  73. package/src/plugins/consumers/index.js +10 -2
  74. package/src/plugins/consumers/sqs-consumer.js +12 -2
  75. package/src/plugins/cookie-farm-suite.plugin.js +278 -0
  76. package/src/plugins/cookie-farm.errors.js +73 -0
  77. package/src/plugins/cookie-farm.plugin.js +869 -0
  78. package/src/plugins/costs.plugin.js +7 -1
  79. package/src/plugins/eventual-consistency/analytics.js +94 -19
  80. package/src/plugins/eventual-consistency/config.js +15 -7
  81. package/src/plugins/eventual-consistency/consolidation.js +29 -11
  82. package/src/plugins/eventual-consistency/garbage-collection.js +3 -1
  83. package/src/plugins/eventual-consistency/helpers.js +39 -14
  84. package/src/plugins/eventual-consistency/install.js +21 -2
  85. package/src/plugins/eventual-consistency/utils.js +32 -10
  86. package/src/plugins/fulltext.plugin.js +38 -11
  87. package/src/plugins/geo.plugin.js +61 -9
  88. package/src/plugins/identity/concerns/config.js +61 -0
  89. package/src/plugins/identity/concerns/mfa-manager.js +15 -2
  90. package/src/plugins/identity/concerns/rate-limit.js +124 -0
  91. package/src/plugins/identity/concerns/resource-schemas.js +9 -1
  92. package/src/plugins/identity/concerns/token-generator.js +29 -4
  93. package/src/plugins/identity/drivers/auth-driver.interface.js +76 -0
  94. package/src/plugins/identity/drivers/client-credentials-driver.js +127 -0
  95. package/src/plugins/identity/drivers/index.js +18 -0
  96. package/src/plugins/identity/drivers/password-driver.js +122 -0
  97. package/src/plugins/identity/email-service.js +17 -2
  98. package/src/plugins/identity/index.js +413 -69
  99. package/src/plugins/identity/oauth2-server.js +413 -30
  100. package/src/plugins/identity/oidc-discovery.js +16 -8
  101. package/src/plugins/identity/rsa-keys.js +115 -35
  102. package/src/plugins/identity/server.js +166 -45
  103. package/src/plugins/identity/session-manager.js +53 -7
  104. package/src/plugins/identity/ui/pages/mfa-verification.js +17 -15
  105. package/src/plugins/identity/ui/routes.js +363 -255
  106. package/src/plugins/importer/index.js +153 -20
  107. package/src/plugins/index.js +9 -2
  108. package/src/plugins/kubernetes-inventory/index.js +6 -0
  109. package/src/plugins/kubernetes-inventory/k8s-driver.js +867 -0
  110. package/src/plugins/kubernetes-inventory/resource-types.js +274 -0
  111. package/src/plugins/kubernetes-inventory.plugin.js +980 -0
  112. package/src/plugins/metrics.plugin.js +64 -16
  113. package/src/plugins/ml/base-model.class.js +25 -15
  114. package/src/plugins/ml/regression-model.class.js +1 -1
  115. package/src/plugins/ml.errors.js +57 -25
  116. package/src/plugins/ml.plugin.js +28 -4
  117. package/src/plugins/namespace.js +210 -0
  118. package/src/plugins/plugin.class.js +180 -8
  119. package/src/plugins/puppeteer/console-monitor.js +729 -0
  120. package/src/plugins/puppeteer/cookie-manager.js +492 -0
  121. package/src/plugins/puppeteer/network-monitor.js +816 -0
  122. package/src/plugins/puppeteer/performance-manager.js +746 -0
  123. package/src/plugins/puppeteer/proxy-manager.js +478 -0
  124. package/src/plugins/puppeteer/stealth-manager.js +556 -0
  125. package/src/plugins/puppeteer.errors.js +81 -0
  126. package/src/plugins/puppeteer.plugin.js +1327 -0
  127. package/src/plugins/queue-consumer.plugin.js +69 -14
  128. package/src/plugins/recon/behaviors/uptime-behavior.js +691 -0
  129. package/src/plugins/recon/concerns/command-runner.js +148 -0
  130. package/src/plugins/recon/concerns/diff-detector.js +372 -0
  131. package/src/plugins/recon/concerns/fingerprint-builder.js +307 -0
  132. package/src/plugins/recon/concerns/process-manager.js +338 -0
  133. package/src/plugins/recon/concerns/report-generator.js +478 -0
  134. package/src/plugins/recon/concerns/security-analyzer.js +571 -0
  135. package/src/plugins/recon/concerns/target-normalizer.js +68 -0
  136. package/src/plugins/recon/config/defaults.js +321 -0
  137. package/src/plugins/recon/config/resources.js +370 -0
  138. package/src/plugins/recon/index.js +778 -0
  139. package/src/plugins/recon/managers/dependency-manager.js +174 -0
  140. package/src/plugins/recon/managers/scheduler-manager.js +179 -0
  141. package/src/plugins/recon/managers/storage-manager.js +745 -0
  142. package/src/plugins/recon/managers/target-manager.js +274 -0
  143. package/src/plugins/recon/stages/asn-stage.js +314 -0
  144. package/src/plugins/recon/stages/certificate-stage.js +84 -0
  145. package/src/plugins/recon/stages/dns-stage.js +107 -0
  146. package/src/plugins/recon/stages/dnsdumpster-stage.js +362 -0
  147. package/src/plugins/recon/stages/fingerprint-stage.js +71 -0
  148. package/src/plugins/recon/stages/google-dorks-stage.js +440 -0
  149. package/src/plugins/recon/stages/http-stage.js +89 -0
  150. package/src/plugins/recon/stages/latency-stage.js +148 -0
  151. package/src/plugins/recon/stages/massdns-stage.js +302 -0
  152. package/src/plugins/recon/stages/osint-stage.js +1373 -0
  153. package/src/plugins/recon/stages/ports-stage.js +169 -0
  154. package/src/plugins/recon/stages/screenshot-stage.js +94 -0
  155. package/src/plugins/recon/stages/secrets-stage.js +514 -0
  156. package/src/plugins/recon/stages/subdomains-stage.js +295 -0
  157. package/src/plugins/recon/stages/tls-audit-stage.js +78 -0
  158. package/src/plugins/recon/stages/vulnerability-stage.js +78 -0
  159. package/src/plugins/recon/stages/web-discovery-stage.js +113 -0
  160. package/src/plugins/recon/stages/whois-stage.js +349 -0
  161. package/src/plugins/recon.plugin.js +75 -0
  162. package/src/plugins/recon.plugin.js.backup +2635 -0
  163. package/src/plugins/relation.errors.js +87 -14
  164. package/src/plugins/replicator.plugin.js +514 -137
  165. package/src/plugins/replicators/base-replicator.class.js +89 -1
  166. package/src/plugins/replicators/bigquery-replicator.class.js +66 -22
  167. package/src/plugins/replicators/dynamodb-replicator.class.js +22 -15
  168. package/src/plugins/replicators/mongodb-replicator.class.js +22 -15
  169. package/src/plugins/replicators/mysql-replicator.class.js +52 -17
  170. package/src/plugins/replicators/planetscale-replicator.class.js +30 -4
  171. package/src/plugins/replicators/postgres-replicator.class.js +62 -27
  172. package/src/plugins/replicators/s3db-replicator.class.js +25 -18
  173. package/src/plugins/replicators/schema-sync.helper.js +3 -3
  174. package/src/plugins/replicators/sqs-replicator.class.js +8 -2
  175. package/src/plugins/replicators/turso-replicator.class.js +23 -3
  176. package/src/plugins/replicators/webhook-replicator.class.js +42 -4
  177. package/src/plugins/s3-queue.plugin.js +464 -65
  178. package/src/plugins/scheduler.plugin.js +20 -6
  179. package/src/plugins/state-machine.plugin.js +40 -9
  180. package/src/plugins/tfstate/README.md +126 -126
  181. package/src/plugins/tfstate/base-driver.js +28 -4
  182. package/src/plugins/tfstate/errors.js +65 -10
  183. package/src/plugins/tfstate/filesystem-driver.js +52 -8
  184. package/src/plugins/tfstate/index.js +163 -90
  185. package/src/plugins/tfstate/s3-driver.js +64 -6
  186. package/src/plugins/ttl.plugin.js +72 -17
  187. package/src/plugins/vector/distances.js +18 -12
  188. package/src/plugins/vector/kmeans.js +26 -4
  189. package/src/resource.class.js +115 -19
  190. package/src/testing/factory.class.js +20 -3
  191. package/src/testing/seeder.class.js +7 -1
  192. package/src/clients/memory-client.md +0 -917
  193. package/src/plugins/cloud-inventory/drivers/mock-drivers.js +0 -449
@@ -0,0 +1,302 @@
1
+ /**
2
+ * MassDNS Stage
3
+ *
4
+ * High-performance DNS resolver for mass subdomain enumeration
5
+ *
6
+ * Discovers:
7
+ * - Subdomains via wordlist-based brute force
8
+ * - A/AAAA records
9
+ * - Fast resolution (1000s of queries per second)
10
+ *
11
+ * Uses 100% free CLI tool:
12
+ * - massdns (https://github.com/blechschmidt/massdns)
13
+ */
14
+
15
+ export class MassDNSStage {
16
+ constructor(plugin) {
17
+ this.plugin = plugin;
18
+ this.commandRunner = plugin.commandRunner;
19
+ this.config = plugin.config;
20
+ }
21
+
22
+ /**
23
+ * Execute MassDNS lookup
24
+ * @param {Object} target - Target object with host property
25
+ * @param {Object} options - MassDNS options
26
+ * @returns {Promise<Object>} MassDNS results
27
+ */
28
+ async execute(target, options = {}) {
29
+ const result = {
30
+ status: 'ok',
31
+ host: target.host,
32
+ subdomains: [],
33
+ resolvedCount: 0,
34
+ totalAttempts: 0,
35
+ errors: {}
36
+ };
37
+
38
+ // Track individual tool results for artifact persistence
39
+ const individual = {
40
+ massdns: { status: 'ok', raw: null, subdomains: [], resolvedCount: 0 },
41
+ wordlist: { status: 'ok', path: null, entriesUsed: 0 }
42
+ };
43
+
44
+ // Check if massdns is available
45
+ const isAvailable = await this.commandRunner.isAvailable('massdns');
46
+
47
+ if (!isAvailable) {
48
+ result.status = 'unavailable';
49
+ result.errors.massdns = 'massdns not found in PATH';
50
+ individual.massdns.status = 'unavailable';
51
+
52
+ return {
53
+ _individual: individual,
54
+ _aggregated: result,
55
+ ...result
56
+ };
57
+ }
58
+
59
+ // Check if wordlist is provided
60
+ const wordlist = options.wordlist || this.config.massdns?.wordlist;
61
+
62
+ if (!wordlist) {
63
+ result.status = 'error';
64
+ result.errors.wordlist = 'No wordlist provided for massdns';
65
+ individual.wordlist.status = 'error';
66
+
67
+ return {
68
+ _individual: individual,
69
+ _aggregated: result,
70
+ ...result
71
+ };
72
+ }
73
+
74
+ individual.wordlist.path = wordlist;
75
+
76
+ // Check if resolvers file exists
77
+ const resolvers = options.resolvers || this.config.massdns?.resolvers || '/etc/resolv.conf';
78
+
79
+ try {
80
+ // Generate domain list from wordlist
81
+ const domainList = await this.generateDomainList(target.host, wordlist, options);
82
+
83
+ if (domainList.length === 0) {
84
+ result.status = 'empty';
85
+ result.errors.domains = 'No domains generated from wordlist';
86
+ individual.wordlist.status = 'empty';
87
+
88
+ return {
89
+ _individual: individual,
90
+ _aggregated: result,
91
+ ...result
92
+ };
93
+ }
94
+
95
+ result.totalAttempts = domainList.length;
96
+ individual.wordlist.entriesUsed = domainList.length;
97
+
98
+ // Run massdns
99
+ const massdnsResults = await this.runMassDNS(domainList, resolvers, options);
100
+
101
+ result.subdomains = massdnsResults.subdomains;
102
+ result.resolvedCount = massdnsResults.resolvedCount;
103
+
104
+ individual.massdns.subdomains = massdnsResults.subdomains;
105
+ individual.massdns.resolvedCount = massdnsResults.resolvedCount;
106
+
107
+ // Save raw output if persistRawOutput is enabled
108
+ if (this.config?.storage?.persistRawOutput && massdnsResults.raw) {
109
+ individual.massdns.raw = massdnsResults.raw;
110
+ }
111
+
112
+ if (result.resolvedCount === 0) {
113
+ result.status = 'empty';
114
+ }
115
+
116
+ } catch (error) {
117
+ result.status = 'error';
118
+ result.errors.general = error.message;
119
+ individual.massdns.status = 'error';
120
+ }
121
+
122
+ return {
123
+ _individual: individual,
124
+ _aggregated: result,
125
+ ...result // Root level for compatibility
126
+ };
127
+ }
128
+
129
+ /**
130
+ * Generate domain list from wordlist
131
+ * Reads wordlist and appends target domain to each entry
132
+ */
133
+ async generateDomainList(domain, wordlistPath, options = {}) {
134
+ const maxSubdomains = options.maxSubdomains || 1000;
135
+
136
+ try {
137
+ // Use cat to read wordlist
138
+ const catRun = await this.commandRunner.run('cat', [wordlistPath], {
139
+ timeout: 5000,
140
+ maxBuffer: 10 * 1024 * 1024
141
+ });
142
+
143
+ if (!catRun.ok || !catRun.stdout) {
144
+ return [];
145
+ }
146
+
147
+ // Parse wordlist and append domain
148
+ const words = catRun.stdout
149
+ .split('\n')
150
+ .map(line => line.trim())
151
+ .filter(line => line.length > 0 && !line.startsWith('#'))
152
+ .slice(0, maxSubdomains);
153
+
154
+ return words.map(word => `${word}.${domain}`);
155
+
156
+ } catch (error) {
157
+ return [];
158
+ }
159
+ }
160
+
161
+ /**
162
+ * Run massdns with domain list
163
+ */
164
+ async runMassDNS(domainList, resolversFile, options = {}) {
165
+ const result = {
166
+ subdomains: [],
167
+ resolvedCount: 0
168
+ };
169
+
170
+ try {
171
+ // Create temporary file with domain list
172
+ const tempFile = `/tmp/massdns-domains-${Date.now()}.txt`;
173
+ const domainsContent = domainList.join('\n');
174
+
175
+ // Write domains to temp file
176
+ const writeRun = await this.commandRunner.run('sh', [
177
+ '-c',
178
+ `echo "${domainsContent.replace(/"/g, '\\"')}" > ${tempFile}`
179
+ ], { timeout: 5000 });
180
+
181
+ if (!writeRun.ok) {
182
+ return result;
183
+ }
184
+
185
+ // Run massdns
186
+ // -r: resolvers file
187
+ // -t: record type (A)
188
+ // -o: output format (simple)
189
+ // -q: quiet
190
+ const massdnsArgs = [
191
+ '-r', resolversFile,
192
+ '-t', 'A',
193
+ '-o', 'S',
194
+ '-q',
195
+ tempFile
196
+ ];
197
+
198
+ // Add rate limit if specified
199
+ if (options.rate) {
200
+ massdnsArgs.unshift('-s', options.rate.toString());
201
+ }
202
+
203
+ const massdnsRun = await this.commandRunner.run('massdns', massdnsArgs, {
204
+ timeout: options.timeout || 60000,
205
+ maxBuffer: 10 * 1024 * 1024
206
+ });
207
+
208
+ // Cleanup temp file
209
+ await this.commandRunner.run('rm', ['-f', tempFile], { timeout: 1000 });
210
+
211
+ if (!massdnsRun.ok || !massdnsRun.stdout) {
212
+ return result;
213
+ }
214
+
215
+ // Parse massdns output
216
+ // Format: domain. A ip
217
+ const subdomains = [];
218
+ const lines = massdnsRun.stdout.split('\n');
219
+
220
+ for (const line of lines) {
221
+ const trimmed = line.trim();
222
+ if (!trimmed) continue;
223
+
224
+ // Parse: subdomain.domain.com. A 1.2.3.4
225
+ const match = trimmed.match(/^([\w\-\.]+)\.\s+A\s+([\d\.]+)$/);
226
+
227
+ if (match) {
228
+ const subdomain = match[1].replace(/\.$/, '');
229
+ const ip = match[2];
230
+
231
+ if (subdomain && ip) {
232
+ subdomains.push({ subdomain, ip });
233
+ }
234
+ }
235
+ }
236
+
237
+ result.subdomains = subdomains;
238
+ result.resolvedCount = subdomains.length;
239
+ result.raw = massdnsRun.stdout;
240
+
241
+ } catch (error) {
242
+ // Return empty result on error
243
+ }
244
+
245
+ return result;
246
+ }
247
+
248
+ /**
249
+ * Fallback: Use traditional dig-based subdomain enumeration
250
+ * This is used if massdns is not available
251
+ */
252
+ async fallbackDigEnum(domain, wordlist, options = {}) {
253
+ const result = {
254
+ subdomains: [],
255
+ resolvedCount: 0
256
+ };
257
+
258
+ try {
259
+ const domainList = await this.generateDomainList(domain, wordlist, options);
260
+ const maxConcurrent = 10;
261
+
262
+ // Process in batches to avoid overwhelming DNS
263
+ for (let i = 0; i < domainList.length; i += maxConcurrent) {
264
+ const batch = domainList.slice(i, i + maxConcurrent);
265
+
266
+ const batchResults = await Promise.all(
267
+ batch.map(async (subdomain) => {
268
+ const digRun = await this.commandRunner.run('dig', ['+short', 'A', subdomain], {
269
+ timeout: 3000
270
+ });
271
+
272
+ if (digRun.ok && digRun.stdout) {
273
+ const ips = digRun.stdout
274
+ .split('\n')
275
+ .map(line => line.trim())
276
+ .filter(line => /^\d+\.\d+\.\d+\.\d+$/.test(line));
277
+
278
+ if (ips.length > 0) {
279
+ return ips.map(ip => ({ subdomain, ip }));
280
+ }
281
+ }
282
+
283
+ return [];
284
+ })
285
+ );
286
+
287
+ // Flatten and add to result
288
+ result.subdomains.push(...batchResults.flat());
289
+
290
+ // Small delay between batches
291
+ await new Promise(resolve => setTimeout(resolve, 100));
292
+ }
293
+
294
+ result.resolvedCount = result.subdomains.length;
295
+
296
+ } catch (error) {
297
+ // Return empty result on error
298
+ }
299
+
300
+ return result;
301
+ }
302
+ }