s3db.js 13.6.0 → 14.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/README.md +139 -43
  2. package/dist/s3db.cjs +72425 -38970
  3. package/dist/s3db.cjs.map +1 -1
  4. package/dist/s3db.es.js +72177 -38764
  5. package/dist/s3db.es.js.map +1 -1
  6. package/mcp/lib/base-handler.js +157 -0
  7. package/mcp/lib/handlers/connection-handler.js +280 -0
  8. package/mcp/lib/handlers/query-handler.js +533 -0
  9. package/mcp/lib/handlers/resource-handler.js +428 -0
  10. package/mcp/lib/tool-registry.js +336 -0
  11. package/mcp/lib/tools/connection-tools.js +161 -0
  12. package/mcp/lib/tools/query-tools.js +267 -0
  13. package/mcp/lib/tools/resource-tools.js +404 -0
  14. package/package.json +94 -49
  15. package/src/clients/memory-client.class.js +346 -191
  16. package/src/clients/memory-storage.class.js +300 -84
  17. package/src/clients/s3-client.class.js +7 -6
  18. package/src/concerns/geo-encoding.js +19 -2
  19. package/src/concerns/ip.js +59 -9
  20. package/src/concerns/money.js +8 -1
  21. package/src/concerns/password-hashing.js +49 -8
  22. package/src/concerns/plugin-storage.js +186 -18
  23. package/src/concerns/storage-drivers/filesystem-driver.js +284 -0
  24. package/src/database.class.js +139 -29
  25. package/src/errors.js +332 -42
  26. package/src/plugins/api/auth/oidc-auth.js +66 -17
  27. package/src/plugins/api/auth/strategies/base-strategy.class.js +74 -0
  28. package/src/plugins/api/auth/strategies/factory.class.js +63 -0
  29. package/src/plugins/api/auth/strategies/global-strategy.class.js +44 -0
  30. package/src/plugins/api/auth/strategies/path-based-strategy.class.js +83 -0
  31. package/src/plugins/api/auth/strategies/path-rules-strategy.class.js +118 -0
  32. package/src/plugins/api/concerns/failban-manager.js +106 -57
  33. package/src/plugins/api/concerns/opengraph-helper.js +116 -0
  34. package/src/plugins/api/concerns/route-context.js +601 -0
  35. package/src/plugins/api/concerns/state-machine.js +288 -0
  36. package/src/plugins/api/index.js +180 -41
  37. package/src/plugins/api/routes/auth-routes.js +198 -30
  38. package/src/plugins/api/routes/resource-routes.js +19 -4
  39. package/src/plugins/api/server/health-manager.class.js +163 -0
  40. package/src/plugins/api/server/middleware-chain.class.js +310 -0
  41. package/src/plugins/api/server/router.class.js +472 -0
  42. package/src/plugins/api/server.js +280 -1303
  43. package/src/plugins/api/utils/custom-routes.js +17 -5
  44. package/src/plugins/api/utils/guards.js +76 -17
  45. package/src/plugins/api/utils/openapi-generator-cached.class.js +133 -0
  46. package/src/plugins/api/utils/openapi-generator.js +7 -6
  47. package/src/plugins/api/utils/template-engine.js +77 -3
  48. package/src/plugins/audit.plugin.js +30 -8
  49. package/src/plugins/backup.plugin.js +110 -14
  50. package/src/plugins/cache/cache.class.js +22 -5
  51. package/src/plugins/cache/filesystem-cache.class.js +116 -19
  52. package/src/plugins/cache/memory-cache.class.js +211 -57
  53. package/src/plugins/cache/multi-tier-cache.class.js +371 -0
  54. package/src/plugins/cache/partition-aware-filesystem-cache.class.js +168 -47
  55. package/src/plugins/cache/redis-cache.class.js +552 -0
  56. package/src/plugins/cache/s3-cache.class.js +17 -8
  57. package/src/plugins/cache.plugin.js +176 -61
  58. package/src/plugins/cloud-inventory/drivers/alibaba-driver.js +8 -1
  59. package/src/plugins/cloud-inventory/drivers/aws-driver.js +60 -29
  60. package/src/plugins/cloud-inventory/drivers/azure-driver.js +8 -1
  61. package/src/plugins/cloud-inventory/drivers/base-driver.js +16 -2
  62. package/src/plugins/cloud-inventory/drivers/cloudflare-driver.js +8 -1
  63. package/src/plugins/cloud-inventory/drivers/digitalocean-driver.js +8 -1
  64. package/src/plugins/cloud-inventory/drivers/hetzner-driver.js +8 -1
  65. package/src/plugins/cloud-inventory/drivers/linode-driver.js +8 -1
  66. package/src/plugins/cloud-inventory/drivers/mongodb-atlas-driver.js +8 -1
  67. package/src/plugins/cloud-inventory/drivers/vultr-driver.js +8 -1
  68. package/src/plugins/cloud-inventory/index.js +29 -8
  69. package/src/plugins/cloud-inventory/registry.js +64 -42
  70. package/src/plugins/cloud-inventory.plugin.js +240 -138
  71. package/src/plugins/concerns/plugin-dependencies.js +54 -0
  72. package/src/plugins/concerns/resource-names.js +100 -0
  73. package/src/plugins/consumers/index.js +10 -2
  74. package/src/plugins/consumers/sqs-consumer.js +12 -2
  75. package/src/plugins/cookie-farm-suite.plugin.js +278 -0
  76. package/src/plugins/cookie-farm.errors.js +73 -0
  77. package/src/plugins/cookie-farm.plugin.js +869 -0
  78. package/src/plugins/costs.plugin.js +7 -1
  79. package/src/plugins/eventual-consistency/analytics.js +94 -19
  80. package/src/plugins/eventual-consistency/config.js +15 -7
  81. package/src/plugins/eventual-consistency/consolidation.js +29 -11
  82. package/src/plugins/eventual-consistency/garbage-collection.js +3 -1
  83. package/src/plugins/eventual-consistency/helpers.js +39 -14
  84. package/src/plugins/eventual-consistency/install.js +21 -2
  85. package/src/plugins/eventual-consistency/utils.js +32 -10
  86. package/src/plugins/fulltext.plugin.js +38 -11
  87. package/src/plugins/geo.plugin.js +61 -9
  88. package/src/plugins/identity/concerns/config.js +61 -0
  89. package/src/plugins/identity/concerns/mfa-manager.js +15 -2
  90. package/src/plugins/identity/concerns/rate-limit.js +124 -0
  91. package/src/plugins/identity/concerns/resource-schemas.js +9 -1
  92. package/src/plugins/identity/concerns/token-generator.js +29 -4
  93. package/src/plugins/identity/drivers/auth-driver.interface.js +76 -0
  94. package/src/plugins/identity/drivers/client-credentials-driver.js +127 -0
  95. package/src/plugins/identity/drivers/index.js +18 -0
  96. package/src/plugins/identity/drivers/password-driver.js +122 -0
  97. package/src/plugins/identity/email-service.js +17 -2
  98. package/src/plugins/identity/index.js +413 -69
  99. package/src/plugins/identity/oauth2-server.js +413 -30
  100. package/src/plugins/identity/oidc-discovery.js +16 -8
  101. package/src/plugins/identity/rsa-keys.js +115 -35
  102. package/src/plugins/identity/server.js +166 -45
  103. package/src/plugins/identity/session-manager.js +53 -7
  104. package/src/plugins/identity/ui/pages/mfa-verification.js +17 -15
  105. package/src/plugins/identity/ui/routes.js +363 -255
  106. package/src/plugins/importer/index.js +153 -20
  107. package/src/plugins/index.js +9 -2
  108. package/src/plugins/kubernetes-inventory/index.js +6 -0
  109. package/src/plugins/kubernetes-inventory/k8s-driver.js +867 -0
  110. package/src/plugins/kubernetes-inventory/resource-types.js +274 -0
  111. package/src/plugins/kubernetes-inventory.plugin.js +980 -0
  112. package/src/plugins/metrics.plugin.js +64 -16
  113. package/src/plugins/ml/base-model.class.js +25 -15
  114. package/src/plugins/ml/regression-model.class.js +1 -1
  115. package/src/plugins/ml.errors.js +57 -25
  116. package/src/plugins/ml.plugin.js +28 -4
  117. package/src/plugins/namespace.js +210 -0
  118. package/src/plugins/plugin.class.js +180 -8
  119. package/src/plugins/puppeteer/console-monitor.js +729 -0
  120. package/src/plugins/puppeteer/cookie-manager.js +492 -0
  121. package/src/plugins/puppeteer/network-monitor.js +816 -0
  122. package/src/plugins/puppeteer/performance-manager.js +746 -0
  123. package/src/plugins/puppeteer/proxy-manager.js +478 -0
  124. package/src/plugins/puppeteer/stealth-manager.js +556 -0
  125. package/src/plugins/puppeteer.errors.js +81 -0
  126. package/src/plugins/puppeteer.plugin.js +1327 -0
  127. package/src/plugins/queue-consumer.plugin.js +69 -14
  128. package/src/plugins/recon/behaviors/uptime-behavior.js +691 -0
  129. package/src/plugins/recon/concerns/command-runner.js +148 -0
  130. package/src/plugins/recon/concerns/diff-detector.js +372 -0
  131. package/src/plugins/recon/concerns/fingerprint-builder.js +307 -0
  132. package/src/plugins/recon/concerns/process-manager.js +338 -0
  133. package/src/plugins/recon/concerns/report-generator.js +478 -0
  134. package/src/plugins/recon/concerns/security-analyzer.js +571 -0
  135. package/src/plugins/recon/concerns/target-normalizer.js +68 -0
  136. package/src/plugins/recon/config/defaults.js +321 -0
  137. package/src/plugins/recon/config/resources.js +370 -0
  138. package/src/plugins/recon/index.js +778 -0
  139. package/src/plugins/recon/managers/dependency-manager.js +174 -0
  140. package/src/plugins/recon/managers/scheduler-manager.js +179 -0
  141. package/src/plugins/recon/managers/storage-manager.js +745 -0
  142. package/src/plugins/recon/managers/target-manager.js +274 -0
  143. package/src/plugins/recon/stages/asn-stage.js +314 -0
  144. package/src/plugins/recon/stages/certificate-stage.js +84 -0
  145. package/src/plugins/recon/stages/dns-stage.js +107 -0
  146. package/src/plugins/recon/stages/dnsdumpster-stage.js +362 -0
  147. package/src/plugins/recon/stages/fingerprint-stage.js +71 -0
  148. package/src/plugins/recon/stages/google-dorks-stage.js +440 -0
  149. package/src/plugins/recon/stages/http-stage.js +89 -0
  150. package/src/plugins/recon/stages/latency-stage.js +148 -0
  151. package/src/plugins/recon/stages/massdns-stage.js +302 -0
  152. package/src/plugins/recon/stages/osint-stage.js +1373 -0
  153. package/src/plugins/recon/stages/ports-stage.js +169 -0
  154. package/src/plugins/recon/stages/screenshot-stage.js +94 -0
  155. package/src/plugins/recon/stages/secrets-stage.js +514 -0
  156. package/src/plugins/recon/stages/subdomains-stage.js +295 -0
  157. package/src/plugins/recon/stages/tls-audit-stage.js +78 -0
  158. package/src/plugins/recon/stages/vulnerability-stage.js +78 -0
  159. package/src/plugins/recon/stages/web-discovery-stage.js +113 -0
  160. package/src/plugins/recon/stages/whois-stage.js +349 -0
  161. package/src/plugins/recon.plugin.js +75 -0
  162. package/src/plugins/recon.plugin.js.backup +2635 -0
  163. package/src/plugins/relation.errors.js +87 -14
  164. package/src/plugins/replicator.plugin.js +514 -137
  165. package/src/plugins/replicators/base-replicator.class.js +89 -1
  166. package/src/plugins/replicators/bigquery-replicator.class.js +66 -22
  167. package/src/plugins/replicators/dynamodb-replicator.class.js +22 -15
  168. package/src/plugins/replicators/mongodb-replicator.class.js +22 -15
  169. package/src/plugins/replicators/mysql-replicator.class.js +52 -17
  170. package/src/plugins/replicators/planetscale-replicator.class.js +30 -4
  171. package/src/plugins/replicators/postgres-replicator.class.js +62 -27
  172. package/src/plugins/replicators/s3db-replicator.class.js +25 -18
  173. package/src/plugins/replicators/schema-sync.helper.js +3 -3
  174. package/src/plugins/replicators/sqs-replicator.class.js +8 -2
  175. package/src/plugins/replicators/turso-replicator.class.js +23 -3
  176. package/src/plugins/replicators/webhook-replicator.class.js +42 -4
  177. package/src/plugins/s3-queue.plugin.js +464 -65
  178. package/src/plugins/scheduler.plugin.js +20 -6
  179. package/src/plugins/state-machine.plugin.js +40 -9
  180. package/src/plugins/tfstate/README.md +126 -126
  181. package/src/plugins/tfstate/base-driver.js +28 -4
  182. package/src/plugins/tfstate/errors.js +65 -10
  183. package/src/plugins/tfstate/filesystem-driver.js +52 -8
  184. package/src/plugins/tfstate/index.js +163 -90
  185. package/src/plugins/tfstate/s3-driver.js +64 -6
  186. package/src/plugins/ttl.plugin.js +72 -17
  187. package/src/plugins/vector/distances.js +18 -12
  188. package/src/plugins/vector/kmeans.js +26 -4
  189. package/src/resource.class.js +115 -19
  190. package/src/testing/factory.class.js +20 -3
  191. package/src/testing/seeder.class.js +7 -1
  192. package/src/clients/memory-client.md +0 -917
  193. package/src/plugins/cloud-inventory/drivers/mock-drivers.js +0 -449
@@ -0,0 +1,169 @@
1
+ /**
2
+ * PortsStage
3
+ *
4
+ * Port scanning with multiple tools:
5
+ * - nmap (fast, detailed service detection)
6
+ * - masscan (ultra-fast, full port range)
7
+ * - Aggregates results from both scanners
8
+ */
9
+
10
+ export class PortsStage {
11
+ constructor(plugin) {
12
+ this.plugin = plugin;
13
+ this.commandRunner = plugin.commandRunner;
14
+ this.config = plugin.config;
15
+ }
16
+
17
+ async execute(target, featureConfig = {}) {
18
+ const scanners = {};
19
+ const openPorts = new Map();
20
+
21
+ if (featureConfig.nmap) {
22
+ const result = await this.executeNmap(target, { extraArgs: featureConfig.nmapArgs });
23
+ scanners.nmap = result;
24
+ if (result.status === 'ok' && Array.isArray(result.summary?.openPorts)) {
25
+ for (const entry of result.summary.openPorts) {
26
+ openPorts.set(entry.port, entry);
27
+ }
28
+ }
29
+ }
30
+
31
+ if (featureConfig.masscan) {
32
+ const result = await this.executeMasscan(target, featureConfig.masscan);
33
+ scanners.masscan = result;
34
+ if (result.status === 'ok' && Array.isArray(result.openPorts)) {
35
+ for (const entry of result.openPorts) {
36
+ if (!openPorts.has(entry.port)) {
37
+ openPorts.set(entry.port, entry);
38
+ }
39
+ }
40
+ }
41
+ }
42
+
43
+ return {
44
+ _individual: scanners,
45
+ _aggregated: {
46
+ status: openPorts.size > 0 ? 'ok' : 'empty',
47
+ openPorts: Array.from(openPorts.values()),
48
+ scanners
49
+ },
50
+ status: openPorts.size > 0 ? 'ok' : 'empty',
51
+ openPorts: Array.from(openPorts.values()),
52
+ scanners
53
+ };
54
+ }
55
+
56
+ async executeNmap(target, options = {}) {
57
+ if (!(await this.commandRunner.isAvailable('nmap'))) {
58
+ return {
59
+ status: 'unavailable',
60
+ message: 'nmap is not available on this system'
61
+ };
62
+ }
63
+
64
+ const topPorts = options.topPorts ?? this.config.nmap.topPorts;
65
+ const extraArgs = options.extraArgs ?? this.config.nmap.extraArgs;
66
+
67
+ const args = [
68
+ '-Pn',
69
+ '--top-ports',
70
+ String(topPorts),
71
+ target.host,
72
+ ...extraArgs
73
+ ];
74
+
75
+ const result = await this.commandRunner.run('nmap', args, {
76
+ timeout: 20000,
77
+ maxBuffer: 4 * 1024 * 1024
78
+ });
79
+
80
+ if (!result.ok) {
81
+ return {
82
+ status: 'error',
83
+ message: result.error?.message || 'nmap scan failed',
84
+ stderr: result.stderr
85
+ };
86
+ }
87
+
88
+ return {
89
+ status: 'ok',
90
+ summary: this._parseNmapOutput(result.stdout),
91
+ raw: this.config.storage.persistRawOutput ? this._truncateOutput(result.stdout) : undefined
92
+ };
93
+ }
94
+
95
+ async executeMasscan(target, featureConfig = {}) {
96
+ if (!(await this.commandRunner.isAvailable('masscan'))) {
97
+ return {
98
+ status: 'unavailable',
99
+ message: 'masscan is not available on this system'
100
+ };
101
+ }
102
+
103
+ const ports = featureConfig.ports ?? '1-65535';
104
+ const rate = featureConfig.rate ?? 1000;
105
+
106
+ const args = ['-p', ports, target.host, '--rate', String(rate), '--wait', '0'];
107
+ const result = await this.commandRunner.run('masscan', args, {
108
+ timeout: featureConfig.timeout ?? 30000,
109
+ maxBuffer: 4 * 1024 * 1024
110
+ });
111
+
112
+ if (!result.ok) {
113
+ return {
114
+ status: result.error?.code === 'ENOENT' ? 'unavailable' : 'error',
115
+ message: result.error?.message || 'masscan scan failed',
116
+ stderr: result.stderr
117
+ };
118
+ }
119
+
120
+ const openPorts = result.stdout
121
+ .split(/\r?\n/)
122
+ .map((line) => line.trim())
123
+ .filter((line) => line.toLowerCase().startsWith('discovered open port'))
124
+ .map((line) => {
125
+ const parts = line.split(' ');
126
+ const portProto = parts[3];
127
+ const ip = parts[5];
128
+ return {
129
+ port: portProto,
130
+ ip
131
+ };
132
+ });
133
+
134
+ return {
135
+ status: openPorts.length ? 'ok' : 'empty',
136
+ openPorts,
137
+ raw: this.config.storage.persistRawOutput ? this._truncateOutput(result.stdout) : undefined
138
+ };
139
+ }
140
+
141
+ _parseNmapOutput(raw) {
142
+ const lines = raw.split('\n');
143
+ const openPorts = [];
144
+ const detectedServices = [];
145
+
146
+ for (const line of lines) {
147
+ const match = line.match(/^(\d+\/[a-z]+)\s+(open|filtered|closed)\s+([^\s]+)(.*)$/);
148
+ if (match && match[2] === 'open') {
149
+ const port = match[1];
150
+ const service = match[3];
151
+ const detail = match[4]?.trim();
152
+ openPorts.push({ port, service, detail });
153
+ detectedServices.push(`${service}${detail ? ` ${detail}` : ''}`.trim());
154
+ }
155
+ }
156
+
157
+ return {
158
+ openPorts,
159
+ detectedServices: Array.from(new Set(detectedServices))
160
+ };
161
+ }
162
+
163
+ _truncateOutput(text, maxLength = 10000) {
164
+ if (!text || text.length <= maxLength) {
165
+ return text;
166
+ }
167
+ return text.substring(0, maxLength) + '\n... (truncated)';
168
+ }
169
+ }
@@ -0,0 +1,94 @@
1
+ /**
2
+ * ScreenshotStage
3
+ *
4
+ * Visual reconnaissance:
5
+ * - aquatone (screenshot capture with clustering)
6
+ * - EyeWitness (screenshot + report generation)
7
+ */
8
+
9
+ import fs from 'fs/promises';
10
+ import path from 'path';
11
+ import os from 'os';
12
+ import { randomUUID } from 'crypto';
13
+
14
+ export class ScreenshotStage {
15
+ constructor(plugin) {
16
+ this.plugin = plugin;
17
+ this.commandRunner = plugin.commandRunner;
18
+ this.config = plugin.config;
19
+ }
20
+
21
+ async execute(target, featureConfig = {}) {
22
+ if (!featureConfig.aquatone && !featureConfig.eyewitness) {
23
+ return { status: 'skipped' };
24
+ }
25
+
26
+ const screenshots = {};
27
+ const hostsFile = await this._writeTempHostsFile([this._buildUrl(target)]);
28
+
29
+ const executeCapture = async (name, command, args) => {
30
+ const run = await this.commandRunner.run(command, args, {
31
+ timeout: featureConfig.timeout ?? 60000,
32
+ maxBuffer: 4 * 1024 * 1024
33
+ });
34
+ if (!run.ok) {
35
+ screenshots[name] = {
36
+ status: run.error?.code === 'ENOENT' ? 'unavailable' : 'error',
37
+ message: run.error?.message || `${command} failed`
38
+ };
39
+ return;
40
+ }
41
+ screenshots[name] = { status: 'ok' };
42
+ };
43
+
44
+ if (featureConfig.aquatone) {
45
+ const outputDir = featureConfig.outputDir || path.join(os.tmpdir(), `aquatone-${randomUUID()}`);
46
+ await fs.mkdir(outputDir, { recursive: true });
47
+ await executeCapture('aquatone', 'aquatone', ['-scan-timeout', '20000', '-out', outputDir, '-list', hostsFile]);
48
+ screenshots.aquatone.outputDir = outputDir;
49
+ }
50
+
51
+ if (featureConfig.eyewitness) {
52
+ const outputDir = featureConfig.outputDir || path.join(os.tmpdir(), `eyewitness-${randomUUID()}`);
53
+ await fs.mkdir(outputDir, { recursive: true });
54
+ await executeCapture('eyewitness', 'EyeWitness', ['--web', '--timeout', '20', '--threads', '5', '--headless', '-f', hostsFile, '-d', outputDir]);
55
+ screenshots.eyewitness = { status: 'ok', outputDir };
56
+ }
57
+
58
+ await fs.rm(hostsFile, { force: true });
59
+
60
+ if (Object.values(screenshots).some((entry) => entry.status === 'ok')) {
61
+ return {
62
+ _individual: screenshots,
63
+ _aggregated: {
64
+ status: 'ok',
65
+ tools: screenshots
66
+ },
67
+ status: 'ok',
68
+ tools: screenshots
69
+ };
70
+ }
71
+
72
+ return {
73
+ _individual: screenshots,
74
+ _aggregated: {
75
+ status: 'empty',
76
+ tools: screenshots
77
+ },
78
+ status: 'empty',
79
+ tools: screenshots
80
+ };
81
+ }
82
+
83
+ async _writeTempHostsFile(hosts) {
84
+ const filePath = path.join(os.tmpdir(), `recon-plugin-${randomUUID()}.txt`);
85
+ await fs.writeFile(filePath, hosts.join('\n'), { encoding: 'utf8' });
86
+ return filePath;
87
+ }
88
+
89
+ _buildUrl(target) {
90
+ const protocol = target.protocol || 'https';
91
+ const port = target.port && target.port !== (protocol === 'http' ? 80 : 443) ? `:${target.port}` : '';
92
+ return `${protocol}://${target.host}${port}${target.path || ''}`;
93
+ }
94
+ }