s3db.js 13.6.0 → 14.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/README.md +139 -43
  2. package/dist/s3db.cjs +72425 -38970
  3. package/dist/s3db.cjs.map +1 -1
  4. package/dist/s3db.es.js +72177 -38764
  5. package/dist/s3db.es.js.map +1 -1
  6. package/mcp/lib/base-handler.js +157 -0
  7. package/mcp/lib/handlers/connection-handler.js +280 -0
  8. package/mcp/lib/handlers/query-handler.js +533 -0
  9. package/mcp/lib/handlers/resource-handler.js +428 -0
  10. package/mcp/lib/tool-registry.js +336 -0
  11. package/mcp/lib/tools/connection-tools.js +161 -0
  12. package/mcp/lib/tools/query-tools.js +267 -0
  13. package/mcp/lib/tools/resource-tools.js +404 -0
  14. package/package.json +94 -49
  15. package/src/clients/memory-client.class.js +346 -191
  16. package/src/clients/memory-storage.class.js +300 -84
  17. package/src/clients/s3-client.class.js +7 -6
  18. package/src/concerns/geo-encoding.js +19 -2
  19. package/src/concerns/ip.js +59 -9
  20. package/src/concerns/money.js +8 -1
  21. package/src/concerns/password-hashing.js +49 -8
  22. package/src/concerns/plugin-storage.js +186 -18
  23. package/src/concerns/storage-drivers/filesystem-driver.js +284 -0
  24. package/src/database.class.js +139 -29
  25. package/src/errors.js +332 -42
  26. package/src/plugins/api/auth/oidc-auth.js +66 -17
  27. package/src/plugins/api/auth/strategies/base-strategy.class.js +74 -0
  28. package/src/plugins/api/auth/strategies/factory.class.js +63 -0
  29. package/src/plugins/api/auth/strategies/global-strategy.class.js +44 -0
  30. package/src/plugins/api/auth/strategies/path-based-strategy.class.js +83 -0
  31. package/src/plugins/api/auth/strategies/path-rules-strategy.class.js +118 -0
  32. package/src/plugins/api/concerns/failban-manager.js +106 -57
  33. package/src/plugins/api/concerns/opengraph-helper.js +116 -0
  34. package/src/plugins/api/concerns/route-context.js +601 -0
  35. package/src/plugins/api/concerns/state-machine.js +288 -0
  36. package/src/plugins/api/index.js +180 -41
  37. package/src/plugins/api/routes/auth-routes.js +198 -30
  38. package/src/plugins/api/routes/resource-routes.js +19 -4
  39. package/src/plugins/api/server/health-manager.class.js +163 -0
  40. package/src/plugins/api/server/middleware-chain.class.js +310 -0
  41. package/src/plugins/api/server/router.class.js +472 -0
  42. package/src/plugins/api/server.js +280 -1303
  43. package/src/plugins/api/utils/custom-routes.js +17 -5
  44. package/src/plugins/api/utils/guards.js +76 -17
  45. package/src/plugins/api/utils/openapi-generator-cached.class.js +133 -0
  46. package/src/plugins/api/utils/openapi-generator.js +7 -6
  47. package/src/plugins/api/utils/template-engine.js +77 -3
  48. package/src/plugins/audit.plugin.js +30 -8
  49. package/src/plugins/backup.plugin.js +110 -14
  50. package/src/plugins/cache/cache.class.js +22 -5
  51. package/src/plugins/cache/filesystem-cache.class.js +116 -19
  52. package/src/plugins/cache/memory-cache.class.js +211 -57
  53. package/src/plugins/cache/multi-tier-cache.class.js +371 -0
  54. package/src/plugins/cache/partition-aware-filesystem-cache.class.js +168 -47
  55. package/src/plugins/cache/redis-cache.class.js +552 -0
  56. package/src/plugins/cache/s3-cache.class.js +17 -8
  57. package/src/plugins/cache.plugin.js +176 -61
  58. package/src/plugins/cloud-inventory/drivers/alibaba-driver.js +8 -1
  59. package/src/plugins/cloud-inventory/drivers/aws-driver.js +60 -29
  60. package/src/plugins/cloud-inventory/drivers/azure-driver.js +8 -1
  61. package/src/plugins/cloud-inventory/drivers/base-driver.js +16 -2
  62. package/src/plugins/cloud-inventory/drivers/cloudflare-driver.js +8 -1
  63. package/src/plugins/cloud-inventory/drivers/digitalocean-driver.js +8 -1
  64. package/src/plugins/cloud-inventory/drivers/hetzner-driver.js +8 -1
  65. package/src/plugins/cloud-inventory/drivers/linode-driver.js +8 -1
  66. package/src/plugins/cloud-inventory/drivers/mongodb-atlas-driver.js +8 -1
  67. package/src/plugins/cloud-inventory/drivers/vultr-driver.js +8 -1
  68. package/src/plugins/cloud-inventory/index.js +29 -8
  69. package/src/plugins/cloud-inventory/registry.js +64 -42
  70. package/src/plugins/cloud-inventory.plugin.js +240 -138
  71. package/src/plugins/concerns/plugin-dependencies.js +54 -0
  72. package/src/plugins/concerns/resource-names.js +100 -0
  73. package/src/plugins/consumers/index.js +10 -2
  74. package/src/plugins/consumers/sqs-consumer.js +12 -2
  75. package/src/plugins/cookie-farm-suite.plugin.js +278 -0
  76. package/src/plugins/cookie-farm.errors.js +73 -0
  77. package/src/plugins/cookie-farm.plugin.js +869 -0
  78. package/src/plugins/costs.plugin.js +7 -1
  79. package/src/plugins/eventual-consistency/analytics.js +94 -19
  80. package/src/plugins/eventual-consistency/config.js +15 -7
  81. package/src/plugins/eventual-consistency/consolidation.js +29 -11
  82. package/src/plugins/eventual-consistency/garbage-collection.js +3 -1
  83. package/src/plugins/eventual-consistency/helpers.js +39 -14
  84. package/src/plugins/eventual-consistency/install.js +21 -2
  85. package/src/plugins/eventual-consistency/utils.js +32 -10
  86. package/src/plugins/fulltext.plugin.js +38 -11
  87. package/src/plugins/geo.plugin.js +61 -9
  88. package/src/plugins/identity/concerns/config.js +61 -0
  89. package/src/plugins/identity/concerns/mfa-manager.js +15 -2
  90. package/src/plugins/identity/concerns/rate-limit.js +124 -0
  91. package/src/plugins/identity/concerns/resource-schemas.js +9 -1
  92. package/src/plugins/identity/concerns/token-generator.js +29 -4
  93. package/src/plugins/identity/drivers/auth-driver.interface.js +76 -0
  94. package/src/plugins/identity/drivers/client-credentials-driver.js +127 -0
  95. package/src/plugins/identity/drivers/index.js +18 -0
  96. package/src/plugins/identity/drivers/password-driver.js +122 -0
  97. package/src/plugins/identity/email-service.js +17 -2
  98. package/src/plugins/identity/index.js +413 -69
  99. package/src/plugins/identity/oauth2-server.js +413 -30
  100. package/src/plugins/identity/oidc-discovery.js +16 -8
  101. package/src/plugins/identity/rsa-keys.js +115 -35
  102. package/src/plugins/identity/server.js +166 -45
  103. package/src/plugins/identity/session-manager.js +53 -7
  104. package/src/plugins/identity/ui/pages/mfa-verification.js +17 -15
  105. package/src/plugins/identity/ui/routes.js +363 -255
  106. package/src/plugins/importer/index.js +153 -20
  107. package/src/plugins/index.js +9 -2
  108. package/src/plugins/kubernetes-inventory/index.js +6 -0
  109. package/src/plugins/kubernetes-inventory/k8s-driver.js +867 -0
  110. package/src/plugins/kubernetes-inventory/resource-types.js +274 -0
  111. package/src/plugins/kubernetes-inventory.plugin.js +980 -0
  112. package/src/plugins/metrics.plugin.js +64 -16
  113. package/src/plugins/ml/base-model.class.js +25 -15
  114. package/src/plugins/ml/regression-model.class.js +1 -1
  115. package/src/plugins/ml.errors.js +57 -25
  116. package/src/plugins/ml.plugin.js +28 -4
  117. package/src/plugins/namespace.js +210 -0
  118. package/src/plugins/plugin.class.js +180 -8
  119. package/src/plugins/puppeteer/console-monitor.js +729 -0
  120. package/src/plugins/puppeteer/cookie-manager.js +492 -0
  121. package/src/plugins/puppeteer/network-monitor.js +816 -0
  122. package/src/plugins/puppeteer/performance-manager.js +746 -0
  123. package/src/plugins/puppeteer/proxy-manager.js +478 -0
  124. package/src/plugins/puppeteer/stealth-manager.js +556 -0
  125. package/src/plugins/puppeteer.errors.js +81 -0
  126. package/src/plugins/puppeteer.plugin.js +1327 -0
  127. package/src/plugins/queue-consumer.plugin.js +69 -14
  128. package/src/plugins/recon/behaviors/uptime-behavior.js +691 -0
  129. package/src/plugins/recon/concerns/command-runner.js +148 -0
  130. package/src/plugins/recon/concerns/diff-detector.js +372 -0
  131. package/src/plugins/recon/concerns/fingerprint-builder.js +307 -0
  132. package/src/plugins/recon/concerns/process-manager.js +338 -0
  133. package/src/plugins/recon/concerns/report-generator.js +478 -0
  134. package/src/plugins/recon/concerns/security-analyzer.js +571 -0
  135. package/src/plugins/recon/concerns/target-normalizer.js +68 -0
  136. package/src/plugins/recon/config/defaults.js +321 -0
  137. package/src/plugins/recon/config/resources.js +370 -0
  138. package/src/plugins/recon/index.js +778 -0
  139. package/src/plugins/recon/managers/dependency-manager.js +174 -0
  140. package/src/plugins/recon/managers/scheduler-manager.js +179 -0
  141. package/src/plugins/recon/managers/storage-manager.js +745 -0
  142. package/src/plugins/recon/managers/target-manager.js +274 -0
  143. package/src/plugins/recon/stages/asn-stage.js +314 -0
  144. package/src/plugins/recon/stages/certificate-stage.js +84 -0
  145. package/src/plugins/recon/stages/dns-stage.js +107 -0
  146. package/src/plugins/recon/stages/dnsdumpster-stage.js +362 -0
  147. package/src/plugins/recon/stages/fingerprint-stage.js +71 -0
  148. package/src/plugins/recon/stages/google-dorks-stage.js +440 -0
  149. package/src/plugins/recon/stages/http-stage.js +89 -0
  150. package/src/plugins/recon/stages/latency-stage.js +148 -0
  151. package/src/plugins/recon/stages/massdns-stage.js +302 -0
  152. package/src/plugins/recon/stages/osint-stage.js +1373 -0
  153. package/src/plugins/recon/stages/ports-stage.js +169 -0
  154. package/src/plugins/recon/stages/screenshot-stage.js +94 -0
  155. package/src/plugins/recon/stages/secrets-stage.js +514 -0
  156. package/src/plugins/recon/stages/subdomains-stage.js +295 -0
  157. package/src/plugins/recon/stages/tls-audit-stage.js +78 -0
  158. package/src/plugins/recon/stages/vulnerability-stage.js +78 -0
  159. package/src/plugins/recon/stages/web-discovery-stage.js +113 -0
  160. package/src/plugins/recon/stages/whois-stage.js +349 -0
  161. package/src/plugins/recon.plugin.js +75 -0
  162. package/src/plugins/recon.plugin.js.backup +2635 -0
  163. package/src/plugins/relation.errors.js +87 -14
  164. package/src/plugins/replicator.plugin.js +514 -137
  165. package/src/plugins/replicators/base-replicator.class.js +89 -1
  166. package/src/plugins/replicators/bigquery-replicator.class.js +66 -22
  167. package/src/plugins/replicators/dynamodb-replicator.class.js +22 -15
  168. package/src/plugins/replicators/mongodb-replicator.class.js +22 -15
  169. package/src/plugins/replicators/mysql-replicator.class.js +52 -17
  170. package/src/plugins/replicators/planetscale-replicator.class.js +30 -4
  171. package/src/plugins/replicators/postgres-replicator.class.js +62 -27
  172. package/src/plugins/replicators/s3db-replicator.class.js +25 -18
  173. package/src/plugins/replicators/schema-sync.helper.js +3 -3
  174. package/src/plugins/replicators/sqs-replicator.class.js +8 -2
  175. package/src/plugins/replicators/turso-replicator.class.js +23 -3
  176. package/src/plugins/replicators/webhook-replicator.class.js +42 -4
  177. package/src/plugins/s3-queue.plugin.js +464 -65
  178. package/src/plugins/scheduler.plugin.js +20 -6
  179. package/src/plugins/state-machine.plugin.js +40 -9
  180. package/src/plugins/tfstate/README.md +126 -126
  181. package/src/plugins/tfstate/base-driver.js +28 -4
  182. package/src/plugins/tfstate/errors.js +65 -10
  183. package/src/plugins/tfstate/filesystem-driver.js +52 -8
  184. package/src/plugins/tfstate/index.js +163 -90
  185. package/src/plugins/tfstate/s3-driver.js +64 -6
  186. package/src/plugins/ttl.plugin.js +72 -17
  187. package/src/plugins/vector/distances.js +18 -12
  188. package/src/plugins/vector/kmeans.js +26 -4
  189. package/src/resource.class.js +115 -19
  190. package/src/testing/factory.class.js +20 -3
  191. package/src/testing/seeder.class.js +7 -1
  192. package/src/clients/memory-client.md +0 -917
  193. package/src/plugins/cloud-inventory/drivers/mock-drivers.js +0 -449
@@ -0,0 +1,310 @@
1
+ /**
2
+ * MiddlewareChain - Manages middleware application order
3
+ *
4
+ * Applies middlewares in correct order for security and performance:
5
+ * 1. Request tracking (for graceful shutdown)
6
+ * 2. Failban (block banned IPs early)
7
+ * 3. Request ID (before all logging)
8
+ * 4. CORS (before auth checks)
9
+ * 5. Security headers
10
+ * 6. Session tracking
11
+ * 7. Custom middlewares
12
+ * 8. Templates
13
+ * 9. Body size limits
14
+ */
15
+
16
+ import { idGenerator } from '../../../concerns/id.js';
17
+ import { createRequestIdMiddleware } from '../middlewares/request-id.js';
18
+ import { createSecurityHeadersMiddleware } from '../middlewares/security-headers.js';
19
+ import { createSessionTrackingMiddleware } from '../middlewares/session-tracking.js';
20
+ import { createFailbanMiddleware, setupFailbanViolationListener } from '../middlewares/failban.js';
21
+ import { setupTemplateEngine } from '../utils/template-engine.js';
22
+ import * as formatter from '../../shared/response-formatter.js';
23
+
24
+ export class MiddlewareChain {
25
+ constructor({
26
+ requestId,
27
+ cors,
28
+ security,
29
+ sessionTracking,
30
+ middlewares,
31
+ templates,
32
+ maxBodySize,
33
+ failban,
34
+ events,
35
+ verbose,
36
+ database,
37
+ inFlightRequests,
38
+ acceptingRequests,
39
+ corsMiddleware
40
+ }) {
41
+ this.requestId = requestId;
42
+ this.cors = cors;
43
+ this.security = security;
44
+ this.sessionTracking = sessionTracking;
45
+ this.middlewares = middlewares || [];
46
+ this.templates = templates;
47
+ this.maxBodySize = maxBodySize;
48
+ this.failban = failban;
49
+ this.events = events;
50
+ this.verbose = verbose;
51
+ this.database = database;
52
+ this.inFlightRequests = inFlightRequests;
53
+ this.acceptingRequests = acceptingRequests;
54
+ this.corsMiddleware = corsMiddleware;
55
+ }
56
+
57
+ /**
58
+ * Apply all middlewares to Hono app in correct order
59
+ * @param {Hono} app - Hono application instance
60
+ */
61
+ apply(app) {
62
+ // 1. Request tracking (must be first!)
63
+ this.applyRequestTracking(app);
64
+
65
+ // 2. Failban (check banned IPs early)
66
+ this.applyFailban(app);
67
+
68
+ // 3. Request ID
69
+ this.applyRequestId(app);
70
+
71
+ // 4. CORS
72
+ this.applyCors(app);
73
+
74
+ // 5. Security headers
75
+ this.applySecurity(app);
76
+
77
+ // 6. Session tracking
78
+ this.applySessionTracking(app);
79
+
80
+ // 7. Custom middlewares
81
+ this.applyCustomMiddlewares(app);
82
+
83
+ // 8. Template engine
84
+ this.applyTemplates(app);
85
+
86
+ // 9. Body size limits
87
+ this.applyBodySizeLimits(app);
88
+ }
89
+
90
+ /**
91
+ * Apply request tracking middleware (for graceful shutdown)
92
+ * @private
93
+ */
94
+ applyRequestTracking(app) {
95
+ app.use('*', async (c, next) => {
96
+ // Check if we're still accepting requests
97
+ if (!this.acceptingRequests()) {
98
+ return c.json({ error: 'Server is shutting down' }, 503);
99
+ }
100
+
101
+ // Track this request
102
+ const requestId = Symbol('request');
103
+ this.inFlightRequests.add(requestId);
104
+
105
+ const startTime = Date.now();
106
+ const requestInfo = {
107
+ requestId: c.get('requestId') || requestId.toString(),
108
+ method: c.req.method,
109
+ path: c.req.path,
110
+ userAgent: c.req.header('user-agent'),
111
+ ip: c.req.header('x-forwarded-for') || c.req.header('x-real-ip')
112
+ };
113
+
114
+ // Emit request:start
115
+ this.events.emitRequestEvent('start', requestInfo);
116
+
117
+ try {
118
+ await next();
119
+
120
+ // Emit request:end
121
+ this.events.emitRequestEvent('end', {
122
+ ...requestInfo,
123
+ duration: Date.now() - startTime,
124
+ status: c.res.status
125
+ });
126
+ } catch (err) {
127
+ // Emit request:error
128
+ this.events.emitRequestEvent('error', {
129
+ ...requestInfo,
130
+ duration: Date.now() - startTime,
131
+ error: err.message,
132
+ stack: err.stack
133
+ });
134
+ throw err;
135
+ } finally {
136
+ // Remove from tracking
137
+ this.inFlightRequests.delete(requestId);
138
+ }
139
+ });
140
+ }
141
+
142
+ /**
143
+ * Apply failban middleware
144
+ * @private
145
+ */
146
+ applyFailban(app) {
147
+ if (!this.failban) {
148
+ return;
149
+ }
150
+
151
+ const failbanMiddleware = createFailbanMiddleware({
152
+ plugin: this.failban,
153
+ events: this.events
154
+ });
155
+
156
+ app.use('*', failbanMiddleware);
157
+
158
+ // Setup violation listeners
159
+ setupFailbanViolationListener({
160
+ plugin: this.failban,
161
+ events: this.events
162
+ });
163
+
164
+ if (this.verbose) {
165
+ console.log('[MiddlewareChain] Failban protection enabled');
166
+ }
167
+ }
168
+
169
+ /**
170
+ * Apply request ID middleware
171
+ * @private
172
+ */
173
+ applyRequestId(app) {
174
+ if (!this.requestId?.enabled) {
175
+ // Always set requestId and verbose, even if not officially enabled
176
+ app.use('*', async (c, next) => {
177
+ c.set('requestId', idGenerator());
178
+ c.set('verbose', this.verbose);
179
+ await next();
180
+ });
181
+ return;
182
+ }
183
+
184
+ const requestIdMiddleware = createRequestIdMiddleware(this.requestId);
185
+ app.use('*', requestIdMiddleware);
186
+
187
+ if (this.verbose) {
188
+ console.log(`[MiddlewareChain] Request ID tracking enabled (header: ${this.requestId.headerName || 'X-Request-ID'})`);
189
+ }
190
+ }
191
+
192
+ /**
193
+ * Apply CORS middleware
194
+ * @private
195
+ */
196
+ applyCors(app) {
197
+ if (!this.cors?.enabled || !this.corsMiddleware) {
198
+ return;
199
+ }
200
+
201
+ const corsConfig = this.cors;
202
+ app.use('*', this.corsMiddleware({
203
+ origin: corsConfig.origin || '*',
204
+ allowMethods: corsConfig.allowMethods || ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'],
205
+ allowHeaders: corsConfig.allowHeaders || ['Content-Type', 'Authorization', 'X-Request-ID'],
206
+ exposeHeaders: corsConfig.exposeHeaders || ['X-Request-ID'],
207
+ credentials: corsConfig.credentials || false,
208
+ maxAge: corsConfig.maxAge || 86400
209
+ }));
210
+
211
+ if (this.verbose) {
212
+ console.log(`[MiddlewareChain] CORS enabled (maxAge: ${corsConfig.maxAge || 86400}s, origin: ${corsConfig.origin || '*'})`);
213
+ }
214
+ }
215
+
216
+ /**
217
+ * Apply security headers middleware
218
+ * @private
219
+ */
220
+ applySecurity(app) {
221
+ if (!this.security?.enabled) {
222
+ return;
223
+ }
224
+
225
+ const securityMiddleware = createSecurityHeadersMiddleware(this.security);
226
+ app.use('*', securityMiddleware);
227
+
228
+ if (this.verbose) {
229
+ console.log('[MiddlewareChain] Security headers enabled');
230
+ }
231
+ }
232
+
233
+ /**
234
+ * Apply session tracking middleware
235
+ * @private
236
+ */
237
+ applySessionTracking(app) {
238
+ if (!this.sessionTracking?.enabled) {
239
+ return;
240
+ }
241
+
242
+ const sessionMiddleware = createSessionTrackingMiddleware(
243
+ this.sessionTracking,
244
+ this.database
245
+ );
246
+ app.use('*', sessionMiddleware);
247
+
248
+ if (this.verbose) {
249
+ const resource = this.sessionTracking.resource ? ` (resource: ${this.sessionTracking.resource})` : ' (in-memory)';
250
+ console.log(`[MiddlewareChain] Session tracking enabled${resource}`);
251
+ }
252
+ }
253
+
254
+ /**
255
+ * Apply custom middlewares
256
+ * @private
257
+ */
258
+ applyCustomMiddlewares(app) {
259
+ this.middlewares.forEach(middleware => {
260
+ app.use('*', middleware);
261
+ });
262
+
263
+ if (this.verbose && this.middlewares.length > 0) {
264
+ console.log(`[MiddlewareChain] Applied ${this.middlewares.length} custom middleware(s)`);
265
+ }
266
+ }
267
+
268
+ /**
269
+ * Apply template engine middleware
270
+ * @private
271
+ */
272
+ applyTemplates(app) {
273
+ if (!this.templates?.enabled) {
274
+ return;
275
+ }
276
+
277
+ const templateMiddleware = setupTemplateEngine(this.templates);
278
+ app.use('*', templateMiddleware);
279
+
280
+ if (this.verbose) {
281
+ console.log(`[MiddlewareChain] Template engine enabled: ${this.templates.engine}`);
282
+ }
283
+ }
284
+
285
+ /**
286
+ * Apply body size limits
287
+ * @private
288
+ */
289
+ applyBodySizeLimits(app) {
290
+ app.use('*', async (c, next) => {
291
+ const method = c.req.method;
292
+
293
+ if (['POST', 'PUT', 'PATCH'].includes(method)) {
294
+ const contentLength = c.req.header('content-length');
295
+
296
+ if (contentLength) {
297
+ const size = parseInt(contentLength);
298
+
299
+ if (size > this.maxBodySize) {
300
+ const response = formatter.payloadTooLarge(size, this.maxBodySize);
301
+ c.header('Connection', 'close');
302
+ return c.json(response, response._status);
303
+ }
304
+ }
305
+ }
306
+
307
+ await next();
308
+ });
309
+ }
310
+ }