s3db.js 13.6.1 → 14.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/README.md +56 -15
  2. package/dist/s3db.cjs +72446 -39022
  3. package/dist/s3db.cjs.map +1 -1
  4. package/dist/s3db.es.js +72172 -38790
  5. package/dist/s3db.es.js.map +1 -1
  6. package/mcp/lib/base-handler.js +157 -0
  7. package/mcp/lib/handlers/connection-handler.js +280 -0
  8. package/mcp/lib/handlers/query-handler.js +533 -0
  9. package/mcp/lib/handlers/resource-handler.js +428 -0
  10. package/mcp/lib/tool-registry.js +336 -0
  11. package/mcp/lib/tools/connection-tools.js +161 -0
  12. package/mcp/lib/tools/query-tools.js +267 -0
  13. package/mcp/lib/tools/resource-tools.js +404 -0
  14. package/package.json +85 -50
  15. package/src/clients/memory-client.class.js +346 -191
  16. package/src/clients/memory-storage.class.js +300 -84
  17. package/src/clients/s3-client.class.js +7 -6
  18. package/src/concerns/geo-encoding.js +19 -2
  19. package/src/concerns/ip.js +59 -9
  20. package/src/concerns/money.js +8 -1
  21. package/src/concerns/password-hashing.js +49 -8
  22. package/src/concerns/plugin-storage.js +186 -18
  23. package/src/concerns/storage-drivers/filesystem-driver.js +284 -0
  24. package/src/database.class.js +139 -29
  25. package/src/errors.js +332 -42
  26. package/src/plugins/api/auth/oidc-auth.js +66 -17
  27. package/src/plugins/api/auth/strategies/base-strategy.class.js +74 -0
  28. package/src/plugins/api/auth/strategies/factory.class.js +63 -0
  29. package/src/plugins/api/auth/strategies/global-strategy.class.js +44 -0
  30. package/src/plugins/api/auth/strategies/path-based-strategy.class.js +83 -0
  31. package/src/plugins/api/auth/strategies/path-rules-strategy.class.js +118 -0
  32. package/src/plugins/api/concerns/failban-manager.js +106 -57
  33. package/src/plugins/api/concerns/route-context.js +601 -0
  34. package/src/plugins/api/index.js +168 -40
  35. package/src/plugins/api/routes/auth-routes.js +198 -30
  36. package/src/plugins/api/routes/resource-routes.js +19 -4
  37. package/src/plugins/api/server/health-manager.class.js +163 -0
  38. package/src/plugins/api/server/middleware-chain.class.js +310 -0
  39. package/src/plugins/api/server/router.class.js +472 -0
  40. package/src/plugins/api/server.js +280 -1303
  41. package/src/plugins/api/utils/custom-routes.js +17 -5
  42. package/src/plugins/api/utils/guards.js +76 -17
  43. package/src/plugins/api/utils/openapi-generator-cached.class.js +133 -0
  44. package/src/plugins/api/utils/openapi-generator.js +7 -6
  45. package/src/plugins/audit.plugin.js +30 -8
  46. package/src/plugins/backup.plugin.js +110 -14
  47. package/src/plugins/cache/cache.class.js +22 -5
  48. package/src/plugins/cache/filesystem-cache.class.js +116 -19
  49. package/src/plugins/cache/memory-cache.class.js +211 -57
  50. package/src/plugins/cache/multi-tier-cache.class.js +371 -0
  51. package/src/plugins/cache/partition-aware-filesystem-cache.class.js +168 -47
  52. package/src/plugins/cache/redis-cache.class.js +552 -0
  53. package/src/plugins/cache/s3-cache.class.js +17 -8
  54. package/src/plugins/cache.plugin.js +176 -61
  55. package/src/plugins/cloud-inventory/drivers/alibaba-driver.js +8 -1
  56. package/src/plugins/cloud-inventory/drivers/aws-driver.js +60 -29
  57. package/src/plugins/cloud-inventory/drivers/azure-driver.js +8 -1
  58. package/src/plugins/cloud-inventory/drivers/base-driver.js +16 -2
  59. package/src/plugins/cloud-inventory/drivers/cloudflare-driver.js +8 -1
  60. package/src/plugins/cloud-inventory/drivers/digitalocean-driver.js +8 -1
  61. package/src/plugins/cloud-inventory/drivers/hetzner-driver.js +8 -1
  62. package/src/plugins/cloud-inventory/drivers/linode-driver.js +8 -1
  63. package/src/plugins/cloud-inventory/drivers/mongodb-atlas-driver.js +8 -1
  64. package/src/plugins/cloud-inventory/drivers/vultr-driver.js +8 -1
  65. package/src/plugins/cloud-inventory/index.js +29 -8
  66. package/src/plugins/cloud-inventory/registry.js +64 -42
  67. package/src/plugins/cloud-inventory.plugin.js +240 -138
  68. package/src/plugins/concerns/plugin-dependencies.js +54 -0
  69. package/src/plugins/concerns/resource-names.js +100 -0
  70. package/src/plugins/consumers/index.js +10 -2
  71. package/src/plugins/consumers/sqs-consumer.js +12 -2
  72. package/src/plugins/cookie-farm-suite.plugin.js +278 -0
  73. package/src/plugins/cookie-farm.errors.js +73 -0
  74. package/src/plugins/cookie-farm.plugin.js +869 -0
  75. package/src/plugins/costs.plugin.js +7 -1
  76. package/src/plugins/eventual-consistency/analytics.js +94 -19
  77. package/src/plugins/eventual-consistency/config.js +15 -7
  78. package/src/plugins/eventual-consistency/consolidation.js +29 -11
  79. package/src/plugins/eventual-consistency/garbage-collection.js +3 -1
  80. package/src/plugins/eventual-consistency/helpers.js +39 -14
  81. package/src/plugins/eventual-consistency/install.js +21 -2
  82. package/src/plugins/eventual-consistency/utils.js +32 -10
  83. package/src/plugins/fulltext.plugin.js +38 -11
  84. package/src/plugins/geo.plugin.js +61 -9
  85. package/src/plugins/identity/concerns/config.js +61 -0
  86. package/src/plugins/identity/concerns/mfa-manager.js +15 -2
  87. package/src/plugins/identity/concerns/rate-limit.js +124 -0
  88. package/src/plugins/identity/concerns/resource-schemas.js +9 -1
  89. package/src/plugins/identity/concerns/token-generator.js +29 -4
  90. package/src/plugins/identity/drivers/auth-driver.interface.js +76 -0
  91. package/src/plugins/identity/drivers/client-credentials-driver.js +127 -0
  92. package/src/plugins/identity/drivers/index.js +18 -0
  93. package/src/plugins/identity/drivers/password-driver.js +122 -0
  94. package/src/plugins/identity/email-service.js +17 -2
  95. package/src/plugins/identity/index.js +413 -69
  96. package/src/plugins/identity/oauth2-server.js +413 -30
  97. package/src/plugins/identity/oidc-discovery.js +16 -8
  98. package/src/plugins/identity/rsa-keys.js +115 -35
  99. package/src/plugins/identity/server.js +166 -45
  100. package/src/plugins/identity/session-manager.js +53 -7
  101. package/src/plugins/identity/ui/pages/mfa-verification.js +17 -15
  102. package/src/plugins/identity/ui/routes.js +363 -255
  103. package/src/plugins/importer/index.js +153 -20
  104. package/src/plugins/index.js +9 -2
  105. package/src/plugins/kubernetes-inventory/index.js +6 -0
  106. package/src/plugins/kubernetes-inventory/k8s-driver.js +867 -0
  107. package/src/plugins/kubernetes-inventory/resource-types.js +274 -0
  108. package/src/plugins/kubernetes-inventory.plugin.js +980 -0
  109. package/src/plugins/metrics.plugin.js +64 -16
  110. package/src/plugins/ml/base-model.class.js +25 -15
  111. package/src/plugins/ml/regression-model.class.js +1 -1
  112. package/src/plugins/ml.errors.js +57 -25
  113. package/src/plugins/ml.plugin.js +28 -4
  114. package/src/plugins/namespace.js +210 -0
  115. package/src/plugins/plugin.class.js +180 -8
  116. package/src/plugins/puppeteer/console-monitor.js +729 -0
  117. package/src/plugins/puppeteer/cookie-manager.js +492 -0
  118. package/src/plugins/puppeteer/network-monitor.js +816 -0
  119. package/src/plugins/puppeteer/performance-manager.js +746 -0
  120. package/src/plugins/puppeteer/proxy-manager.js +478 -0
  121. package/src/plugins/puppeteer/stealth-manager.js +556 -0
  122. package/src/plugins/puppeteer.errors.js +81 -0
  123. package/src/plugins/puppeteer.plugin.js +1327 -0
  124. package/src/plugins/queue-consumer.plugin.js +69 -14
  125. package/src/plugins/recon/behaviors/uptime-behavior.js +691 -0
  126. package/src/plugins/recon/concerns/command-runner.js +148 -0
  127. package/src/plugins/recon/concerns/diff-detector.js +372 -0
  128. package/src/plugins/recon/concerns/fingerprint-builder.js +307 -0
  129. package/src/plugins/recon/concerns/process-manager.js +338 -0
  130. package/src/plugins/recon/concerns/report-generator.js +478 -0
  131. package/src/plugins/recon/concerns/security-analyzer.js +571 -0
  132. package/src/plugins/recon/concerns/target-normalizer.js +68 -0
  133. package/src/plugins/recon/config/defaults.js +321 -0
  134. package/src/plugins/recon/config/resources.js +370 -0
  135. package/src/plugins/recon/index.js +778 -0
  136. package/src/plugins/recon/managers/dependency-manager.js +174 -0
  137. package/src/plugins/recon/managers/scheduler-manager.js +179 -0
  138. package/src/plugins/recon/managers/storage-manager.js +745 -0
  139. package/src/plugins/recon/managers/target-manager.js +274 -0
  140. package/src/plugins/recon/stages/asn-stage.js +314 -0
  141. package/src/plugins/recon/stages/certificate-stage.js +84 -0
  142. package/src/plugins/recon/stages/dns-stage.js +107 -0
  143. package/src/plugins/recon/stages/dnsdumpster-stage.js +362 -0
  144. package/src/plugins/recon/stages/fingerprint-stage.js +71 -0
  145. package/src/plugins/recon/stages/google-dorks-stage.js +440 -0
  146. package/src/plugins/recon/stages/http-stage.js +89 -0
  147. package/src/plugins/recon/stages/latency-stage.js +148 -0
  148. package/src/plugins/recon/stages/massdns-stage.js +302 -0
  149. package/src/plugins/recon/stages/osint-stage.js +1373 -0
  150. package/src/plugins/recon/stages/ports-stage.js +169 -0
  151. package/src/plugins/recon/stages/screenshot-stage.js +94 -0
  152. package/src/plugins/recon/stages/secrets-stage.js +514 -0
  153. package/src/plugins/recon/stages/subdomains-stage.js +295 -0
  154. package/src/plugins/recon/stages/tls-audit-stage.js +78 -0
  155. package/src/plugins/recon/stages/vulnerability-stage.js +78 -0
  156. package/src/plugins/recon/stages/web-discovery-stage.js +113 -0
  157. package/src/plugins/recon/stages/whois-stage.js +349 -0
  158. package/src/plugins/recon.plugin.js +75 -0
  159. package/src/plugins/recon.plugin.js.backup +2635 -0
  160. package/src/plugins/relation.errors.js +87 -14
  161. package/src/plugins/replicator.plugin.js +514 -137
  162. package/src/plugins/replicators/base-replicator.class.js +89 -1
  163. package/src/plugins/replicators/bigquery-replicator.class.js +66 -22
  164. package/src/plugins/replicators/dynamodb-replicator.class.js +22 -15
  165. package/src/plugins/replicators/mongodb-replicator.class.js +22 -15
  166. package/src/plugins/replicators/mysql-replicator.class.js +52 -17
  167. package/src/plugins/replicators/planetscale-replicator.class.js +30 -4
  168. package/src/plugins/replicators/postgres-replicator.class.js +62 -27
  169. package/src/plugins/replicators/s3db-replicator.class.js +25 -18
  170. package/src/plugins/replicators/schema-sync.helper.js +3 -3
  171. package/src/plugins/replicators/sqs-replicator.class.js +8 -2
  172. package/src/plugins/replicators/turso-replicator.class.js +23 -3
  173. package/src/plugins/replicators/webhook-replicator.class.js +42 -4
  174. package/src/plugins/s3-queue.plugin.js +464 -65
  175. package/src/plugins/scheduler.plugin.js +20 -6
  176. package/src/plugins/state-machine.plugin.js +40 -9
  177. package/src/plugins/tfstate/base-driver.js +28 -4
  178. package/src/plugins/tfstate/errors.js +65 -10
  179. package/src/plugins/tfstate/filesystem-driver.js +52 -8
  180. package/src/plugins/tfstate/index.js +163 -90
  181. package/src/plugins/tfstate/s3-driver.js +64 -6
  182. package/src/plugins/ttl.plugin.js +72 -17
  183. package/src/plugins/vector/distances.js +18 -12
  184. package/src/plugins/vector/kmeans.js +26 -4
  185. package/src/resource.class.js +115 -19
  186. package/src/testing/factory.class.js +20 -3
  187. package/src/testing/seeder.class.js +7 -1
  188. package/src/clients/memory-client.md +0 -917
  189. package/src/plugins/cloud-inventory/drivers/mock-drivers.js +0 -449
@@ -0,0 +1,816 @@
1
+ /**
2
+ * NetworkMonitor - Comprehensive Network Activity Tracking
3
+ *
4
+ * Captures all network requests/responses using Chrome DevTools Protocol (CDP):
5
+ * - Request/Response headers, timing, sizes
6
+ * - Status codes, errors, redirects
7
+ * - Resource types (image, script, stylesheet, xhr, fetch, etc.)
8
+ * - Compression detection (gzip, brotli)
9
+ * - Cache behavior
10
+ * - Failed requests
11
+ *
12
+ * Persistence Strategy:
13
+ * - Optional S3DB storage with intelligent partitioning
14
+ * - Compression for large payloads
15
+ * - Filtering by resource type, status, size
16
+ * - Separate resources for sessions, requests, and errors
17
+ *
18
+ * Use cases:
19
+ * - SEO analysis (image sizes, script sizes, load times)
20
+ * - Performance debugging (slow requests, failed requests)
21
+ * - Security auditing (CSP violations, mixed content)
22
+ * - Cost analysis (bandwidth usage)
23
+ * - A/B testing (compare network behavior)
24
+ */
25
+ import tryFn from '../../concerns/try-fn.js';
26
+ import { PuppeteerError } from '../puppeteer.errors.js';
27
+
28
+ export class NetworkMonitor {
29
+ constructor(plugin) {
30
+ this.plugin = plugin;
31
+ this.config = plugin.config.networkMonitor || {
32
+ enabled: false,
33
+ persist: false,
34
+ filters: {
35
+ types: null, // ['image', 'script', 'stylesheet'] or null for all
36
+ statuses: null, // [404, 500] or null for all
37
+ minSize: null, // Only requests >= this size (bytes)
38
+ maxSize: null, // Only requests <= this size (bytes)
39
+ saveErrors: true, // Always save failed requests
40
+ saveLargeAssets: true // Always save assets > 1MB
41
+ },
42
+ compression: {
43
+ enabled: true,
44
+ threshold: 10240 // Compress payloads > 10KB
45
+ }
46
+ };
47
+
48
+ // Resources for persistence (lazy-initialized)
49
+ this.sessionsResource = null;
50
+ this.requestsResource = null;
51
+ this.errorsResource = null;
52
+
53
+ // Resource type mapping (CDP -> simplified)
54
+ this.resourceTypes = {
55
+ 'Document': 'document',
56
+ 'Stylesheet': 'stylesheet',
57
+ 'Image': 'image',
58
+ 'Media': 'media',
59
+ 'Font': 'font',
60
+ 'Script': 'script',
61
+ 'TextTrack': 'texttrack',
62
+ 'XHR': 'xhr',
63
+ 'Fetch': 'fetch',
64
+ 'EventSource': 'eventsource',
65
+ 'WebSocket': 'websocket',
66
+ 'Manifest': 'manifest',
67
+ 'SignedExchange': 'signedexchange',
68
+ 'Ping': 'ping',
69
+ 'CSPViolationReport': 'cspviolation',
70
+ 'Preflight': 'preflight',
71
+ 'Other': 'other'
72
+ };
73
+ }
74
+
75
+ /**
76
+ * Initialize network monitoring resources
77
+ */
78
+ async initialize() {
79
+ if (!this.config.persist) {
80
+ return;
81
+ }
82
+
83
+ // Create sessions resource (metadata about each crawl session)
84
+ const resourceNames = this.plugin.resourceNames || {};
85
+ const sessionsName = resourceNames.networkSessions || 'plg_puppeteer_network_sessions';
86
+ const requestsName = resourceNames.networkRequests || 'plg_puppeteer_network_requests';
87
+ const errorsName = resourceNames.networkErrors || 'plg_puppeteer_network_errors';
88
+
89
+ const [sessionsCreated, sessionsErr, sessionsResource] = await tryFn(() => this.plugin.database.createResource({
90
+ name: sessionsName,
91
+ attributes: {
92
+ sessionId: 'string|required',
93
+ url: 'string|required',
94
+ domain: 'string|required',
95
+ date: 'string|required', // YYYY-MM-DD for partitioning
96
+ startTime: 'number|required',
97
+ endTime: 'number',
98
+ duration: 'number',
99
+
100
+ // Summary statistics
101
+ totalRequests: 'number',
102
+ successfulRequests: 'number',
103
+ failedRequests: 'number',
104
+ totalBytes: 'number',
105
+ transferredBytes: 'number',
106
+ cachedBytes: 'number',
107
+
108
+ // By type
109
+ byType: 'object', // { image: { count, size }, script: {...} }
110
+
111
+ // Performance metrics
112
+ performance: 'object', // From PerformanceManager
113
+
114
+ // User agent
115
+ userAgent: 'string'
116
+ },
117
+ behavior: 'body-overflow',
118
+ timestamps: true,
119
+ partitions: {
120
+ byUrl: { fields: { url: 'string' } },
121
+ byDate: { fields: { date: 'string' } },
122
+ byDomain: { fields: { domain: 'string' } }
123
+ }
124
+ }));
125
+
126
+ if (sessionsCreated) {
127
+ this.sessionsResource = sessionsResource;
128
+ } else if (this.plugin.database.resources?.[sessionsName]) {
129
+ this.sessionsResource = this.plugin.database.resources[sessionsName];
130
+ } else {
131
+ throw sessionsErr;
132
+ }
133
+
134
+ // Create requests resource (detailed info about each request)
135
+ const [requestsCreated, requestsErr, requestsResource] = await tryFn(() => this.plugin.database.createResource({
136
+ name: requestsName,
137
+ attributes: {
138
+ requestId: 'string|required',
139
+ sessionId: 'string|required',
140
+ url: 'string|required',
141
+ domain: 'string|required',
142
+ path: 'string',
143
+
144
+ // Type and categorization
145
+ type: 'string|required', // image, script, stylesheet, xhr, etc.
146
+ statusCode: 'number',
147
+ statusText: 'string',
148
+ method: 'string', // GET, POST, etc.
149
+
150
+ // Size information
151
+ size: 'number', // Total size (bytes)
152
+ transferredSize: 'number', // Bytes transferred (after compression)
153
+ resourceSize: 'number', // Uncompressed size
154
+ fromCache: 'boolean',
155
+
156
+ // Timing information (ms)
157
+ timing: 'object', // { dns, tcp, ssl, request, response, total }
158
+ startTime: 'number',
159
+ endTime: 'number',
160
+ duration: 'number',
161
+
162
+ // Headers (compressed if large)
163
+ requestHeaders: 'object',
164
+ responseHeaders: 'object',
165
+
166
+ // Compression
167
+ compression: 'string', // gzip, br (brotli), deflate, none
168
+
169
+ // Cache
170
+ cacheControl: 'string',
171
+ expires: 'string',
172
+
173
+ // Error information (if failed)
174
+ failed: 'boolean',
175
+ errorText: 'string',
176
+ blockedReason: 'string', // CSP, mixed-content, etc.
177
+
178
+ // Redirects
179
+ redirected: 'boolean',
180
+ redirectUrl: 'string',
181
+
182
+ // CDN detection
183
+ cdn: 'string', // cloudflare, cloudfront, fastly, etc.
184
+ cdnDetected: 'boolean',
185
+
186
+ // Metadata
187
+ mimeType: 'string',
188
+ priority: 'string' // VeryHigh, High, Medium, Low, VeryLow
189
+ },
190
+ behavior: 'body-overflow',
191
+ timestamps: true,
192
+ partitions: {
193
+ bySession: { fields: { sessionId: 'string' } },
194
+ byType: { fields: { type: 'string' } },
195
+ byStatus: { fields: { statusCode: 'number' } },
196
+ bySize: { fields: { size: 'number' } },
197
+ byDomain: { fields: { domain: 'string' } }
198
+ }
199
+ }));
200
+
201
+ if (requestsCreated) {
202
+ this.requestsResource = requestsResource;
203
+ } else if (this.plugin.database.resources?.[requestsName]) {
204
+ this.requestsResource = this.plugin.database.resources[requestsName];
205
+ } else {
206
+ throw requestsErr;
207
+ }
208
+
209
+ // Create errors resource (failed requests only)
210
+ const [errorsCreated, errorsErr, errorsResource] = await tryFn(() => this.plugin.database.createResource({
211
+ name: errorsName,
212
+ attributes: {
213
+ errorId: 'string|required',
214
+ sessionId: 'string|required',
215
+ requestId: 'string|required',
216
+ url: 'string|required',
217
+ domain: 'string|required',
218
+ date: 'string|required', // YYYY-MM-DD
219
+
220
+ // Error details
221
+ errorType: 'string|required', // net::ERR_*, failed, timeout, blocked
222
+ errorText: 'string',
223
+ statusCode: 'number',
224
+
225
+ // Context
226
+ type: 'string', // Resource type
227
+ method: 'string',
228
+ timing: 'object',
229
+
230
+ // Additional info
231
+ blockedReason: 'string',
232
+ consoleMessages: 'array' // Related console errors
233
+ },
234
+ behavior: 'body-overflow',
235
+ timestamps: true,
236
+ partitions: {
237
+ bySession: { fields: { sessionId: 'string' } },
238
+ byErrorType: { fields: { errorType: 'string' } },
239
+ byDate: { fields: { date: 'string' } },
240
+ byDomain: { fields: { domain: 'string' } }
241
+ }
242
+ }));
243
+
244
+ if (errorsCreated) {
245
+ this.errorsResource = errorsResource;
246
+ } else if (this.plugin.database.resources?.[errorsName]) {
247
+ this.errorsResource = this.plugin.database.resources[errorsName];
248
+ } else {
249
+ throw errorsErr;
250
+ }
251
+
252
+ this.plugin.emit('networkMonitor.initialized', {
253
+ persist: this.config.persist
254
+ });
255
+ }
256
+
257
+ /**
258
+ * Start monitoring network activity for a page
259
+ * @param {Page} page - Puppeteer page
260
+ * @param {Object} options - Monitoring options
261
+ * @returns {Object} Session object with methods
262
+ */
263
+ async startMonitoring(page, options = {}) {
264
+ const {
265
+ sessionId = `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
266
+ persist = this.config.persist,
267
+ filters = this.config.filters
268
+ } = options;
269
+
270
+ const session = {
271
+ sessionId,
272
+ url: page.url(),
273
+ domain: this._extractDomain(page.url()),
274
+ date: new Date().toISOString().split('T')[0],
275
+ startTime: Date.now(),
276
+ endTime: null,
277
+ duration: null,
278
+
279
+ // Tracked data
280
+ requests: new Map(), // requestId -> request data
281
+ responses: new Map(), // requestId -> response data
282
+ failures: [],
283
+ consoleMessages: [],
284
+
285
+ // Statistics
286
+ stats: {
287
+ totalRequests: 0,
288
+ successfulRequests: 0,
289
+ failedRequests: 0,
290
+ totalBytes: 0,
291
+ transferredBytes: 0,
292
+ cachedBytes: 0,
293
+ byType: {}
294
+ }
295
+ };
296
+
297
+ // Get CDP session
298
+ const client = await page.target().createCDPSession();
299
+
300
+ // Enable Network domain
301
+ await client.send('Network.enable');
302
+
303
+ // Request will be sent
304
+ client.on('Network.requestWillBeSent', (params) => {
305
+ const requestData = {
306
+ requestId: params.requestId,
307
+ url: params.request.url,
308
+ domain: this._extractDomain(params.request.url),
309
+ path: this._extractPath(params.request.url),
310
+ type: this.resourceTypes[params.type] || 'other',
311
+ method: params.request.method,
312
+ requestHeaders: params.request.headers,
313
+ priority: params.request.initialPriority,
314
+ startTime: params.timestamp * 1000, // Convert to ms
315
+ redirected: !!params.redirectResponse,
316
+ redirectUrl: params.redirectResponse?.url || null
317
+ };
318
+
319
+ session.requests.set(params.requestId, requestData);
320
+ session.stats.totalRequests++;
321
+ });
322
+
323
+ // Response received
324
+ client.on('Network.responseReceived', (params) => {
325
+ const responseData = {
326
+ requestId: params.requestId,
327
+ statusCode: params.response.status,
328
+ statusText: params.response.statusText,
329
+ mimeType: params.response.mimeType,
330
+ responseHeaders: params.response.headers,
331
+ fromCache: params.response.fromDiskCache || params.response.fromServiceWorker,
332
+ compression: this._detectCompression(params.response.headers),
333
+ cacheControl: params.response.headers['cache-control'] || params.response.headers['Cache-Control'],
334
+ expires: params.response.headers['expires'] || params.response.headers['Expires'],
335
+ timing: params.response.timing ? this._parseTiming(params.response.timing) : null,
336
+ cdn: this._detectCDN(params.response.headers),
337
+ cdnDetected: !!this._detectCDN(params.response.headers)
338
+ };
339
+
340
+ session.responses.set(params.requestId, responseData);
341
+ });
342
+
343
+ // Loading finished
344
+ client.on('Network.loadingFinished', (params) => {
345
+ const request = session.requests.get(params.requestId);
346
+ const response = session.responses.get(params.requestId);
347
+
348
+ if (request && response) {
349
+ const endTime = params.timestamp * 1000;
350
+ const duration = endTime - request.startTime;
351
+
352
+ const combined = {
353
+ ...request,
354
+ ...response,
355
+ endTime,
356
+ duration,
357
+ size: params.encodedDataLength,
358
+ transferredSize: params.encodedDataLength,
359
+ resourceSize: params.decodedBodyLength || params.encodedDataLength,
360
+ failed: false
361
+ };
362
+
363
+ // Check if passes filters
364
+ if (this._passesFilters(combined, filters)) {
365
+ session.requests.set(params.requestId, combined);
366
+
367
+ // Update stats
368
+ session.stats.successfulRequests++;
369
+ session.stats.totalBytes += combined.resourceSize || 0;
370
+ session.stats.transferredBytes += combined.transferredSize || 0;
371
+
372
+ if (combined.fromCache) {
373
+ session.stats.cachedBytes += combined.resourceSize || 0;
374
+ }
375
+
376
+ // By type stats
377
+ const type = combined.type;
378
+ if (!session.stats.byType[type]) {
379
+ session.stats.byType[type] = { count: 0, size: 0, transferredSize: 0 };
380
+ }
381
+ session.stats.byType[type].count++;
382
+ session.stats.byType[type].size += combined.resourceSize || 0;
383
+ session.stats.byType[type].transferredSize += combined.transferredSize || 0;
384
+ } else {
385
+ // Remove if doesn't pass filters
386
+ session.requests.delete(params.requestId);
387
+ }
388
+ }
389
+ });
390
+
391
+ // Loading failed
392
+ client.on('Network.loadingFailed', (params) => {
393
+ const request = session.requests.get(params.requestId);
394
+
395
+ if (request) {
396
+ const errorData = {
397
+ ...request,
398
+ failed: true,
399
+ errorText: params.errorText,
400
+ blockedReason: params.blockedReason,
401
+ endTime: params.timestamp * 1000,
402
+ duration: (params.timestamp * 1000) - request.startTime
403
+ };
404
+
405
+ session.failures.push(errorData);
406
+ session.stats.failedRequests++;
407
+
408
+ // Remove from requests if not saving errors
409
+ if (!filters.saveErrors) {
410
+ session.requests.delete(params.requestId);
411
+ } else {
412
+ session.requests.set(params.requestId, errorData);
413
+ }
414
+ }
415
+ });
416
+
417
+ // Console messages (for correlation with errors)
418
+ page.on('console', (msg) => {
419
+ if (msg.type() === 'error' || msg.type() === 'warning') {
420
+ session.consoleMessages.push({
421
+ type: msg.type(),
422
+ text: msg.text(),
423
+ timestamp: Date.now()
424
+ });
425
+ }
426
+ });
427
+
428
+ // Store CDP session for cleanup
429
+ session._cdpSession = client;
430
+ session._persist = persist;
431
+ session._page = page;
432
+
433
+ this.plugin.emit('networkMonitor.sessionStarted', {
434
+ sessionId,
435
+ url: page.url()
436
+ });
437
+
438
+ return session;
439
+ }
440
+
441
+ /**
442
+ * Stop monitoring and optionally persist data
443
+ * @param {Object} session - Session object from startMonitoring
444
+ * @param {Object} options - Stop options
445
+ * @returns {Object} Final session data
446
+ */
447
+ async stopMonitoring(session, options = {}) {
448
+ const {
449
+ persist = session._persist,
450
+ includePerformance = true
451
+ } = options;
452
+
453
+ session.endTime = Date.now();
454
+ session.duration = session.endTime - session.startTime;
455
+
456
+ // Collect performance metrics if available
457
+ if (includePerformance && this.plugin.performanceManager && session._page) {
458
+ try {
459
+ session.performance = await this.plugin.performanceManager.collectMetrics(session._page, {
460
+ waitForLoad: false,
461
+ collectResources: false, // We already have this from CDP
462
+ collectMemory: true
463
+ });
464
+ } catch (err) {
465
+ this.plugin.emit('networkMonitor.performanceCollectionFailed', {
466
+ sessionId: session.sessionId,
467
+ error: err.message
468
+ });
469
+ }
470
+ }
471
+
472
+ // Disable network tracking
473
+ if (session._cdpSession) {
474
+ try {
475
+ await session._cdpSession.send('Network.disable');
476
+ await session._cdpSession.detach();
477
+ } catch (err) {
478
+ // Ignore - page might be closed
479
+ }
480
+ }
481
+
482
+ // Convert Map to Array for persistence
483
+ const requestsArray = Array.from(session.requests.values());
484
+
485
+ // Persist to S3DB if enabled
486
+ if (persist && this.sessionsResource) {
487
+ try {
488
+ await this._persistSession(session, requestsArray);
489
+ } catch (err) {
490
+ this.plugin.emit('networkMonitor.persistFailed', {
491
+ sessionId: session.sessionId,
492
+ error: err.message
493
+ });
494
+ }
495
+ }
496
+
497
+ this.plugin.emit('networkMonitor.sessionStopped', {
498
+ sessionId: session.sessionId,
499
+ duration: session.duration,
500
+ totalRequests: session.stats.totalRequests,
501
+ failedRequests: session.stats.failedRequests
502
+ });
503
+
504
+ // Clean up references
505
+ delete session._cdpSession;
506
+ delete session._page;
507
+
508
+ return {
509
+ ...session,
510
+ requests: requestsArray
511
+ };
512
+ }
513
+
514
+ /**
515
+ * Persist session data to S3DB
516
+ * @private
517
+ */
518
+ async _persistSession(session, requests) {
519
+ const startPersist = Date.now();
520
+
521
+ // Save session metadata
522
+ await this.sessionsResource.insert({
523
+ sessionId: session.sessionId,
524
+ url: session.url,
525
+ domain: session.domain,
526
+ date: session.date,
527
+ startTime: session.startTime,
528
+ endTime: session.endTime,
529
+ duration: session.duration,
530
+ totalRequests: session.stats.totalRequests,
531
+ successfulRequests: session.stats.successfulRequests,
532
+ failedRequests: session.stats.failedRequests,
533
+ totalBytes: session.stats.totalBytes,
534
+ transferredBytes: session.stats.transferredBytes,
535
+ cachedBytes: session.stats.cachedBytes,
536
+ byType: session.stats.byType,
537
+ performance: session.performance ? {
538
+ score: session.performance.score,
539
+ lcp: session.performance.coreWebVitals.lcp,
540
+ cls: session.performance.coreWebVitals.cls,
541
+ fcp: session.performance.coreWebVitals.fcp
542
+ } : null,
543
+ userAgent: session._page?._userAgent || null
544
+ });
545
+
546
+ // Save requests (batch insert for performance)
547
+ if (requests.length > 0) {
548
+ const requestInserts = requests.map(req => ({
549
+ requestId: req.requestId,
550
+ sessionId: session.sessionId,
551
+ url: req.url,
552
+ domain: req.domain,
553
+ path: req.path,
554
+ type: req.type,
555
+ statusCode: req.statusCode,
556
+ statusText: req.statusText,
557
+ method: req.method,
558
+ size: req.resourceSize,
559
+ transferredSize: req.transferredSize,
560
+ resourceSize: req.resourceSize,
561
+ fromCache: req.fromCache,
562
+ timing: req.timing,
563
+ startTime: req.startTime,
564
+ endTime: req.endTime,
565
+ duration: req.duration,
566
+ requestHeaders: this._compressHeaders(req.requestHeaders),
567
+ responseHeaders: this._compressHeaders(req.responseHeaders),
568
+ compression: req.compression,
569
+ cacheControl: req.cacheControl,
570
+ expires: req.expires,
571
+ failed: req.failed,
572
+ errorText: req.errorText,
573
+ blockedReason: req.blockedReason,
574
+ redirected: req.redirected,
575
+ redirectUrl: req.redirectUrl,
576
+ cdn: req.cdn,
577
+ cdnDetected: req.cdnDetected,
578
+ mimeType: req.mimeType,
579
+ priority: req.priority
580
+ }));
581
+
582
+ // Batch insert (s3db handles chunking)
583
+ for (const request of requestInserts) {
584
+ await this.requestsResource.insert(request);
585
+ }
586
+ }
587
+
588
+ // Save errors separately
589
+ if (session.failures.length > 0) {
590
+ for (const failure of session.failures) {
591
+ await this.errorsResource.insert({
592
+ errorId: `error_${failure.requestId}`,
593
+ sessionId: session.sessionId,
594
+ requestId: failure.requestId,
595
+ url: failure.url,
596
+ domain: failure.domain,
597
+ date: session.date,
598
+ errorType: this._categorizeError(failure.errorText),
599
+ errorText: failure.errorText,
600
+ statusCode: failure.statusCode,
601
+ type: failure.type,
602
+ method: failure.method,
603
+ timing: failure.timing,
604
+ blockedReason: failure.blockedReason,
605
+ consoleMessages: session.consoleMessages
606
+ .filter(msg => Math.abs(msg.timestamp - failure.endTime) < 1000) // Within 1s
607
+ .map(msg => msg.text)
608
+ });
609
+ }
610
+ }
611
+
612
+ const persistDuration = Date.now() - startPersist;
613
+
614
+ this.plugin.emit('networkMonitor.persisted', {
615
+ sessionId: session.sessionId,
616
+ requests: requests.length,
617
+ errors: session.failures.length,
618
+ duration: persistDuration
619
+ });
620
+ }
621
+
622
+ /**
623
+ * Check if request passes filters
624
+ * @private
625
+ */
626
+ _passesFilters(request, filters) {
627
+ // Type filter
628
+ if (filters.types && !filters.types.includes(request.type)) {
629
+ return false;
630
+ }
631
+
632
+ // Status filter
633
+ if (filters.statuses && !filters.statuses.includes(request.statusCode)) {
634
+ return false;
635
+ }
636
+
637
+ // Size filters
638
+ if (filters.minSize && (request.resourceSize || 0) < filters.minSize) {
639
+ return false;
640
+ }
641
+
642
+ if (filters.maxSize && (request.resourceSize || 0) > filters.maxSize) {
643
+ return false;
644
+ }
645
+
646
+ // Always save large assets if configured
647
+ if (filters.saveLargeAssets && (request.resourceSize || 0) > 1024 * 1024) {
648
+ return true;
649
+ }
650
+
651
+ return true;
652
+ }
653
+
654
+ /**
655
+ * Extract domain from URL
656
+ * @private
657
+ */
658
+ _extractDomain(url) {
659
+ try {
660
+ return new URL(url).hostname;
661
+ } catch {
662
+ return 'unknown';
663
+ }
664
+ }
665
+
666
+ /**
667
+ * Extract path from URL
668
+ * @private
669
+ */
670
+ _extractPath(url) {
671
+ try {
672
+ return new URL(url).pathname;
673
+ } catch {
674
+ return '';
675
+ }
676
+ }
677
+
678
+ /**
679
+ * Detect compression algorithm
680
+ * @private
681
+ */
682
+ _detectCompression(headers) {
683
+ const encoding = headers['content-encoding'] || headers['Content-Encoding'] || '';
684
+ if (encoding.includes('br')) return 'brotli';
685
+ if (encoding.includes('gzip')) return 'gzip';
686
+ if (encoding.includes('deflate')) return 'deflate';
687
+ return 'none';
688
+ }
689
+
690
+ /**
691
+ * Detect CDN provider
692
+ * @private
693
+ */
694
+ _detectCDN(headers) {
695
+ const server = headers['server'] || headers['Server'] || '';
696
+ const via = headers['via'] || headers['Via'] || '';
697
+ const cfRay = headers['cf-ray'] || headers['CF-Ray'];
698
+ const xCache = headers['x-cache'] || headers['X-Cache'] || '';
699
+
700
+ if (cfRay || server.includes('cloudflare')) return 'cloudflare';
701
+ if (xCache.includes('cloudfront') || headers['x-amz-cf-id']) return 'cloudfront';
702
+ if (server.includes('fastly') || via.includes('fastly')) return 'fastly';
703
+ if (headers['x-akamai-transformed'] || headers['x-akamai-staging']) return 'akamai';
704
+ if (headers['x-cdn'] || headers['X-CDN']) return headers['x-cdn'] || headers['X-CDN'];
705
+
706
+ return null;
707
+ }
708
+
709
+ /**
710
+ * Parse timing data
711
+ * @private
712
+ */
713
+ _parseTiming(timing) {
714
+ if (!timing) return null;
715
+
716
+ return {
717
+ dns: timing.dnsEnd - timing.dnsStart,
718
+ tcp: timing.connectEnd - timing.connectStart,
719
+ ssl: timing.sslEnd - timing.sslStart,
720
+ request: timing.sendEnd - timing.sendStart,
721
+ response: timing.receiveHeadersEnd - timing.sendEnd,
722
+ total: timing.receiveHeadersEnd
723
+ };
724
+ }
725
+
726
+ /**
727
+ * Compress headers (remove unnecessary data)
728
+ * @private
729
+ */
730
+ _compressHeaders(headers) {
731
+ if (!headers) return {};
732
+
733
+ // Remove common unnecessary headers
734
+ const compressed = { ...headers };
735
+ const toRemove = ['cookie', 'Cookie', 'set-cookie', 'Set-Cookie'];
736
+
737
+ toRemove.forEach(key => delete compressed[key]);
738
+
739
+ return compressed;
740
+ }
741
+
742
+ /**
743
+ * Categorize error type
744
+ * @private
745
+ */
746
+ _categorizeError(errorText) {
747
+ if (!errorText) return 'unknown';
748
+
749
+ if (errorText.includes('ERR_NAME_NOT_RESOLVED')) return 'dns';
750
+ if (errorText.includes('ERR_CONNECTION')) return 'connection';
751
+ if (errorText.includes('ERR_TIMED_OUT')) return 'timeout';
752
+ if (errorText.includes('ERR_SSL')) return 'ssl';
753
+ if (errorText.includes('ERR_CERT')) return 'certificate';
754
+ if (errorText.includes('ERR_BLOCKED')) return 'blocked';
755
+ if (errorText.includes('ERR_FAILED')) return 'failed';
756
+ if (errorText.includes('ERR_ABORTED')) return 'aborted';
757
+
758
+ return 'other';
759
+ }
760
+
761
+ /**
762
+ * Get session statistics
763
+ * @param {string} sessionId - Session ID
764
+ * @returns {Promise<Object>} Statistics
765
+ */
766
+ async getSessionStats(sessionId) {
767
+ if (!this.sessionsResource) {
768
+ throw new PuppeteerError('Network monitoring persistence not enabled', {
769
+ operation: 'getSessionStats',
770
+ retriable: false,
771
+ suggestion: 'Enable persistence in NetworkMonitor configuration to query stored stats.'
772
+ });
773
+ }
774
+
775
+ const session = await this.sessionsResource.get(sessionId);
776
+ return session;
777
+ }
778
+
779
+ /**
780
+ * Query requests for a session
781
+ * @param {string} sessionId - Session ID
782
+ * @param {Object} filters - Query filters
783
+ * @returns {Promise<Array>} Requests
784
+ */
785
+ async getSessionRequests(sessionId, filters = {}) {
786
+ if (!this.requestsResource) {
787
+ throw new PuppeteerError('Network monitoring persistence not enabled', {
788
+ operation: 'getSessionRequests',
789
+ retriable: false,
790
+ suggestion: 'Enable persistence in NetworkMonitor configuration to query stored requests.'
791
+ });
792
+ }
793
+
794
+ // Use partition for fast lookup
795
+ return await this.requestsResource.listPartition('bySession', { sessionId }, filters);
796
+ }
797
+
798
+ /**
799
+ * Query errors for a session
800
+ * @param {string} sessionId - Session ID
801
+ * @returns {Promise<Array>} Errors
802
+ */
803
+ async getSessionErrors(sessionId) {
804
+ if (!this.errorsResource) {
805
+ throw new PuppeteerError('Network monitoring persistence not enabled', {
806
+ operation: 'getSessionErrors',
807
+ retriable: false,
808
+ suggestion: 'Enable persistence in NetworkMonitor configuration to query stored errors.'
809
+ });
810
+ }
811
+
812
+ return await this.errorsResource.listPartition('bySession', { sessionId });
813
+ }
814
+ }
815
+
816
+ export default NetworkMonitor;