s3db.js 13.6.1 → 14.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -15
- package/dist/s3db.cjs +72446 -39022
- package/dist/s3db.cjs.map +1 -1
- package/dist/s3db.es.js +72172 -38790
- package/dist/s3db.es.js.map +1 -1
- package/mcp/lib/base-handler.js +157 -0
- package/mcp/lib/handlers/connection-handler.js +280 -0
- package/mcp/lib/handlers/query-handler.js +533 -0
- package/mcp/lib/handlers/resource-handler.js +428 -0
- package/mcp/lib/tool-registry.js +336 -0
- package/mcp/lib/tools/connection-tools.js +161 -0
- package/mcp/lib/tools/query-tools.js +267 -0
- package/mcp/lib/tools/resource-tools.js +404 -0
- package/package.json +85 -50
- package/src/clients/memory-client.class.js +346 -191
- package/src/clients/memory-storage.class.js +300 -84
- package/src/clients/s3-client.class.js +7 -6
- package/src/concerns/geo-encoding.js +19 -2
- package/src/concerns/ip.js +59 -9
- package/src/concerns/money.js +8 -1
- package/src/concerns/password-hashing.js +49 -8
- package/src/concerns/plugin-storage.js +186 -18
- package/src/concerns/storage-drivers/filesystem-driver.js +284 -0
- package/src/database.class.js +139 -29
- package/src/errors.js +332 -42
- package/src/plugins/api/auth/oidc-auth.js +66 -17
- package/src/plugins/api/auth/strategies/base-strategy.class.js +74 -0
- package/src/plugins/api/auth/strategies/factory.class.js +63 -0
- package/src/plugins/api/auth/strategies/global-strategy.class.js +44 -0
- package/src/plugins/api/auth/strategies/path-based-strategy.class.js +83 -0
- package/src/plugins/api/auth/strategies/path-rules-strategy.class.js +118 -0
- package/src/plugins/api/concerns/failban-manager.js +106 -57
- package/src/plugins/api/concerns/route-context.js +601 -0
- package/src/plugins/api/index.js +168 -40
- package/src/plugins/api/routes/auth-routes.js +198 -30
- package/src/plugins/api/routes/resource-routes.js +19 -4
- package/src/plugins/api/server/health-manager.class.js +163 -0
- package/src/plugins/api/server/middleware-chain.class.js +310 -0
- package/src/plugins/api/server/router.class.js +472 -0
- package/src/plugins/api/server.js +280 -1303
- package/src/plugins/api/utils/custom-routes.js +17 -5
- package/src/plugins/api/utils/guards.js +76 -17
- package/src/plugins/api/utils/openapi-generator-cached.class.js +133 -0
- package/src/plugins/api/utils/openapi-generator.js +7 -6
- package/src/plugins/audit.plugin.js +30 -8
- package/src/plugins/backup.plugin.js +110 -14
- package/src/plugins/cache/cache.class.js +22 -5
- package/src/plugins/cache/filesystem-cache.class.js +116 -19
- package/src/plugins/cache/memory-cache.class.js +211 -57
- package/src/plugins/cache/multi-tier-cache.class.js +371 -0
- package/src/plugins/cache/partition-aware-filesystem-cache.class.js +168 -47
- package/src/plugins/cache/redis-cache.class.js +552 -0
- package/src/plugins/cache/s3-cache.class.js +17 -8
- package/src/plugins/cache.plugin.js +176 -61
- package/src/plugins/cloud-inventory/drivers/alibaba-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/aws-driver.js +60 -29
- package/src/plugins/cloud-inventory/drivers/azure-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/base-driver.js +16 -2
- package/src/plugins/cloud-inventory/drivers/cloudflare-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/digitalocean-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/hetzner-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/linode-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/mongodb-atlas-driver.js +8 -1
- package/src/plugins/cloud-inventory/drivers/vultr-driver.js +8 -1
- package/src/plugins/cloud-inventory/index.js +29 -8
- package/src/plugins/cloud-inventory/registry.js +64 -42
- package/src/plugins/cloud-inventory.plugin.js +240 -138
- package/src/plugins/concerns/plugin-dependencies.js +54 -0
- package/src/plugins/concerns/resource-names.js +100 -0
- package/src/plugins/consumers/index.js +10 -2
- package/src/plugins/consumers/sqs-consumer.js +12 -2
- package/src/plugins/cookie-farm-suite.plugin.js +278 -0
- package/src/plugins/cookie-farm.errors.js +73 -0
- package/src/plugins/cookie-farm.plugin.js +869 -0
- package/src/plugins/costs.plugin.js +7 -1
- package/src/plugins/eventual-consistency/analytics.js +94 -19
- package/src/plugins/eventual-consistency/config.js +15 -7
- package/src/plugins/eventual-consistency/consolidation.js +29 -11
- package/src/plugins/eventual-consistency/garbage-collection.js +3 -1
- package/src/plugins/eventual-consistency/helpers.js +39 -14
- package/src/plugins/eventual-consistency/install.js +21 -2
- package/src/plugins/eventual-consistency/utils.js +32 -10
- package/src/plugins/fulltext.plugin.js +38 -11
- package/src/plugins/geo.plugin.js +61 -9
- package/src/plugins/identity/concerns/config.js +61 -0
- package/src/plugins/identity/concerns/mfa-manager.js +15 -2
- package/src/plugins/identity/concerns/rate-limit.js +124 -0
- package/src/plugins/identity/concerns/resource-schemas.js +9 -1
- package/src/plugins/identity/concerns/token-generator.js +29 -4
- package/src/plugins/identity/drivers/auth-driver.interface.js +76 -0
- package/src/plugins/identity/drivers/client-credentials-driver.js +127 -0
- package/src/plugins/identity/drivers/index.js +18 -0
- package/src/plugins/identity/drivers/password-driver.js +122 -0
- package/src/plugins/identity/email-service.js +17 -2
- package/src/plugins/identity/index.js +413 -69
- package/src/plugins/identity/oauth2-server.js +413 -30
- package/src/plugins/identity/oidc-discovery.js +16 -8
- package/src/plugins/identity/rsa-keys.js +115 -35
- package/src/plugins/identity/server.js +166 -45
- package/src/plugins/identity/session-manager.js +53 -7
- package/src/plugins/identity/ui/pages/mfa-verification.js +17 -15
- package/src/plugins/identity/ui/routes.js +363 -255
- package/src/plugins/importer/index.js +153 -20
- package/src/plugins/index.js +9 -2
- package/src/plugins/kubernetes-inventory/index.js +6 -0
- package/src/plugins/kubernetes-inventory/k8s-driver.js +867 -0
- package/src/plugins/kubernetes-inventory/resource-types.js +274 -0
- package/src/plugins/kubernetes-inventory.plugin.js +980 -0
- package/src/plugins/metrics.plugin.js +64 -16
- package/src/plugins/ml/base-model.class.js +25 -15
- package/src/plugins/ml/regression-model.class.js +1 -1
- package/src/plugins/ml.errors.js +57 -25
- package/src/plugins/ml.plugin.js +28 -4
- package/src/plugins/namespace.js +210 -0
- package/src/plugins/plugin.class.js +180 -8
- package/src/plugins/puppeteer/console-monitor.js +729 -0
- package/src/plugins/puppeteer/cookie-manager.js +492 -0
- package/src/plugins/puppeteer/network-monitor.js +816 -0
- package/src/plugins/puppeteer/performance-manager.js +746 -0
- package/src/plugins/puppeteer/proxy-manager.js +478 -0
- package/src/plugins/puppeteer/stealth-manager.js +556 -0
- package/src/plugins/puppeteer.errors.js +81 -0
- package/src/plugins/puppeteer.plugin.js +1327 -0
- package/src/plugins/queue-consumer.plugin.js +69 -14
- package/src/plugins/recon/behaviors/uptime-behavior.js +691 -0
- package/src/plugins/recon/concerns/command-runner.js +148 -0
- package/src/plugins/recon/concerns/diff-detector.js +372 -0
- package/src/plugins/recon/concerns/fingerprint-builder.js +307 -0
- package/src/plugins/recon/concerns/process-manager.js +338 -0
- package/src/plugins/recon/concerns/report-generator.js +478 -0
- package/src/plugins/recon/concerns/security-analyzer.js +571 -0
- package/src/plugins/recon/concerns/target-normalizer.js +68 -0
- package/src/plugins/recon/config/defaults.js +321 -0
- package/src/plugins/recon/config/resources.js +370 -0
- package/src/plugins/recon/index.js +778 -0
- package/src/plugins/recon/managers/dependency-manager.js +174 -0
- package/src/plugins/recon/managers/scheduler-manager.js +179 -0
- package/src/plugins/recon/managers/storage-manager.js +745 -0
- package/src/plugins/recon/managers/target-manager.js +274 -0
- package/src/plugins/recon/stages/asn-stage.js +314 -0
- package/src/plugins/recon/stages/certificate-stage.js +84 -0
- package/src/plugins/recon/stages/dns-stage.js +107 -0
- package/src/plugins/recon/stages/dnsdumpster-stage.js +362 -0
- package/src/plugins/recon/stages/fingerprint-stage.js +71 -0
- package/src/plugins/recon/stages/google-dorks-stage.js +440 -0
- package/src/plugins/recon/stages/http-stage.js +89 -0
- package/src/plugins/recon/stages/latency-stage.js +148 -0
- package/src/plugins/recon/stages/massdns-stage.js +302 -0
- package/src/plugins/recon/stages/osint-stage.js +1373 -0
- package/src/plugins/recon/stages/ports-stage.js +169 -0
- package/src/plugins/recon/stages/screenshot-stage.js +94 -0
- package/src/plugins/recon/stages/secrets-stage.js +514 -0
- package/src/plugins/recon/stages/subdomains-stage.js +295 -0
- package/src/plugins/recon/stages/tls-audit-stage.js +78 -0
- package/src/plugins/recon/stages/vulnerability-stage.js +78 -0
- package/src/plugins/recon/stages/web-discovery-stage.js +113 -0
- package/src/plugins/recon/stages/whois-stage.js +349 -0
- package/src/plugins/recon.plugin.js +75 -0
- package/src/plugins/recon.plugin.js.backup +2635 -0
- package/src/plugins/relation.errors.js +87 -14
- package/src/plugins/replicator.plugin.js +514 -137
- package/src/plugins/replicators/base-replicator.class.js +89 -1
- package/src/plugins/replicators/bigquery-replicator.class.js +66 -22
- package/src/plugins/replicators/dynamodb-replicator.class.js +22 -15
- package/src/plugins/replicators/mongodb-replicator.class.js +22 -15
- package/src/plugins/replicators/mysql-replicator.class.js +52 -17
- package/src/plugins/replicators/planetscale-replicator.class.js +30 -4
- package/src/plugins/replicators/postgres-replicator.class.js +62 -27
- package/src/plugins/replicators/s3db-replicator.class.js +25 -18
- package/src/plugins/replicators/schema-sync.helper.js +3 -3
- package/src/plugins/replicators/sqs-replicator.class.js +8 -2
- package/src/plugins/replicators/turso-replicator.class.js +23 -3
- package/src/plugins/replicators/webhook-replicator.class.js +42 -4
- package/src/plugins/s3-queue.plugin.js +464 -65
- package/src/plugins/scheduler.plugin.js +20 -6
- package/src/plugins/state-machine.plugin.js +40 -9
- package/src/plugins/tfstate/base-driver.js +28 -4
- package/src/plugins/tfstate/errors.js +65 -10
- package/src/plugins/tfstate/filesystem-driver.js +52 -8
- package/src/plugins/tfstate/index.js +163 -90
- package/src/plugins/tfstate/s3-driver.js +64 -6
- package/src/plugins/ttl.plugin.js +72 -17
- package/src/plugins/vector/distances.js +18 -12
- package/src/plugins/vector/kmeans.js +26 -4
- package/src/resource.class.js +115 -19
- package/src/testing/factory.class.js +20 -3
- package/src/testing/seeder.class.js +7 -1
- package/src/clients/memory-client.md +0 -917
- package/src/plugins/cloud-inventory/drivers/mock-drivers.js +0 -449
|
@@ -0,0 +1,816 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NetworkMonitor - Comprehensive Network Activity Tracking
|
|
3
|
+
*
|
|
4
|
+
* Captures all network requests/responses using Chrome DevTools Protocol (CDP):
|
|
5
|
+
* - Request/Response headers, timing, sizes
|
|
6
|
+
* - Status codes, errors, redirects
|
|
7
|
+
* - Resource types (image, script, stylesheet, xhr, fetch, etc.)
|
|
8
|
+
* - Compression detection (gzip, brotli)
|
|
9
|
+
* - Cache behavior
|
|
10
|
+
* - Failed requests
|
|
11
|
+
*
|
|
12
|
+
* Persistence Strategy:
|
|
13
|
+
* - Optional S3DB storage with intelligent partitioning
|
|
14
|
+
* - Compression for large payloads
|
|
15
|
+
* - Filtering by resource type, status, size
|
|
16
|
+
* - Separate resources for sessions, requests, and errors
|
|
17
|
+
*
|
|
18
|
+
* Use cases:
|
|
19
|
+
* - SEO analysis (image sizes, script sizes, load times)
|
|
20
|
+
* - Performance debugging (slow requests, failed requests)
|
|
21
|
+
* - Security auditing (CSP violations, mixed content)
|
|
22
|
+
* - Cost analysis (bandwidth usage)
|
|
23
|
+
* - A/B testing (compare network behavior)
|
|
24
|
+
*/
|
|
25
|
+
import tryFn from '../../concerns/try-fn.js';
|
|
26
|
+
import { PuppeteerError } from '../puppeteer.errors.js';
|
|
27
|
+
|
|
28
|
+
export class NetworkMonitor {
|
|
29
|
+
constructor(plugin) {
|
|
30
|
+
this.plugin = plugin;
|
|
31
|
+
this.config = plugin.config.networkMonitor || {
|
|
32
|
+
enabled: false,
|
|
33
|
+
persist: false,
|
|
34
|
+
filters: {
|
|
35
|
+
types: null, // ['image', 'script', 'stylesheet'] or null for all
|
|
36
|
+
statuses: null, // [404, 500] or null for all
|
|
37
|
+
minSize: null, // Only requests >= this size (bytes)
|
|
38
|
+
maxSize: null, // Only requests <= this size (bytes)
|
|
39
|
+
saveErrors: true, // Always save failed requests
|
|
40
|
+
saveLargeAssets: true // Always save assets > 1MB
|
|
41
|
+
},
|
|
42
|
+
compression: {
|
|
43
|
+
enabled: true,
|
|
44
|
+
threshold: 10240 // Compress payloads > 10KB
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
// Resources for persistence (lazy-initialized)
|
|
49
|
+
this.sessionsResource = null;
|
|
50
|
+
this.requestsResource = null;
|
|
51
|
+
this.errorsResource = null;
|
|
52
|
+
|
|
53
|
+
// Resource type mapping (CDP -> simplified)
|
|
54
|
+
this.resourceTypes = {
|
|
55
|
+
'Document': 'document',
|
|
56
|
+
'Stylesheet': 'stylesheet',
|
|
57
|
+
'Image': 'image',
|
|
58
|
+
'Media': 'media',
|
|
59
|
+
'Font': 'font',
|
|
60
|
+
'Script': 'script',
|
|
61
|
+
'TextTrack': 'texttrack',
|
|
62
|
+
'XHR': 'xhr',
|
|
63
|
+
'Fetch': 'fetch',
|
|
64
|
+
'EventSource': 'eventsource',
|
|
65
|
+
'WebSocket': 'websocket',
|
|
66
|
+
'Manifest': 'manifest',
|
|
67
|
+
'SignedExchange': 'signedexchange',
|
|
68
|
+
'Ping': 'ping',
|
|
69
|
+
'CSPViolationReport': 'cspviolation',
|
|
70
|
+
'Preflight': 'preflight',
|
|
71
|
+
'Other': 'other'
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Initialize network monitoring resources
|
|
77
|
+
*/
|
|
78
|
+
async initialize() {
|
|
79
|
+
if (!this.config.persist) {
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Create sessions resource (metadata about each crawl session)
|
|
84
|
+
const resourceNames = this.plugin.resourceNames || {};
|
|
85
|
+
const sessionsName = resourceNames.networkSessions || 'plg_puppeteer_network_sessions';
|
|
86
|
+
const requestsName = resourceNames.networkRequests || 'plg_puppeteer_network_requests';
|
|
87
|
+
const errorsName = resourceNames.networkErrors || 'plg_puppeteer_network_errors';
|
|
88
|
+
|
|
89
|
+
const [sessionsCreated, sessionsErr, sessionsResource] = await tryFn(() => this.plugin.database.createResource({
|
|
90
|
+
name: sessionsName,
|
|
91
|
+
attributes: {
|
|
92
|
+
sessionId: 'string|required',
|
|
93
|
+
url: 'string|required',
|
|
94
|
+
domain: 'string|required',
|
|
95
|
+
date: 'string|required', // YYYY-MM-DD for partitioning
|
|
96
|
+
startTime: 'number|required',
|
|
97
|
+
endTime: 'number',
|
|
98
|
+
duration: 'number',
|
|
99
|
+
|
|
100
|
+
// Summary statistics
|
|
101
|
+
totalRequests: 'number',
|
|
102
|
+
successfulRequests: 'number',
|
|
103
|
+
failedRequests: 'number',
|
|
104
|
+
totalBytes: 'number',
|
|
105
|
+
transferredBytes: 'number',
|
|
106
|
+
cachedBytes: 'number',
|
|
107
|
+
|
|
108
|
+
// By type
|
|
109
|
+
byType: 'object', // { image: { count, size }, script: {...} }
|
|
110
|
+
|
|
111
|
+
// Performance metrics
|
|
112
|
+
performance: 'object', // From PerformanceManager
|
|
113
|
+
|
|
114
|
+
// User agent
|
|
115
|
+
userAgent: 'string'
|
|
116
|
+
},
|
|
117
|
+
behavior: 'body-overflow',
|
|
118
|
+
timestamps: true,
|
|
119
|
+
partitions: {
|
|
120
|
+
byUrl: { fields: { url: 'string' } },
|
|
121
|
+
byDate: { fields: { date: 'string' } },
|
|
122
|
+
byDomain: { fields: { domain: 'string' } }
|
|
123
|
+
}
|
|
124
|
+
}));
|
|
125
|
+
|
|
126
|
+
if (sessionsCreated) {
|
|
127
|
+
this.sessionsResource = sessionsResource;
|
|
128
|
+
} else if (this.plugin.database.resources?.[sessionsName]) {
|
|
129
|
+
this.sessionsResource = this.plugin.database.resources[sessionsName];
|
|
130
|
+
} else {
|
|
131
|
+
throw sessionsErr;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// Create requests resource (detailed info about each request)
|
|
135
|
+
const [requestsCreated, requestsErr, requestsResource] = await tryFn(() => this.plugin.database.createResource({
|
|
136
|
+
name: requestsName,
|
|
137
|
+
attributes: {
|
|
138
|
+
requestId: 'string|required',
|
|
139
|
+
sessionId: 'string|required',
|
|
140
|
+
url: 'string|required',
|
|
141
|
+
domain: 'string|required',
|
|
142
|
+
path: 'string',
|
|
143
|
+
|
|
144
|
+
// Type and categorization
|
|
145
|
+
type: 'string|required', // image, script, stylesheet, xhr, etc.
|
|
146
|
+
statusCode: 'number',
|
|
147
|
+
statusText: 'string',
|
|
148
|
+
method: 'string', // GET, POST, etc.
|
|
149
|
+
|
|
150
|
+
// Size information
|
|
151
|
+
size: 'number', // Total size (bytes)
|
|
152
|
+
transferredSize: 'number', // Bytes transferred (after compression)
|
|
153
|
+
resourceSize: 'number', // Uncompressed size
|
|
154
|
+
fromCache: 'boolean',
|
|
155
|
+
|
|
156
|
+
// Timing information (ms)
|
|
157
|
+
timing: 'object', // { dns, tcp, ssl, request, response, total }
|
|
158
|
+
startTime: 'number',
|
|
159
|
+
endTime: 'number',
|
|
160
|
+
duration: 'number',
|
|
161
|
+
|
|
162
|
+
// Headers (compressed if large)
|
|
163
|
+
requestHeaders: 'object',
|
|
164
|
+
responseHeaders: 'object',
|
|
165
|
+
|
|
166
|
+
// Compression
|
|
167
|
+
compression: 'string', // gzip, br (brotli), deflate, none
|
|
168
|
+
|
|
169
|
+
// Cache
|
|
170
|
+
cacheControl: 'string',
|
|
171
|
+
expires: 'string',
|
|
172
|
+
|
|
173
|
+
// Error information (if failed)
|
|
174
|
+
failed: 'boolean',
|
|
175
|
+
errorText: 'string',
|
|
176
|
+
blockedReason: 'string', // CSP, mixed-content, etc.
|
|
177
|
+
|
|
178
|
+
// Redirects
|
|
179
|
+
redirected: 'boolean',
|
|
180
|
+
redirectUrl: 'string',
|
|
181
|
+
|
|
182
|
+
// CDN detection
|
|
183
|
+
cdn: 'string', // cloudflare, cloudfront, fastly, etc.
|
|
184
|
+
cdnDetected: 'boolean',
|
|
185
|
+
|
|
186
|
+
// Metadata
|
|
187
|
+
mimeType: 'string',
|
|
188
|
+
priority: 'string' // VeryHigh, High, Medium, Low, VeryLow
|
|
189
|
+
},
|
|
190
|
+
behavior: 'body-overflow',
|
|
191
|
+
timestamps: true,
|
|
192
|
+
partitions: {
|
|
193
|
+
bySession: { fields: { sessionId: 'string' } },
|
|
194
|
+
byType: { fields: { type: 'string' } },
|
|
195
|
+
byStatus: { fields: { statusCode: 'number' } },
|
|
196
|
+
bySize: { fields: { size: 'number' } },
|
|
197
|
+
byDomain: { fields: { domain: 'string' } }
|
|
198
|
+
}
|
|
199
|
+
}));
|
|
200
|
+
|
|
201
|
+
if (requestsCreated) {
|
|
202
|
+
this.requestsResource = requestsResource;
|
|
203
|
+
} else if (this.plugin.database.resources?.[requestsName]) {
|
|
204
|
+
this.requestsResource = this.plugin.database.resources[requestsName];
|
|
205
|
+
} else {
|
|
206
|
+
throw requestsErr;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Create errors resource (failed requests only)
|
|
210
|
+
const [errorsCreated, errorsErr, errorsResource] = await tryFn(() => this.plugin.database.createResource({
|
|
211
|
+
name: errorsName,
|
|
212
|
+
attributes: {
|
|
213
|
+
errorId: 'string|required',
|
|
214
|
+
sessionId: 'string|required',
|
|
215
|
+
requestId: 'string|required',
|
|
216
|
+
url: 'string|required',
|
|
217
|
+
domain: 'string|required',
|
|
218
|
+
date: 'string|required', // YYYY-MM-DD
|
|
219
|
+
|
|
220
|
+
// Error details
|
|
221
|
+
errorType: 'string|required', // net::ERR_*, failed, timeout, blocked
|
|
222
|
+
errorText: 'string',
|
|
223
|
+
statusCode: 'number',
|
|
224
|
+
|
|
225
|
+
// Context
|
|
226
|
+
type: 'string', // Resource type
|
|
227
|
+
method: 'string',
|
|
228
|
+
timing: 'object',
|
|
229
|
+
|
|
230
|
+
// Additional info
|
|
231
|
+
blockedReason: 'string',
|
|
232
|
+
consoleMessages: 'array' // Related console errors
|
|
233
|
+
},
|
|
234
|
+
behavior: 'body-overflow',
|
|
235
|
+
timestamps: true,
|
|
236
|
+
partitions: {
|
|
237
|
+
bySession: { fields: { sessionId: 'string' } },
|
|
238
|
+
byErrorType: { fields: { errorType: 'string' } },
|
|
239
|
+
byDate: { fields: { date: 'string' } },
|
|
240
|
+
byDomain: { fields: { domain: 'string' } }
|
|
241
|
+
}
|
|
242
|
+
}));
|
|
243
|
+
|
|
244
|
+
if (errorsCreated) {
|
|
245
|
+
this.errorsResource = errorsResource;
|
|
246
|
+
} else if (this.plugin.database.resources?.[errorsName]) {
|
|
247
|
+
this.errorsResource = this.plugin.database.resources[errorsName];
|
|
248
|
+
} else {
|
|
249
|
+
throw errorsErr;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
this.plugin.emit('networkMonitor.initialized', {
|
|
253
|
+
persist: this.config.persist
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Start monitoring network activity for a page
|
|
259
|
+
* @param {Page} page - Puppeteer page
|
|
260
|
+
* @param {Object} options - Monitoring options
|
|
261
|
+
* @returns {Object} Session object with methods
|
|
262
|
+
*/
|
|
263
|
+
async startMonitoring(page, options = {}) {
|
|
264
|
+
const {
|
|
265
|
+
sessionId = `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
|
|
266
|
+
persist = this.config.persist,
|
|
267
|
+
filters = this.config.filters
|
|
268
|
+
} = options;
|
|
269
|
+
|
|
270
|
+
const session = {
|
|
271
|
+
sessionId,
|
|
272
|
+
url: page.url(),
|
|
273
|
+
domain: this._extractDomain(page.url()),
|
|
274
|
+
date: new Date().toISOString().split('T')[0],
|
|
275
|
+
startTime: Date.now(),
|
|
276
|
+
endTime: null,
|
|
277
|
+
duration: null,
|
|
278
|
+
|
|
279
|
+
// Tracked data
|
|
280
|
+
requests: new Map(), // requestId -> request data
|
|
281
|
+
responses: new Map(), // requestId -> response data
|
|
282
|
+
failures: [],
|
|
283
|
+
consoleMessages: [],
|
|
284
|
+
|
|
285
|
+
// Statistics
|
|
286
|
+
stats: {
|
|
287
|
+
totalRequests: 0,
|
|
288
|
+
successfulRequests: 0,
|
|
289
|
+
failedRequests: 0,
|
|
290
|
+
totalBytes: 0,
|
|
291
|
+
transferredBytes: 0,
|
|
292
|
+
cachedBytes: 0,
|
|
293
|
+
byType: {}
|
|
294
|
+
}
|
|
295
|
+
};
|
|
296
|
+
|
|
297
|
+
// Get CDP session
|
|
298
|
+
const client = await page.target().createCDPSession();
|
|
299
|
+
|
|
300
|
+
// Enable Network domain
|
|
301
|
+
await client.send('Network.enable');
|
|
302
|
+
|
|
303
|
+
// Request will be sent
|
|
304
|
+
client.on('Network.requestWillBeSent', (params) => {
|
|
305
|
+
const requestData = {
|
|
306
|
+
requestId: params.requestId,
|
|
307
|
+
url: params.request.url,
|
|
308
|
+
domain: this._extractDomain(params.request.url),
|
|
309
|
+
path: this._extractPath(params.request.url),
|
|
310
|
+
type: this.resourceTypes[params.type] || 'other',
|
|
311
|
+
method: params.request.method,
|
|
312
|
+
requestHeaders: params.request.headers,
|
|
313
|
+
priority: params.request.initialPriority,
|
|
314
|
+
startTime: params.timestamp * 1000, // Convert to ms
|
|
315
|
+
redirected: !!params.redirectResponse,
|
|
316
|
+
redirectUrl: params.redirectResponse?.url || null
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
session.requests.set(params.requestId, requestData);
|
|
320
|
+
session.stats.totalRequests++;
|
|
321
|
+
});
|
|
322
|
+
|
|
323
|
+
// Response received
|
|
324
|
+
client.on('Network.responseReceived', (params) => {
|
|
325
|
+
const responseData = {
|
|
326
|
+
requestId: params.requestId,
|
|
327
|
+
statusCode: params.response.status,
|
|
328
|
+
statusText: params.response.statusText,
|
|
329
|
+
mimeType: params.response.mimeType,
|
|
330
|
+
responseHeaders: params.response.headers,
|
|
331
|
+
fromCache: params.response.fromDiskCache || params.response.fromServiceWorker,
|
|
332
|
+
compression: this._detectCompression(params.response.headers),
|
|
333
|
+
cacheControl: params.response.headers['cache-control'] || params.response.headers['Cache-Control'],
|
|
334
|
+
expires: params.response.headers['expires'] || params.response.headers['Expires'],
|
|
335
|
+
timing: params.response.timing ? this._parseTiming(params.response.timing) : null,
|
|
336
|
+
cdn: this._detectCDN(params.response.headers),
|
|
337
|
+
cdnDetected: !!this._detectCDN(params.response.headers)
|
|
338
|
+
};
|
|
339
|
+
|
|
340
|
+
session.responses.set(params.requestId, responseData);
|
|
341
|
+
});
|
|
342
|
+
|
|
343
|
+
// Loading finished
|
|
344
|
+
client.on('Network.loadingFinished', (params) => {
|
|
345
|
+
const request = session.requests.get(params.requestId);
|
|
346
|
+
const response = session.responses.get(params.requestId);
|
|
347
|
+
|
|
348
|
+
if (request && response) {
|
|
349
|
+
const endTime = params.timestamp * 1000;
|
|
350
|
+
const duration = endTime - request.startTime;
|
|
351
|
+
|
|
352
|
+
const combined = {
|
|
353
|
+
...request,
|
|
354
|
+
...response,
|
|
355
|
+
endTime,
|
|
356
|
+
duration,
|
|
357
|
+
size: params.encodedDataLength,
|
|
358
|
+
transferredSize: params.encodedDataLength,
|
|
359
|
+
resourceSize: params.decodedBodyLength || params.encodedDataLength,
|
|
360
|
+
failed: false
|
|
361
|
+
};
|
|
362
|
+
|
|
363
|
+
// Check if passes filters
|
|
364
|
+
if (this._passesFilters(combined, filters)) {
|
|
365
|
+
session.requests.set(params.requestId, combined);
|
|
366
|
+
|
|
367
|
+
// Update stats
|
|
368
|
+
session.stats.successfulRequests++;
|
|
369
|
+
session.stats.totalBytes += combined.resourceSize || 0;
|
|
370
|
+
session.stats.transferredBytes += combined.transferredSize || 0;
|
|
371
|
+
|
|
372
|
+
if (combined.fromCache) {
|
|
373
|
+
session.stats.cachedBytes += combined.resourceSize || 0;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// By type stats
|
|
377
|
+
const type = combined.type;
|
|
378
|
+
if (!session.stats.byType[type]) {
|
|
379
|
+
session.stats.byType[type] = { count: 0, size: 0, transferredSize: 0 };
|
|
380
|
+
}
|
|
381
|
+
session.stats.byType[type].count++;
|
|
382
|
+
session.stats.byType[type].size += combined.resourceSize || 0;
|
|
383
|
+
session.stats.byType[type].transferredSize += combined.transferredSize || 0;
|
|
384
|
+
} else {
|
|
385
|
+
// Remove if doesn't pass filters
|
|
386
|
+
session.requests.delete(params.requestId);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
});
|
|
390
|
+
|
|
391
|
+
// Loading failed
|
|
392
|
+
client.on('Network.loadingFailed', (params) => {
|
|
393
|
+
const request = session.requests.get(params.requestId);
|
|
394
|
+
|
|
395
|
+
if (request) {
|
|
396
|
+
const errorData = {
|
|
397
|
+
...request,
|
|
398
|
+
failed: true,
|
|
399
|
+
errorText: params.errorText,
|
|
400
|
+
blockedReason: params.blockedReason,
|
|
401
|
+
endTime: params.timestamp * 1000,
|
|
402
|
+
duration: (params.timestamp * 1000) - request.startTime
|
|
403
|
+
};
|
|
404
|
+
|
|
405
|
+
session.failures.push(errorData);
|
|
406
|
+
session.stats.failedRequests++;
|
|
407
|
+
|
|
408
|
+
// Remove from requests if not saving errors
|
|
409
|
+
if (!filters.saveErrors) {
|
|
410
|
+
session.requests.delete(params.requestId);
|
|
411
|
+
} else {
|
|
412
|
+
session.requests.set(params.requestId, errorData);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
// Console messages (for correlation with errors)
|
|
418
|
+
page.on('console', (msg) => {
|
|
419
|
+
if (msg.type() === 'error' || msg.type() === 'warning') {
|
|
420
|
+
session.consoleMessages.push({
|
|
421
|
+
type: msg.type(),
|
|
422
|
+
text: msg.text(),
|
|
423
|
+
timestamp: Date.now()
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
});
|
|
427
|
+
|
|
428
|
+
// Store CDP session for cleanup
|
|
429
|
+
session._cdpSession = client;
|
|
430
|
+
session._persist = persist;
|
|
431
|
+
session._page = page;
|
|
432
|
+
|
|
433
|
+
this.plugin.emit('networkMonitor.sessionStarted', {
|
|
434
|
+
sessionId,
|
|
435
|
+
url: page.url()
|
|
436
|
+
});
|
|
437
|
+
|
|
438
|
+
return session;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
/**
|
|
442
|
+
* Stop monitoring and optionally persist data
|
|
443
|
+
* @param {Object} session - Session object from startMonitoring
|
|
444
|
+
* @param {Object} options - Stop options
|
|
445
|
+
* @returns {Object} Final session data
|
|
446
|
+
*/
|
|
447
|
+
async stopMonitoring(session, options = {}) {
|
|
448
|
+
const {
|
|
449
|
+
persist = session._persist,
|
|
450
|
+
includePerformance = true
|
|
451
|
+
} = options;
|
|
452
|
+
|
|
453
|
+
session.endTime = Date.now();
|
|
454
|
+
session.duration = session.endTime - session.startTime;
|
|
455
|
+
|
|
456
|
+
// Collect performance metrics if available
|
|
457
|
+
if (includePerformance && this.plugin.performanceManager && session._page) {
|
|
458
|
+
try {
|
|
459
|
+
session.performance = await this.plugin.performanceManager.collectMetrics(session._page, {
|
|
460
|
+
waitForLoad: false,
|
|
461
|
+
collectResources: false, // We already have this from CDP
|
|
462
|
+
collectMemory: true
|
|
463
|
+
});
|
|
464
|
+
} catch (err) {
|
|
465
|
+
this.plugin.emit('networkMonitor.performanceCollectionFailed', {
|
|
466
|
+
sessionId: session.sessionId,
|
|
467
|
+
error: err.message
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
// Disable network tracking
|
|
473
|
+
if (session._cdpSession) {
|
|
474
|
+
try {
|
|
475
|
+
await session._cdpSession.send('Network.disable');
|
|
476
|
+
await session._cdpSession.detach();
|
|
477
|
+
} catch (err) {
|
|
478
|
+
// Ignore - page might be closed
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
// Convert Map to Array for persistence
|
|
483
|
+
const requestsArray = Array.from(session.requests.values());
|
|
484
|
+
|
|
485
|
+
// Persist to S3DB if enabled
|
|
486
|
+
if (persist && this.sessionsResource) {
|
|
487
|
+
try {
|
|
488
|
+
await this._persistSession(session, requestsArray);
|
|
489
|
+
} catch (err) {
|
|
490
|
+
this.plugin.emit('networkMonitor.persistFailed', {
|
|
491
|
+
sessionId: session.sessionId,
|
|
492
|
+
error: err.message
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
this.plugin.emit('networkMonitor.sessionStopped', {
|
|
498
|
+
sessionId: session.sessionId,
|
|
499
|
+
duration: session.duration,
|
|
500
|
+
totalRequests: session.stats.totalRequests,
|
|
501
|
+
failedRequests: session.stats.failedRequests
|
|
502
|
+
});
|
|
503
|
+
|
|
504
|
+
// Clean up references
|
|
505
|
+
delete session._cdpSession;
|
|
506
|
+
delete session._page;
|
|
507
|
+
|
|
508
|
+
return {
|
|
509
|
+
...session,
|
|
510
|
+
requests: requestsArray
|
|
511
|
+
};
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/**
|
|
515
|
+
* Persist session data to S3DB
|
|
516
|
+
* @private
|
|
517
|
+
*/
|
|
518
|
+
async _persistSession(session, requests) {
|
|
519
|
+
const startPersist = Date.now();
|
|
520
|
+
|
|
521
|
+
// Save session metadata
|
|
522
|
+
await this.sessionsResource.insert({
|
|
523
|
+
sessionId: session.sessionId,
|
|
524
|
+
url: session.url,
|
|
525
|
+
domain: session.domain,
|
|
526
|
+
date: session.date,
|
|
527
|
+
startTime: session.startTime,
|
|
528
|
+
endTime: session.endTime,
|
|
529
|
+
duration: session.duration,
|
|
530
|
+
totalRequests: session.stats.totalRequests,
|
|
531
|
+
successfulRequests: session.stats.successfulRequests,
|
|
532
|
+
failedRequests: session.stats.failedRequests,
|
|
533
|
+
totalBytes: session.stats.totalBytes,
|
|
534
|
+
transferredBytes: session.stats.transferredBytes,
|
|
535
|
+
cachedBytes: session.stats.cachedBytes,
|
|
536
|
+
byType: session.stats.byType,
|
|
537
|
+
performance: session.performance ? {
|
|
538
|
+
score: session.performance.score,
|
|
539
|
+
lcp: session.performance.coreWebVitals.lcp,
|
|
540
|
+
cls: session.performance.coreWebVitals.cls,
|
|
541
|
+
fcp: session.performance.coreWebVitals.fcp
|
|
542
|
+
} : null,
|
|
543
|
+
userAgent: session._page?._userAgent || null
|
|
544
|
+
});
|
|
545
|
+
|
|
546
|
+
// Save requests (batch insert for performance)
|
|
547
|
+
if (requests.length > 0) {
|
|
548
|
+
const requestInserts = requests.map(req => ({
|
|
549
|
+
requestId: req.requestId,
|
|
550
|
+
sessionId: session.sessionId,
|
|
551
|
+
url: req.url,
|
|
552
|
+
domain: req.domain,
|
|
553
|
+
path: req.path,
|
|
554
|
+
type: req.type,
|
|
555
|
+
statusCode: req.statusCode,
|
|
556
|
+
statusText: req.statusText,
|
|
557
|
+
method: req.method,
|
|
558
|
+
size: req.resourceSize,
|
|
559
|
+
transferredSize: req.transferredSize,
|
|
560
|
+
resourceSize: req.resourceSize,
|
|
561
|
+
fromCache: req.fromCache,
|
|
562
|
+
timing: req.timing,
|
|
563
|
+
startTime: req.startTime,
|
|
564
|
+
endTime: req.endTime,
|
|
565
|
+
duration: req.duration,
|
|
566
|
+
requestHeaders: this._compressHeaders(req.requestHeaders),
|
|
567
|
+
responseHeaders: this._compressHeaders(req.responseHeaders),
|
|
568
|
+
compression: req.compression,
|
|
569
|
+
cacheControl: req.cacheControl,
|
|
570
|
+
expires: req.expires,
|
|
571
|
+
failed: req.failed,
|
|
572
|
+
errorText: req.errorText,
|
|
573
|
+
blockedReason: req.blockedReason,
|
|
574
|
+
redirected: req.redirected,
|
|
575
|
+
redirectUrl: req.redirectUrl,
|
|
576
|
+
cdn: req.cdn,
|
|
577
|
+
cdnDetected: req.cdnDetected,
|
|
578
|
+
mimeType: req.mimeType,
|
|
579
|
+
priority: req.priority
|
|
580
|
+
}));
|
|
581
|
+
|
|
582
|
+
// Batch insert (s3db handles chunking)
|
|
583
|
+
for (const request of requestInserts) {
|
|
584
|
+
await this.requestsResource.insert(request);
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
// Save errors separately
|
|
589
|
+
if (session.failures.length > 0) {
|
|
590
|
+
for (const failure of session.failures) {
|
|
591
|
+
await this.errorsResource.insert({
|
|
592
|
+
errorId: `error_${failure.requestId}`,
|
|
593
|
+
sessionId: session.sessionId,
|
|
594
|
+
requestId: failure.requestId,
|
|
595
|
+
url: failure.url,
|
|
596
|
+
domain: failure.domain,
|
|
597
|
+
date: session.date,
|
|
598
|
+
errorType: this._categorizeError(failure.errorText),
|
|
599
|
+
errorText: failure.errorText,
|
|
600
|
+
statusCode: failure.statusCode,
|
|
601
|
+
type: failure.type,
|
|
602
|
+
method: failure.method,
|
|
603
|
+
timing: failure.timing,
|
|
604
|
+
blockedReason: failure.blockedReason,
|
|
605
|
+
consoleMessages: session.consoleMessages
|
|
606
|
+
.filter(msg => Math.abs(msg.timestamp - failure.endTime) < 1000) // Within 1s
|
|
607
|
+
.map(msg => msg.text)
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
const persistDuration = Date.now() - startPersist;
|
|
613
|
+
|
|
614
|
+
this.plugin.emit('networkMonitor.persisted', {
|
|
615
|
+
sessionId: session.sessionId,
|
|
616
|
+
requests: requests.length,
|
|
617
|
+
errors: session.failures.length,
|
|
618
|
+
duration: persistDuration
|
|
619
|
+
});
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
/**
|
|
623
|
+
* Check if request passes filters
|
|
624
|
+
* @private
|
|
625
|
+
*/
|
|
626
|
+
_passesFilters(request, filters) {
|
|
627
|
+
// Type filter
|
|
628
|
+
if (filters.types && !filters.types.includes(request.type)) {
|
|
629
|
+
return false;
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
// Status filter
|
|
633
|
+
if (filters.statuses && !filters.statuses.includes(request.statusCode)) {
|
|
634
|
+
return false;
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
// Size filters
|
|
638
|
+
if (filters.minSize && (request.resourceSize || 0) < filters.minSize) {
|
|
639
|
+
return false;
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
if (filters.maxSize && (request.resourceSize || 0) > filters.maxSize) {
|
|
643
|
+
return false;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// Always save large assets if configured
|
|
647
|
+
if (filters.saveLargeAssets && (request.resourceSize || 0) > 1024 * 1024) {
|
|
648
|
+
return true;
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
return true;
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
/**
|
|
655
|
+
* Extract domain from URL
|
|
656
|
+
* @private
|
|
657
|
+
*/
|
|
658
|
+
_extractDomain(url) {
|
|
659
|
+
try {
|
|
660
|
+
return new URL(url).hostname;
|
|
661
|
+
} catch {
|
|
662
|
+
return 'unknown';
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
/**
|
|
667
|
+
* Extract path from URL
|
|
668
|
+
* @private
|
|
669
|
+
*/
|
|
670
|
+
_extractPath(url) {
|
|
671
|
+
try {
|
|
672
|
+
return new URL(url).pathname;
|
|
673
|
+
} catch {
|
|
674
|
+
return '';
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
/**
|
|
679
|
+
* Detect compression algorithm
|
|
680
|
+
* @private
|
|
681
|
+
*/
|
|
682
|
+
_detectCompression(headers) {
|
|
683
|
+
const encoding = headers['content-encoding'] || headers['Content-Encoding'] || '';
|
|
684
|
+
if (encoding.includes('br')) return 'brotli';
|
|
685
|
+
if (encoding.includes('gzip')) return 'gzip';
|
|
686
|
+
if (encoding.includes('deflate')) return 'deflate';
|
|
687
|
+
return 'none';
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
/**
|
|
691
|
+
* Detect CDN provider
|
|
692
|
+
* @private
|
|
693
|
+
*/
|
|
694
|
+
_detectCDN(headers) {
|
|
695
|
+
const server = headers['server'] || headers['Server'] || '';
|
|
696
|
+
const via = headers['via'] || headers['Via'] || '';
|
|
697
|
+
const cfRay = headers['cf-ray'] || headers['CF-Ray'];
|
|
698
|
+
const xCache = headers['x-cache'] || headers['X-Cache'] || '';
|
|
699
|
+
|
|
700
|
+
if (cfRay || server.includes('cloudflare')) return 'cloudflare';
|
|
701
|
+
if (xCache.includes('cloudfront') || headers['x-amz-cf-id']) return 'cloudfront';
|
|
702
|
+
if (server.includes('fastly') || via.includes('fastly')) return 'fastly';
|
|
703
|
+
if (headers['x-akamai-transformed'] || headers['x-akamai-staging']) return 'akamai';
|
|
704
|
+
if (headers['x-cdn'] || headers['X-CDN']) return headers['x-cdn'] || headers['X-CDN'];
|
|
705
|
+
|
|
706
|
+
return null;
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
/**
|
|
710
|
+
* Parse timing data
|
|
711
|
+
* @private
|
|
712
|
+
*/
|
|
713
|
+
_parseTiming(timing) {
|
|
714
|
+
if (!timing) return null;
|
|
715
|
+
|
|
716
|
+
return {
|
|
717
|
+
dns: timing.dnsEnd - timing.dnsStart,
|
|
718
|
+
tcp: timing.connectEnd - timing.connectStart,
|
|
719
|
+
ssl: timing.sslEnd - timing.sslStart,
|
|
720
|
+
request: timing.sendEnd - timing.sendStart,
|
|
721
|
+
response: timing.receiveHeadersEnd - timing.sendEnd,
|
|
722
|
+
total: timing.receiveHeadersEnd
|
|
723
|
+
};
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
/**
|
|
727
|
+
* Compress headers (remove unnecessary data)
|
|
728
|
+
* @private
|
|
729
|
+
*/
|
|
730
|
+
_compressHeaders(headers) {
|
|
731
|
+
if (!headers) return {};
|
|
732
|
+
|
|
733
|
+
// Remove common unnecessary headers
|
|
734
|
+
const compressed = { ...headers };
|
|
735
|
+
const toRemove = ['cookie', 'Cookie', 'set-cookie', 'Set-Cookie'];
|
|
736
|
+
|
|
737
|
+
toRemove.forEach(key => delete compressed[key]);
|
|
738
|
+
|
|
739
|
+
return compressed;
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
/**
|
|
743
|
+
* Categorize error type
|
|
744
|
+
* @private
|
|
745
|
+
*/
|
|
746
|
+
_categorizeError(errorText) {
|
|
747
|
+
if (!errorText) return 'unknown';
|
|
748
|
+
|
|
749
|
+
if (errorText.includes('ERR_NAME_NOT_RESOLVED')) return 'dns';
|
|
750
|
+
if (errorText.includes('ERR_CONNECTION')) return 'connection';
|
|
751
|
+
if (errorText.includes('ERR_TIMED_OUT')) return 'timeout';
|
|
752
|
+
if (errorText.includes('ERR_SSL')) return 'ssl';
|
|
753
|
+
if (errorText.includes('ERR_CERT')) return 'certificate';
|
|
754
|
+
if (errorText.includes('ERR_BLOCKED')) return 'blocked';
|
|
755
|
+
if (errorText.includes('ERR_FAILED')) return 'failed';
|
|
756
|
+
if (errorText.includes('ERR_ABORTED')) return 'aborted';
|
|
757
|
+
|
|
758
|
+
return 'other';
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
/**
|
|
762
|
+
* Get session statistics
|
|
763
|
+
* @param {string} sessionId - Session ID
|
|
764
|
+
* @returns {Promise<Object>} Statistics
|
|
765
|
+
*/
|
|
766
|
+
async getSessionStats(sessionId) {
|
|
767
|
+
if (!this.sessionsResource) {
|
|
768
|
+
throw new PuppeteerError('Network monitoring persistence not enabled', {
|
|
769
|
+
operation: 'getSessionStats',
|
|
770
|
+
retriable: false,
|
|
771
|
+
suggestion: 'Enable persistence in NetworkMonitor configuration to query stored stats.'
|
|
772
|
+
});
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
const session = await this.sessionsResource.get(sessionId);
|
|
776
|
+
return session;
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
/**
|
|
780
|
+
* Query requests for a session
|
|
781
|
+
* @param {string} sessionId - Session ID
|
|
782
|
+
* @param {Object} filters - Query filters
|
|
783
|
+
* @returns {Promise<Array>} Requests
|
|
784
|
+
*/
|
|
785
|
+
async getSessionRequests(sessionId, filters = {}) {
|
|
786
|
+
if (!this.requestsResource) {
|
|
787
|
+
throw new PuppeteerError('Network monitoring persistence not enabled', {
|
|
788
|
+
operation: 'getSessionRequests',
|
|
789
|
+
retriable: false,
|
|
790
|
+
suggestion: 'Enable persistence in NetworkMonitor configuration to query stored requests.'
|
|
791
|
+
});
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
// Use partition for fast lookup
|
|
795
|
+
return await this.requestsResource.listPartition('bySession', { sessionId }, filters);
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
/**
|
|
799
|
+
* Query errors for a session
|
|
800
|
+
* @param {string} sessionId - Session ID
|
|
801
|
+
* @returns {Promise<Array>} Errors
|
|
802
|
+
*/
|
|
803
|
+
async getSessionErrors(sessionId) {
|
|
804
|
+
if (!this.errorsResource) {
|
|
805
|
+
throw new PuppeteerError('Network monitoring persistence not enabled', {
|
|
806
|
+
operation: 'getSessionErrors',
|
|
807
|
+
retriable: false,
|
|
808
|
+
suggestion: 'Enable persistence in NetworkMonitor configuration to query stored errors.'
|
|
809
|
+
});
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
return await this.errorsResource.listPartition('bySession', { sessionId });
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
export default NetworkMonitor;
|