recker 1.0.2-0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +109 -0
- package/dist/cache/file-storage.d.ts +13 -0
- package/dist/cache/file-storage.d.ts.map +1 -0
- package/dist/cache/file-storage.js +50 -0
- package/dist/cache/memory-storage.d.ts +10 -0
- package/dist/cache/memory-storage.d.ts.map +1 -0
- package/dist/cache/memory-storage.js +29 -0
- package/dist/cache/redis-storage.d.ts +16 -0
- package/dist/cache/redis-storage.d.ts.map +1 -0
- package/dist/cache/redis-storage.js +25 -0
- package/dist/constants.d.ts +19 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +18 -0
- package/dist/contract/index.d.ts +32 -0
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +67 -0
- package/dist/core/client.d.ts +107 -0
- package/dist/core/client.d.ts.map +1 -0
- package/dist/core/client.js +475 -0
- package/dist/core/errors.d.ts +19 -0
- package/dist/core/errors.d.ts.map +1 -0
- package/dist/core/errors.js +34 -0
- package/dist/core/request-promise.d.ts +24 -0
- package/dist/core/request-promise.d.ts.map +1 -0
- package/dist/core/request-promise.js +77 -0
- package/dist/core/request.d.ts +15 -0
- package/dist/core/request.d.ts.map +1 -0
- package/dist/core/request.js +44 -0
- package/dist/core/response.d.ts +33 -0
- package/dist/core/response.d.ts.map +1 -0
- package/dist/core/response.js +154 -0
- package/dist/index.d.ts +40 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +39 -0
- package/dist/mcp/client.d.ts +59 -0
- package/dist/mcp/client.d.ts.map +1 -0
- package/dist/mcp/client.js +195 -0
- package/dist/mcp/index.d.ts +3 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +2 -0
- package/dist/mcp/types.d.ts +151 -0
- package/dist/mcp/types.d.ts.map +1 -0
- package/dist/mcp/types.js +1 -0
- package/dist/plugins/cache.d.ts +10 -0
- package/dist/plugins/cache.d.ts.map +1 -0
- package/dist/plugins/cache.js +72 -0
- package/dist/plugins/circuit-breaker.d.ts +14 -0
- package/dist/plugins/circuit-breaker.d.ts.map +1 -0
- package/dist/plugins/circuit-breaker.js +100 -0
- package/dist/plugins/compression.d.ts +5 -0
- package/dist/plugins/compression.d.ts.map +1 -0
- package/dist/plugins/compression.js +128 -0
- package/dist/plugins/cookie-jar.d.ts +6 -0
- package/dist/plugins/cookie-jar.d.ts.map +1 -0
- package/dist/plugins/cookie-jar.js +72 -0
- package/dist/plugins/dedup.d.ts +6 -0
- package/dist/plugins/dedup.d.ts.map +1 -0
- package/dist/plugins/dedup.js +34 -0
- package/dist/plugins/graphql.d.ts +13 -0
- package/dist/plugins/graphql.d.ts.map +1 -0
- package/dist/plugins/graphql.js +39 -0
- package/dist/plugins/har-player.d.ts +7 -0
- package/dist/plugins/har-player.d.ts.map +1 -0
- package/dist/plugins/har-player.js +53 -0
- package/dist/plugins/har-recorder.d.ts +7 -0
- package/dist/plugins/har-recorder.d.ts.map +1 -0
- package/dist/plugins/har-recorder.js +67 -0
- package/dist/plugins/logger.d.ts +11 -0
- package/dist/plugins/logger.d.ts.map +1 -0
- package/dist/plugins/logger.js +72 -0
- package/dist/plugins/pagination.d.ts +17 -0
- package/dist/plugins/pagination.d.ts.map +1 -0
- package/dist/plugins/pagination.js +105 -0
- package/dist/plugins/proxy-rotator.d.ts +8 -0
- package/dist/plugins/proxy-rotator.d.ts.map +1 -0
- package/dist/plugins/proxy-rotator.js +35 -0
- package/dist/plugins/rate-limit.d.ts +8 -0
- package/dist/plugins/rate-limit.d.ts.map +1 -0
- package/dist/plugins/rate-limit.js +57 -0
- package/dist/plugins/retry.d.ts +14 -0
- package/dist/plugins/retry.d.ts.map +1 -0
- package/dist/plugins/retry.js +92 -0
- package/dist/plugins/server-timing.d.ts +8 -0
- package/dist/plugins/server-timing.d.ts.map +1 -0
- package/dist/plugins/server-timing.js +24 -0
- package/dist/plugins/xsrf.d.ts +10 -0
- package/dist/plugins/xsrf.d.ts.map +1 -0
- package/dist/plugins/xsrf.js +48 -0
- package/dist/runner/request-runner.d.ts +47 -0
- package/dist/runner/request-runner.d.ts.map +1 -0
- package/dist/runner/request-runner.js +89 -0
- package/dist/transport/fetch.d.ts +6 -0
- package/dist/transport/fetch.d.ts.map +1 -0
- package/dist/transport/fetch.js +153 -0
- package/dist/transport/undici.d.ts +23 -0
- package/dist/transport/undici.d.ts.map +1 -0
- package/dist/transport/undici.js +218 -0
- package/dist/types/index.d.ts +251 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +1 -0
- package/dist/utils/agent-manager.d.ts +29 -0
- package/dist/utils/agent-manager.d.ts.map +1 -0
- package/dist/utils/agent-manager.js +133 -0
- package/dist/utils/body.d.ts +11 -0
- package/dist/utils/body.d.ts.map +1 -0
- package/dist/utils/body.js +136 -0
- package/dist/utils/cert.d.ts +12 -0
- package/dist/utils/cert.d.ts.map +1 -0
- package/dist/utils/cert.js +32 -0
- package/dist/utils/concurrency.d.ts +21 -0
- package/dist/utils/concurrency.d.ts.map +1 -0
- package/dist/utils/concurrency.js +116 -0
- package/dist/utils/dns.d.ts +7 -0
- package/dist/utils/dns.d.ts.map +1 -0
- package/dist/utils/dns.js +26 -0
- package/dist/utils/doh.d.ts +3 -0
- package/dist/utils/doh.d.ts.map +1 -0
- package/dist/utils/doh.js +35 -0
- package/dist/utils/header-parser.d.ts +81 -0
- package/dist/utils/header-parser.d.ts.map +1 -0
- package/dist/utils/header-parser.js +457 -0
- package/dist/utils/html-cleaner.d.ts +2 -0
- package/dist/utils/html-cleaner.d.ts.map +1 -0
- package/dist/utils/html-cleaner.js +21 -0
- package/dist/utils/logger.d.ts +33 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +160 -0
- package/dist/utils/progress.d.ts +4 -0
- package/dist/utils/progress.d.ts.map +1 -0
- package/dist/utils/progress.js +49 -0
- package/dist/utils/request-pool.d.ts +23 -0
- package/dist/utils/request-pool.d.ts.map +1 -0
- package/dist/utils/request-pool.js +100 -0
- package/dist/utils/sse.d.ts +8 -0
- package/dist/utils/sse.d.ts.map +1 -0
- package/dist/utils/sse.js +62 -0
- package/dist/utils/streaming.d.ts +18 -0
- package/dist/utils/streaming.d.ts.map +1 -0
- package/dist/utils/streaming.js +83 -0
- package/dist/utils/task-pool.d.ts +38 -0
- package/dist/utils/task-pool.js +104 -0
- package/dist/utils/try-fn.d.ts +4 -0
- package/dist/utils/try-fn.d.ts.map +1 -0
- package/dist/utils/try-fn.js +53 -0
- package/dist/utils/upload.d.ts +10 -0
- package/dist/utils/upload.d.ts.map +1 -0
- package/dist/utils/upload.js +45 -0
- package/dist/utils/user-agent.d.ts +45 -0
- package/dist/utils/user-agent.d.ts.map +1 -0
- package/dist/utils/user-agent.js +100 -0
- package/dist/utils/whois.d.ts +15 -0
- package/dist/utils/whois.d.ts.map +1 -0
- package/dist/utils/whois.js +159 -0
- package/dist/websocket/client.d.ts +38 -0
- package/dist/websocket/client.d.ts.map +1 -0
- package/dist/websocket/client.js +184 -0
- package/package.json +100 -0
|
@@ -0,0 +1,457 @@
|
|
|
1
|
+
export function parseCacheInfo(headers) {
|
|
2
|
+
const info = { hit: false };
|
|
3
|
+
const cfCacheStatus = headers.get('cf-cache-status');
|
|
4
|
+
if (cfCacheStatus) {
|
|
5
|
+
info.provider = 'cloudflare';
|
|
6
|
+
info.status = cfCacheStatus.toUpperCase();
|
|
7
|
+
info.hit = cfCacheStatus.toUpperCase() === 'HIT';
|
|
8
|
+
}
|
|
9
|
+
const xCache = headers.get('x-cache');
|
|
10
|
+
if (xCache) {
|
|
11
|
+
const upper = xCache.toUpperCase();
|
|
12
|
+
info.hit = upper.includes('HIT');
|
|
13
|
+
if (upper.includes('FASTLY'))
|
|
14
|
+
info.provider = 'fastly';
|
|
15
|
+
else if (upper.includes('VARNISH'))
|
|
16
|
+
info.provider = 'varnish';
|
|
17
|
+
if (upper.includes('HIT'))
|
|
18
|
+
info.status = 'HIT';
|
|
19
|
+
else if (upper.includes('MISS'))
|
|
20
|
+
info.status = 'MISS';
|
|
21
|
+
else if (upper.includes('EXPIRED'))
|
|
22
|
+
info.status = 'EXPIRED';
|
|
23
|
+
else if (upper.includes('STALE'))
|
|
24
|
+
info.status = 'STALE';
|
|
25
|
+
}
|
|
26
|
+
const akamaiCache = headers.get('x-akamai-cache-status');
|
|
27
|
+
if (akamaiCache) {
|
|
28
|
+
info.provider = 'akamai';
|
|
29
|
+
info.hit = akamaiCache.toUpperCase().includes('HIT');
|
|
30
|
+
}
|
|
31
|
+
const cloudFrontCache = headers.get('x-cache');
|
|
32
|
+
const cloudFrontId = headers.get('x-amz-cf-id');
|
|
33
|
+
if (cloudFrontId) {
|
|
34
|
+
info.provider = 'cloudfront';
|
|
35
|
+
}
|
|
36
|
+
const cacheControl = headers.get('cache-control');
|
|
37
|
+
if (cacheControl) {
|
|
38
|
+
const maxAgeMatch = cacheControl.match(/max-age=(\d+)/);
|
|
39
|
+
if (maxAgeMatch) {
|
|
40
|
+
info.maxAge = parseInt(maxAgeMatch[1], 10);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const age = headers.get('age');
|
|
44
|
+
if (age) {
|
|
45
|
+
info.age = parseInt(age, 10);
|
|
46
|
+
}
|
|
47
|
+
return info;
|
|
48
|
+
}
|
|
49
|
+
const PLATFORM_DETECTORS = [
|
|
50
|
+
{
|
|
51
|
+
name: 'cloudflare',
|
|
52
|
+
category: 'cdn',
|
|
53
|
+
headers: ['cf-ray', 'cf-cache-status'],
|
|
54
|
+
detect: (headers) => headers.get('cf-ray') !== null,
|
|
55
|
+
extract: (headers) => {
|
|
56
|
+
const ray = headers.get('cf-ray');
|
|
57
|
+
const metadata = {};
|
|
58
|
+
if (ray) {
|
|
59
|
+
metadata.ray = ray;
|
|
60
|
+
const parts = ray.split('-');
|
|
61
|
+
if (parts.length > 1) {
|
|
62
|
+
return { region: parts[1], metadata };
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return { metadata };
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
name: 'fastly',
|
|
70
|
+
category: 'cdn',
|
|
71
|
+
headers: ['fastly-debug-digest', 'x-served-by'],
|
|
72
|
+
detect: (headers) => {
|
|
73
|
+
const servedBy = headers.get('x-served-by');
|
|
74
|
+
return servedBy?.includes('cache-') || headers.get('fastly-debug-digest') !== null;
|
|
75
|
+
},
|
|
76
|
+
extract: (headers) => ({
|
|
77
|
+
server: headers.get('x-served-by') || undefined,
|
|
78
|
+
metadata: {
|
|
79
|
+
digest: headers.get('fastly-debug-digest')
|
|
80
|
+
}
|
|
81
|
+
})
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
name: 'akamai',
|
|
85
|
+
category: 'cdn',
|
|
86
|
+
headers: ['x-akamai-request-id', 'x-akamai-cache-status'],
|
|
87
|
+
detect: (headers) => headers.get('x-akamai-request-id') !== null,
|
|
88
|
+
extract: (headers) => ({
|
|
89
|
+
metadata: {
|
|
90
|
+
requestId: headers.get('x-akamai-request-id'),
|
|
91
|
+
cacheId: headers.get('x-akamai-cache-status')
|
|
92
|
+
}
|
|
93
|
+
})
|
|
94
|
+
},
|
|
95
|
+
{
|
|
96
|
+
name: 'aws-cloudfront',
|
|
97
|
+
category: 'cloud',
|
|
98
|
+
headers: ['x-amz-cf-id', 'x-amz-cf-pop'],
|
|
99
|
+
detect: (headers) => headers.get('x-amz-cf-id') !== null,
|
|
100
|
+
extract: (headers) => ({
|
|
101
|
+
region: headers.get('x-amz-cf-pop') || undefined,
|
|
102
|
+
metadata: {
|
|
103
|
+
requestId: headers.get('x-amz-cf-id'),
|
|
104
|
+
pop: headers.get('x-amz-cf-pop')
|
|
105
|
+
}
|
|
106
|
+
})
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
name: 'gcp',
|
|
110
|
+
category: 'cloud',
|
|
111
|
+
headers: ['x-cloud-trace-context', 'x-goog-request-id'],
|
|
112
|
+
detect: (headers) => headers.get('x-cloud-trace-context') !== null || headers.get('x-goog-request-id') !== null,
|
|
113
|
+
extract: (headers) => {
|
|
114
|
+
const trace = headers.get('x-cloud-trace-context');
|
|
115
|
+
return {
|
|
116
|
+
metadata: {
|
|
117
|
+
traceId: trace ? trace.split('/')[0] : undefined,
|
|
118
|
+
requestId: headers.get('x-goog-request-id')
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
name: 'azure',
|
|
125
|
+
category: 'cloud',
|
|
126
|
+
headers: ['x-ms-request-id', 'x-azure-ref'],
|
|
127
|
+
detect: (headers) => headers.get('x-ms-request-id') !== null,
|
|
128
|
+
extract: (headers) => ({
|
|
129
|
+
metadata: {
|
|
130
|
+
requestId: headers.get('x-ms-request-id'),
|
|
131
|
+
ref: headers.get('x-azure-ref')
|
|
132
|
+
}
|
|
133
|
+
})
|
|
134
|
+
},
|
|
135
|
+
{
|
|
136
|
+
name: 'oracle-cloud',
|
|
137
|
+
category: 'cloud',
|
|
138
|
+
headers: ['x-oracle-dms-rid', 'x-oracle-dms-ecid'],
|
|
139
|
+
detect: (headers) => headers.get('x-oracle-dms-rid') !== null,
|
|
140
|
+
extract: (headers) => ({
|
|
141
|
+
metadata: {
|
|
142
|
+
requestId: headers.get('x-oracle-dms-rid'),
|
|
143
|
+
ecid: headers.get('x-oracle-dms-ecid')
|
|
144
|
+
}
|
|
145
|
+
})
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
name: 'alibaba-cloud',
|
|
149
|
+
category: 'cloud',
|
|
150
|
+
headers: ['ali-swift-global-savetime', 'eagleeye-traceid'],
|
|
151
|
+
detect: (headers) => headers.get('eagleeye-traceid') !== null,
|
|
152
|
+
extract: (headers) => ({
|
|
153
|
+
metadata: {
|
|
154
|
+
traceId: headers.get('eagleeye-traceid')
|
|
155
|
+
}
|
|
156
|
+
})
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
name: 'vercel',
|
|
160
|
+
category: 'hosting',
|
|
161
|
+
headers: ['x-vercel-id', 'x-vercel-cache'],
|
|
162
|
+
detect: (headers) => headers.get('x-vercel-id') !== null,
|
|
163
|
+
extract: (headers) => ({
|
|
164
|
+
metadata: {
|
|
165
|
+
requestId: headers.get('x-vercel-id'),
|
|
166
|
+
cache: headers.get('x-vercel-cache')
|
|
167
|
+
}
|
|
168
|
+
})
|
|
169
|
+
},
|
|
170
|
+
{
|
|
171
|
+
name: 'netlify',
|
|
172
|
+
category: 'hosting',
|
|
173
|
+
headers: ['x-nf-request-id', 'x-nf-trace-id'],
|
|
174
|
+
detect: (headers) => headers.get('x-nf-request-id') !== null,
|
|
175
|
+
extract: (headers) => ({
|
|
176
|
+
metadata: {
|
|
177
|
+
requestId: headers.get('x-nf-request-id'),
|
|
178
|
+
traceId: headers.get('x-nf-trace-id')
|
|
179
|
+
}
|
|
180
|
+
})
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
name: 'render',
|
|
184
|
+
category: 'hosting',
|
|
185
|
+
headers: ['x-render-origin-server'],
|
|
186
|
+
detect: (headers) => headers.get('x-render-origin-server') !== null,
|
|
187
|
+
extract: (headers) => ({
|
|
188
|
+
metadata: {
|
|
189
|
+
origin: headers.get('x-render-origin-server')
|
|
190
|
+
}
|
|
191
|
+
})
|
|
192
|
+
}
|
|
193
|
+
];
|
|
194
|
+
export function parsePlatformInfo(headers) {
|
|
195
|
+
const info = {
|
|
196
|
+
metadata: {}
|
|
197
|
+
};
|
|
198
|
+
for (const detector of PLATFORM_DETECTORS) {
|
|
199
|
+
if (detector.detect(headers)) {
|
|
200
|
+
info.provider = detector.name;
|
|
201
|
+
info.category = detector.category;
|
|
202
|
+
const extracted = detector.extract(headers);
|
|
203
|
+
if (extracted.region)
|
|
204
|
+
info.region = extracted.region;
|
|
205
|
+
if (extracted.server)
|
|
206
|
+
info.server = extracted.server;
|
|
207
|
+
if (extracted.metadata) {
|
|
208
|
+
Object.assign(info.metadata, extracted.metadata);
|
|
209
|
+
}
|
|
210
|
+
break;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
if (!info.provider) {
|
|
214
|
+
const server = headers.get('server');
|
|
215
|
+
if (server) {
|
|
216
|
+
info.server = server;
|
|
217
|
+
const serverLower = server.toLowerCase();
|
|
218
|
+
if (serverLower.includes('cloudflare')) {
|
|
219
|
+
info.provider = 'cloudflare';
|
|
220
|
+
info.category = 'cdn';
|
|
221
|
+
}
|
|
222
|
+
else if (serverLower.includes('cloudfront')) {
|
|
223
|
+
info.provider = 'aws-cloudfront';
|
|
224
|
+
info.category = 'cloud';
|
|
225
|
+
}
|
|
226
|
+
else if (serverLower.includes('fastly')) {
|
|
227
|
+
info.provider = 'fastly';
|
|
228
|
+
info.category = 'cdn';
|
|
229
|
+
}
|
|
230
|
+
else if (serverLower.includes('akamai')) {
|
|
231
|
+
info.provider = 'akamai';
|
|
232
|
+
info.category = 'cdn';
|
|
233
|
+
}
|
|
234
|
+
else if (serverLower.includes('nginx')) {
|
|
235
|
+
info.provider = 'nginx';
|
|
236
|
+
info.category = 'server';
|
|
237
|
+
}
|
|
238
|
+
else if (serverLower.includes('apache')) {
|
|
239
|
+
info.provider = 'apache';
|
|
240
|
+
info.category = 'server';
|
|
241
|
+
}
|
|
242
|
+
else if (serverLower.includes('varnish')) {
|
|
243
|
+
info.provider = 'varnish';
|
|
244
|
+
info.category = 'proxy';
|
|
245
|
+
}
|
|
246
|
+
else {
|
|
247
|
+
info.provider = 'unknown';
|
|
248
|
+
info.category = 'unknown';
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
if (!info.provider) {
|
|
253
|
+
const via = headers.get('via');
|
|
254
|
+
if (via) {
|
|
255
|
+
const viaLower = via.toLowerCase();
|
|
256
|
+
if (viaLower.includes('cloudflare')) {
|
|
257
|
+
info.provider = 'cloudflare';
|
|
258
|
+
info.category = 'cdn';
|
|
259
|
+
}
|
|
260
|
+
else if (viaLower.includes('akamai')) {
|
|
261
|
+
info.provider = 'akamai';
|
|
262
|
+
info.category = 'cdn';
|
|
263
|
+
}
|
|
264
|
+
else if (viaLower.includes('varnish')) {
|
|
265
|
+
info.provider = 'varnish';
|
|
266
|
+
info.category = 'proxy';
|
|
267
|
+
}
|
|
268
|
+
else if (viaLower.includes('squid')) {
|
|
269
|
+
info.provider = 'squid';
|
|
270
|
+
info.category = 'proxy';
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
if (Object.keys(info.metadata || {}).length === 0) {
|
|
275
|
+
delete info.metadata;
|
|
276
|
+
}
|
|
277
|
+
return info;
|
|
278
|
+
}
|
|
279
|
+
export function parseRateLimitInfo(headers, status) {
|
|
280
|
+
const info = { limited: status === 429 };
|
|
281
|
+
const limit = headers.get('x-ratelimit-limit') || headers.get('ratelimit-limit');
|
|
282
|
+
const remaining = headers.get('x-ratelimit-remaining') || headers.get('ratelimit-remaining');
|
|
283
|
+
const reset = headers.get('x-ratelimit-reset') || headers.get('ratelimit-reset');
|
|
284
|
+
if (limit)
|
|
285
|
+
info.limit = parseInt(limit, 10);
|
|
286
|
+
if (remaining)
|
|
287
|
+
info.remaining = parseInt(remaining, 10);
|
|
288
|
+
if (reset) {
|
|
289
|
+
const resetValue = parseInt(reset, 10);
|
|
290
|
+
if (resetValue > 1000000000) {
|
|
291
|
+
info.reset = new Date(resetValue * 1000);
|
|
292
|
+
}
|
|
293
|
+
else {
|
|
294
|
+
info.reset = new Date(Date.now() + resetValue * 1000);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
const retryAfter = headers.get('retry-after');
|
|
298
|
+
if (retryAfter) {
|
|
299
|
+
const seconds = parseInt(retryAfter, 10);
|
|
300
|
+
if (!isNaN(seconds)) {
|
|
301
|
+
info.retryAfter = seconds;
|
|
302
|
+
}
|
|
303
|
+
else {
|
|
304
|
+
const retryDate = new Date(retryAfter);
|
|
305
|
+
if (!isNaN(retryDate.getTime())) {
|
|
306
|
+
info.retryAfter = Math.ceil((retryDate.getTime() - Date.now()) / 1000);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
const policy = headers.get('ratelimit-policy');
|
|
311
|
+
if (policy) {
|
|
312
|
+
info.policy = policy;
|
|
313
|
+
}
|
|
314
|
+
if (info.remaining !== undefined && info.remaining === 0) {
|
|
315
|
+
info.limited = true;
|
|
316
|
+
}
|
|
317
|
+
return info;
|
|
318
|
+
}
|
|
319
|
+
export function parseCompressionInfo(headers) {
|
|
320
|
+
const info = {};
|
|
321
|
+
const encoding = headers.get('content-encoding');
|
|
322
|
+
if (encoding) {
|
|
323
|
+
info.encoding = encoding.toLowerCase();
|
|
324
|
+
}
|
|
325
|
+
const xOriginalSize = headers.get('x-original-size') || headers.get('x-uncompressed-size');
|
|
326
|
+
if (xOriginalSize) {
|
|
327
|
+
info.originalSize = parseInt(xOriginalSize, 10);
|
|
328
|
+
}
|
|
329
|
+
const contentLength = headers.get('content-length');
|
|
330
|
+
if (contentLength) {
|
|
331
|
+
info.compressedSize = parseInt(contentLength, 10);
|
|
332
|
+
}
|
|
333
|
+
if (info.originalSize && info.compressedSize) {
|
|
334
|
+
info.ratio = info.originalSize / info.compressedSize;
|
|
335
|
+
}
|
|
336
|
+
return info;
|
|
337
|
+
}
|
|
338
|
+
export function parseCSPInfo(headers) {
|
|
339
|
+
const info = {
|
|
340
|
+
directives: {},
|
|
341
|
+
reportOnly: false
|
|
342
|
+
};
|
|
343
|
+
let cspHeader = headers.get('content-security-policy');
|
|
344
|
+
if (!cspHeader) {
|
|
345
|
+
cspHeader = headers.get('content-security-policy-report-only');
|
|
346
|
+
if (cspHeader) {
|
|
347
|
+
info.reportOnly = true;
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
if (cspHeader) {
|
|
351
|
+
info.policy = cspHeader;
|
|
352
|
+
const directives = cspHeader.split(';').map(d => d.trim()).filter(Boolean);
|
|
353
|
+
for (const directive of directives) {
|
|
354
|
+
const [name, ...values] = directive.split(/\s+/);
|
|
355
|
+
if (name) {
|
|
356
|
+
info.directives[name] = values;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
return info;
|
|
361
|
+
}
|
|
362
|
+
export function parseContentType(headers) {
|
|
363
|
+
const info = {};
|
|
364
|
+
const contentType = headers.get('content-type');
|
|
365
|
+
if (!contentType)
|
|
366
|
+
return info;
|
|
367
|
+
const parts = contentType.split(';').map(p => p.trim());
|
|
368
|
+
if (parts[0]) {
|
|
369
|
+
info.mediaType = parts[0];
|
|
370
|
+
const [type, subtype] = parts[0].split('/');
|
|
371
|
+
if (type)
|
|
372
|
+
info.type = type;
|
|
373
|
+
if (subtype)
|
|
374
|
+
info.subtype = subtype;
|
|
375
|
+
}
|
|
376
|
+
for (let i = 1; i < parts.length; i++) {
|
|
377
|
+
const param = parts[i];
|
|
378
|
+
const [key, value] = param.split('=').map(s => s.trim());
|
|
379
|
+
if (key === 'charset' && value) {
|
|
380
|
+
info.charset = value.replace(/['"]/g, '');
|
|
381
|
+
}
|
|
382
|
+
else if (key === 'boundary' && value) {
|
|
383
|
+
info.boundary = value.replace(/['"]/g, '');
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
return info;
|
|
387
|
+
}
|
|
388
|
+
export function parseAcceptInfo(headers) {
|
|
389
|
+
const info = {
|
|
390
|
+
types: [],
|
|
391
|
+
encodings: [],
|
|
392
|
+
languages: []
|
|
393
|
+
};
|
|
394
|
+
const accept = headers.get('accept');
|
|
395
|
+
if (accept) {
|
|
396
|
+
const types = accept.split(',').map(t => t.trim());
|
|
397
|
+
for (const typeStr of types) {
|
|
398
|
+
const [mediaType, ...params] = typeStr.split(';').map(s => s.trim());
|
|
399
|
+
let q = 1.0;
|
|
400
|
+
for (const param of params) {
|
|
401
|
+
if (param.startsWith('q=')) {
|
|
402
|
+
q = parseFloat(param.substring(2)) || 1.0;
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
const [type, subtype] = mediaType.split('/');
|
|
406
|
+
info.types.push({
|
|
407
|
+
mediaType,
|
|
408
|
+
q,
|
|
409
|
+
type: type || undefined,
|
|
410
|
+
subtype: subtype || undefined
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
info.types.sort((a, b) => b.q - a.q);
|
|
414
|
+
}
|
|
415
|
+
const acceptEncoding = headers.get('accept-encoding');
|
|
416
|
+
if (acceptEncoding) {
|
|
417
|
+
const encodings = acceptEncoding.split(',').map(e => e.trim());
|
|
418
|
+
for (const encodingStr of encodings) {
|
|
419
|
+
const [encoding, ...params] = encodingStr.split(';').map(s => s.trim());
|
|
420
|
+
let q = 1.0;
|
|
421
|
+
for (const param of params) {
|
|
422
|
+
if (param.startsWith('q=')) {
|
|
423
|
+
q = parseFloat(param.substring(2)) || 1.0;
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
info.encodings.push({ encoding, q });
|
|
427
|
+
}
|
|
428
|
+
info.encodings.sort((a, b) => b.q - a.q);
|
|
429
|
+
}
|
|
430
|
+
const acceptLanguage = headers.get('accept-language');
|
|
431
|
+
if (acceptLanguage) {
|
|
432
|
+
const languages = acceptLanguage.split(',').map(l => l.trim());
|
|
433
|
+
for (const languageStr of languages) {
|
|
434
|
+
const [language, ...params] = languageStr.split(';').map(s => s.trim());
|
|
435
|
+
let q = 1.0;
|
|
436
|
+
for (const param of params) {
|
|
437
|
+
if (param.startsWith('q=')) {
|
|
438
|
+
q = parseFloat(param.substring(2)) || 1.0;
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
info.languages.push({ language, q });
|
|
442
|
+
}
|
|
443
|
+
info.languages.sort((a, b) => b.q - a.q);
|
|
444
|
+
}
|
|
445
|
+
return info;
|
|
446
|
+
}
|
|
447
|
+
export function parseHeaders(headers, status) {
|
|
448
|
+
return {
|
|
449
|
+
cache: parseCacheInfo(headers),
|
|
450
|
+
platform: parsePlatformInfo(headers),
|
|
451
|
+
rateLimit: parseRateLimitInfo(headers, status),
|
|
452
|
+
compression: parseCompressionInfo(headers),
|
|
453
|
+
csp: parseCSPInfo(headers),
|
|
454
|
+
contentType: parseContentType(headers),
|
|
455
|
+
accept: parseAcceptInfo(headers)
|
|
456
|
+
};
|
|
457
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"html-cleaner.d.ts","sourceRoot":"","sources":["../../src/utils/html-cleaner.ts"],"names":[],"mappings":"AAIA,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAmC9C"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export function cleanHtml(html) {
|
|
2
|
+
if (!html)
|
|
3
|
+
return '';
|
|
4
|
+
let text = html;
|
|
5
|
+
text = text.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '');
|
|
6
|
+
text = text.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '');
|
|
7
|
+
text = text.replace(/<!--[\s\S]*?-->/g, '');
|
|
8
|
+
text = text.replace(/<\/(div|p|h[1-6]|li|ul|ol|tr|table|section|article|main|header|footer|nav)>/gi, '\n');
|
|
9
|
+
text = text.replace(/<(br|hr)\s*\/?>/gi, '\n');
|
|
10
|
+
text = text.replace(/<[^>]+>/g, ' ');
|
|
11
|
+
text = text
|
|
12
|
+
.replace(/ /g, ' ')
|
|
13
|
+
.replace(/&/g, '&')
|
|
14
|
+
.replace(/</g, '<')
|
|
15
|
+
.replace(/>/g, '>')
|
|
16
|
+
.replace(/"/g, '"')
|
|
17
|
+
.replace(/'/g, "'");
|
|
18
|
+
text = text.replace(/[ \t]+/g, ' ');
|
|
19
|
+
text = text.replace(/\n\s*\n\s*\n+/g, '\n\n');
|
|
20
|
+
return text.trim();
|
|
21
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { ReckerRequest, ReckerResponse, Timings } from '../types/index.js';
|
|
2
|
+
export type LogLevel = 'debug' | 'info' | 'warn' | 'error' | 'none';
|
|
3
|
+
export interface LoggerOptions {
|
|
4
|
+
level?: LogLevel;
|
|
5
|
+
prefix?: string;
|
|
6
|
+
timestamp?: boolean;
|
|
7
|
+
colors?: boolean;
|
|
8
|
+
}
|
|
9
|
+
export declare class Logger {
|
|
10
|
+
private level;
|
|
11
|
+
private prefix;
|
|
12
|
+
private useTimestamp;
|
|
13
|
+
private useColors;
|
|
14
|
+
constructor(options?: LoggerOptions);
|
|
15
|
+
private detectLogLevel;
|
|
16
|
+
private supportsColors;
|
|
17
|
+
private colorize;
|
|
18
|
+
private formatTimestamp;
|
|
19
|
+
private shouldLog;
|
|
20
|
+
private log;
|
|
21
|
+
debug(message: string): void;
|
|
22
|
+
info(message: string): void;
|
|
23
|
+
warn(message: string): void;
|
|
24
|
+
error(message: string): void;
|
|
25
|
+
logRequest(req: ReckerRequest): void;
|
|
26
|
+
logResponse(req: ReckerRequest, res: ReckerResponse, startTime: number): void;
|
|
27
|
+
logTimings(timings: Timings): void;
|
|
28
|
+
logError(req: ReckerRequest, error: Error): void;
|
|
29
|
+
private formatBytes;
|
|
30
|
+
}
|
|
31
|
+
export declare function getLogger(): Logger;
|
|
32
|
+
export declare function setLogger(logger: Logger): void;
|
|
33
|
+
//# sourceMappingURL=logger.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE3E,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;AAEpE,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,QAAQ,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AA2BD,qBAAa,MAAM;IACjB,OAAO,CAAC,KAAK,CAAW;IACxB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAU;IAC9B,OAAO,CAAC,SAAS,CAAU;gBAEf,OAAO,GAAE,aAAkB;IAOvC,OAAO,CAAC,cAAc;IAQtB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,SAAS;IAIjB,OAAO,CAAC,GAAG;IAQX,KAAK,CAAC,OAAO,EAAE,MAAM;IAIrB,IAAI,CAAC,OAAO,EAAE,MAAM;IAIpB,IAAI,CAAC,OAAO,EAAE,MAAM;IAIpB,KAAK,CAAC,OAAO,EAAE,MAAM;IAOrB,UAAU,CAAC,GAAG,EAAE,aAAa;IAmB7B,WAAW,CAAC,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM;IA4BtE,UAAU,CAAC,OAAO,EAAE,OAAO;IAoB3B,QAAQ,CAAC,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK;IAoBzC,OAAO,CAAC,WAAW;CAOpB;AAKD,wBAAgB,SAAS,IAAI,MAAM,CAKlC;AAED,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,QAEvC"}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
const colors = {
|
|
2
|
+
reset: '\x1b[0m',
|
|
3
|
+
bright: '\x1b[1m',
|
|
4
|
+
dim: '\x1b[2m',
|
|
5
|
+
red: '\x1b[31m',
|
|
6
|
+
green: '\x1b[32m',
|
|
7
|
+
yellow: '\x1b[33m',
|
|
8
|
+
blue: '\x1b[34m',
|
|
9
|
+
magenta: '\x1b[35m',
|
|
10
|
+
cyan: '\x1b[36m',
|
|
11
|
+
white: '\x1b[37m',
|
|
12
|
+
gray: '\x1b[90m',
|
|
13
|
+
};
|
|
14
|
+
const levels = {
|
|
15
|
+
debug: 0,
|
|
16
|
+
info: 1,
|
|
17
|
+
warn: 2,
|
|
18
|
+
error: 3,
|
|
19
|
+
none: 999,
|
|
20
|
+
};
|
|
21
|
+
export class Logger {
|
|
22
|
+
level;
|
|
23
|
+
prefix;
|
|
24
|
+
useTimestamp;
|
|
25
|
+
useColors;
|
|
26
|
+
constructor(options = {}) {
|
|
27
|
+
this.level = options.level || this.detectLogLevel();
|
|
28
|
+
this.prefix = options.prefix || 'recker';
|
|
29
|
+
this.useTimestamp = options.timestamp !== false;
|
|
30
|
+
this.useColors = options.colors !== false && this.supportsColors();
|
|
31
|
+
}
|
|
32
|
+
detectLogLevel() {
|
|
33
|
+
const env = process.env.DEBUG || '';
|
|
34
|
+
if (env === '*' || env.includes('recker') || env.includes('*')) {
|
|
35
|
+
return 'debug';
|
|
36
|
+
}
|
|
37
|
+
return 'none';
|
|
38
|
+
}
|
|
39
|
+
supportsColors() {
|
|
40
|
+
return (process.stdout.isTTY &&
|
|
41
|
+
!process.env.NO_COLOR &&
|
|
42
|
+
process.env.TERM !== 'dumb');
|
|
43
|
+
}
|
|
44
|
+
colorize(text, color) {
|
|
45
|
+
if (!this.useColors)
|
|
46
|
+
return text;
|
|
47
|
+
return `${colors[color]}${text}${colors.reset}`;
|
|
48
|
+
}
|
|
49
|
+
formatTimestamp() {
|
|
50
|
+
if (!this.useTimestamp)
|
|
51
|
+
return '';
|
|
52
|
+
const now = new Date();
|
|
53
|
+
const time = now.toTimeString().split(' ')[0];
|
|
54
|
+
return this.colorize(`[${time}]`, 'gray') + ' ';
|
|
55
|
+
}
|
|
56
|
+
shouldLog(level) {
|
|
57
|
+
return levels[level] >= levels[this.level];
|
|
58
|
+
}
|
|
59
|
+
log(level, message) {
|
|
60
|
+
if (!this.shouldLog(level))
|
|
61
|
+
return;
|
|
62
|
+
const timestamp = this.formatTimestamp();
|
|
63
|
+
const prefix = this.colorize(`[${this.prefix}]`, 'cyan');
|
|
64
|
+
console.log(`${timestamp}${prefix} ${message}`);
|
|
65
|
+
}
|
|
66
|
+
debug(message) {
|
|
67
|
+
this.log('debug', message);
|
|
68
|
+
}
|
|
69
|
+
info(message) {
|
|
70
|
+
this.log('info', message);
|
|
71
|
+
}
|
|
72
|
+
warn(message) {
|
|
73
|
+
this.log('warn', this.colorize(message, 'yellow'));
|
|
74
|
+
}
|
|
75
|
+
error(message) {
|
|
76
|
+
this.log('error', this.colorize(message, 'red'));
|
|
77
|
+
}
|
|
78
|
+
logRequest(req) {
|
|
79
|
+
if (!this.shouldLog('debug'))
|
|
80
|
+
return;
|
|
81
|
+
const method = this.colorize(req.method, 'blue');
|
|
82
|
+
const url = this.colorize(req.url, 'bright');
|
|
83
|
+
this.debug(`${this.colorize('→', 'green')} ${method} ${url}`);
|
|
84
|
+
if (req.headers && Array.from(req.headers.keys()).length > 0) {
|
|
85
|
+
const headerStr = Array.from(req.headers.entries())
|
|
86
|
+
.map(([k, v]) => ` ${this.colorize(k, 'gray')}: ${v}`)
|
|
87
|
+
.join('\n');
|
|
88
|
+
console.log(headerStr);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
logResponse(req, res, startTime) {
|
|
92
|
+
if (!this.shouldLog('debug'))
|
|
93
|
+
return;
|
|
94
|
+
const duration = Date.now() - startTime;
|
|
95
|
+
const statusColor = res.ok ? 'green' : 'red';
|
|
96
|
+
const status = this.colorize(String(res.status), statusColor);
|
|
97
|
+
const method = this.colorize(req.method, 'gray');
|
|
98
|
+
this.debug(`${this.colorize('←', 'green')} ${status} ${method} ${req.url} ${this.colorize(`(${duration}ms)`, 'gray')}`);
|
|
99
|
+
if (res.timings) {
|
|
100
|
+
this.logTimings(res.timings);
|
|
101
|
+
}
|
|
102
|
+
const contentLength = res.headers.get('content-length');
|
|
103
|
+
if (contentLength) {
|
|
104
|
+
const size = this.formatBytes(parseInt(contentLength, 10));
|
|
105
|
+
console.log(` ${this.colorize('Size:', 'gray')} ${size}`);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
logTimings(timings) {
|
|
109
|
+
if (!this.shouldLog('debug'))
|
|
110
|
+
return;
|
|
111
|
+
const parts = [];
|
|
112
|
+
if (timings.dns)
|
|
113
|
+
parts.push(`DNS: ${timings.dns.toFixed(0)}ms`);
|
|
114
|
+
if (timings.tcp)
|
|
115
|
+
parts.push(`TCP: ${timings.tcp.toFixed(0)}ms`);
|
|
116
|
+
if (timings.tls)
|
|
117
|
+
parts.push(`TLS: ${timings.tls.toFixed(0)}ms`);
|
|
118
|
+
if (timings.firstByte)
|
|
119
|
+
parts.push(`TTFB: ${timings.firstByte.toFixed(0)}ms`);
|
|
120
|
+
if (timings.total)
|
|
121
|
+
parts.push(`Total: ${timings.total.toFixed(0)}ms`);
|
|
122
|
+
if (parts.length > 0) {
|
|
123
|
+
const timelinesStr = parts.join(', ');
|
|
124
|
+
console.log(` ${this.colorize('├─', 'gray')} ${timelinesStr}`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
logError(req, error) {
|
|
128
|
+
if (!this.shouldLog('error'))
|
|
129
|
+
return;
|
|
130
|
+
const method = this.colorize(req.method, 'gray');
|
|
131
|
+
this.error(`${this.colorize('✖', 'red')} ${method} ${req.url}`);
|
|
132
|
+
console.log(` ${this.colorize('Error:', 'red')} ${error.message}`);
|
|
133
|
+
if (error.stack && this.shouldLog('debug')) {
|
|
134
|
+
const stack = error.stack
|
|
135
|
+
.split('\n')
|
|
136
|
+
.slice(1, 4)
|
|
137
|
+
.map((line) => ` ${this.colorize('│', 'gray')} ${line.trim()}`)
|
|
138
|
+
.join('\n');
|
|
139
|
+
console.log(stack);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
formatBytes(bytes) {
|
|
143
|
+
if (bytes === 0)
|
|
144
|
+
return '0 B';
|
|
145
|
+
const k = 1024;
|
|
146
|
+
const sizes = ['B', 'KB', 'MB', 'GB'];
|
|
147
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
148
|
+
return `${(bytes / Math.pow(k, i)).toFixed(1)} ${sizes[i]}`;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
let globalLogger = null;
|
|
152
|
+
export function getLogger() {
|
|
153
|
+
if (!globalLogger) {
|
|
154
|
+
globalLogger = new Logger();
|
|
155
|
+
}
|
|
156
|
+
return globalLogger;
|
|
157
|
+
}
|
|
158
|
+
export function setLogger(logger) {
|
|
159
|
+
globalLogger = logger;
|
|
160
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { ProgressEvent, ProgressCallback } from '../types/index.js';
|
|
2
|
+
export declare function createProgressStream(stream: ReadableStream<Uint8Array>, onProgress: ProgressCallback, total?: number): ReadableStream<Uint8Array>;
|
|
3
|
+
export declare function calculateProgress(loaded: number, total?: number): ProgressEvent;
|
|
4
|
+
//# sourceMappingURL=progress.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"progress.d.ts","sourceRoot":"","sources":["../../src/utils/progress.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAKpE,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,EAClC,UAAU,EAAE,gBAAgB,EAC5B,KAAK,CAAC,EAAE,MAAM,GACb,cAAc,CAAC,UAAU,CAAC,CA6C5B;AAKD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,CAW/E"}
|