recker 1.0.2-0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +109 -0
  3. package/dist/cache/file-storage.d.ts +13 -0
  4. package/dist/cache/file-storage.d.ts.map +1 -0
  5. package/dist/cache/file-storage.js +50 -0
  6. package/dist/cache/memory-storage.d.ts +10 -0
  7. package/dist/cache/memory-storage.d.ts.map +1 -0
  8. package/dist/cache/memory-storage.js +29 -0
  9. package/dist/cache/redis-storage.d.ts +16 -0
  10. package/dist/cache/redis-storage.d.ts.map +1 -0
  11. package/dist/cache/redis-storage.js +25 -0
  12. package/dist/constants.d.ts +19 -0
  13. package/dist/constants.d.ts.map +1 -0
  14. package/dist/constants.js +18 -0
  15. package/dist/contract/index.d.ts +32 -0
  16. package/dist/contract/index.d.ts.map +1 -0
  17. package/dist/contract/index.js +67 -0
  18. package/dist/core/client.d.ts +107 -0
  19. package/dist/core/client.d.ts.map +1 -0
  20. package/dist/core/client.js +475 -0
  21. package/dist/core/errors.d.ts +19 -0
  22. package/dist/core/errors.d.ts.map +1 -0
  23. package/dist/core/errors.js +34 -0
  24. package/dist/core/request-promise.d.ts +24 -0
  25. package/dist/core/request-promise.d.ts.map +1 -0
  26. package/dist/core/request-promise.js +77 -0
  27. package/dist/core/request.d.ts +15 -0
  28. package/dist/core/request.d.ts.map +1 -0
  29. package/dist/core/request.js +44 -0
  30. package/dist/core/response.d.ts +33 -0
  31. package/dist/core/response.d.ts.map +1 -0
  32. package/dist/core/response.js +154 -0
  33. package/dist/index.d.ts +40 -0
  34. package/dist/index.d.ts.map +1 -0
  35. package/dist/index.js +39 -0
  36. package/dist/mcp/client.d.ts +59 -0
  37. package/dist/mcp/client.d.ts.map +1 -0
  38. package/dist/mcp/client.js +195 -0
  39. package/dist/mcp/index.d.ts +3 -0
  40. package/dist/mcp/index.d.ts.map +1 -0
  41. package/dist/mcp/index.js +2 -0
  42. package/dist/mcp/types.d.ts +151 -0
  43. package/dist/mcp/types.d.ts.map +1 -0
  44. package/dist/mcp/types.js +1 -0
  45. package/dist/plugins/cache.d.ts +10 -0
  46. package/dist/plugins/cache.d.ts.map +1 -0
  47. package/dist/plugins/cache.js +72 -0
  48. package/dist/plugins/circuit-breaker.d.ts +14 -0
  49. package/dist/plugins/circuit-breaker.d.ts.map +1 -0
  50. package/dist/plugins/circuit-breaker.js +100 -0
  51. package/dist/plugins/compression.d.ts +5 -0
  52. package/dist/plugins/compression.d.ts.map +1 -0
  53. package/dist/plugins/compression.js +128 -0
  54. package/dist/plugins/cookie-jar.d.ts +6 -0
  55. package/dist/plugins/cookie-jar.d.ts.map +1 -0
  56. package/dist/plugins/cookie-jar.js +72 -0
  57. package/dist/plugins/dedup.d.ts +6 -0
  58. package/dist/plugins/dedup.d.ts.map +1 -0
  59. package/dist/plugins/dedup.js +34 -0
  60. package/dist/plugins/graphql.d.ts +13 -0
  61. package/dist/plugins/graphql.d.ts.map +1 -0
  62. package/dist/plugins/graphql.js +39 -0
  63. package/dist/plugins/har-player.d.ts +7 -0
  64. package/dist/plugins/har-player.d.ts.map +1 -0
  65. package/dist/plugins/har-player.js +53 -0
  66. package/dist/plugins/har-recorder.d.ts +7 -0
  67. package/dist/plugins/har-recorder.d.ts.map +1 -0
  68. package/dist/plugins/har-recorder.js +67 -0
  69. package/dist/plugins/logger.d.ts +11 -0
  70. package/dist/plugins/logger.d.ts.map +1 -0
  71. package/dist/plugins/logger.js +72 -0
  72. package/dist/plugins/pagination.d.ts +17 -0
  73. package/dist/plugins/pagination.d.ts.map +1 -0
  74. package/dist/plugins/pagination.js +105 -0
  75. package/dist/plugins/proxy-rotator.d.ts +8 -0
  76. package/dist/plugins/proxy-rotator.d.ts.map +1 -0
  77. package/dist/plugins/proxy-rotator.js +35 -0
  78. package/dist/plugins/rate-limit.d.ts +8 -0
  79. package/dist/plugins/rate-limit.d.ts.map +1 -0
  80. package/dist/plugins/rate-limit.js +57 -0
  81. package/dist/plugins/retry.d.ts +14 -0
  82. package/dist/plugins/retry.d.ts.map +1 -0
  83. package/dist/plugins/retry.js +92 -0
  84. package/dist/plugins/server-timing.d.ts +8 -0
  85. package/dist/plugins/server-timing.d.ts.map +1 -0
  86. package/dist/plugins/server-timing.js +24 -0
  87. package/dist/plugins/xsrf.d.ts +10 -0
  88. package/dist/plugins/xsrf.d.ts.map +1 -0
  89. package/dist/plugins/xsrf.js +48 -0
  90. package/dist/runner/request-runner.d.ts +47 -0
  91. package/dist/runner/request-runner.d.ts.map +1 -0
  92. package/dist/runner/request-runner.js +89 -0
  93. package/dist/transport/fetch.d.ts +6 -0
  94. package/dist/transport/fetch.d.ts.map +1 -0
  95. package/dist/transport/fetch.js +153 -0
  96. package/dist/transport/undici.d.ts +23 -0
  97. package/dist/transport/undici.d.ts.map +1 -0
  98. package/dist/transport/undici.js +218 -0
  99. package/dist/types/index.d.ts +251 -0
  100. package/dist/types/index.d.ts.map +1 -0
  101. package/dist/types/index.js +1 -0
  102. package/dist/utils/agent-manager.d.ts +29 -0
  103. package/dist/utils/agent-manager.d.ts.map +1 -0
  104. package/dist/utils/agent-manager.js +133 -0
  105. package/dist/utils/body.d.ts +11 -0
  106. package/dist/utils/body.d.ts.map +1 -0
  107. package/dist/utils/body.js +136 -0
  108. package/dist/utils/cert.d.ts +12 -0
  109. package/dist/utils/cert.d.ts.map +1 -0
  110. package/dist/utils/cert.js +32 -0
  111. package/dist/utils/concurrency.d.ts +21 -0
  112. package/dist/utils/concurrency.d.ts.map +1 -0
  113. package/dist/utils/concurrency.js +116 -0
  114. package/dist/utils/dns.d.ts +7 -0
  115. package/dist/utils/dns.d.ts.map +1 -0
  116. package/dist/utils/dns.js +26 -0
  117. package/dist/utils/doh.d.ts +3 -0
  118. package/dist/utils/doh.d.ts.map +1 -0
  119. package/dist/utils/doh.js +35 -0
  120. package/dist/utils/header-parser.d.ts +81 -0
  121. package/dist/utils/header-parser.d.ts.map +1 -0
  122. package/dist/utils/header-parser.js +457 -0
  123. package/dist/utils/html-cleaner.d.ts +2 -0
  124. package/dist/utils/html-cleaner.d.ts.map +1 -0
  125. package/dist/utils/html-cleaner.js +21 -0
  126. package/dist/utils/logger.d.ts +33 -0
  127. package/dist/utils/logger.d.ts.map +1 -0
  128. package/dist/utils/logger.js +160 -0
  129. package/dist/utils/progress.d.ts +4 -0
  130. package/dist/utils/progress.d.ts.map +1 -0
  131. package/dist/utils/progress.js +49 -0
  132. package/dist/utils/request-pool.d.ts +23 -0
  133. package/dist/utils/request-pool.d.ts.map +1 -0
  134. package/dist/utils/request-pool.js +100 -0
  135. package/dist/utils/sse.d.ts +8 -0
  136. package/dist/utils/sse.d.ts.map +1 -0
  137. package/dist/utils/sse.js +62 -0
  138. package/dist/utils/streaming.d.ts +18 -0
  139. package/dist/utils/streaming.d.ts.map +1 -0
  140. package/dist/utils/streaming.js +83 -0
  141. package/dist/utils/task-pool.d.ts +38 -0
  142. package/dist/utils/task-pool.js +104 -0
  143. package/dist/utils/try-fn.d.ts +4 -0
  144. package/dist/utils/try-fn.d.ts.map +1 -0
  145. package/dist/utils/try-fn.js +53 -0
  146. package/dist/utils/upload.d.ts +10 -0
  147. package/dist/utils/upload.d.ts.map +1 -0
  148. package/dist/utils/upload.js +45 -0
  149. package/dist/utils/user-agent.d.ts +45 -0
  150. package/dist/utils/user-agent.d.ts.map +1 -0
  151. package/dist/utils/user-agent.js +100 -0
  152. package/dist/utils/whois.d.ts +15 -0
  153. package/dist/utils/whois.d.ts.map +1 -0
  154. package/dist/utils/whois.js +159 -0
  155. package/dist/websocket/client.d.ts +38 -0
  156. package/dist/websocket/client.d.ts.map +1 -0
  157. package/dist/websocket/client.js +184 -0
  158. package/package.json +100 -0
@@ -0,0 +1,457 @@
1
+ export function parseCacheInfo(headers) {
2
+ const info = { hit: false };
3
+ const cfCacheStatus = headers.get('cf-cache-status');
4
+ if (cfCacheStatus) {
5
+ info.provider = 'cloudflare';
6
+ info.status = cfCacheStatus.toUpperCase();
7
+ info.hit = cfCacheStatus.toUpperCase() === 'HIT';
8
+ }
9
+ const xCache = headers.get('x-cache');
10
+ if (xCache) {
11
+ const upper = xCache.toUpperCase();
12
+ info.hit = upper.includes('HIT');
13
+ if (upper.includes('FASTLY'))
14
+ info.provider = 'fastly';
15
+ else if (upper.includes('VARNISH'))
16
+ info.provider = 'varnish';
17
+ if (upper.includes('HIT'))
18
+ info.status = 'HIT';
19
+ else if (upper.includes('MISS'))
20
+ info.status = 'MISS';
21
+ else if (upper.includes('EXPIRED'))
22
+ info.status = 'EXPIRED';
23
+ else if (upper.includes('STALE'))
24
+ info.status = 'STALE';
25
+ }
26
+ const akamaiCache = headers.get('x-akamai-cache-status');
27
+ if (akamaiCache) {
28
+ info.provider = 'akamai';
29
+ info.hit = akamaiCache.toUpperCase().includes('HIT');
30
+ }
31
+ const cloudFrontCache = headers.get('x-cache');
32
+ const cloudFrontId = headers.get('x-amz-cf-id');
33
+ if (cloudFrontId) {
34
+ info.provider = 'cloudfront';
35
+ }
36
+ const cacheControl = headers.get('cache-control');
37
+ if (cacheControl) {
38
+ const maxAgeMatch = cacheControl.match(/max-age=(\d+)/);
39
+ if (maxAgeMatch) {
40
+ info.maxAge = parseInt(maxAgeMatch[1], 10);
41
+ }
42
+ }
43
+ const age = headers.get('age');
44
+ if (age) {
45
+ info.age = parseInt(age, 10);
46
+ }
47
+ return info;
48
+ }
49
+ const PLATFORM_DETECTORS = [
50
+ {
51
+ name: 'cloudflare',
52
+ category: 'cdn',
53
+ headers: ['cf-ray', 'cf-cache-status'],
54
+ detect: (headers) => headers.get('cf-ray') !== null,
55
+ extract: (headers) => {
56
+ const ray = headers.get('cf-ray');
57
+ const metadata = {};
58
+ if (ray) {
59
+ metadata.ray = ray;
60
+ const parts = ray.split('-');
61
+ if (parts.length > 1) {
62
+ return { region: parts[1], metadata };
63
+ }
64
+ }
65
+ return { metadata };
66
+ }
67
+ },
68
+ {
69
+ name: 'fastly',
70
+ category: 'cdn',
71
+ headers: ['fastly-debug-digest', 'x-served-by'],
72
+ detect: (headers) => {
73
+ const servedBy = headers.get('x-served-by');
74
+ return servedBy?.includes('cache-') || headers.get('fastly-debug-digest') !== null;
75
+ },
76
+ extract: (headers) => ({
77
+ server: headers.get('x-served-by') || undefined,
78
+ metadata: {
79
+ digest: headers.get('fastly-debug-digest')
80
+ }
81
+ })
82
+ },
83
+ {
84
+ name: 'akamai',
85
+ category: 'cdn',
86
+ headers: ['x-akamai-request-id', 'x-akamai-cache-status'],
87
+ detect: (headers) => headers.get('x-akamai-request-id') !== null,
88
+ extract: (headers) => ({
89
+ metadata: {
90
+ requestId: headers.get('x-akamai-request-id'),
91
+ cacheId: headers.get('x-akamai-cache-status')
92
+ }
93
+ })
94
+ },
95
+ {
96
+ name: 'aws-cloudfront',
97
+ category: 'cloud',
98
+ headers: ['x-amz-cf-id', 'x-amz-cf-pop'],
99
+ detect: (headers) => headers.get('x-amz-cf-id') !== null,
100
+ extract: (headers) => ({
101
+ region: headers.get('x-amz-cf-pop') || undefined,
102
+ metadata: {
103
+ requestId: headers.get('x-amz-cf-id'),
104
+ pop: headers.get('x-amz-cf-pop')
105
+ }
106
+ })
107
+ },
108
+ {
109
+ name: 'gcp',
110
+ category: 'cloud',
111
+ headers: ['x-cloud-trace-context', 'x-goog-request-id'],
112
+ detect: (headers) => headers.get('x-cloud-trace-context') !== null || headers.get('x-goog-request-id') !== null,
113
+ extract: (headers) => {
114
+ const trace = headers.get('x-cloud-trace-context');
115
+ return {
116
+ metadata: {
117
+ traceId: trace ? trace.split('/')[0] : undefined,
118
+ requestId: headers.get('x-goog-request-id')
119
+ }
120
+ };
121
+ }
122
+ },
123
+ {
124
+ name: 'azure',
125
+ category: 'cloud',
126
+ headers: ['x-ms-request-id', 'x-azure-ref'],
127
+ detect: (headers) => headers.get('x-ms-request-id') !== null,
128
+ extract: (headers) => ({
129
+ metadata: {
130
+ requestId: headers.get('x-ms-request-id'),
131
+ ref: headers.get('x-azure-ref')
132
+ }
133
+ })
134
+ },
135
+ {
136
+ name: 'oracle-cloud',
137
+ category: 'cloud',
138
+ headers: ['x-oracle-dms-rid', 'x-oracle-dms-ecid'],
139
+ detect: (headers) => headers.get('x-oracle-dms-rid') !== null,
140
+ extract: (headers) => ({
141
+ metadata: {
142
+ requestId: headers.get('x-oracle-dms-rid'),
143
+ ecid: headers.get('x-oracle-dms-ecid')
144
+ }
145
+ })
146
+ },
147
+ {
148
+ name: 'alibaba-cloud',
149
+ category: 'cloud',
150
+ headers: ['ali-swift-global-savetime', 'eagleeye-traceid'],
151
+ detect: (headers) => headers.get('eagleeye-traceid') !== null,
152
+ extract: (headers) => ({
153
+ metadata: {
154
+ traceId: headers.get('eagleeye-traceid')
155
+ }
156
+ })
157
+ },
158
+ {
159
+ name: 'vercel',
160
+ category: 'hosting',
161
+ headers: ['x-vercel-id', 'x-vercel-cache'],
162
+ detect: (headers) => headers.get('x-vercel-id') !== null,
163
+ extract: (headers) => ({
164
+ metadata: {
165
+ requestId: headers.get('x-vercel-id'),
166
+ cache: headers.get('x-vercel-cache')
167
+ }
168
+ })
169
+ },
170
+ {
171
+ name: 'netlify',
172
+ category: 'hosting',
173
+ headers: ['x-nf-request-id', 'x-nf-trace-id'],
174
+ detect: (headers) => headers.get('x-nf-request-id') !== null,
175
+ extract: (headers) => ({
176
+ metadata: {
177
+ requestId: headers.get('x-nf-request-id'),
178
+ traceId: headers.get('x-nf-trace-id')
179
+ }
180
+ })
181
+ },
182
+ {
183
+ name: 'render',
184
+ category: 'hosting',
185
+ headers: ['x-render-origin-server'],
186
+ detect: (headers) => headers.get('x-render-origin-server') !== null,
187
+ extract: (headers) => ({
188
+ metadata: {
189
+ origin: headers.get('x-render-origin-server')
190
+ }
191
+ })
192
+ }
193
+ ];
194
+ export function parsePlatformInfo(headers) {
195
+ const info = {
196
+ metadata: {}
197
+ };
198
+ for (const detector of PLATFORM_DETECTORS) {
199
+ if (detector.detect(headers)) {
200
+ info.provider = detector.name;
201
+ info.category = detector.category;
202
+ const extracted = detector.extract(headers);
203
+ if (extracted.region)
204
+ info.region = extracted.region;
205
+ if (extracted.server)
206
+ info.server = extracted.server;
207
+ if (extracted.metadata) {
208
+ Object.assign(info.metadata, extracted.metadata);
209
+ }
210
+ break;
211
+ }
212
+ }
213
+ if (!info.provider) {
214
+ const server = headers.get('server');
215
+ if (server) {
216
+ info.server = server;
217
+ const serverLower = server.toLowerCase();
218
+ if (serverLower.includes('cloudflare')) {
219
+ info.provider = 'cloudflare';
220
+ info.category = 'cdn';
221
+ }
222
+ else if (serverLower.includes('cloudfront')) {
223
+ info.provider = 'aws-cloudfront';
224
+ info.category = 'cloud';
225
+ }
226
+ else if (serverLower.includes('fastly')) {
227
+ info.provider = 'fastly';
228
+ info.category = 'cdn';
229
+ }
230
+ else if (serverLower.includes('akamai')) {
231
+ info.provider = 'akamai';
232
+ info.category = 'cdn';
233
+ }
234
+ else if (serverLower.includes('nginx')) {
235
+ info.provider = 'nginx';
236
+ info.category = 'server';
237
+ }
238
+ else if (serverLower.includes('apache')) {
239
+ info.provider = 'apache';
240
+ info.category = 'server';
241
+ }
242
+ else if (serverLower.includes('varnish')) {
243
+ info.provider = 'varnish';
244
+ info.category = 'proxy';
245
+ }
246
+ else {
247
+ info.provider = 'unknown';
248
+ info.category = 'unknown';
249
+ }
250
+ }
251
+ }
252
+ if (!info.provider) {
253
+ const via = headers.get('via');
254
+ if (via) {
255
+ const viaLower = via.toLowerCase();
256
+ if (viaLower.includes('cloudflare')) {
257
+ info.provider = 'cloudflare';
258
+ info.category = 'cdn';
259
+ }
260
+ else if (viaLower.includes('akamai')) {
261
+ info.provider = 'akamai';
262
+ info.category = 'cdn';
263
+ }
264
+ else if (viaLower.includes('varnish')) {
265
+ info.provider = 'varnish';
266
+ info.category = 'proxy';
267
+ }
268
+ else if (viaLower.includes('squid')) {
269
+ info.provider = 'squid';
270
+ info.category = 'proxy';
271
+ }
272
+ }
273
+ }
274
+ if (Object.keys(info.metadata || {}).length === 0) {
275
+ delete info.metadata;
276
+ }
277
+ return info;
278
+ }
279
+ export function parseRateLimitInfo(headers, status) {
280
+ const info = { limited: status === 429 };
281
+ const limit = headers.get('x-ratelimit-limit') || headers.get('ratelimit-limit');
282
+ const remaining = headers.get('x-ratelimit-remaining') || headers.get('ratelimit-remaining');
283
+ const reset = headers.get('x-ratelimit-reset') || headers.get('ratelimit-reset');
284
+ if (limit)
285
+ info.limit = parseInt(limit, 10);
286
+ if (remaining)
287
+ info.remaining = parseInt(remaining, 10);
288
+ if (reset) {
289
+ const resetValue = parseInt(reset, 10);
290
+ if (resetValue > 1000000000) {
291
+ info.reset = new Date(resetValue * 1000);
292
+ }
293
+ else {
294
+ info.reset = new Date(Date.now() + resetValue * 1000);
295
+ }
296
+ }
297
+ const retryAfter = headers.get('retry-after');
298
+ if (retryAfter) {
299
+ const seconds = parseInt(retryAfter, 10);
300
+ if (!isNaN(seconds)) {
301
+ info.retryAfter = seconds;
302
+ }
303
+ else {
304
+ const retryDate = new Date(retryAfter);
305
+ if (!isNaN(retryDate.getTime())) {
306
+ info.retryAfter = Math.ceil((retryDate.getTime() - Date.now()) / 1000);
307
+ }
308
+ }
309
+ }
310
+ const policy = headers.get('ratelimit-policy');
311
+ if (policy) {
312
+ info.policy = policy;
313
+ }
314
+ if (info.remaining !== undefined && info.remaining === 0) {
315
+ info.limited = true;
316
+ }
317
+ return info;
318
+ }
319
+ export function parseCompressionInfo(headers) {
320
+ const info = {};
321
+ const encoding = headers.get('content-encoding');
322
+ if (encoding) {
323
+ info.encoding = encoding.toLowerCase();
324
+ }
325
+ const xOriginalSize = headers.get('x-original-size') || headers.get('x-uncompressed-size');
326
+ if (xOriginalSize) {
327
+ info.originalSize = parseInt(xOriginalSize, 10);
328
+ }
329
+ const contentLength = headers.get('content-length');
330
+ if (contentLength) {
331
+ info.compressedSize = parseInt(contentLength, 10);
332
+ }
333
+ if (info.originalSize && info.compressedSize) {
334
+ info.ratio = info.originalSize / info.compressedSize;
335
+ }
336
+ return info;
337
+ }
338
+ export function parseCSPInfo(headers) {
339
+ const info = {
340
+ directives: {},
341
+ reportOnly: false
342
+ };
343
+ let cspHeader = headers.get('content-security-policy');
344
+ if (!cspHeader) {
345
+ cspHeader = headers.get('content-security-policy-report-only');
346
+ if (cspHeader) {
347
+ info.reportOnly = true;
348
+ }
349
+ }
350
+ if (cspHeader) {
351
+ info.policy = cspHeader;
352
+ const directives = cspHeader.split(';').map(d => d.trim()).filter(Boolean);
353
+ for (const directive of directives) {
354
+ const [name, ...values] = directive.split(/\s+/);
355
+ if (name) {
356
+ info.directives[name] = values;
357
+ }
358
+ }
359
+ }
360
+ return info;
361
+ }
362
+ export function parseContentType(headers) {
363
+ const info = {};
364
+ const contentType = headers.get('content-type');
365
+ if (!contentType)
366
+ return info;
367
+ const parts = contentType.split(';').map(p => p.trim());
368
+ if (parts[0]) {
369
+ info.mediaType = parts[0];
370
+ const [type, subtype] = parts[0].split('/');
371
+ if (type)
372
+ info.type = type;
373
+ if (subtype)
374
+ info.subtype = subtype;
375
+ }
376
+ for (let i = 1; i < parts.length; i++) {
377
+ const param = parts[i];
378
+ const [key, value] = param.split('=').map(s => s.trim());
379
+ if (key === 'charset' && value) {
380
+ info.charset = value.replace(/['"]/g, '');
381
+ }
382
+ else if (key === 'boundary' && value) {
383
+ info.boundary = value.replace(/['"]/g, '');
384
+ }
385
+ }
386
+ return info;
387
+ }
388
+ export function parseAcceptInfo(headers) {
389
+ const info = {
390
+ types: [],
391
+ encodings: [],
392
+ languages: []
393
+ };
394
+ const accept = headers.get('accept');
395
+ if (accept) {
396
+ const types = accept.split(',').map(t => t.trim());
397
+ for (const typeStr of types) {
398
+ const [mediaType, ...params] = typeStr.split(';').map(s => s.trim());
399
+ let q = 1.0;
400
+ for (const param of params) {
401
+ if (param.startsWith('q=')) {
402
+ q = parseFloat(param.substring(2)) || 1.0;
403
+ }
404
+ }
405
+ const [type, subtype] = mediaType.split('/');
406
+ info.types.push({
407
+ mediaType,
408
+ q,
409
+ type: type || undefined,
410
+ subtype: subtype || undefined
411
+ });
412
+ }
413
+ info.types.sort((a, b) => b.q - a.q);
414
+ }
415
+ const acceptEncoding = headers.get('accept-encoding');
416
+ if (acceptEncoding) {
417
+ const encodings = acceptEncoding.split(',').map(e => e.trim());
418
+ for (const encodingStr of encodings) {
419
+ const [encoding, ...params] = encodingStr.split(';').map(s => s.trim());
420
+ let q = 1.0;
421
+ for (const param of params) {
422
+ if (param.startsWith('q=')) {
423
+ q = parseFloat(param.substring(2)) || 1.0;
424
+ }
425
+ }
426
+ info.encodings.push({ encoding, q });
427
+ }
428
+ info.encodings.sort((a, b) => b.q - a.q);
429
+ }
430
+ const acceptLanguage = headers.get('accept-language');
431
+ if (acceptLanguage) {
432
+ const languages = acceptLanguage.split(',').map(l => l.trim());
433
+ for (const languageStr of languages) {
434
+ const [language, ...params] = languageStr.split(';').map(s => s.trim());
435
+ let q = 1.0;
436
+ for (const param of params) {
437
+ if (param.startsWith('q=')) {
438
+ q = parseFloat(param.substring(2)) || 1.0;
439
+ }
440
+ }
441
+ info.languages.push({ language, q });
442
+ }
443
+ info.languages.sort((a, b) => b.q - a.q);
444
+ }
445
+ return info;
446
+ }
447
+ export function parseHeaders(headers, status) {
448
+ return {
449
+ cache: parseCacheInfo(headers),
450
+ platform: parsePlatformInfo(headers),
451
+ rateLimit: parseRateLimitInfo(headers, status),
452
+ compression: parseCompressionInfo(headers),
453
+ csp: parseCSPInfo(headers),
454
+ contentType: parseContentType(headers),
455
+ accept: parseAcceptInfo(headers)
456
+ };
457
+ }
@@ -0,0 +1,2 @@
1
+ export declare function cleanHtml(html: string): string;
2
+ //# sourceMappingURL=html-cleaner.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"html-cleaner.d.ts","sourceRoot":"","sources":["../../src/utils/html-cleaner.ts"],"names":[],"mappings":"AAIA,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAmC9C"}
@@ -0,0 +1,21 @@
1
+ export function cleanHtml(html) {
2
+ if (!html)
3
+ return '';
4
+ let text = html;
5
+ text = text.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '');
6
+ text = text.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '');
7
+ text = text.replace(/<!--[\s\S]*?-->/g, '');
8
+ text = text.replace(/<\/(div|p|h[1-6]|li|ul|ol|tr|table|section|article|main|header|footer|nav)>/gi, '\n');
9
+ text = text.replace(/<(br|hr)\s*\/?>/gi, '\n');
10
+ text = text.replace(/<[^>]+>/g, ' ');
11
+ text = text
12
+ .replace(/&nbsp;/g, ' ')
13
+ .replace(/&amp;/g, '&')
14
+ .replace(/&lt;/g, '<')
15
+ .replace(/&gt;/g, '>')
16
+ .replace(/&quot;/g, '"')
17
+ .replace(/&#39;/g, "'");
18
+ text = text.replace(/[ \t]+/g, ' ');
19
+ text = text.replace(/\n\s*\n\s*\n+/g, '\n\n');
20
+ return text.trim();
21
+ }
@@ -0,0 +1,33 @@
1
+ import { ReckerRequest, ReckerResponse, Timings } from '../types/index.js';
2
+ export type LogLevel = 'debug' | 'info' | 'warn' | 'error' | 'none';
3
+ export interface LoggerOptions {
4
+ level?: LogLevel;
5
+ prefix?: string;
6
+ timestamp?: boolean;
7
+ colors?: boolean;
8
+ }
9
+ export declare class Logger {
10
+ private level;
11
+ private prefix;
12
+ private useTimestamp;
13
+ private useColors;
14
+ constructor(options?: LoggerOptions);
15
+ private detectLogLevel;
16
+ private supportsColors;
17
+ private colorize;
18
+ private formatTimestamp;
19
+ private shouldLog;
20
+ private log;
21
+ debug(message: string): void;
22
+ info(message: string): void;
23
+ warn(message: string): void;
24
+ error(message: string): void;
25
+ logRequest(req: ReckerRequest): void;
26
+ logResponse(req: ReckerRequest, res: ReckerResponse, startTime: number): void;
27
+ logTimings(timings: Timings): void;
28
+ logError(req: ReckerRequest, error: Error): void;
29
+ private formatBytes;
30
+ }
31
+ export declare function getLogger(): Logger;
32
+ export declare function setLogger(logger: Logger): void;
33
+ //# sourceMappingURL=logger.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAE3E,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC;AAEpE,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,QAAQ,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AA2BD,qBAAa,MAAM;IACjB,OAAO,CAAC,KAAK,CAAW;IACxB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAU;IAC9B,OAAO,CAAC,SAAS,CAAU;gBAEf,OAAO,GAAE,aAAkB;IAOvC,OAAO,CAAC,cAAc;IAQtB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,QAAQ;IAKhB,OAAO,CAAC,eAAe;IAOvB,OAAO,CAAC,SAAS;IAIjB,OAAO,CAAC,GAAG;IAQX,KAAK,CAAC,OAAO,EAAE,MAAM;IAIrB,IAAI,CAAC,OAAO,EAAE,MAAM;IAIpB,IAAI,CAAC,OAAO,EAAE,MAAM;IAIpB,KAAK,CAAC,OAAO,EAAE,MAAM;IAOrB,UAAU,CAAC,GAAG,EAAE,aAAa;IAmB7B,WAAW,CAAC,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM;IA4BtE,UAAU,CAAC,OAAO,EAAE,OAAO;IAoB3B,QAAQ,CAAC,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK;IAoBzC,OAAO,CAAC,WAAW;CAOpB;AAKD,wBAAgB,SAAS,IAAI,MAAM,CAKlC;AAED,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,QAEvC"}
@@ -0,0 +1,160 @@
1
+ const colors = {
2
+ reset: '\x1b[0m',
3
+ bright: '\x1b[1m',
4
+ dim: '\x1b[2m',
5
+ red: '\x1b[31m',
6
+ green: '\x1b[32m',
7
+ yellow: '\x1b[33m',
8
+ blue: '\x1b[34m',
9
+ magenta: '\x1b[35m',
10
+ cyan: '\x1b[36m',
11
+ white: '\x1b[37m',
12
+ gray: '\x1b[90m',
13
+ };
14
+ const levels = {
15
+ debug: 0,
16
+ info: 1,
17
+ warn: 2,
18
+ error: 3,
19
+ none: 999,
20
+ };
21
+ export class Logger {
22
+ level;
23
+ prefix;
24
+ useTimestamp;
25
+ useColors;
26
+ constructor(options = {}) {
27
+ this.level = options.level || this.detectLogLevel();
28
+ this.prefix = options.prefix || 'recker';
29
+ this.useTimestamp = options.timestamp !== false;
30
+ this.useColors = options.colors !== false && this.supportsColors();
31
+ }
32
+ detectLogLevel() {
33
+ const env = process.env.DEBUG || '';
34
+ if (env === '*' || env.includes('recker') || env.includes('*')) {
35
+ return 'debug';
36
+ }
37
+ return 'none';
38
+ }
39
+ supportsColors() {
40
+ return (process.stdout.isTTY &&
41
+ !process.env.NO_COLOR &&
42
+ process.env.TERM !== 'dumb');
43
+ }
44
+ colorize(text, color) {
45
+ if (!this.useColors)
46
+ return text;
47
+ return `${colors[color]}${text}${colors.reset}`;
48
+ }
49
+ formatTimestamp() {
50
+ if (!this.useTimestamp)
51
+ return '';
52
+ const now = new Date();
53
+ const time = now.toTimeString().split(' ')[0];
54
+ return this.colorize(`[${time}]`, 'gray') + ' ';
55
+ }
56
+ shouldLog(level) {
57
+ return levels[level] >= levels[this.level];
58
+ }
59
+ log(level, message) {
60
+ if (!this.shouldLog(level))
61
+ return;
62
+ const timestamp = this.formatTimestamp();
63
+ const prefix = this.colorize(`[${this.prefix}]`, 'cyan');
64
+ console.log(`${timestamp}${prefix} ${message}`);
65
+ }
66
+ debug(message) {
67
+ this.log('debug', message);
68
+ }
69
+ info(message) {
70
+ this.log('info', message);
71
+ }
72
+ warn(message) {
73
+ this.log('warn', this.colorize(message, 'yellow'));
74
+ }
75
+ error(message) {
76
+ this.log('error', this.colorize(message, 'red'));
77
+ }
78
+ logRequest(req) {
79
+ if (!this.shouldLog('debug'))
80
+ return;
81
+ const method = this.colorize(req.method, 'blue');
82
+ const url = this.colorize(req.url, 'bright');
83
+ this.debug(`${this.colorize('→', 'green')} ${method} ${url}`);
84
+ if (req.headers && Array.from(req.headers.keys()).length > 0) {
85
+ const headerStr = Array.from(req.headers.entries())
86
+ .map(([k, v]) => ` ${this.colorize(k, 'gray')}: ${v}`)
87
+ .join('\n');
88
+ console.log(headerStr);
89
+ }
90
+ }
91
+ logResponse(req, res, startTime) {
92
+ if (!this.shouldLog('debug'))
93
+ return;
94
+ const duration = Date.now() - startTime;
95
+ const statusColor = res.ok ? 'green' : 'red';
96
+ const status = this.colorize(String(res.status), statusColor);
97
+ const method = this.colorize(req.method, 'gray');
98
+ this.debug(`${this.colorize('←', 'green')} ${status} ${method} ${req.url} ${this.colorize(`(${duration}ms)`, 'gray')}`);
99
+ if (res.timings) {
100
+ this.logTimings(res.timings);
101
+ }
102
+ const contentLength = res.headers.get('content-length');
103
+ if (contentLength) {
104
+ const size = this.formatBytes(parseInt(contentLength, 10));
105
+ console.log(` ${this.colorize('Size:', 'gray')} ${size}`);
106
+ }
107
+ }
108
+ logTimings(timings) {
109
+ if (!this.shouldLog('debug'))
110
+ return;
111
+ const parts = [];
112
+ if (timings.dns)
113
+ parts.push(`DNS: ${timings.dns.toFixed(0)}ms`);
114
+ if (timings.tcp)
115
+ parts.push(`TCP: ${timings.tcp.toFixed(0)}ms`);
116
+ if (timings.tls)
117
+ parts.push(`TLS: ${timings.tls.toFixed(0)}ms`);
118
+ if (timings.firstByte)
119
+ parts.push(`TTFB: ${timings.firstByte.toFixed(0)}ms`);
120
+ if (timings.total)
121
+ parts.push(`Total: ${timings.total.toFixed(0)}ms`);
122
+ if (parts.length > 0) {
123
+ const timelinesStr = parts.join(', ');
124
+ console.log(` ${this.colorize('├─', 'gray')} ${timelinesStr}`);
125
+ }
126
+ }
127
+ logError(req, error) {
128
+ if (!this.shouldLog('error'))
129
+ return;
130
+ const method = this.colorize(req.method, 'gray');
131
+ this.error(`${this.colorize('✖', 'red')} ${method} ${req.url}`);
132
+ console.log(` ${this.colorize('Error:', 'red')} ${error.message}`);
133
+ if (error.stack && this.shouldLog('debug')) {
134
+ const stack = error.stack
135
+ .split('\n')
136
+ .slice(1, 4)
137
+ .map((line) => ` ${this.colorize('│', 'gray')} ${line.trim()}`)
138
+ .join('\n');
139
+ console.log(stack);
140
+ }
141
+ }
142
+ formatBytes(bytes) {
143
+ if (bytes === 0)
144
+ return '0 B';
145
+ const k = 1024;
146
+ const sizes = ['B', 'KB', 'MB', 'GB'];
147
+ const i = Math.floor(Math.log(bytes) / Math.log(k));
148
+ return `${(bytes / Math.pow(k, i)).toFixed(1)} ${sizes[i]}`;
149
+ }
150
+ }
151
+ let globalLogger = null;
152
+ export function getLogger() {
153
+ if (!globalLogger) {
154
+ globalLogger = new Logger();
155
+ }
156
+ return globalLogger;
157
+ }
158
+ export function setLogger(logger) {
159
+ globalLogger = logger;
160
+ }
@@ -0,0 +1,4 @@
1
+ import { ProgressEvent, ProgressCallback } from '../types/index.js';
2
+ export declare function createProgressStream(stream: ReadableStream<Uint8Array>, onProgress: ProgressCallback, total?: number): ReadableStream<Uint8Array>;
3
+ export declare function calculateProgress(loaded: number, total?: number): ProgressEvent;
4
+ //# sourceMappingURL=progress.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"progress.d.ts","sourceRoot":"","sources":["../../src/utils/progress.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAKpE,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,EAClC,UAAU,EAAE,gBAAgB,EAC5B,KAAK,CAAC,EAAE,MAAM,GACb,cAAc,CAAC,UAAU,CAAC,CA6C5B;AAKD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,CAW/E"}