rezo 1.0.26 → 1.0.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/curl.cjs +4 -2
- package/dist/adapters/curl.js +4 -2
- package/dist/adapters/entries/curl.d.ts +45 -0
- package/dist/adapters/entries/fetch.d.ts +45 -0
- package/dist/adapters/entries/http.d.ts +45 -0
- package/dist/adapters/entries/http2.d.ts +45 -0
- package/dist/adapters/entries/react-native.d.ts +45 -0
- package/dist/adapters/entries/xhr.d.ts +45 -0
- package/dist/adapters/fetch.cjs +90 -1
- package/dist/adapters/fetch.js +90 -1
- package/dist/adapters/http.cjs +1 -1
- package/dist/adapters/http.js +1 -1
- package/dist/adapters/http2.cjs +1 -1
- package/dist/adapters/http2.js +1 -1
- package/dist/adapters/index.cjs +6 -6
- package/dist/adapters/react-native.cjs +162 -3
- package/dist/adapters/react-native.js +162 -3
- package/dist/cache/index.cjs +13 -13
- package/dist/crawler.d.ts +45 -0
- package/dist/entries/crawler.cjs +5 -5
- package/dist/index.cjs +24 -24
- package/dist/index.d.ts +45 -0
- package/dist/platform/browser.d.ts +45 -0
- package/dist/platform/bun.d.ts +45 -0
- package/dist/platform/deno.d.ts +45 -0
- package/dist/platform/node.d.ts +45 -0
- package/dist/platform/react-native.d.ts +45 -0
- package/dist/platform/worker.d.ts +45 -0
- package/dist/plugin/index.cjs +36 -36
- package/dist/proxy/index.cjs +2 -2
- package/dist/queue/index.cjs +8 -8
- package/dist/utils/cookies.cjs +39 -0
- package/dist/utils/cookies.js +39 -0
- package/dist/utils/http-config.cjs +16 -4
- package/dist/utils/http-config.js +16 -4
- package/package.json +1 -1
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
const { RezoError } = require('../errors/rezo-error.cjs');
|
|
2
2
|
const { buildSmartError, builErrorFromResponse } = require('../responses/buildError.cjs');
|
|
3
|
+
const { RezoCookieJar } = require('../utils/cookies.cjs');
|
|
3
4
|
const RezoFormData = require('../utils/form-data.cjs');
|
|
4
5
|
const { getDefaultConfig, prepareHTTPOptions } = require('../utils/http-config.cjs');
|
|
5
6
|
const { RezoHeaders } = require('../utils/headers.cjs');
|
|
@@ -17,6 +18,85 @@ const Environment = {
|
|
|
17
18
|
hasFormData: typeof FormData !== "undefined",
|
|
18
19
|
hasAbortController: typeof AbortController !== "undefined"
|
|
19
20
|
};
|
|
21
|
+
const debugLog = {
|
|
22
|
+
requestStart: (config, url, method) => {
|
|
23
|
+
if (config.debug) {
|
|
24
|
+
console.log(`
|
|
25
|
+
[Rezo Debug] ─────────────────────────────────────`);
|
|
26
|
+
console.log(`[Rezo Debug] ${method} ${url}`);
|
|
27
|
+
console.log(`[Rezo Debug] Request ID: ${config.requestId}`);
|
|
28
|
+
console.log(`[Rezo Debug] Adapter: react-native`);
|
|
29
|
+
if (config.originalRequest?.headers) {
|
|
30
|
+
const headers = config.originalRequest.headers instanceof RezoHeaders ? config.originalRequest.headers.toObject() : config.originalRequest.headers;
|
|
31
|
+
console.log(`[Rezo Debug] Request Headers:`, JSON.stringify(headers, null, 2));
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (config.trackUrl) {
|
|
35
|
+
console.log(`[Rezo Track] → ${method} ${url}`);
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
retry: (config, attempt, maxRetries, statusCode, delay) => {
|
|
39
|
+
if (config.debug) {
|
|
40
|
+
console.log(`[Rezo Debug] Retry ${attempt}/${maxRetries} after status ${statusCode}${delay > 0 ? ` (waiting ${delay}ms)` : ""}`);
|
|
41
|
+
}
|
|
42
|
+
if (config.trackUrl) {
|
|
43
|
+
console.log(`[Rezo Track] ⟳ Retry ${attempt}/${maxRetries} (status ${statusCode})`);
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
maxRetries: (config, maxRetries) => {
|
|
47
|
+
if (config.debug) {
|
|
48
|
+
console.log(`[Rezo Debug] Max retries (${maxRetries}) reached, throwing error`);
|
|
49
|
+
}
|
|
50
|
+
if (config.trackUrl) {
|
|
51
|
+
console.log(`[Rezo Track] ✗ Max retries reached`);
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
response: (config, status, statusText, duration) => {
|
|
55
|
+
if (config.debug) {
|
|
56
|
+
console.log(`[Rezo Debug] Response: ${status} ${statusText} (${duration.toFixed(2)}ms)`);
|
|
57
|
+
}
|
|
58
|
+
if (config.trackUrl) {
|
|
59
|
+
console.log(`[Rezo Track] ✓ ${status} ${statusText}`);
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
responseHeaders: (config, headers) => {
|
|
63
|
+
if (config.debug) {
|
|
64
|
+
console.log(`[Rezo Debug] Response Headers:`, JSON.stringify(headers, null, 2));
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
cookies: (config, cookieCount) => {
|
|
68
|
+
if (config.debug && cookieCount > 0) {
|
|
69
|
+
console.log(`[Rezo Debug] Cookies received: ${cookieCount}`);
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
timing: (config, timing) => {
|
|
73
|
+
if (config.debug) {
|
|
74
|
+
const parts = [];
|
|
75
|
+
if (timing.ttfb)
|
|
76
|
+
parts.push(`TTFB: ${timing.ttfb.toFixed(2)}ms`);
|
|
77
|
+
if (timing.total)
|
|
78
|
+
parts.push(`Total: ${timing.total.toFixed(2)}ms`);
|
|
79
|
+
if (parts.length > 0) {
|
|
80
|
+
console.log(`[Rezo Debug] Timing: ${parts.join(" | ")}`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
},
|
|
84
|
+
complete: (config, url) => {
|
|
85
|
+
if (config.debug) {
|
|
86
|
+
console.log(`[Rezo Debug] Request complete: ${url}`);
|
|
87
|
+
console.log(`[Rezo Debug] ─────────────────────────────────────
|
|
88
|
+
`);
|
|
89
|
+
}
|
|
90
|
+
},
|
|
91
|
+
error: (config, error) => {
|
|
92
|
+
if (config.debug) {
|
|
93
|
+
console.log(`[Rezo Debug] Error: ${error instanceof Error ? error.message : error}`);
|
|
94
|
+
}
|
|
95
|
+
if (config.trackUrl) {
|
|
96
|
+
console.log(`[Rezo Track] ✗ Error: ${error instanceof Error ? error.message : error}`);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
};
|
|
20
100
|
function updateTiming(config, timing, bodySize) {
|
|
21
101
|
const now = performance.now();
|
|
22
102
|
config.timing.domainLookupStart = config.timing.startTime;
|
|
@@ -291,11 +371,25 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
291
371
|
throw response;
|
|
292
372
|
}
|
|
293
373
|
if (maxRetries <= retries) {
|
|
374
|
+
debugLog.maxRetries(config, maxRetries);
|
|
294
375
|
throw response;
|
|
295
376
|
}
|
|
296
377
|
retries++;
|
|
297
|
-
|
|
298
|
-
|
|
378
|
+
const currentDelay = incrementDelay ? retryDelay * retries : retryDelay;
|
|
379
|
+
debugLog.retry(config, retries, maxRetries, response.status || 0, currentDelay);
|
|
380
|
+
if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
|
|
381
|
+
for (const hook of config.hooks.beforeRetry) {
|
|
382
|
+
try {
|
|
383
|
+
await hook(config, response, retries);
|
|
384
|
+
} catch (hookErr) {
|
|
385
|
+
if (config.debug) {
|
|
386
|
+
console.log("[Rezo Debug] beforeRetry hook error:", hookErr);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
if (currentDelay > 0) {
|
|
392
|
+
await new Promise((resolve) => setTimeout(resolve, currentDelay));
|
|
299
393
|
}
|
|
300
394
|
}
|
|
301
395
|
config.retryAttempts++;
|
|
@@ -319,6 +413,7 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
319
413
|
config.isSecure = isSecure;
|
|
320
414
|
config.finalUrl = url;
|
|
321
415
|
config.network.protocol = isSecure ? "https" : "http";
|
|
416
|
+
debugLog.requestStart(config, url, fetchOptions.method?.toUpperCase() || "GET");
|
|
322
417
|
const reqHeaders = fetchOptions.headers instanceof RezoHeaders ? fetchOptions.headers.toObject() : fetchOptions.headers || {};
|
|
323
418
|
const headers = toFetchHeaders(reqHeaders);
|
|
324
419
|
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
@@ -380,13 +475,68 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
380
475
|
const responseHeaders = fromFetchHeaders(response.headers);
|
|
381
476
|
const contentType = response.headers.get("content-type") || "";
|
|
382
477
|
const contentLength = response.headers.get("content-length");
|
|
383
|
-
|
|
478
|
+
let cookies = {
|
|
384
479
|
array: [],
|
|
385
480
|
serialized: [],
|
|
386
481
|
netscape: "",
|
|
387
482
|
string: "",
|
|
388
483
|
setCookiesString: []
|
|
389
484
|
};
|
|
485
|
+
const setCookieHeader = response.headers.get("set-cookie");
|
|
486
|
+
if (setCookieHeader && config.useCookies) {
|
|
487
|
+
const cookieStrings = setCookieHeader.split(/,(?=[^;]+=[^;]+)/).map((s) => s.trim());
|
|
488
|
+
const tempJar = new RezoCookieJar;
|
|
489
|
+
tempJar.setCookiesSync(cookieStrings, url);
|
|
490
|
+
const parsedCookies = tempJar.cookies();
|
|
491
|
+
const acceptedCookies = [];
|
|
492
|
+
let hookError = null;
|
|
493
|
+
if (config.hooks?.beforeCookie && config.hooks.beforeCookie.length > 0) {
|
|
494
|
+
for (const cookie of parsedCookies.array) {
|
|
495
|
+
let shouldAccept = true;
|
|
496
|
+
for (const hook of config.hooks.beforeCookie) {
|
|
497
|
+
try {
|
|
498
|
+
const result = await hook({
|
|
499
|
+
cookie,
|
|
500
|
+
source: "response",
|
|
501
|
+
url,
|
|
502
|
+
isValid: true
|
|
503
|
+
}, config);
|
|
504
|
+
if (result === false) {
|
|
505
|
+
shouldAccept = false;
|
|
506
|
+
break;
|
|
507
|
+
}
|
|
508
|
+
} catch (err) {
|
|
509
|
+
hookError = err;
|
|
510
|
+
if (config.debug) {
|
|
511
|
+
console.log("[Rezo Debug] beforeCookie hook error:", err);
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
if (shouldAccept) {
|
|
516
|
+
acceptedCookies.push(cookie);
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
} else {
|
|
520
|
+
acceptedCookies.push(...parsedCookies.array);
|
|
521
|
+
}
|
|
522
|
+
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
523
|
+
if (config.enableCookieJar && config.cookieJar) {
|
|
524
|
+
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
525
|
+
}
|
|
526
|
+
const cookieJar = new RezoCookieJar(acceptedCookies, url);
|
|
527
|
+
cookies = cookieJar.cookies();
|
|
528
|
+
if (!hookError && config.hooks?.afterCookie && config.hooks.afterCookie.length > 0) {
|
|
529
|
+
for (const hook of config.hooks.afterCookie) {
|
|
530
|
+
try {
|
|
531
|
+
await hook(acceptedCookies, config);
|
|
532
|
+
} catch (err) {
|
|
533
|
+
if (config.debug) {
|
|
534
|
+
console.log("[Rezo Debug] afterCookie hook error:", err);
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
390
540
|
config.responseCookies = cookies;
|
|
391
541
|
if (eventEmitter) {
|
|
392
542
|
const headersEvent = {
|
|
@@ -464,6 +614,14 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
464
614
|
eventEmitter.emit("error", error);
|
|
465
615
|
return error;
|
|
466
616
|
}
|
|
617
|
+
const duration = performance.now() - timing.startTime;
|
|
618
|
+
debugLog.response(config, status, statusText, duration);
|
|
619
|
+
debugLog.responseHeaders(config, responseHeaders.toObject());
|
|
620
|
+
debugLog.cookies(config, cookies.array.length);
|
|
621
|
+
debugLog.timing(config, {
|
|
622
|
+
ttfb: timing.firstByteTime ? timing.firstByteTime - timing.startTime : undefined,
|
|
623
|
+
total: duration
|
|
624
|
+
});
|
|
467
625
|
const finalResponse = {
|
|
468
626
|
data: responseData,
|
|
469
627
|
status,
|
|
@@ -476,6 +634,7 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
476
634
|
finalUrl: url,
|
|
477
635
|
urls: buildUrlTree(config, url)
|
|
478
636
|
};
|
|
637
|
+
debugLog.complete(config, url);
|
|
479
638
|
if (streamResult) {
|
|
480
639
|
const streamFinishEvent = {
|
|
481
640
|
status,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { RezoError } from '../errors/rezo-error.js';
|
|
2
2
|
import { buildSmartError, builErrorFromResponse } from '../responses/buildError.js';
|
|
3
|
+
import { RezoCookieJar } from '../utils/cookies.js';
|
|
3
4
|
import RezoFormData from '../utils/form-data.js';
|
|
4
5
|
import { getDefaultConfig, prepareHTTPOptions } from '../utils/http-config.js';
|
|
5
6
|
import { RezoHeaders } from '../utils/headers.js';
|
|
@@ -17,6 +18,85 @@ const Environment = {
|
|
|
17
18
|
hasFormData: typeof FormData !== "undefined",
|
|
18
19
|
hasAbortController: typeof AbortController !== "undefined"
|
|
19
20
|
};
|
|
21
|
+
const debugLog = {
|
|
22
|
+
requestStart: (config, url, method) => {
|
|
23
|
+
if (config.debug) {
|
|
24
|
+
console.log(`
|
|
25
|
+
[Rezo Debug] ─────────────────────────────────────`);
|
|
26
|
+
console.log(`[Rezo Debug] ${method} ${url}`);
|
|
27
|
+
console.log(`[Rezo Debug] Request ID: ${config.requestId}`);
|
|
28
|
+
console.log(`[Rezo Debug] Adapter: react-native`);
|
|
29
|
+
if (config.originalRequest?.headers) {
|
|
30
|
+
const headers = config.originalRequest.headers instanceof RezoHeaders ? config.originalRequest.headers.toObject() : config.originalRequest.headers;
|
|
31
|
+
console.log(`[Rezo Debug] Request Headers:`, JSON.stringify(headers, null, 2));
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (config.trackUrl) {
|
|
35
|
+
console.log(`[Rezo Track] → ${method} ${url}`);
|
|
36
|
+
}
|
|
37
|
+
},
|
|
38
|
+
retry: (config, attempt, maxRetries, statusCode, delay) => {
|
|
39
|
+
if (config.debug) {
|
|
40
|
+
console.log(`[Rezo Debug] Retry ${attempt}/${maxRetries} after status ${statusCode}${delay > 0 ? ` (waiting ${delay}ms)` : ""}`);
|
|
41
|
+
}
|
|
42
|
+
if (config.trackUrl) {
|
|
43
|
+
console.log(`[Rezo Track] ⟳ Retry ${attempt}/${maxRetries} (status ${statusCode})`);
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
maxRetries: (config, maxRetries) => {
|
|
47
|
+
if (config.debug) {
|
|
48
|
+
console.log(`[Rezo Debug] Max retries (${maxRetries}) reached, throwing error`);
|
|
49
|
+
}
|
|
50
|
+
if (config.trackUrl) {
|
|
51
|
+
console.log(`[Rezo Track] ✗ Max retries reached`);
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
response: (config, status, statusText, duration) => {
|
|
55
|
+
if (config.debug) {
|
|
56
|
+
console.log(`[Rezo Debug] Response: ${status} ${statusText} (${duration.toFixed(2)}ms)`);
|
|
57
|
+
}
|
|
58
|
+
if (config.trackUrl) {
|
|
59
|
+
console.log(`[Rezo Track] ✓ ${status} ${statusText}`);
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
responseHeaders: (config, headers) => {
|
|
63
|
+
if (config.debug) {
|
|
64
|
+
console.log(`[Rezo Debug] Response Headers:`, JSON.stringify(headers, null, 2));
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
cookies: (config, cookieCount) => {
|
|
68
|
+
if (config.debug && cookieCount > 0) {
|
|
69
|
+
console.log(`[Rezo Debug] Cookies received: ${cookieCount}`);
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
timing: (config, timing) => {
|
|
73
|
+
if (config.debug) {
|
|
74
|
+
const parts = [];
|
|
75
|
+
if (timing.ttfb)
|
|
76
|
+
parts.push(`TTFB: ${timing.ttfb.toFixed(2)}ms`);
|
|
77
|
+
if (timing.total)
|
|
78
|
+
parts.push(`Total: ${timing.total.toFixed(2)}ms`);
|
|
79
|
+
if (parts.length > 0) {
|
|
80
|
+
console.log(`[Rezo Debug] Timing: ${parts.join(" | ")}`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
},
|
|
84
|
+
complete: (config, url) => {
|
|
85
|
+
if (config.debug) {
|
|
86
|
+
console.log(`[Rezo Debug] Request complete: ${url}`);
|
|
87
|
+
console.log(`[Rezo Debug] ─────────────────────────────────────
|
|
88
|
+
`);
|
|
89
|
+
}
|
|
90
|
+
},
|
|
91
|
+
error: (config, error) => {
|
|
92
|
+
if (config.debug) {
|
|
93
|
+
console.log(`[Rezo Debug] Error: ${error instanceof Error ? error.message : error}`);
|
|
94
|
+
}
|
|
95
|
+
if (config.trackUrl) {
|
|
96
|
+
console.log(`[Rezo Track] ✗ Error: ${error instanceof Error ? error.message : error}`);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
};
|
|
20
100
|
function updateTiming(config, timing, bodySize) {
|
|
21
101
|
const now = performance.now();
|
|
22
102
|
config.timing.domainLookupStart = config.timing.startTime;
|
|
@@ -291,11 +371,25 @@ async function executeFetchRequest(fetchOptions, config, options, perform, strea
|
|
|
291
371
|
throw response;
|
|
292
372
|
}
|
|
293
373
|
if (maxRetries <= retries) {
|
|
374
|
+
debugLog.maxRetries(config, maxRetries);
|
|
294
375
|
throw response;
|
|
295
376
|
}
|
|
296
377
|
retries++;
|
|
297
|
-
|
|
298
|
-
|
|
378
|
+
const currentDelay = incrementDelay ? retryDelay * retries : retryDelay;
|
|
379
|
+
debugLog.retry(config, retries, maxRetries, response.status || 0, currentDelay);
|
|
380
|
+
if (config.hooks?.beforeRetry && config.hooks.beforeRetry.length > 0) {
|
|
381
|
+
for (const hook of config.hooks.beforeRetry) {
|
|
382
|
+
try {
|
|
383
|
+
await hook(config, response, retries);
|
|
384
|
+
} catch (hookErr) {
|
|
385
|
+
if (config.debug) {
|
|
386
|
+
console.log("[Rezo Debug] beforeRetry hook error:", hookErr);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
if (currentDelay > 0) {
|
|
392
|
+
await new Promise((resolve) => setTimeout(resolve, currentDelay));
|
|
299
393
|
}
|
|
300
394
|
}
|
|
301
395
|
config.retryAttempts++;
|
|
@@ -319,6 +413,7 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
319
413
|
config.isSecure = isSecure;
|
|
320
414
|
config.finalUrl = url;
|
|
321
415
|
config.network.protocol = isSecure ? "https" : "http";
|
|
416
|
+
debugLog.requestStart(config, url, fetchOptions.method?.toUpperCase() || "GET");
|
|
322
417
|
const reqHeaders = fetchOptions.headers instanceof RezoHeaders ? fetchOptions.headers.toObject() : fetchOptions.headers || {};
|
|
323
418
|
const headers = toFetchHeaders(reqHeaders);
|
|
324
419
|
const eventEmitter = streamResult || downloadResult || uploadResult;
|
|
@@ -380,13 +475,68 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
380
475
|
const responseHeaders = fromFetchHeaders(response.headers);
|
|
381
476
|
const contentType = response.headers.get("content-type") || "";
|
|
382
477
|
const contentLength = response.headers.get("content-length");
|
|
383
|
-
|
|
478
|
+
let cookies = {
|
|
384
479
|
array: [],
|
|
385
480
|
serialized: [],
|
|
386
481
|
netscape: "",
|
|
387
482
|
string: "",
|
|
388
483
|
setCookiesString: []
|
|
389
484
|
};
|
|
485
|
+
const setCookieHeader = response.headers.get("set-cookie");
|
|
486
|
+
if (setCookieHeader && config.useCookies) {
|
|
487
|
+
const cookieStrings = setCookieHeader.split(/,(?=[^;]+=[^;]+)/).map((s) => s.trim());
|
|
488
|
+
const tempJar = new RezoCookieJar;
|
|
489
|
+
tempJar.setCookiesSync(cookieStrings, url);
|
|
490
|
+
const parsedCookies = tempJar.cookies();
|
|
491
|
+
const acceptedCookies = [];
|
|
492
|
+
let hookError = null;
|
|
493
|
+
if (config.hooks?.beforeCookie && config.hooks.beforeCookie.length > 0) {
|
|
494
|
+
for (const cookie of parsedCookies.array) {
|
|
495
|
+
let shouldAccept = true;
|
|
496
|
+
for (const hook of config.hooks.beforeCookie) {
|
|
497
|
+
try {
|
|
498
|
+
const result = await hook({
|
|
499
|
+
cookie,
|
|
500
|
+
source: "response",
|
|
501
|
+
url,
|
|
502
|
+
isValid: true
|
|
503
|
+
}, config);
|
|
504
|
+
if (result === false) {
|
|
505
|
+
shouldAccept = false;
|
|
506
|
+
break;
|
|
507
|
+
}
|
|
508
|
+
} catch (err) {
|
|
509
|
+
hookError = err;
|
|
510
|
+
if (config.debug) {
|
|
511
|
+
console.log("[Rezo Debug] beforeCookie hook error:", err);
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
if (shouldAccept) {
|
|
516
|
+
acceptedCookies.push(cookie);
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
} else {
|
|
520
|
+
acceptedCookies.push(...parsedCookies.array);
|
|
521
|
+
}
|
|
522
|
+
const acceptedCookieStrings = acceptedCookies.map((c) => c.toSetCookieString());
|
|
523
|
+
if (config.enableCookieJar && config.cookieJar) {
|
|
524
|
+
config.cookieJar.setCookiesSync(acceptedCookieStrings, url);
|
|
525
|
+
}
|
|
526
|
+
const cookieJar = new RezoCookieJar(acceptedCookies, url);
|
|
527
|
+
cookies = cookieJar.cookies();
|
|
528
|
+
if (!hookError && config.hooks?.afterCookie && config.hooks.afterCookie.length > 0) {
|
|
529
|
+
for (const hook of config.hooks.afterCookie) {
|
|
530
|
+
try {
|
|
531
|
+
await hook(acceptedCookies, config);
|
|
532
|
+
} catch (err) {
|
|
533
|
+
if (config.debug) {
|
|
534
|
+
console.log("[Rezo Debug] afterCookie hook error:", err);
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
390
540
|
config.responseCookies = cookies;
|
|
391
541
|
if (eventEmitter) {
|
|
392
542
|
const headersEvent = {
|
|
@@ -464,6 +614,14 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
464
614
|
eventEmitter.emit("error", error);
|
|
465
615
|
return error;
|
|
466
616
|
}
|
|
617
|
+
const duration = performance.now() - timing.startTime;
|
|
618
|
+
debugLog.response(config, status, statusText, duration);
|
|
619
|
+
debugLog.responseHeaders(config, responseHeaders.toObject());
|
|
620
|
+
debugLog.cookies(config, cookies.array.length);
|
|
621
|
+
debugLog.timing(config, {
|
|
622
|
+
ttfb: timing.firstByteTime ? timing.firstByteTime - timing.startTime : undefined,
|
|
623
|
+
total: duration
|
|
624
|
+
});
|
|
467
625
|
const finalResponse = {
|
|
468
626
|
data: responseData,
|
|
469
627
|
status,
|
|
@@ -476,6 +634,7 @@ async function executeSingleRequest(config, fetchOptions, timing, streamResult,
|
|
|
476
634
|
finalUrl: url,
|
|
477
635
|
urls: buildUrlTree(config, url)
|
|
478
636
|
};
|
|
637
|
+
debugLog.complete(config, url);
|
|
479
638
|
if (streamResult) {
|
|
480
639
|
const streamFinishEvent = {
|
|
481
640
|
status,
|
package/dist/cache/index.cjs
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.LRUCache =
|
|
3
|
-
const
|
|
4
|
-
exports.DNSCache =
|
|
5
|
-
exports.getGlobalDNSCache =
|
|
6
|
-
exports.resetGlobalDNSCache =
|
|
7
|
-
const
|
|
8
|
-
exports.ResponseCache =
|
|
9
|
-
exports.normalizeResponseCacheConfig =
|
|
10
|
-
const
|
|
11
|
-
exports.FileCacher =
|
|
12
|
-
const
|
|
13
|
-
exports.UrlStore =
|
|
1
|
+
const _mod_2qk1gr = require('./lru-cache.cjs');
|
|
2
|
+
exports.LRUCache = _mod_2qk1gr.LRUCache;;
|
|
3
|
+
const _mod_4k7izn = require('./dns-cache.cjs');
|
|
4
|
+
exports.DNSCache = _mod_4k7izn.DNSCache;
|
|
5
|
+
exports.getGlobalDNSCache = _mod_4k7izn.getGlobalDNSCache;
|
|
6
|
+
exports.resetGlobalDNSCache = _mod_4k7izn.resetGlobalDNSCache;;
|
|
7
|
+
const _mod_21d2ne = require('./response-cache.cjs');
|
|
8
|
+
exports.ResponseCache = _mod_21d2ne.ResponseCache;
|
|
9
|
+
exports.normalizeResponseCacheConfig = _mod_21d2ne.normalizeResponseCacheConfig;;
|
|
10
|
+
const _mod_n5qqdj = require('./file-cacher.cjs');
|
|
11
|
+
exports.FileCacher = _mod_n5qqdj.FileCacher;;
|
|
12
|
+
const _mod_4ly8p8 = require('./url-store.cjs');
|
|
13
|
+
exports.UrlStore = _mod_4ly8p8.UrlStore;;
|
package/dist/crawler.d.ts
CHANGED
|
@@ -428,6 +428,51 @@ declare class RezoCookieJar extends TouchCookieJar {
|
|
|
428
428
|
toArray(): Cookie[];
|
|
429
429
|
toSetCookies(): string[];
|
|
430
430
|
toSerializedCookies(): SerializedCookie[];
|
|
431
|
+
/**
|
|
432
|
+
* Get cookies for a request URL with proper browser-like matching.
|
|
433
|
+
* This method properly handles:
|
|
434
|
+
* - Domain matching (exact or parent domain)
|
|
435
|
+
* - Path matching (cookie path must be prefix of request path)
|
|
436
|
+
* - Secure flag (secure cookies only over HTTPS)
|
|
437
|
+
* - Expiry (expired cookies not returned)
|
|
438
|
+
*
|
|
439
|
+
* @param requestUrl - The full request URL including path (e.g., 'https://example.com/api/users')
|
|
440
|
+
* @returns Array of Cookie objects that should be sent with the request
|
|
441
|
+
*/
|
|
442
|
+
getCookiesForRequest(requestUrl: string | URL): Cookie[];
|
|
443
|
+
/**
|
|
444
|
+
* Get the Cookie header value for a request URL with proper browser-like matching.
|
|
445
|
+
* Returns cookies in the format: "key1=value1; key2=value2"
|
|
446
|
+
*
|
|
447
|
+
* This is the browser-accurate way to build the Cookie header, properly filtering
|
|
448
|
+
* cookies by domain, path, secure flag, and expiry.
|
|
449
|
+
*
|
|
450
|
+
* @param requestUrl - The full request URL including path (e.g., 'https://example.com/api/users')
|
|
451
|
+
* @returns Cookie header string in "key=value; key=value" format
|
|
452
|
+
*/
|
|
453
|
+
getCookieHeader(requestUrl: string | URL): string;
|
|
454
|
+
/**
|
|
455
|
+
* Debug method to show which cookies would be sent for a given URL.
|
|
456
|
+
* Useful for troubleshooting cookie matching issues.
|
|
457
|
+
*
|
|
458
|
+
* @param requestUrl - The full request URL including path
|
|
459
|
+
* @returns Object with matching cookies and the Cookie header that would be sent
|
|
460
|
+
*/
|
|
461
|
+
debugCookiesForRequest(requestUrl: string | URL): {
|
|
462
|
+
url: string;
|
|
463
|
+
matchingCookies: Array<{
|
|
464
|
+
key: string;
|
|
465
|
+
value: string;
|
|
466
|
+
domain: string;
|
|
467
|
+
path: string;
|
|
468
|
+
}>;
|
|
469
|
+
cookieHeader: string;
|
|
470
|
+
allCookies: Array<{
|
|
471
|
+
key: string;
|
|
472
|
+
domain: string;
|
|
473
|
+
path: string;
|
|
474
|
+
}>;
|
|
475
|
+
};
|
|
431
476
|
setCookiesSync(setCookieArray: string[]): Cookies;
|
|
432
477
|
setCookiesSync(setCookieArray: string[], url: string): Cookies;
|
|
433
478
|
setCookiesSync(cookiesString: string): Cookies;
|
package/dist/entries/crawler.cjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Crawler =
|
|
3
|
-
const
|
|
4
|
-
exports.CrawlerOptions =
|
|
5
|
-
exports.Domain =
|
|
1
|
+
const _mod_x66gaq = require('../plugin/crawler.cjs');
|
|
2
|
+
exports.Crawler = _mod_x66gaq.Crawler;;
|
|
3
|
+
const _mod_cnlc8b = require('../plugin/crawler-options.cjs');
|
|
4
|
+
exports.CrawlerOptions = _mod_cnlc8b.CrawlerOptions;
|
|
5
|
+
exports.Domain = _mod_cnlc8b.Domain;;
|
package/dist/index.cjs
CHANGED
|
@@ -1,27 +1,27 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Rezo =
|
|
3
|
-
exports.createRezoInstance =
|
|
4
|
-
exports.createDefaultInstance =
|
|
5
|
-
const
|
|
6
|
-
exports.RezoError =
|
|
7
|
-
exports.RezoErrorCode =
|
|
8
|
-
const
|
|
9
|
-
exports.RezoHeaders =
|
|
10
|
-
const
|
|
11
|
-
exports.RezoFormData =
|
|
12
|
-
const
|
|
13
|
-
exports.RezoCookieJar =
|
|
14
|
-
exports.Cookie =
|
|
15
|
-
const
|
|
16
|
-
exports.createDefaultHooks =
|
|
17
|
-
exports.mergeHooks =
|
|
18
|
-
const
|
|
19
|
-
exports.ProxyManager =
|
|
20
|
-
const
|
|
21
|
-
exports.RezoQueue =
|
|
22
|
-
exports.HttpQueue =
|
|
23
|
-
exports.Priority =
|
|
24
|
-
exports.HttpMethodPriority =
|
|
1
|
+
const _mod_04noq8 = require('./core/rezo.cjs');
|
|
2
|
+
exports.Rezo = _mod_04noq8.Rezo;
|
|
3
|
+
exports.createRezoInstance = _mod_04noq8.createRezoInstance;
|
|
4
|
+
exports.createDefaultInstance = _mod_04noq8.createDefaultInstance;;
|
|
5
|
+
const _mod_gt2ibf = require('./errors/rezo-error.cjs');
|
|
6
|
+
exports.RezoError = _mod_gt2ibf.RezoError;
|
|
7
|
+
exports.RezoErrorCode = _mod_gt2ibf.RezoErrorCode;;
|
|
8
|
+
const _mod_2hepie = require('./utils/headers.cjs');
|
|
9
|
+
exports.RezoHeaders = _mod_2hepie.RezoHeaders;;
|
|
10
|
+
const _mod_99p45u = require('./utils/form-data.cjs');
|
|
11
|
+
exports.RezoFormData = _mod_99p45u.RezoFormData;;
|
|
12
|
+
const _mod_7vtdxn = require('./utils/cookies.cjs');
|
|
13
|
+
exports.RezoCookieJar = _mod_7vtdxn.RezoCookieJar;
|
|
14
|
+
exports.Cookie = _mod_7vtdxn.Cookie;;
|
|
15
|
+
const _mod_q6z263 = require('./core/hooks.cjs');
|
|
16
|
+
exports.createDefaultHooks = _mod_q6z263.createDefaultHooks;
|
|
17
|
+
exports.mergeHooks = _mod_q6z263.mergeHooks;;
|
|
18
|
+
const _mod_5cwf7e = require('./proxy/manager.cjs');
|
|
19
|
+
exports.ProxyManager = _mod_5cwf7e.ProxyManager;;
|
|
20
|
+
const _mod_ewxgow = require('./queue/index.cjs');
|
|
21
|
+
exports.RezoQueue = _mod_ewxgow.RezoQueue;
|
|
22
|
+
exports.HttpQueue = _mod_ewxgow.HttpQueue;
|
|
23
|
+
exports.Priority = _mod_ewxgow.Priority;
|
|
24
|
+
exports.HttpMethodPriority = _mod_ewxgow.HttpMethodPriority;;
|
|
25
25
|
const { RezoError } = require('./errors/rezo-error.cjs');
|
|
26
26
|
const isRezoError = exports.isRezoError = RezoError.isRezoError;
|
|
27
27
|
const Cancel = exports.Cancel = RezoError;
|
package/dist/index.d.ts
CHANGED
|
@@ -205,6 +205,51 @@ export declare class RezoCookieJar extends TouchCookieJar {
|
|
|
205
205
|
toArray(): Cookie[];
|
|
206
206
|
toSetCookies(): string[];
|
|
207
207
|
toSerializedCookies(): SerializedCookie[];
|
|
208
|
+
/**
|
|
209
|
+
* Get cookies for a request URL with proper browser-like matching.
|
|
210
|
+
* This method properly handles:
|
|
211
|
+
* - Domain matching (exact or parent domain)
|
|
212
|
+
* - Path matching (cookie path must be prefix of request path)
|
|
213
|
+
* - Secure flag (secure cookies only over HTTPS)
|
|
214
|
+
* - Expiry (expired cookies not returned)
|
|
215
|
+
*
|
|
216
|
+
* @param requestUrl - The full request URL including path (e.g., 'https://example.com/api/users')
|
|
217
|
+
* @returns Array of Cookie objects that should be sent with the request
|
|
218
|
+
*/
|
|
219
|
+
getCookiesForRequest(requestUrl: string | URL): Cookie[];
|
|
220
|
+
/**
|
|
221
|
+
* Get the Cookie header value for a request URL with proper browser-like matching.
|
|
222
|
+
* Returns cookies in the format: "key1=value1; key2=value2"
|
|
223
|
+
*
|
|
224
|
+
* This is the browser-accurate way to build the Cookie header, properly filtering
|
|
225
|
+
* cookies by domain, path, secure flag, and expiry.
|
|
226
|
+
*
|
|
227
|
+
* @param requestUrl - The full request URL including path (e.g., 'https://example.com/api/users')
|
|
228
|
+
* @returns Cookie header string in "key=value; key=value" format
|
|
229
|
+
*/
|
|
230
|
+
getCookieHeader(requestUrl: string | URL): string;
|
|
231
|
+
/**
|
|
232
|
+
* Debug method to show which cookies would be sent for a given URL.
|
|
233
|
+
* Useful for troubleshooting cookie matching issues.
|
|
234
|
+
*
|
|
235
|
+
* @param requestUrl - The full request URL including path
|
|
236
|
+
* @returns Object with matching cookies and the Cookie header that would be sent
|
|
237
|
+
*/
|
|
238
|
+
debugCookiesForRequest(requestUrl: string | URL): {
|
|
239
|
+
url: string;
|
|
240
|
+
matchingCookies: Array<{
|
|
241
|
+
key: string;
|
|
242
|
+
value: string;
|
|
243
|
+
domain: string;
|
|
244
|
+
path: string;
|
|
245
|
+
}>;
|
|
246
|
+
cookieHeader: string;
|
|
247
|
+
allCookies: Array<{
|
|
248
|
+
key: string;
|
|
249
|
+
domain: string;
|
|
250
|
+
path: string;
|
|
251
|
+
}>;
|
|
252
|
+
};
|
|
208
253
|
setCookiesSync(setCookieArray: string[]): Cookies;
|
|
209
254
|
setCookiesSync(setCookieArray: string[], url: string): Cookies;
|
|
210
255
|
setCookiesSync(cookiesString: string): Cookies;
|