rezo 1.0.43 → 1.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/entries/curl.d.ts +115 -0
- package/dist/adapters/entries/fetch.d.ts +115 -0
- package/dist/adapters/entries/http.d.ts +115 -0
- package/dist/adapters/entries/http2.d.ts +115 -0
- package/dist/adapters/entries/react-native.d.ts +115 -0
- package/dist/adapters/entries/xhr.d.ts +115 -0
- package/dist/adapters/fetch.cjs +18 -0
- package/dist/adapters/fetch.js +18 -0
- package/dist/adapters/http.cjs +18 -0
- package/dist/adapters/http.js +18 -0
- package/dist/adapters/http2.cjs +18 -0
- package/dist/adapters/http2.js +18 -0
- package/dist/adapters/index.cjs +6 -6
- package/dist/adapters/xhr.cjs +19 -0
- package/dist/adapters/xhr.js +19 -0
- package/dist/cache/index.cjs +9 -15
- package/dist/cache/index.js +0 -3
- package/dist/core/hooks.cjs +4 -2
- package/dist/core/hooks.js +4 -2
- package/dist/crawler/addon/decodo/index.cjs +1 -0
- package/dist/crawler/addon/decodo/index.js +1 -0
- package/dist/crawler/crawler-options.cjs +1 -0
- package/dist/crawler/crawler-options.js +1 -0
- package/dist/{plugin → crawler}/crawler.cjs +392 -32
- package/dist/{plugin → crawler}/crawler.js +392 -32
- package/dist/crawler/index.cjs +40 -0
- package/dist/{plugin → crawler}/index.js +4 -2
- package/dist/crawler/plugin/file-cacher.cjs +19 -0
- package/dist/crawler/plugin/file-cacher.js +19 -0
- package/dist/crawler/plugin/index.cjs +1 -0
- package/dist/crawler/plugin/index.js +1 -0
- package/dist/crawler/plugin/navigation-history.cjs +43 -0
- package/dist/crawler/plugin/navigation-history.js +43 -0
- package/dist/crawler/plugin/robots-txt.cjs +2 -0
- package/dist/crawler/plugin/robots-txt.js +2 -0
- package/dist/crawler/plugin/url-store.cjs +18 -0
- package/dist/crawler/plugin/url-store.js +18 -0
- package/dist/crawler.d.ts +430 -172
- package/dist/entries/crawler.cjs +5 -5
- package/dist/entries/crawler.js +2 -2
- package/dist/index.cjs +27 -27
- package/dist/index.d.ts +115 -0
- package/dist/internal/agents/index.cjs +10 -10
- package/dist/platform/browser.d.ts +115 -0
- package/dist/platform/bun.d.ts +115 -0
- package/dist/platform/deno.d.ts +115 -0
- package/dist/platform/node.d.ts +115 -0
- package/dist/platform/react-native.d.ts +115 -0
- package/dist/platform/worker.d.ts +115 -0
- package/dist/proxy/index.cjs +5 -5
- package/dist/proxy/index.js +1 -1
- package/dist/queue/index.cjs +8 -8
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/utils/rate-limit-wait.cjs +217 -0
- package/dist/utils/rate-limit-wait.js +208 -0
- package/package.json +2 -6
- package/dist/cache/file-cacher.cjs +0 -270
- package/dist/cache/file-cacher.js +0 -267
- package/dist/cache/navigation-history.cjs +0 -298
- package/dist/cache/navigation-history.js +0 -296
- package/dist/cache/url-store.cjs +0 -294
- package/dist/cache/url-store.js +0 -291
- package/dist/plugin/addon/decodo/index.cjs +0 -1
- package/dist/plugin/addon/decodo/index.js +0 -1
- package/dist/plugin/crawler-options.cjs +0 -1
- package/dist/plugin/crawler-options.js +0 -1
- package/dist/plugin/index.cjs +0 -36
- /package/dist/{plugin → crawler}/addon/decodo/options.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/decodo/options.js +0 -0
- /package/dist/{plugin → crawler}/addon/decodo/types.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/decodo/types.js +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/index.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/index.js +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/options.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/options.js +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/types.cjs +0 -0
- /package/dist/{plugin → crawler}/addon/oxylabs/types.js +0 -0
- /package/dist/{plugin → crawler}/scraper.cjs +0 -0
- /package/dist/{plugin → crawler}/scraper.js +0 -0
package/dist/platform/node.d.ts
CHANGED
|
@@ -1416,6 +1416,35 @@ export type OnTimeoutHook = (event: TimeoutEvent, config: RezoConfig) => void;
|
|
|
1416
1416
|
* Use for cleanup, logging
|
|
1417
1417
|
*/
|
|
1418
1418
|
export type OnAbortHook = (event: AbortEvent, config: RezoConfig) => void;
|
|
1419
|
+
/**
|
|
1420
|
+
* Rate limit wait event data - fired when waiting due to rate limiting
|
|
1421
|
+
*/
|
|
1422
|
+
export interface RateLimitWaitEvent {
|
|
1423
|
+
/** HTTP status code that triggered the wait (e.g., 429, 503) */
|
|
1424
|
+
status: number;
|
|
1425
|
+
/** Time to wait in milliseconds */
|
|
1426
|
+
waitTime: number;
|
|
1427
|
+
/** Current wait attempt number (1-indexed) */
|
|
1428
|
+
attempt: number;
|
|
1429
|
+
/** Maximum wait attempts configured */
|
|
1430
|
+
maxAttempts: number;
|
|
1431
|
+
/** Where the wait time was extracted from */
|
|
1432
|
+
source: "header" | "body" | "function" | "default";
|
|
1433
|
+
/** The header or body path used (if applicable) */
|
|
1434
|
+
sourcePath?: string;
|
|
1435
|
+
/** URL being requested */
|
|
1436
|
+
url: string;
|
|
1437
|
+
/** HTTP method of the request */
|
|
1438
|
+
method: string;
|
|
1439
|
+
/** Timestamp when the wait started */
|
|
1440
|
+
timestamp: number;
|
|
1441
|
+
}
|
|
1442
|
+
/**
|
|
1443
|
+
* Hook called when rate limit wait occurs
|
|
1444
|
+
* Informational only - cannot abort the wait
|
|
1445
|
+
* Use for logging, monitoring, alerting
|
|
1446
|
+
*/
|
|
1447
|
+
export type OnRateLimitWaitHook = (event: RateLimitWaitEvent, config: RezoConfig) => void | Promise<void>;
|
|
1419
1448
|
/**
|
|
1420
1449
|
* Hook called before a proxy is selected
|
|
1421
1450
|
* Can return a specific proxy to override selection
|
|
@@ -1496,6 +1525,7 @@ export interface RezoHooks {
|
|
|
1496
1525
|
onTls: OnTlsHook[];
|
|
1497
1526
|
onTimeout: OnTimeoutHook[];
|
|
1498
1527
|
onAbort: OnAbortHook[];
|
|
1528
|
+
onRateLimitWait: OnRateLimitWaitHook[];
|
|
1499
1529
|
}
|
|
1500
1530
|
/**
|
|
1501
1531
|
* Create empty hooks object with all arrays initialized
|
|
@@ -2426,6 +2456,91 @@ export interface RezoRequestConfig<D = any> {
|
|
|
2426
2456
|
/** Weather to stop or continue retry when certain condition is met*/
|
|
2427
2457
|
condition?: (error: RezoError) => boolean | Promise<boolean>;
|
|
2428
2458
|
};
|
|
2459
|
+
/**
|
|
2460
|
+
* Rate limit wait configuration - wait and retry when receiving rate limit responses.
|
|
2461
|
+
*
|
|
2462
|
+
* This feature runs BEFORE the retry system. When a rate-limiting status code is received,
|
|
2463
|
+
* the client will wait for the specified time and automatically retry the request.
|
|
2464
|
+
*
|
|
2465
|
+
* **Basic Usage:**
|
|
2466
|
+
* - `waitOnStatus: true` - Enable waiting on 429 status (default behavior)
|
|
2467
|
+
* - `waitOnStatus: [429, 503]` - Enable waiting on specific status codes
|
|
2468
|
+
*
|
|
2469
|
+
* **Wait Time Sources:**
|
|
2470
|
+
* - `'retry-after'` - Use standard Retry-After header (default)
|
|
2471
|
+
* - `{ header: 'X-RateLimit-Reset' }` - Use custom header
|
|
2472
|
+
* - `{ body: 'retry_after' }` - Extract from JSON response body
|
|
2473
|
+
* - Custom function for complex logic
|
|
2474
|
+
*
|
|
2475
|
+
* @example
|
|
2476
|
+
* ```typescript
|
|
2477
|
+
* // Wait on 429 using Retry-After header
|
|
2478
|
+
* await rezo.get(url, { waitOnStatus: true });
|
|
2479
|
+
*
|
|
2480
|
+
* // Wait on 429 using custom header
|
|
2481
|
+
* await rezo.get(url, {
|
|
2482
|
+
* waitOnStatus: true,
|
|
2483
|
+
* waitTimeSource: { header: 'X-RateLimit-Reset' }
|
|
2484
|
+
* });
|
|
2485
|
+
*
|
|
2486
|
+
* // Wait on 429 extracting time from JSON body
|
|
2487
|
+
* await rezo.get(url, {
|
|
2488
|
+
* waitOnStatus: true,
|
|
2489
|
+
* waitTimeSource: { body: 'data.retry_after' }
|
|
2490
|
+
* });
|
|
2491
|
+
*
|
|
2492
|
+
* // Custom function for complex APIs
|
|
2493
|
+
* await rezo.get(url, {
|
|
2494
|
+
* waitOnStatus: [429, 503],
|
|
2495
|
+
* waitTimeSource: (response) => {
|
|
2496
|
+
* const reset = response.headers.get('x-ratelimit-reset');
|
|
2497
|
+
* return reset ? parseInt(reset) - Math.floor(Date.now() / 1000) : null;
|
|
2498
|
+
* }
|
|
2499
|
+
* });
|
|
2500
|
+
* ```
|
|
2501
|
+
*/
|
|
2502
|
+
waitOnStatus?: boolean | number[];
|
|
2503
|
+
/**
|
|
2504
|
+
* Where to extract the wait time from when rate-limited.
|
|
2505
|
+
*
|
|
2506
|
+
* - `'retry-after'` - Standard Retry-After header (default)
|
|
2507
|
+
* - `{ header: string }` - Custom header name (e.g., 'X-RateLimit-Reset')
|
|
2508
|
+
* - `{ body: string }` - JSON path in response body (e.g., 'data.retry_after', 'wait_seconds')
|
|
2509
|
+
* - Function - Custom logic receiving the response, return seconds to wait or null
|
|
2510
|
+
*
|
|
2511
|
+
* @default 'retry-after'
|
|
2512
|
+
*/
|
|
2513
|
+
waitTimeSource?: "retry-after" | {
|
|
2514
|
+
header: string;
|
|
2515
|
+
} | {
|
|
2516
|
+
body: string;
|
|
2517
|
+
} | ((response: {
|
|
2518
|
+
status: number;
|
|
2519
|
+
headers: RezoHeaders;
|
|
2520
|
+
data?: any;
|
|
2521
|
+
}) => number | null);
|
|
2522
|
+
/**
|
|
2523
|
+
* Maximum time to wait for rate limit in milliseconds.
|
|
2524
|
+
* If the extracted wait time exceeds this, the request will fail instead of waiting.
|
|
2525
|
+
* Set to 0 for unlimited wait time.
|
|
2526
|
+
*
|
|
2527
|
+
* @default 60000 (60 seconds)
|
|
2528
|
+
*/
|
|
2529
|
+
maxWaitTime?: number;
|
|
2530
|
+
/**
|
|
2531
|
+
* Default wait time in milliseconds if the wait time source returns nothing.
|
|
2532
|
+
* Used as fallback when Retry-After header or body path is not present.
|
|
2533
|
+
*
|
|
2534
|
+
* @default 1000 (1 second)
|
|
2535
|
+
*/
|
|
2536
|
+
defaultWaitTime?: number;
|
|
2537
|
+
/**
|
|
2538
|
+
* Maximum number of wait attempts before giving up.
|
|
2539
|
+
* After this many waits, the request will proceed to retry logic or fail.
|
|
2540
|
+
*
|
|
2541
|
+
* @default 3
|
|
2542
|
+
*/
|
|
2543
|
+
maxWaitAttempts?: number;
|
|
2429
2544
|
/** Whether to use a secure context for HTTPS requests */
|
|
2430
2545
|
useSecureContext?: boolean;
|
|
2431
2546
|
/** Custom secure context for TLS connections */
|
|
@@ -1416,6 +1416,35 @@ export type OnTimeoutHook = (event: TimeoutEvent, config: RezoConfig) => void;
|
|
|
1416
1416
|
* Use for cleanup, logging
|
|
1417
1417
|
*/
|
|
1418
1418
|
export type OnAbortHook = (event: AbortEvent, config: RezoConfig) => void;
|
|
1419
|
+
/**
|
|
1420
|
+
* Rate limit wait event data - fired when waiting due to rate limiting
|
|
1421
|
+
*/
|
|
1422
|
+
export interface RateLimitWaitEvent {
|
|
1423
|
+
/** HTTP status code that triggered the wait (e.g., 429, 503) */
|
|
1424
|
+
status: number;
|
|
1425
|
+
/** Time to wait in milliseconds */
|
|
1426
|
+
waitTime: number;
|
|
1427
|
+
/** Current wait attempt number (1-indexed) */
|
|
1428
|
+
attempt: number;
|
|
1429
|
+
/** Maximum wait attempts configured */
|
|
1430
|
+
maxAttempts: number;
|
|
1431
|
+
/** Where the wait time was extracted from */
|
|
1432
|
+
source: "header" | "body" | "function" | "default";
|
|
1433
|
+
/** The header or body path used (if applicable) */
|
|
1434
|
+
sourcePath?: string;
|
|
1435
|
+
/** URL being requested */
|
|
1436
|
+
url: string;
|
|
1437
|
+
/** HTTP method of the request */
|
|
1438
|
+
method: string;
|
|
1439
|
+
/** Timestamp when the wait started */
|
|
1440
|
+
timestamp: number;
|
|
1441
|
+
}
|
|
1442
|
+
/**
|
|
1443
|
+
* Hook called when rate limit wait occurs
|
|
1444
|
+
* Informational only - cannot abort the wait
|
|
1445
|
+
* Use for logging, monitoring, alerting
|
|
1446
|
+
*/
|
|
1447
|
+
export type OnRateLimitWaitHook = (event: RateLimitWaitEvent, config: RezoConfig) => void | Promise<void>;
|
|
1419
1448
|
/**
|
|
1420
1449
|
* Hook called before a proxy is selected
|
|
1421
1450
|
* Can return a specific proxy to override selection
|
|
@@ -1496,6 +1525,7 @@ export interface RezoHooks {
|
|
|
1496
1525
|
onTls: OnTlsHook[];
|
|
1497
1526
|
onTimeout: OnTimeoutHook[];
|
|
1498
1527
|
onAbort: OnAbortHook[];
|
|
1528
|
+
onRateLimitWait: OnRateLimitWaitHook[];
|
|
1499
1529
|
}
|
|
1500
1530
|
/**
|
|
1501
1531
|
* Create empty hooks object with all arrays initialized
|
|
@@ -2426,6 +2456,91 @@ export interface RezoRequestConfig<D = any> {
|
|
|
2426
2456
|
/** Weather to stop or continue retry when certain condition is met*/
|
|
2427
2457
|
condition?: (error: RezoError) => boolean | Promise<boolean>;
|
|
2428
2458
|
};
|
|
2459
|
+
/**
|
|
2460
|
+
* Rate limit wait configuration - wait and retry when receiving rate limit responses.
|
|
2461
|
+
*
|
|
2462
|
+
* This feature runs BEFORE the retry system. When a rate-limiting status code is received,
|
|
2463
|
+
* the client will wait for the specified time and automatically retry the request.
|
|
2464
|
+
*
|
|
2465
|
+
* **Basic Usage:**
|
|
2466
|
+
* - `waitOnStatus: true` - Enable waiting on 429 status (default behavior)
|
|
2467
|
+
* - `waitOnStatus: [429, 503]` - Enable waiting on specific status codes
|
|
2468
|
+
*
|
|
2469
|
+
* **Wait Time Sources:**
|
|
2470
|
+
* - `'retry-after'` - Use standard Retry-After header (default)
|
|
2471
|
+
* - `{ header: 'X-RateLimit-Reset' }` - Use custom header
|
|
2472
|
+
* - `{ body: 'retry_after' }` - Extract from JSON response body
|
|
2473
|
+
* - Custom function for complex logic
|
|
2474
|
+
*
|
|
2475
|
+
* @example
|
|
2476
|
+
* ```typescript
|
|
2477
|
+
* // Wait on 429 using Retry-After header
|
|
2478
|
+
* await rezo.get(url, { waitOnStatus: true });
|
|
2479
|
+
*
|
|
2480
|
+
* // Wait on 429 using custom header
|
|
2481
|
+
* await rezo.get(url, {
|
|
2482
|
+
* waitOnStatus: true,
|
|
2483
|
+
* waitTimeSource: { header: 'X-RateLimit-Reset' }
|
|
2484
|
+
* });
|
|
2485
|
+
*
|
|
2486
|
+
* // Wait on 429 extracting time from JSON body
|
|
2487
|
+
* await rezo.get(url, {
|
|
2488
|
+
* waitOnStatus: true,
|
|
2489
|
+
* waitTimeSource: { body: 'data.retry_after' }
|
|
2490
|
+
* });
|
|
2491
|
+
*
|
|
2492
|
+
* // Custom function for complex APIs
|
|
2493
|
+
* await rezo.get(url, {
|
|
2494
|
+
* waitOnStatus: [429, 503],
|
|
2495
|
+
* waitTimeSource: (response) => {
|
|
2496
|
+
* const reset = response.headers.get('x-ratelimit-reset');
|
|
2497
|
+
* return reset ? parseInt(reset) - Math.floor(Date.now() / 1000) : null;
|
|
2498
|
+
* }
|
|
2499
|
+
* });
|
|
2500
|
+
* ```
|
|
2501
|
+
*/
|
|
2502
|
+
waitOnStatus?: boolean | number[];
|
|
2503
|
+
/**
|
|
2504
|
+
* Where to extract the wait time from when rate-limited.
|
|
2505
|
+
*
|
|
2506
|
+
* - `'retry-after'` - Standard Retry-After header (default)
|
|
2507
|
+
* - `{ header: string }` - Custom header name (e.g., 'X-RateLimit-Reset')
|
|
2508
|
+
* - `{ body: string }` - JSON path in response body (e.g., 'data.retry_after', 'wait_seconds')
|
|
2509
|
+
* - Function - Custom logic receiving the response, return seconds to wait or null
|
|
2510
|
+
*
|
|
2511
|
+
* @default 'retry-after'
|
|
2512
|
+
*/
|
|
2513
|
+
waitTimeSource?: "retry-after" | {
|
|
2514
|
+
header: string;
|
|
2515
|
+
} | {
|
|
2516
|
+
body: string;
|
|
2517
|
+
} | ((response: {
|
|
2518
|
+
status: number;
|
|
2519
|
+
headers: RezoHeaders;
|
|
2520
|
+
data?: any;
|
|
2521
|
+
}) => number | null);
|
|
2522
|
+
/**
|
|
2523
|
+
* Maximum time to wait for rate limit in milliseconds.
|
|
2524
|
+
* If the extracted wait time exceeds this, the request will fail instead of waiting.
|
|
2525
|
+
* Set to 0 for unlimited wait time.
|
|
2526
|
+
*
|
|
2527
|
+
* @default 60000 (60 seconds)
|
|
2528
|
+
*/
|
|
2529
|
+
maxWaitTime?: number;
|
|
2530
|
+
/**
|
|
2531
|
+
* Default wait time in milliseconds if the wait time source returns nothing.
|
|
2532
|
+
* Used as fallback when Retry-After header or body path is not present.
|
|
2533
|
+
*
|
|
2534
|
+
* @default 1000 (1 second)
|
|
2535
|
+
*/
|
|
2536
|
+
defaultWaitTime?: number;
|
|
2537
|
+
/**
|
|
2538
|
+
* Maximum number of wait attempts before giving up.
|
|
2539
|
+
* After this many waits, the request will proceed to retry logic or fail.
|
|
2540
|
+
*
|
|
2541
|
+
* @default 3
|
|
2542
|
+
*/
|
|
2543
|
+
maxWaitAttempts?: number;
|
|
2429
2544
|
/** Whether to use a secure context for HTTPS requests */
|
|
2430
2545
|
useSecureContext?: boolean;
|
|
2431
2546
|
/** Custom secure context for TLS connections */
|
|
@@ -1416,6 +1416,35 @@ export type OnTimeoutHook = (event: TimeoutEvent, config: RezoConfig) => void;
|
|
|
1416
1416
|
* Use for cleanup, logging
|
|
1417
1417
|
*/
|
|
1418
1418
|
export type OnAbortHook = (event: AbortEvent, config: RezoConfig) => void;
|
|
1419
|
+
/**
|
|
1420
|
+
* Rate limit wait event data - fired when waiting due to rate limiting
|
|
1421
|
+
*/
|
|
1422
|
+
export interface RateLimitWaitEvent {
|
|
1423
|
+
/** HTTP status code that triggered the wait (e.g., 429, 503) */
|
|
1424
|
+
status: number;
|
|
1425
|
+
/** Time to wait in milliseconds */
|
|
1426
|
+
waitTime: number;
|
|
1427
|
+
/** Current wait attempt number (1-indexed) */
|
|
1428
|
+
attempt: number;
|
|
1429
|
+
/** Maximum wait attempts configured */
|
|
1430
|
+
maxAttempts: number;
|
|
1431
|
+
/** Where the wait time was extracted from */
|
|
1432
|
+
source: "header" | "body" | "function" | "default";
|
|
1433
|
+
/** The header or body path used (if applicable) */
|
|
1434
|
+
sourcePath?: string;
|
|
1435
|
+
/** URL being requested */
|
|
1436
|
+
url: string;
|
|
1437
|
+
/** HTTP method of the request */
|
|
1438
|
+
method: string;
|
|
1439
|
+
/** Timestamp when the wait started */
|
|
1440
|
+
timestamp: number;
|
|
1441
|
+
}
|
|
1442
|
+
/**
|
|
1443
|
+
* Hook called when rate limit wait occurs
|
|
1444
|
+
* Informational only - cannot abort the wait
|
|
1445
|
+
* Use for logging, monitoring, alerting
|
|
1446
|
+
*/
|
|
1447
|
+
export type OnRateLimitWaitHook = (event: RateLimitWaitEvent, config: RezoConfig) => void | Promise<void>;
|
|
1419
1448
|
/**
|
|
1420
1449
|
* Hook called before a proxy is selected
|
|
1421
1450
|
* Can return a specific proxy to override selection
|
|
@@ -1496,6 +1525,7 @@ export interface RezoHooks {
|
|
|
1496
1525
|
onTls: OnTlsHook[];
|
|
1497
1526
|
onTimeout: OnTimeoutHook[];
|
|
1498
1527
|
onAbort: OnAbortHook[];
|
|
1528
|
+
onRateLimitWait: OnRateLimitWaitHook[];
|
|
1499
1529
|
}
|
|
1500
1530
|
/**
|
|
1501
1531
|
* Create empty hooks object with all arrays initialized
|
|
@@ -2426,6 +2456,91 @@ export interface RezoRequestConfig<D = any> {
|
|
|
2426
2456
|
/** Weather to stop or continue retry when certain condition is met*/
|
|
2427
2457
|
condition?: (error: RezoError) => boolean | Promise<boolean>;
|
|
2428
2458
|
};
|
|
2459
|
+
/**
|
|
2460
|
+
* Rate limit wait configuration - wait and retry when receiving rate limit responses.
|
|
2461
|
+
*
|
|
2462
|
+
* This feature runs BEFORE the retry system. When a rate-limiting status code is received,
|
|
2463
|
+
* the client will wait for the specified time and automatically retry the request.
|
|
2464
|
+
*
|
|
2465
|
+
* **Basic Usage:**
|
|
2466
|
+
* - `waitOnStatus: true` - Enable waiting on 429 status (default behavior)
|
|
2467
|
+
* - `waitOnStatus: [429, 503]` - Enable waiting on specific status codes
|
|
2468
|
+
*
|
|
2469
|
+
* **Wait Time Sources:**
|
|
2470
|
+
* - `'retry-after'` - Use standard Retry-After header (default)
|
|
2471
|
+
* - `{ header: 'X-RateLimit-Reset' }` - Use custom header
|
|
2472
|
+
* - `{ body: 'retry_after' }` - Extract from JSON response body
|
|
2473
|
+
* - Custom function for complex logic
|
|
2474
|
+
*
|
|
2475
|
+
* @example
|
|
2476
|
+
* ```typescript
|
|
2477
|
+
* // Wait on 429 using Retry-After header
|
|
2478
|
+
* await rezo.get(url, { waitOnStatus: true });
|
|
2479
|
+
*
|
|
2480
|
+
* // Wait on 429 using custom header
|
|
2481
|
+
* await rezo.get(url, {
|
|
2482
|
+
* waitOnStatus: true,
|
|
2483
|
+
* waitTimeSource: { header: 'X-RateLimit-Reset' }
|
|
2484
|
+
* });
|
|
2485
|
+
*
|
|
2486
|
+
* // Wait on 429 extracting time from JSON body
|
|
2487
|
+
* await rezo.get(url, {
|
|
2488
|
+
* waitOnStatus: true,
|
|
2489
|
+
* waitTimeSource: { body: 'data.retry_after' }
|
|
2490
|
+
* });
|
|
2491
|
+
*
|
|
2492
|
+
* // Custom function for complex APIs
|
|
2493
|
+
* await rezo.get(url, {
|
|
2494
|
+
* waitOnStatus: [429, 503],
|
|
2495
|
+
* waitTimeSource: (response) => {
|
|
2496
|
+
* const reset = response.headers.get('x-ratelimit-reset');
|
|
2497
|
+
* return reset ? parseInt(reset) - Math.floor(Date.now() / 1000) : null;
|
|
2498
|
+
* }
|
|
2499
|
+
* });
|
|
2500
|
+
* ```
|
|
2501
|
+
*/
|
|
2502
|
+
waitOnStatus?: boolean | number[];
|
|
2503
|
+
/**
|
|
2504
|
+
* Where to extract the wait time from when rate-limited.
|
|
2505
|
+
*
|
|
2506
|
+
* - `'retry-after'` - Standard Retry-After header (default)
|
|
2507
|
+
* - `{ header: string }` - Custom header name (e.g., 'X-RateLimit-Reset')
|
|
2508
|
+
* - `{ body: string }` - JSON path in response body (e.g., 'data.retry_after', 'wait_seconds')
|
|
2509
|
+
* - Function - Custom logic receiving the response, return seconds to wait or null
|
|
2510
|
+
*
|
|
2511
|
+
* @default 'retry-after'
|
|
2512
|
+
*/
|
|
2513
|
+
waitTimeSource?: "retry-after" | {
|
|
2514
|
+
header: string;
|
|
2515
|
+
} | {
|
|
2516
|
+
body: string;
|
|
2517
|
+
} | ((response: {
|
|
2518
|
+
status: number;
|
|
2519
|
+
headers: RezoHeaders;
|
|
2520
|
+
data?: any;
|
|
2521
|
+
}) => number | null);
|
|
2522
|
+
/**
|
|
2523
|
+
* Maximum time to wait for rate limit in milliseconds.
|
|
2524
|
+
* If the extracted wait time exceeds this, the request will fail instead of waiting.
|
|
2525
|
+
* Set to 0 for unlimited wait time.
|
|
2526
|
+
*
|
|
2527
|
+
* @default 60000 (60 seconds)
|
|
2528
|
+
*/
|
|
2529
|
+
maxWaitTime?: number;
|
|
2530
|
+
/**
|
|
2531
|
+
* Default wait time in milliseconds if the wait time source returns nothing.
|
|
2532
|
+
* Used as fallback when Retry-After header or body path is not present.
|
|
2533
|
+
*
|
|
2534
|
+
* @default 1000 (1 second)
|
|
2535
|
+
*/
|
|
2536
|
+
defaultWaitTime?: number;
|
|
2537
|
+
/**
|
|
2538
|
+
* Maximum number of wait attempts before giving up.
|
|
2539
|
+
* After this many waits, the request will proceed to retry logic or fail.
|
|
2540
|
+
*
|
|
2541
|
+
* @default 3
|
|
2542
|
+
*/
|
|
2543
|
+
maxWaitAttempts?: number;
|
|
2429
2544
|
/** Whether to use a secure context for HTTPS requests */
|
|
2430
2545
|
useSecureContext?: boolean;
|
|
2431
2546
|
/** Custom secure context for TLS connections */
|
package/dist/proxy/index.cjs
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
const { Agent, HttpProxyAgent, HttpsProxyAgent, SocksProxyAgent } = require('../internal/agents.cjs');
|
|
1
|
+
const { Agent, HttpProxyAgent, HttpsProxyAgent, SocksProxyAgent } = require('../internal/agents/index.cjs');
|
|
2
2
|
const { parseProxyString } = require('./parse.cjs');
|
|
3
|
-
const
|
|
4
|
-
exports.ProxyManager =
|
|
5
|
-
const
|
|
6
|
-
exports.parseProxyString =
|
|
3
|
+
const _mod_0wfzka = require('./manager.cjs');
|
|
4
|
+
exports.ProxyManager = _mod_0wfzka.ProxyManager;;
|
|
5
|
+
const _mod_bwzrrk = require('./parse.cjs');
|
|
6
|
+
exports.parseProxyString = _mod_bwzrrk.parseProxyString;;
|
|
7
7
|
function createOptions(uri, opts) {
|
|
8
8
|
if (uri instanceof URL || typeof uri === "string") {
|
|
9
9
|
return {
|
package/dist/proxy/index.js
CHANGED
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
HttpProxyAgent,
|
|
4
4
|
HttpsProxyAgent,
|
|
5
5
|
SocksProxyAgent
|
|
6
|
-
} from '../internal/agents.js';
|
|
6
|
+
} from '../internal/agents/index.js';
|
|
7
7
|
import { parseProxyString } from './parse.js';
|
|
8
8
|
export { ProxyManager } from './manager.js';
|
|
9
9
|
export { parseProxyString } from './parse.js';
|
package/dist/queue/index.cjs
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.RezoQueue =
|
|
3
|
-
const
|
|
4
|
-
exports.HttpQueue =
|
|
5
|
-
exports.extractDomain =
|
|
6
|
-
const
|
|
7
|
-
exports.Priority =
|
|
8
|
-
exports.HttpMethodPriority =
|
|
1
|
+
const _mod_cfqi4l = require('./queue.cjs');
|
|
2
|
+
exports.RezoQueue = _mod_cfqi4l.RezoQueue;;
|
|
3
|
+
const _mod_9m2if8 = require('./http-queue.cjs');
|
|
4
|
+
exports.HttpQueue = _mod_9m2if8.HttpQueue;
|
|
5
|
+
exports.extractDomain = _mod_9m2if8.extractDomain;;
|
|
6
|
+
const _mod_w9of0o = require('./types.cjs');
|
|
7
|
+
exports.Priority = _mod_w9of0o.Priority;
|
|
8
|
+
exports.HttpMethodPriority = _mod_w9of0o.HttpMethodPriority;;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.UniversalEventEmitter =
|
|
3
|
-
const
|
|
4
|
-
exports.UniversalStreamResponse =
|
|
5
|
-
exports.StreamResponse =
|
|
6
|
-
const
|
|
7
|
-
exports.UniversalDownloadResponse =
|
|
8
|
-
exports.DownloadResponse =
|
|
9
|
-
const
|
|
10
|
-
exports.UniversalUploadResponse =
|
|
11
|
-
exports.UploadResponse =
|
|
1
|
+
const _mod_rxkza8 = require('./event-emitter.cjs');
|
|
2
|
+
exports.UniversalEventEmitter = _mod_rxkza8.UniversalEventEmitter;;
|
|
3
|
+
const _mod_gre5wb = require('./stream.cjs');
|
|
4
|
+
exports.UniversalStreamResponse = _mod_gre5wb.UniversalStreamResponse;
|
|
5
|
+
exports.StreamResponse = _mod_gre5wb.StreamResponse;;
|
|
6
|
+
const _mod_xea59w = require('./download.cjs');
|
|
7
|
+
exports.UniversalDownloadResponse = _mod_xea59w.UniversalDownloadResponse;
|
|
8
|
+
exports.DownloadResponse = _mod_xea59w.DownloadResponse;;
|
|
9
|
+
const _mod_1q9fpp = require('./upload.cjs');
|
|
10
|
+
exports.UniversalUploadResponse = _mod_1q9fpp.UniversalUploadResponse;
|
|
11
|
+
exports.UploadResponse = _mod_1q9fpp.UploadResponse;;
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
const { RezoHeaders } = require('./headers.cjs');
|
|
2
|
+
const DEFAULT_WAIT_STATUS_CODES = exports.DEFAULT_WAIT_STATUS_CODES = [429];
|
|
3
|
+
const DEFAULT_MAX_WAIT_TIME = exports.DEFAULT_MAX_WAIT_TIME = 60000;
|
|
4
|
+
const DEFAULT_WAIT_TIME = exports.DEFAULT_WAIT_TIME = 1000;
|
|
5
|
+
const DEFAULT_MAX_WAIT_ATTEMPTS = exports.DEFAULT_MAX_WAIT_ATTEMPTS = 3;
|
|
6
|
+
function shouldWaitOnStatus(status, waitOnStatus) {
|
|
7
|
+
if (!waitOnStatus)
|
|
8
|
+
return false;
|
|
9
|
+
if (waitOnStatus === true) {
|
|
10
|
+
return DEFAULT_WAIT_STATUS_CODES.includes(status);
|
|
11
|
+
}
|
|
12
|
+
if (Array.isArray(waitOnStatus)) {
|
|
13
|
+
return waitOnStatus.includes(status);
|
|
14
|
+
}
|
|
15
|
+
return false;
|
|
16
|
+
}
|
|
17
|
+
function parseRetryAfterHeader(value) {
|
|
18
|
+
if (!value)
|
|
19
|
+
return null;
|
|
20
|
+
const trimmed = value.trim();
|
|
21
|
+
const seconds = parseInt(trimmed, 10);
|
|
22
|
+
if (!isNaN(seconds) && seconds >= 0) {
|
|
23
|
+
return seconds * 1000;
|
|
24
|
+
}
|
|
25
|
+
const date = new Date(trimmed);
|
|
26
|
+
if (!isNaN(date.getTime())) {
|
|
27
|
+
const delayMs = date.getTime() - Date.now();
|
|
28
|
+
return delayMs > 0 ? delayMs : 0;
|
|
29
|
+
}
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
function getNestedValue(obj, path) {
|
|
33
|
+
if (!obj || typeof path !== "string")
|
|
34
|
+
return;
|
|
35
|
+
const parts = path.split(".");
|
|
36
|
+
let current = obj;
|
|
37
|
+
for (const part of parts) {
|
|
38
|
+
if (current === null || current === undefined)
|
|
39
|
+
return;
|
|
40
|
+
if (typeof current !== "object")
|
|
41
|
+
return;
|
|
42
|
+
current = current[part];
|
|
43
|
+
}
|
|
44
|
+
return current;
|
|
45
|
+
}
|
|
46
|
+
function extractWaitTime(response, options) {
|
|
47
|
+
const source = options.waitTimeSource ?? "retry-after";
|
|
48
|
+
const defaultWaitTime = options.defaultWaitTime ?? DEFAULT_WAIT_TIME;
|
|
49
|
+
if (typeof source === "function") {
|
|
50
|
+
try {
|
|
51
|
+
const seconds = source(response);
|
|
52
|
+
if (seconds !== null && seconds !== undefined && seconds > 0) {
|
|
53
|
+
return {
|
|
54
|
+
waitTimeMs: seconds * 1000,
|
|
55
|
+
source: "function"
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
} catch {}
|
|
59
|
+
return {
|
|
60
|
+
waitTimeMs: defaultWaitTime,
|
|
61
|
+
source: "default"
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
if (source === "retry-after") {
|
|
65
|
+
const headerValue = response.headers?.get?.("retry-after") ?? response.headers?.get?.("Retry-After") ?? response.headers?.["retry-after"] ?? response.headers?.["Retry-After"];
|
|
66
|
+
const waitTimeMs = parseRetryAfterHeader(headerValue);
|
|
67
|
+
if (waitTimeMs !== null) {
|
|
68
|
+
return {
|
|
69
|
+
waitTimeMs,
|
|
70
|
+
source: "header",
|
|
71
|
+
sourcePath: "retry-after"
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
return {
|
|
75
|
+
waitTimeMs: defaultWaitTime,
|
|
76
|
+
source: "default"
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
if (typeof source === "object" && "header" in source) {
|
|
80
|
+
const headerName = source.header;
|
|
81
|
+
const headerValue = response.headers?.get?.(headerName) ?? response.headers?.get?.(headerName.toLowerCase()) ?? response.headers?.[headerName] ?? response.headers?.[headerName.toLowerCase()];
|
|
82
|
+
if (headerValue) {
|
|
83
|
+
const numValue = parseInt(headerValue, 10);
|
|
84
|
+
if (!isNaN(numValue)) {
|
|
85
|
+
const now = Math.floor(Date.now() / 1000);
|
|
86
|
+
if (numValue > now && numValue < now + 86400 * 365) {
|
|
87
|
+
const waitTimeMs = (numValue - now) * 1000;
|
|
88
|
+
return {
|
|
89
|
+
waitTimeMs: waitTimeMs > 0 ? waitTimeMs : defaultWaitTime,
|
|
90
|
+
source: "header",
|
|
91
|
+
sourcePath: headerName
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
waitTimeMs: numValue * 1000,
|
|
96
|
+
source: "header",
|
|
97
|
+
sourcePath: headerName
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
const waitTimeMs = parseRetryAfterHeader(headerValue);
|
|
101
|
+
if (waitTimeMs !== null) {
|
|
102
|
+
return {
|
|
103
|
+
waitTimeMs,
|
|
104
|
+
source: "header",
|
|
105
|
+
sourcePath: headerName
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
waitTimeMs: defaultWaitTime,
|
|
111
|
+
source: "default"
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
if (typeof source === "object" && "body" in source) {
|
|
115
|
+
const bodyPath = source.body;
|
|
116
|
+
const value = getNestedValue(response.data, bodyPath);
|
|
117
|
+
if (value !== null && value !== undefined) {
|
|
118
|
+
const numValue = typeof value === "number" ? value : parseInt(String(value), 10);
|
|
119
|
+
if (!isNaN(numValue) && numValue > 0) {
|
|
120
|
+
return {
|
|
121
|
+
waitTimeMs: numValue * 1000,
|
|
122
|
+
source: "body",
|
|
123
|
+
sourcePath: bodyPath
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return {
|
|
128
|
+
waitTimeMs: defaultWaitTime,
|
|
129
|
+
source: "default"
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
return {
|
|
133
|
+
waitTimeMs: defaultWaitTime,
|
|
134
|
+
source: "default"
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
function createRateLimitWaitEvent(status, waitTimeMs, attempt, maxAttempts, source, sourcePath, url, method) {
|
|
138
|
+
return {
|
|
139
|
+
status,
|
|
140
|
+
waitTime: waitTimeMs,
|
|
141
|
+
attempt,
|
|
142
|
+
maxAttempts,
|
|
143
|
+
source,
|
|
144
|
+
sourcePath,
|
|
145
|
+
url,
|
|
146
|
+
method,
|
|
147
|
+
timestamp: Date.now()
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
async function executeRateLimitWaitHooks(event, config) {
|
|
151
|
+
const hooks = config.hooks?.onRateLimitWait;
|
|
152
|
+
if (!hooks || hooks.length === 0)
|
|
153
|
+
return;
|
|
154
|
+
for (const hook of hooks) {
|
|
155
|
+
try {
|
|
156
|
+
await hook(event, config);
|
|
157
|
+
} catch (err) {
|
|
158
|
+
if (config.debug) {
|
|
159
|
+
console.log("[Rezo Debug] onRateLimitWait hook error:", err);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
function sleep(ms) {
|
|
165
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
166
|
+
}
|
|
167
|
+
function normalizeHeaders(headers) {
|
|
168
|
+
if (!headers)
|
|
169
|
+
return new RezoHeaders;
|
|
170
|
+
if (headers instanceof RezoHeaders)
|
|
171
|
+
return headers;
|
|
172
|
+
if (typeof headers.get === "function")
|
|
173
|
+
return headers;
|
|
174
|
+
try {
|
|
175
|
+
return new RezoHeaders(headers);
|
|
176
|
+
} catch {
|
|
177
|
+
return new RezoHeaders;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
async function handleRateLimitWait(ctx) {
|
|
181
|
+
const { status, headers, data, url, method, config, options, currentWaitAttempt } = ctx;
|
|
182
|
+
if (!shouldWaitOnStatus(status, options.waitOnStatus)) {
|
|
183
|
+
return { shouldRetry: false, waitAttempt: currentWaitAttempt, waitedMs: 0 };
|
|
184
|
+
}
|
|
185
|
+
const maxAttempts = options.maxWaitAttempts ?? DEFAULT_MAX_WAIT_ATTEMPTS;
|
|
186
|
+
const maxWaitTime = options.maxWaitTime ?? DEFAULT_MAX_WAIT_TIME;
|
|
187
|
+
const nextAttempt = currentWaitAttempt + 1;
|
|
188
|
+
if (nextAttempt > maxAttempts) {
|
|
189
|
+
return { shouldRetry: false, waitAttempt: currentWaitAttempt, waitedMs: 0 };
|
|
190
|
+
}
|
|
191
|
+
const normalizedHeaders = normalizeHeaders(headers);
|
|
192
|
+
const extracted = extractWaitTime({ status, headers: normalizedHeaders, data }, options);
|
|
193
|
+
let waitTimeMs = extracted.waitTimeMs;
|
|
194
|
+
if (maxWaitTime > 0 && waitTimeMs > maxWaitTime) {
|
|
195
|
+
return { shouldRetry: false, waitAttempt: currentWaitAttempt, waitedMs: 0 };
|
|
196
|
+
}
|
|
197
|
+
const event = createRateLimitWaitEvent(status, waitTimeMs, nextAttempt, maxAttempts, extracted.source, extracted.sourcePath, url, method);
|
|
198
|
+
await executeRateLimitWaitHooks(event, config);
|
|
199
|
+
if (config.debug) {
|
|
200
|
+
console.log(`[Rezo Debug] Rate limit (${status}) - waiting ${waitTimeMs}ms (attempt ${nextAttempt}/${maxAttempts}, source: ${extracted.source}${extracted.sourcePath ? `:${extracted.sourcePath}` : ""})`);
|
|
201
|
+
}
|
|
202
|
+
await sleep(waitTimeMs);
|
|
203
|
+
return {
|
|
204
|
+
shouldRetry: true,
|
|
205
|
+
waitAttempt: nextAttempt,
|
|
206
|
+
waitedMs: waitTimeMs
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
exports.shouldWaitOnStatus = shouldWaitOnStatus;
|
|
211
|
+
exports.parseRetryAfterHeader = parseRetryAfterHeader;
|
|
212
|
+
exports.getNestedValue = getNestedValue;
|
|
213
|
+
exports.extractWaitTime = extractWaitTime;
|
|
214
|
+
exports.createRateLimitWaitEvent = createRateLimitWaitEvent;
|
|
215
|
+
exports.executeRateLimitWaitHooks = executeRateLimitWaitHooks;
|
|
216
|
+
exports.sleep = sleep;
|
|
217
|
+
exports.handleRateLimitWait = handleRateLimitWait;
|