rezo 1.0.72 → 1.0.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/entries/curl.d.ts +13 -2
- package/dist/adapters/entries/fetch.d.ts +13 -2
- package/dist/adapters/entries/http.d.ts +13 -2
- package/dist/adapters/entries/http2.d.ts +13 -2
- package/dist/adapters/entries/react-native.d.ts +13 -2
- package/dist/adapters/entries/xhr.d.ts +13 -2
- package/dist/adapters/index.cjs +6 -6
- package/dist/cache/index.cjs +9 -9
- package/dist/crawler/crawler-options.cjs +1 -1
- package/dist/crawler/crawler-options.js +1 -1
- package/dist/crawler/crawler.cjs +320 -89
- package/dist/crawler/crawler.js +320 -89
- package/dist/crawler/index.cjs +40 -40
- package/dist/crawler/plugin/capped-array.cjs +1 -0
- package/dist/crawler/plugin/capped-array.js +1 -0
- package/dist/crawler/plugin/capped-map.cjs +1 -0
- package/dist/crawler/plugin/capped-map.js +1 -0
- package/dist/crawler/plugin/file-cacher.cjs +20 -18
- package/dist/crawler/plugin/file-cacher.js +20 -18
- package/dist/crawler/plugin/health-metrics.cjs +2 -0
- package/dist/crawler/plugin/health-metrics.js +2 -0
- package/dist/crawler/plugin/index.cjs +1 -1
- package/dist/crawler/plugin/index.js +1 -1
- package/dist/crawler/plugin/memory-monitor.cjs +1 -0
- package/dist/crawler/plugin/memory-monitor.js +1 -0
- package/dist/crawler/plugin/navigation-history.cjs +5 -5
- package/dist/crawler/plugin/navigation-history.js +3 -3
- package/dist/crawler/plugin/result-stream.cjs +5 -0
- package/dist/crawler/plugin/result-stream.js +5 -0
- package/dist/crawler/plugin/sqlite-utils.cjs +1 -0
- package/dist/crawler/plugin/sqlite-utils.js +1 -0
- package/dist/crawler/plugin/url-store.cjs +5 -5
- package/dist/crawler/plugin/url-store.js +5 -5
- package/dist/crawler/scraper.cjs +1 -1
- package/dist/crawler/scraper.js +1 -1
- package/dist/crawler.d.ts +152 -25
- package/dist/entries/crawler.cjs +4 -4
- package/dist/errors/rezo-error.cjs +3 -72
- package/dist/errors/rezo-error.js +3 -72
- package/dist/index.cjs +30 -30
- package/dist/index.d.ts +13 -2
- package/dist/internal/agents/index.cjs +10 -10
- package/dist/platform/browser.d.ts +13 -2
- package/dist/platform/bun.d.ts +13 -2
- package/dist/platform/deno.d.ts +13 -2
- package/dist/platform/node.d.ts +13 -2
- package/dist/platform/react-native.d.ts +13 -2
- package/dist/platform/worker.d.ts +13 -2
- package/dist/proxy/index.cjs +4 -4
- package/dist/queue/index.cjs +8 -8
- package/dist/queue/queue.cjs +58 -13
- package/dist/queue/queue.js +58 -13
- package/dist/responses/universal/index.cjs +11 -11
- package/dist/utils/agent-pool.cjs +37 -0
- package/dist/utils/agent-pool.js +37 -0
- package/dist/version.cjs +1 -1
- package/dist/version.js +1 -1
- package/dist/wget/index.cjs +49 -49
- package/dist/wget/index.d.ts +12 -1
- package/package.json +1 -1
package/dist/crawler.d.ts
CHANGED
|
@@ -11,8 +11,9 @@ import { Cookie as TouchCookie, CookieJar as TouchCookieJar, CreateCookieOptions
|
|
|
11
11
|
* Optimized specifically for crawler workloads with:
|
|
12
12
|
* - WAL mode for high-throughput concurrent reads/writes
|
|
13
13
|
* - Batch operations for efficient bulk storage
|
|
14
|
-
* - Domain-based namespacing
|
|
14
|
+
* - Domain-based namespacing via column (single database)
|
|
15
15
|
* - Optional zstd compression for storage efficiency
|
|
16
|
+
* - LRU eviction to prevent unbounded growth
|
|
16
17
|
*
|
|
17
18
|
* @module cache/file-cacher
|
|
18
19
|
* @author Rezo HTTP Client Library
|
|
@@ -22,10 +23,15 @@ import { Cookie as TouchCookie, CookieJar as TouchCookieJar, CreateCookieOptions
|
|
|
22
23
|
*/
|
|
23
24
|
export interface FileCacherOptions {
|
|
24
25
|
/**
|
|
25
|
-
* Directory path for storing cache
|
|
26
|
+
* Directory path for storing the cache database
|
|
26
27
|
* @default '/tmp/rezo-crawler/cache'
|
|
27
28
|
*/
|
|
28
29
|
cacheDir?: string;
|
|
30
|
+
/**
|
|
31
|
+
* Database filename
|
|
32
|
+
* @default 'cache.db'
|
|
33
|
+
*/
|
|
34
|
+
dbFileName?: string;
|
|
29
35
|
/**
|
|
30
36
|
* Default time-to-live in milliseconds
|
|
31
37
|
* @default 604800000 (7 days)
|
|
@@ -38,30 +44,33 @@ export interface FileCacherOptions {
|
|
|
38
44
|
*/
|
|
39
45
|
compression?: boolean;
|
|
40
46
|
/**
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
|
|
44
|
-
encryptNamespace?: boolean;
|
|
45
|
-
/**
|
|
46
|
-
* Maximum entries per namespace (0 = unlimited)
|
|
47
|
-
* @default 0
|
|
47
|
+
* Maximum total entries (0 = unlimited, recommended: 100000)
|
|
48
|
+
* Uses LRU eviction when limit is reached
|
|
49
|
+
* @default 100000
|
|
48
50
|
*/
|
|
49
51
|
maxEntries?: number;
|
|
50
52
|
}
|
|
51
53
|
declare class FileCacher {
|
|
52
|
-
private
|
|
54
|
+
private db;
|
|
53
55
|
private readonly options;
|
|
54
56
|
private readonly cacheDir;
|
|
57
|
+
private readonly dbPath;
|
|
55
58
|
private closed;
|
|
59
|
+
private initPromise;
|
|
60
|
+
private evictionInProgress;
|
|
56
61
|
private constructor();
|
|
57
62
|
/**
|
|
58
63
|
* Create a new FileCacher instance
|
|
59
64
|
*/
|
|
60
65
|
static create(options?: FileCacherOptions): Promise<FileCacher>;
|
|
61
66
|
/**
|
|
62
|
-
*
|
|
67
|
+
* Initialize the single database with optimized settings
|
|
63
68
|
*/
|
|
64
|
-
private
|
|
69
|
+
private initialize;
|
|
70
|
+
/**
|
|
71
|
+
* Ensure database is initialized
|
|
72
|
+
*/
|
|
73
|
+
private ensureDb;
|
|
65
74
|
/**
|
|
66
75
|
* Store a response in the cache
|
|
67
76
|
*/
|
|
@@ -74,6 +83,11 @@ declare class FileCacher {
|
|
|
74
83
|
value: T;
|
|
75
84
|
ttl?: number;
|
|
76
85
|
}>, namespace?: string): Promise<void>;
|
|
86
|
+
/**
|
|
87
|
+
* LRU eviction - removes oldest entries when limit exceeded
|
|
88
|
+
* Runs asynchronously to not block writes
|
|
89
|
+
*/
|
|
90
|
+
private maybeEvict;
|
|
77
91
|
/**
|
|
78
92
|
* Retrieve a cached response
|
|
79
93
|
*/
|
|
@@ -95,22 +109,24 @@ declare class FileCacher {
|
|
|
95
109
|
*/
|
|
96
110
|
clear(namespace?: string): Promise<void>;
|
|
97
111
|
/**
|
|
98
|
-
* Remove all expired entries
|
|
112
|
+
* Remove all expired entries across all namespaces
|
|
99
113
|
*/
|
|
100
|
-
cleanup(
|
|
114
|
+
cleanup(): Promise<number>;
|
|
101
115
|
/**
|
|
102
|
-
* Get cache statistics
|
|
116
|
+
* Get cache statistics
|
|
103
117
|
*/
|
|
104
118
|
stats(namespace?: string): Promise<{
|
|
105
119
|
count: number;
|
|
106
120
|
expired: number;
|
|
121
|
+
namespaces: number;
|
|
107
122
|
}>;
|
|
108
123
|
/**
|
|
109
|
-
* Close
|
|
124
|
+
* Close the database connection
|
|
110
125
|
*/
|
|
111
126
|
close(): Promise<void>;
|
|
112
127
|
get isClosed(): boolean;
|
|
113
128
|
get directory(): string;
|
|
129
|
+
get databasePath(): string;
|
|
114
130
|
}
|
|
115
131
|
export interface CrawlSession {
|
|
116
132
|
sessionId: string;
|
|
@@ -2063,6 +2079,13 @@ export interface QueueConfig {
|
|
|
2063
2079
|
intervalCap?: number;
|
|
2064
2080
|
/** Carry over unused interval capacity to next interval */
|
|
2065
2081
|
carryoverConcurrencyCount?: boolean;
|
|
2082
|
+
/**
|
|
2083
|
+
* Reject the task promise when an error occurs (default: false)
|
|
2084
|
+
* When false, errors are swallowed and task.resolve(undefined) is called.
|
|
2085
|
+
* This prevents unhandled promise rejections but makes error handling harder.
|
|
2086
|
+
* When true, task.reject(error) is called, allowing proper try/catch handling.
|
|
2087
|
+
*/
|
|
2088
|
+
rejectOnError?: boolean;
|
|
2066
2089
|
}
|
|
2067
2090
|
/**
|
|
2068
2091
|
* Task options when adding to queue
|
|
@@ -2268,8 +2291,12 @@ declare class RezoQueue<T = any> {
|
|
|
2268
2291
|
/**
|
|
2269
2292
|
* Wait for queue size to be less than limit
|
|
2270
2293
|
* @param limit - Size threshold
|
|
2294
|
+
* @param timeoutMs - Optional timeout in milliseconds (default: 0 = no timeout)
|
|
2295
|
+
* If timeout occurs, promise resolves (not rejects) to prevent blocking
|
|
2271
2296
|
*/
|
|
2272
|
-
onSizeLessThan(limit: number): Promise<void>;
|
|
2297
|
+
onSizeLessThan(limit: number, timeoutMs?: number): Promise<void>;
|
|
2298
|
+
/** Maximum recommended handlers per event before warning */
|
|
2299
|
+
private static readonly MAX_HANDLERS_WARNING;
|
|
2273
2300
|
/**
|
|
2274
2301
|
* Register an event handler
|
|
2275
2302
|
* @param event - Event name
|
|
@@ -6477,6 +6504,12 @@ export interface ICrawlerOptions {
|
|
|
6477
6504
|
} | {
|
|
6478
6505
|
enable: false;
|
|
6479
6506
|
} | undefined | false;
|
|
6507
|
+
/** Enable graceful shutdown handlers for SIGINT/SIGTERM - saves session state (default: false) */
|
|
6508
|
+
enableSignalHandlers?: boolean;
|
|
6509
|
+
/** Maximum concurrent requests for crawler (default: 100) */
|
|
6510
|
+
concurrency?: number;
|
|
6511
|
+
/** Maximum concurrent requests for scraper - separate queue (default: same as concurrency) */
|
|
6512
|
+
scraperConcurrency?: number;
|
|
6480
6513
|
/** Maximum crawl depth from start URL (0 = unlimited, default: 0) */
|
|
6481
6514
|
maxDepth?: number;
|
|
6482
6515
|
/** Maximum total URLs to crawl (0 = unlimited, default: 0) */
|
|
@@ -6575,6 +6608,12 @@ export declare class CrawlerOptions {
|
|
|
6575
6608
|
throwFatalError?: boolean;
|
|
6576
6609
|
/** Enable debug logging */
|
|
6577
6610
|
debug?: boolean;
|
|
6611
|
+
/** Enable graceful shutdown handlers for SIGINT/SIGTERM - saves session state */
|
|
6612
|
+
enableSignalHandlers: boolean;
|
|
6613
|
+
/** Maximum concurrent requests for crawler (default: 100) */
|
|
6614
|
+
concurrency: number;
|
|
6615
|
+
/** Maximum concurrent requests for scraper (default: same as concurrency) */
|
|
6616
|
+
scraperConcurrency: number;
|
|
6578
6617
|
/** Maximum crawl depth from start URL (0 = unlimited) */
|
|
6579
6618
|
maxDepth: number;
|
|
6580
6619
|
/** Maximum total URLs to crawl (0 = unlimited) */
|
|
@@ -6991,6 +7030,32 @@ export interface EmailDiscoveryEvent<T = Record<string, any>> {
|
|
|
6991
7030
|
timestamp: Date;
|
|
6992
7031
|
metadata: T;
|
|
6993
7032
|
}
|
|
7033
|
+
export interface HealthSnapshot {
|
|
7034
|
+
/** Timestamp of snapshot */
|
|
7035
|
+
timestamp: number;
|
|
7036
|
+
/** Requests per second (rolling average) */
|
|
7037
|
+
requestsPerSecond: number;
|
|
7038
|
+
/** Success rate percentage (0-100) */
|
|
7039
|
+
successRate: number;
|
|
7040
|
+
/** Failure rate percentage (0-100) */
|
|
7041
|
+
failureRate: number;
|
|
7042
|
+
/** Average response time in ms */
|
|
7043
|
+
avgResponseTime: number;
|
|
7044
|
+
/** 95th percentile response time in ms */
|
|
7045
|
+
p95ResponseTime: number;
|
|
7046
|
+
/** Current queue depth */
|
|
7047
|
+
queueDepth: number;
|
|
7048
|
+
/** Active concurrent requests */
|
|
7049
|
+
activeRequests: number;
|
|
7050
|
+
/** Total requests processed */
|
|
7051
|
+
totalRequests: number;
|
|
7052
|
+
/** Total successful requests */
|
|
7053
|
+
totalSuccesses: number;
|
|
7054
|
+
/** Total failed requests */
|
|
7055
|
+
totalFailures: number;
|
|
7056
|
+
/** Uptime in milliseconds */
|
|
7057
|
+
uptimeMs: number;
|
|
7058
|
+
}
|
|
6994
7059
|
interface RedirectEvent$1 {
|
|
6995
7060
|
originalUrl: string;
|
|
6996
7061
|
finalUrl: string;
|
|
@@ -7070,6 +7135,7 @@ export declare class Crawler {
|
|
|
7070
7135
|
*/
|
|
7071
7136
|
cacher: FileCacher;
|
|
7072
7137
|
private queue;
|
|
7138
|
+
private scraperQueue;
|
|
7073
7139
|
private readonly isCacheEnabled;
|
|
7074
7140
|
readonly config: CrawlerOptions;
|
|
7075
7141
|
private urlStorage;
|
|
@@ -7085,15 +7151,10 @@ export declare class Crawler {
|
|
|
7085
7151
|
/** Adapter-specific request executor */
|
|
7086
7152
|
private adapterExecutor;
|
|
7087
7153
|
private adapterType;
|
|
7088
|
-
/** Track pending execute() calls for proper done() behavior */
|
|
7089
|
-
private pendingExecutions;
|
|
7090
|
-
/**
|
|
7091
|
-
* Track pending visits that haven't yet added to the queue.
|
|
7092
|
-
* This prevents waitForAll() from exiting when execute() is still in its async preamble.
|
|
7093
|
-
*/
|
|
7094
|
-
private pendingVisitCount;
|
|
7095
7154
|
/** Track if the crawler has been destroyed */
|
|
7096
7155
|
private isDestroyed;
|
|
7156
|
+
/** Track if graceful shutdown has been requested */
|
|
7157
|
+
private shutdownRequested;
|
|
7097
7158
|
/** Original queue options for restoration */
|
|
7098
7159
|
private readonly queueOptions;
|
|
7099
7160
|
/** robots.txt parser and validator */
|
|
@@ -7105,14 +7166,30 @@ export declare class Crawler {
|
|
|
7105
7166
|
private crawlStats;
|
|
7106
7167
|
/** URL depth tracking for maxDepth limit */
|
|
7107
7168
|
private urlDepthMap;
|
|
7169
|
+
/** Periodic cleanup interval for memory management */
|
|
7170
|
+
private cleanupInterval?;
|
|
7171
|
+
/** Auto-checkpoint interval for session persistence */
|
|
7172
|
+
private checkpointInterval?;
|
|
7173
|
+
/** Last checkpoint timestamp */
|
|
7174
|
+
private lastCheckpointTime;
|
|
7175
|
+
/** Memory monitor for auto-throttling */
|
|
7176
|
+
private memoryMonitor;
|
|
7177
|
+
/** Health metrics for monitoring */
|
|
7178
|
+
private healthMetrics;
|
|
7179
|
+
/** Original concurrency for restoration after memory pressure */
|
|
7180
|
+
private originalConcurrency;
|
|
7181
|
+
/** Shutdown handler reference for cleanup */
|
|
7182
|
+
private shutdownHandler;
|
|
7108
7183
|
/** Lifecycle event handlers */
|
|
7109
7184
|
private startHandlers;
|
|
7110
7185
|
private finishHandlers;
|
|
7111
7186
|
private redirectHandlers;
|
|
7112
|
-
/** Data collection for export */
|
|
7187
|
+
/** Data collection for export - bounded to prevent memory issues */
|
|
7113
7188
|
private collectedData;
|
|
7114
7189
|
/** Flag to track if crawl has started */
|
|
7115
7190
|
private crawlStarted;
|
|
7191
|
+
/** Promise lock for triggerStartHandlers to prevent race condition */
|
|
7192
|
+
private startHandlersPromise;
|
|
7116
7193
|
/**
|
|
7117
7194
|
* Creates a new Crawler instance with the specified configuration.
|
|
7118
7195
|
*
|
|
@@ -7144,6 +7221,18 @@ export declare class Crawler {
|
|
|
7144
7221
|
* ```
|
|
7145
7222
|
*/
|
|
7146
7223
|
constructor(crawlerOptions: ICrawlerOptions, http?: Rezo);
|
|
7224
|
+
/**
|
|
7225
|
+
* Register process signal handlers for graceful shutdown
|
|
7226
|
+
*/
|
|
7227
|
+
private registerShutdownHandlers;
|
|
7228
|
+
/**
|
|
7229
|
+
* Remove process signal handlers
|
|
7230
|
+
*/
|
|
7231
|
+
private removeShutdownHandlers;
|
|
7232
|
+
/**
|
|
7233
|
+
* Graceful shutdown - finish current tasks and save state
|
|
7234
|
+
*/
|
|
7235
|
+
gracefulShutdown(): Promise<void>;
|
|
7147
7236
|
/**
|
|
7148
7237
|
* Initialize the HTTP adapter based on configuration
|
|
7149
7238
|
*/
|
|
@@ -7212,8 +7301,30 @@ export declare class Crawler {
|
|
|
7212
7301
|
*/
|
|
7213
7302
|
setAdapter(adapter: CrawlerAdapterType): Promise<void>;
|
|
7214
7303
|
private rawResponseHandler;
|
|
7304
|
+
/**
|
|
7305
|
+
* Wait for cache to be ready with timeout (non-recursive)
|
|
7306
|
+
* @param timeoutMs - Maximum time to wait in milliseconds (default: 30s)
|
|
7307
|
+
*/
|
|
7215
7308
|
private waitForCache;
|
|
7309
|
+
/**
|
|
7310
|
+
* Wait for storage to be ready with timeout (non-recursive)
|
|
7311
|
+
* @param timeoutMs - Maximum time to wait in milliseconds (default: 30s)
|
|
7312
|
+
*/
|
|
7216
7313
|
private waitForStorage;
|
|
7314
|
+
/**
|
|
7315
|
+
* Start periodic cleanup to prevent unbounded memory growth
|
|
7316
|
+
* Trims urlDepthMap and domainResponseTimes to reasonable sizes
|
|
7317
|
+
* Also monitors memory and auto-throttles if needed
|
|
7318
|
+
*/
|
|
7319
|
+
private startPeriodicCleanup;
|
|
7320
|
+
/**
|
|
7321
|
+
* Start auto-checkpointing to save session progress every 5 minutes
|
|
7322
|
+
*/
|
|
7323
|
+
private startAutoCheckpoint;
|
|
7324
|
+
/**
|
|
7325
|
+
* Save current crawl progress to navigation history
|
|
7326
|
+
*/
|
|
7327
|
+
saveCheckpoint(): Promise<void>;
|
|
7217
7328
|
private saveUrl;
|
|
7218
7329
|
private hasUrlInCache;
|
|
7219
7330
|
private saveCache;
|
|
@@ -7621,8 +7732,24 @@ export declare class Crawler {
|
|
|
7621
7732
|
* Get current crawl statistics
|
|
7622
7733
|
*/
|
|
7623
7734
|
getStats(): CrawlStats;
|
|
7735
|
+
/**
|
|
7736
|
+
* Get health metrics snapshot for monitoring
|
|
7737
|
+
*/
|
|
7738
|
+
getHealthSnapshot(): HealthSnapshot;
|
|
7739
|
+
/**
|
|
7740
|
+
* Check if crawler is healthy based on thresholds
|
|
7741
|
+
*/
|
|
7742
|
+
isHealthy(options?: {
|
|
7743
|
+
minSuccessRate?: number;
|
|
7744
|
+
maxAvgResponseTime?: number;
|
|
7745
|
+
}): boolean;
|
|
7746
|
+
/**
|
|
7747
|
+
* Get Prometheus-formatted metrics for monitoring systems
|
|
7748
|
+
*/
|
|
7749
|
+
getPrometheusMetrics(prefix?: string): string;
|
|
7624
7750
|
/**
|
|
7625
7751
|
* Trigger onStart handlers (called once on first visit)
|
|
7752
|
+
* Uses promise lock to prevent race condition with concurrent visits
|
|
7626
7753
|
*/
|
|
7627
7754
|
private triggerStartHandlers;
|
|
7628
7755
|
/**
|
package/dist/entries/crawler.cjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Crawler =
|
|
3
|
-
const
|
|
4
|
-
exports.CrawlerOptions =
|
|
1
|
+
const _mod_yduaec = require('../crawler/crawler.cjs');
|
|
2
|
+
exports.Crawler = _mod_yduaec.Crawler;;
|
|
3
|
+
const _mod_58t9fo = require('../crawler/crawler-options.cjs');
|
|
4
|
+
exports.CrawlerOptions = _mod_58t9fo.CrawlerOptions;;
|
|
@@ -553,9 +553,9 @@ function cleanStackTrace(stack) {
|
|
|
553
553
|
class RezoError extends Error {
|
|
554
554
|
constructor(message, config, code, request, response) {
|
|
555
555
|
super();
|
|
556
|
-
Object.defineProperty(this, "config", { value: config, enumerable:
|
|
557
|
-
Object.defineProperty(this, "request", { value: request, enumerable:
|
|
558
|
-
Object.defineProperty(this, "response", { value: response, enumerable:
|
|
556
|
+
Object.defineProperty(this, "config", { value: config, enumerable: !!config });
|
|
557
|
+
Object.defineProperty(this, "request", { value: request, enumerable: !!request });
|
|
558
|
+
Object.defineProperty(this, "response", { value: response, enumerable: !!response });
|
|
559
559
|
Object.defineProperty(this, "isRezoError", { value: true, enumerable: false });
|
|
560
560
|
if (code) {
|
|
561
561
|
Object.defineProperty(this, "code", { value: code, enumerable: true });
|
|
@@ -600,75 +600,6 @@ class RezoError extends Error {
|
|
|
600
600
|
}
|
|
601
601
|
}
|
|
602
602
|
}
|
|
603
|
-
[Symbol.for("nodejs.util.inspect.custom")](_depth, options) {
|
|
604
|
-
const parts = [];
|
|
605
|
-
const isDebug = this.config?.debug === true;
|
|
606
|
-
const inspect = options?.stylize ? (v) => require("util").inspect(v, { depth: 3, colors: true }) : JSON.stringify;
|
|
607
|
-
parts.push(`${this.name}: ${this.message}`);
|
|
608
|
-
if (this.code)
|
|
609
|
-
parts.push(` code: '${this.code}'`);
|
|
610
|
-
if (this.method)
|
|
611
|
-
parts.push(` method: '${this.method}'`);
|
|
612
|
-
if (this.url)
|
|
613
|
-
parts.push(` url: '${this.url}'`);
|
|
614
|
-
if (this.finalUrl && this.finalUrl !== this.url) {
|
|
615
|
-
parts.push(` finalUrl: '${this.finalUrl}'`);
|
|
616
|
-
}
|
|
617
|
-
if (this.status)
|
|
618
|
-
parts.push(` status: ${this.status}`);
|
|
619
|
-
if (this.statusText)
|
|
620
|
-
parts.push(` statusText: '${this.statusText}'`);
|
|
621
|
-
if (this.urls && this.urls.length > 1) {
|
|
622
|
-
parts.push(` urls: [${this.urls.map((u) => `'${u}'`).join(", ")}]`);
|
|
623
|
-
}
|
|
624
|
-
if (this.suggestion)
|
|
625
|
-
parts.push(` suggestion: ${this.suggestion}`);
|
|
626
|
-
if (isDebug) {
|
|
627
|
-
parts.push("");
|
|
628
|
-
parts.push(" --- Debug Info ---");
|
|
629
|
-
if (this.cause) {
|
|
630
|
-
const causeMsg = typeof this.cause === "string" ? this.cause : this.cause?.message || String(this.cause);
|
|
631
|
-
parts.push(` cause: ${causeMsg}`);
|
|
632
|
-
}
|
|
633
|
-
if (this.errno)
|
|
634
|
-
parts.push(` errno: ${this.errno}`);
|
|
635
|
-
if (this.hostname)
|
|
636
|
-
parts.push(` hostname: '${this.hostname}'`);
|
|
637
|
-
if (this.port)
|
|
638
|
-
parts.push(` port: ${this.port}`);
|
|
639
|
-
if (this.address)
|
|
640
|
-
parts.push(` address: '${this.address}'`);
|
|
641
|
-
if (this.syscall)
|
|
642
|
-
parts.push(` syscall: '${this.syscall}'`);
|
|
643
|
-
if (this.response) {
|
|
644
|
-
parts.push("");
|
|
645
|
-
parts.push(" --- Response ---");
|
|
646
|
-
parts.push(` response.status: ${this.response.status}`);
|
|
647
|
-
parts.push(` response.statusText: '${this.response.statusText || ""}'`);
|
|
648
|
-
parts.push(` response.finalUrl: '${this.response.finalUrl || ""}'`);
|
|
649
|
-
if (this.response.headers) {
|
|
650
|
-
parts.push(` response.headers: ${inspect(this.response.headers)}`);
|
|
651
|
-
}
|
|
652
|
-
if (this.response.data !== undefined) {
|
|
653
|
-
const dataStr = typeof this.response.data === "string" ? this.response.data.substring(0, 500) + (this.response.data.length > 500 ? "..." : "") : inspect(this.response.data);
|
|
654
|
-
parts.push(` response.data: ${dataStr}`);
|
|
655
|
-
}
|
|
656
|
-
}
|
|
657
|
-
if (this.response?.config) {
|
|
658
|
-
parts.push("");
|
|
659
|
-
parts.push(" --- Request Config ---");
|
|
660
|
-
const { cookieJar, ...configWithoutJar } = this.response.config;
|
|
661
|
-
parts.push(` config: ${inspect(configWithoutJar)}`);
|
|
662
|
-
}
|
|
663
|
-
if (this.stack) {
|
|
664
|
-
parts.push("");
|
|
665
|
-
parts.push(" --- Stack Trace ---");
|
|
666
|
-
parts.push(this.stack);
|
|
667
|
-
}
|
|
668
|
-
}
|
|
669
|
-
return parts.join(`
|
|
670
|
-
`);
|
|
671
|
-
}
|
|
672
603
|
static isRezoError(error) {
|
|
673
604
|
return error instanceof RezoError || error !== null && typeof error === "object" && error.isRezoError === true;
|
|
674
605
|
}
|
|
@@ -553,9 +553,9 @@ function cleanStackTrace(stack) {
|
|
|
553
553
|
export class RezoError extends Error {
|
|
554
554
|
constructor(message, config, code, request, response) {
|
|
555
555
|
super();
|
|
556
|
-
Object.defineProperty(this, "config", { value: config, enumerable:
|
|
557
|
-
Object.defineProperty(this, "request", { value: request, enumerable:
|
|
558
|
-
Object.defineProperty(this, "response", { value: response, enumerable:
|
|
556
|
+
Object.defineProperty(this, "config", { value: config, enumerable: !!config });
|
|
557
|
+
Object.defineProperty(this, "request", { value: request, enumerable: !!request });
|
|
558
|
+
Object.defineProperty(this, "response", { value: response, enumerable: !!response });
|
|
559
559
|
Object.defineProperty(this, "isRezoError", { value: true, enumerable: false });
|
|
560
560
|
if (code) {
|
|
561
561
|
Object.defineProperty(this, "code", { value: code, enumerable: true });
|
|
@@ -600,75 +600,6 @@ export class RezoError extends Error {
|
|
|
600
600
|
}
|
|
601
601
|
}
|
|
602
602
|
}
|
|
603
|
-
[Symbol.for("nodejs.util.inspect.custom")](_depth, options) {
|
|
604
|
-
const parts = [];
|
|
605
|
-
const isDebug = this.config?.debug === true;
|
|
606
|
-
const inspect = options?.stylize ? (v) => require("util").inspect(v, { depth: 3, colors: true }) : JSON.stringify;
|
|
607
|
-
parts.push(`${this.name}: ${this.message}`);
|
|
608
|
-
if (this.code)
|
|
609
|
-
parts.push(` code: '${this.code}'`);
|
|
610
|
-
if (this.method)
|
|
611
|
-
parts.push(` method: '${this.method}'`);
|
|
612
|
-
if (this.url)
|
|
613
|
-
parts.push(` url: '${this.url}'`);
|
|
614
|
-
if (this.finalUrl && this.finalUrl !== this.url) {
|
|
615
|
-
parts.push(` finalUrl: '${this.finalUrl}'`);
|
|
616
|
-
}
|
|
617
|
-
if (this.status)
|
|
618
|
-
parts.push(` status: ${this.status}`);
|
|
619
|
-
if (this.statusText)
|
|
620
|
-
parts.push(` statusText: '${this.statusText}'`);
|
|
621
|
-
if (this.urls && this.urls.length > 1) {
|
|
622
|
-
parts.push(` urls: [${this.urls.map((u) => `'${u}'`).join(", ")}]`);
|
|
623
|
-
}
|
|
624
|
-
if (this.suggestion)
|
|
625
|
-
parts.push(` suggestion: ${this.suggestion}`);
|
|
626
|
-
if (isDebug) {
|
|
627
|
-
parts.push("");
|
|
628
|
-
parts.push(" --- Debug Info ---");
|
|
629
|
-
if (this.cause) {
|
|
630
|
-
const causeMsg = typeof this.cause === "string" ? this.cause : this.cause?.message || String(this.cause);
|
|
631
|
-
parts.push(` cause: ${causeMsg}`);
|
|
632
|
-
}
|
|
633
|
-
if (this.errno)
|
|
634
|
-
parts.push(` errno: ${this.errno}`);
|
|
635
|
-
if (this.hostname)
|
|
636
|
-
parts.push(` hostname: '${this.hostname}'`);
|
|
637
|
-
if (this.port)
|
|
638
|
-
parts.push(` port: ${this.port}`);
|
|
639
|
-
if (this.address)
|
|
640
|
-
parts.push(` address: '${this.address}'`);
|
|
641
|
-
if (this.syscall)
|
|
642
|
-
parts.push(` syscall: '${this.syscall}'`);
|
|
643
|
-
if (this.response) {
|
|
644
|
-
parts.push("");
|
|
645
|
-
parts.push(" --- Response ---");
|
|
646
|
-
parts.push(` response.status: ${this.response.status}`);
|
|
647
|
-
parts.push(` response.statusText: '${this.response.statusText || ""}'`);
|
|
648
|
-
parts.push(` response.finalUrl: '${this.response.finalUrl || ""}'`);
|
|
649
|
-
if (this.response.headers) {
|
|
650
|
-
parts.push(` response.headers: ${inspect(this.response.headers)}`);
|
|
651
|
-
}
|
|
652
|
-
if (this.response.data !== undefined) {
|
|
653
|
-
const dataStr = typeof this.response.data === "string" ? this.response.data.substring(0, 500) + (this.response.data.length > 500 ? "..." : "") : inspect(this.response.data);
|
|
654
|
-
parts.push(` response.data: ${dataStr}`);
|
|
655
|
-
}
|
|
656
|
-
}
|
|
657
|
-
if (this.response?.config) {
|
|
658
|
-
parts.push("");
|
|
659
|
-
parts.push(" --- Request Config ---");
|
|
660
|
-
const { cookieJar, ...configWithoutJar } = this.response.config;
|
|
661
|
-
parts.push(` config: ${inspect(configWithoutJar)}`);
|
|
662
|
-
}
|
|
663
|
-
if (this.stack) {
|
|
664
|
-
parts.push("");
|
|
665
|
-
parts.push(" --- Stack Trace ---");
|
|
666
|
-
parts.push(this.stack);
|
|
667
|
-
}
|
|
668
|
-
}
|
|
669
|
-
return parts.join(`
|
|
670
|
-
`);
|
|
671
|
-
}
|
|
672
603
|
static isRezoError(error) {
|
|
673
604
|
return error instanceof RezoError || error !== null && typeof error === "object" && error.isRezoError === true;
|
|
674
605
|
}
|
package/dist/index.cjs
CHANGED
|
@@ -1,30 +1,30 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Rezo =
|
|
3
|
-
exports.createRezoInstance =
|
|
4
|
-
exports.createDefaultInstance =
|
|
5
|
-
const
|
|
6
|
-
exports.RezoError =
|
|
7
|
-
exports.RezoErrorCode =
|
|
8
|
-
const
|
|
9
|
-
exports.RezoHeaders =
|
|
10
|
-
const
|
|
11
|
-
exports.RezoFormData =
|
|
12
|
-
const
|
|
13
|
-
exports.RezoCookieJar =
|
|
14
|
-
exports.Cookie =
|
|
15
|
-
const
|
|
16
|
-
exports.toCurl =
|
|
17
|
-
exports.fromCurl =
|
|
18
|
-
const
|
|
19
|
-
exports.createDefaultHooks =
|
|
20
|
-
exports.mergeHooks =
|
|
21
|
-
const
|
|
22
|
-
exports.ProxyManager =
|
|
23
|
-
const
|
|
24
|
-
exports.RezoQueue =
|
|
25
|
-
exports.HttpQueue =
|
|
26
|
-
exports.Priority =
|
|
27
|
-
exports.HttpMethodPriority =
|
|
1
|
+
const _mod_f5e2x4 = require('./core/rezo.cjs');
|
|
2
|
+
exports.Rezo = _mod_f5e2x4.Rezo;
|
|
3
|
+
exports.createRezoInstance = _mod_f5e2x4.createRezoInstance;
|
|
4
|
+
exports.createDefaultInstance = _mod_f5e2x4.createDefaultInstance;;
|
|
5
|
+
const _mod_l2n0t8 = require('./errors/rezo-error.cjs');
|
|
6
|
+
exports.RezoError = _mod_l2n0t8.RezoError;
|
|
7
|
+
exports.RezoErrorCode = _mod_l2n0t8.RezoErrorCode;;
|
|
8
|
+
const _mod_pgi4on = require('./utils/headers.cjs');
|
|
9
|
+
exports.RezoHeaders = _mod_pgi4on.RezoHeaders;;
|
|
10
|
+
const _mod_k82fb8 = require('./utils/form-data.cjs');
|
|
11
|
+
exports.RezoFormData = _mod_k82fb8.RezoFormData;;
|
|
12
|
+
const _mod_u8f16d = require('./utils/cookies.cjs');
|
|
13
|
+
exports.RezoCookieJar = _mod_u8f16d.RezoCookieJar;
|
|
14
|
+
exports.Cookie = _mod_u8f16d.Cookie;;
|
|
15
|
+
const _mod_5km04t = require('./utils/curl.cjs');
|
|
16
|
+
exports.toCurl = _mod_5km04t.toCurl;
|
|
17
|
+
exports.fromCurl = _mod_5km04t.fromCurl;;
|
|
18
|
+
const _mod_c99qva = require('./core/hooks.cjs');
|
|
19
|
+
exports.createDefaultHooks = _mod_c99qva.createDefaultHooks;
|
|
20
|
+
exports.mergeHooks = _mod_c99qva.mergeHooks;;
|
|
21
|
+
const _mod_nlh20q = require('./proxy/manager.cjs');
|
|
22
|
+
exports.ProxyManager = _mod_nlh20q.ProxyManager;;
|
|
23
|
+
const _mod_2j9uz9 = require('./queue/index.cjs');
|
|
24
|
+
exports.RezoQueue = _mod_2j9uz9.RezoQueue;
|
|
25
|
+
exports.HttpQueue = _mod_2j9uz9.HttpQueue;
|
|
26
|
+
exports.Priority = _mod_2j9uz9.Priority;
|
|
27
|
+
exports.HttpMethodPriority = _mod_2j9uz9.HttpMethodPriority;;
|
|
28
28
|
const { RezoError } = require('./errors/rezo-error.cjs');
|
|
29
29
|
const isRezoError = exports.isRezoError = RezoError.isRezoError;
|
|
30
30
|
const Cancel = exports.Cancel = RezoError;
|
|
@@ -34,9 +34,9 @@ const isCancel = exports.isCancel = (error) => {
|
|
|
34
34
|
};
|
|
35
35
|
const all = exports.all = Promise.all.bind(Promise);
|
|
36
36
|
const spread = exports.spread = (callback) => (array) => callback(...array);
|
|
37
|
-
const
|
|
38
|
-
exports.VERSION =
|
|
39
|
-
exports.PACKAGE_NAME =
|
|
37
|
+
const _mod_xznh6j = require('./version.cjs');
|
|
38
|
+
exports.VERSION = _mod_xznh6j.VERSION;
|
|
39
|
+
exports.PACKAGE_NAME = _mod_xznh6j.PACKAGE_NAME;;
|
|
40
40
|
const { executeRequest } = require('./adapters/http.cjs');
|
|
41
41
|
const { setGlobalAdapter, createRezoInstance } = require('./core/rezo.cjs');
|
|
42
42
|
setGlobalAdapter(executeRequest);
|
package/dist/index.d.ts
CHANGED
|
@@ -2030,6 +2030,13 @@ export interface QueueConfig {
|
|
|
2030
2030
|
intervalCap?: number;
|
|
2031
2031
|
/** Carry over unused interval capacity to next interval */
|
|
2032
2032
|
carryoverConcurrencyCount?: boolean;
|
|
2033
|
+
/**
|
|
2034
|
+
* Reject the task promise when an error occurs (default: false)
|
|
2035
|
+
* When false, errors are swallowed and task.resolve(undefined) is called.
|
|
2036
|
+
* This prevents unhandled promise rejections but makes error handling harder.
|
|
2037
|
+
* When true, task.reject(error) is called, allowing proper try/catch handling.
|
|
2038
|
+
*/
|
|
2039
|
+
rejectOnError?: boolean;
|
|
2033
2040
|
}
|
|
2034
2041
|
/**
|
|
2035
2042
|
* HTTP-specific queue configuration
|
|
@@ -2342,8 +2349,12 @@ export declare class RezoQueue<T = any> {
|
|
|
2342
2349
|
/**
|
|
2343
2350
|
* Wait for queue size to be less than limit
|
|
2344
2351
|
* @param limit - Size threshold
|
|
2352
|
+
* @param timeoutMs - Optional timeout in milliseconds (default: 0 = no timeout)
|
|
2353
|
+
* If timeout occurs, promise resolves (not rejects) to prevent blocking
|
|
2345
2354
|
*/
|
|
2346
|
-
onSizeLessThan(limit: number): Promise<void>;
|
|
2355
|
+
onSizeLessThan(limit: number, timeoutMs?: number): Promise<void>;
|
|
2356
|
+
/** Maximum recommended handlers per event before warning */
|
|
2357
|
+
private static readonly MAX_HANDLERS_WARNING;
|
|
2347
2358
|
/**
|
|
2348
2359
|
* Register an event handler
|
|
2349
2360
|
* @param event - Event name
|
|
@@ -4872,7 +4883,7 @@ export declare class HttpQueue extends RezoQueue<any> {
|
|
|
4872
4883
|
*
|
|
4873
4884
|
* IMPORTANT: Update these values when bumping package version.
|
|
4874
4885
|
*/
|
|
4875
|
-
export declare const VERSION = "1.0.
|
|
4886
|
+
export declare const VERSION = "1.0.74";
|
|
4876
4887
|
export declare const PACKAGE_NAME = "rezo";
|
|
4877
4888
|
export declare const isRezoError: typeof RezoError.isRezoError;
|
|
4878
4889
|
export declare const Cancel: typeof RezoError;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
const
|
|
2
|
-
exports.Agent =
|
|
3
|
-
const
|
|
4
|
-
exports.HttpProxyAgent =
|
|
5
|
-
const
|
|
6
|
-
exports.HttpsProxyAgent =
|
|
7
|
-
const
|
|
8
|
-
exports.SocksProxyAgent =
|
|
9
|
-
const
|
|
10
|
-
exports.SocksClient =
|
|
1
|
+
const _mod_0a73ie = require('./base.cjs');
|
|
2
|
+
exports.Agent = _mod_0a73ie.Agent;;
|
|
3
|
+
const _mod_rs317e = require('./http-proxy.cjs');
|
|
4
|
+
exports.HttpProxyAgent = _mod_rs317e.HttpProxyAgent;;
|
|
5
|
+
const _mod_syy64b = require('./https-proxy.cjs');
|
|
6
|
+
exports.HttpsProxyAgent = _mod_syy64b.HttpsProxyAgent;;
|
|
7
|
+
const _mod_rciw1o = require('./socks-proxy.cjs');
|
|
8
|
+
exports.SocksProxyAgent = _mod_rciw1o.SocksProxyAgent;;
|
|
9
|
+
const _mod_w9tiwm = require('./socks-client.cjs');
|
|
10
|
+
exports.SocksClient = _mod_w9tiwm.SocksClient;;
|