recker 1.0.2-0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +109 -0
- package/dist/cache/file-storage.d.ts +13 -0
- package/dist/cache/file-storage.d.ts.map +1 -0
- package/dist/cache/file-storage.js +50 -0
- package/dist/cache/memory-storage.d.ts +10 -0
- package/dist/cache/memory-storage.d.ts.map +1 -0
- package/dist/cache/memory-storage.js +29 -0
- package/dist/cache/redis-storage.d.ts +16 -0
- package/dist/cache/redis-storage.d.ts.map +1 -0
- package/dist/cache/redis-storage.js +25 -0
- package/dist/constants.d.ts +19 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +18 -0
- package/dist/contract/index.d.ts +32 -0
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +67 -0
- package/dist/core/client.d.ts +107 -0
- package/dist/core/client.d.ts.map +1 -0
- package/dist/core/client.js +475 -0
- package/dist/core/errors.d.ts +19 -0
- package/dist/core/errors.d.ts.map +1 -0
- package/dist/core/errors.js +34 -0
- package/dist/core/request-promise.d.ts +24 -0
- package/dist/core/request-promise.d.ts.map +1 -0
- package/dist/core/request-promise.js +77 -0
- package/dist/core/request.d.ts +15 -0
- package/dist/core/request.d.ts.map +1 -0
- package/dist/core/request.js +44 -0
- package/dist/core/response.d.ts +33 -0
- package/dist/core/response.d.ts.map +1 -0
- package/dist/core/response.js +154 -0
- package/dist/index.d.ts +40 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +39 -0
- package/dist/mcp/client.d.ts +59 -0
- package/dist/mcp/client.d.ts.map +1 -0
- package/dist/mcp/client.js +195 -0
- package/dist/mcp/index.d.ts +3 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +2 -0
- package/dist/mcp/types.d.ts +151 -0
- package/dist/mcp/types.d.ts.map +1 -0
- package/dist/mcp/types.js +1 -0
- package/dist/plugins/cache.d.ts +10 -0
- package/dist/plugins/cache.d.ts.map +1 -0
- package/dist/plugins/cache.js +72 -0
- package/dist/plugins/circuit-breaker.d.ts +14 -0
- package/dist/plugins/circuit-breaker.d.ts.map +1 -0
- package/dist/plugins/circuit-breaker.js +100 -0
- package/dist/plugins/compression.d.ts +5 -0
- package/dist/plugins/compression.d.ts.map +1 -0
- package/dist/plugins/compression.js +128 -0
- package/dist/plugins/cookie-jar.d.ts +6 -0
- package/dist/plugins/cookie-jar.d.ts.map +1 -0
- package/dist/plugins/cookie-jar.js +72 -0
- package/dist/plugins/dedup.d.ts +6 -0
- package/dist/plugins/dedup.d.ts.map +1 -0
- package/dist/plugins/dedup.js +34 -0
- package/dist/plugins/graphql.d.ts +13 -0
- package/dist/plugins/graphql.d.ts.map +1 -0
- package/dist/plugins/graphql.js +39 -0
- package/dist/plugins/har-player.d.ts +7 -0
- package/dist/plugins/har-player.d.ts.map +1 -0
- package/dist/plugins/har-player.js +53 -0
- package/dist/plugins/har-recorder.d.ts +7 -0
- package/dist/plugins/har-recorder.d.ts.map +1 -0
- package/dist/plugins/har-recorder.js +67 -0
- package/dist/plugins/logger.d.ts +11 -0
- package/dist/plugins/logger.d.ts.map +1 -0
- package/dist/plugins/logger.js +72 -0
- package/dist/plugins/pagination.d.ts +17 -0
- package/dist/plugins/pagination.d.ts.map +1 -0
- package/dist/plugins/pagination.js +105 -0
- package/dist/plugins/proxy-rotator.d.ts +8 -0
- package/dist/plugins/proxy-rotator.d.ts.map +1 -0
- package/dist/plugins/proxy-rotator.js +35 -0
- package/dist/plugins/rate-limit.d.ts +8 -0
- package/dist/plugins/rate-limit.d.ts.map +1 -0
- package/dist/plugins/rate-limit.js +57 -0
- package/dist/plugins/retry.d.ts +14 -0
- package/dist/plugins/retry.d.ts.map +1 -0
- package/dist/plugins/retry.js +92 -0
- package/dist/plugins/server-timing.d.ts +8 -0
- package/dist/plugins/server-timing.d.ts.map +1 -0
- package/dist/plugins/server-timing.js +24 -0
- package/dist/plugins/xsrf.d.ts +10 -0
- package/dist/plugins/xsrf.d.ts.map +1 -0
- package/dist/plugins/xsrf.js +48 -0
- package/dist/runner/request-runner.d.ts +47 -0
- package/dist/runner/request-runner.d.ts.map +1 -0
- package/dist/runner/request-runner.js +89 -0
- package/dist/transport/fetch.d.ts +6 -0
- package/dist/transport/fetch.d.ts.map +1 -0
- package/dist/transport/fetch.js +153 -0
- package/dist/transport/undici.d.ts +23 -0
- package/dist/transport/undici.d.ts.map +1 -0
- package/dist/transport/undici.js +218 -0
- package/dist/types/index.d.ts +251 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +1 -0
- package/dist/utils/agent-manager.d.ts +29 -0
- package/dist/utils/agent-manager.d.ts.map +1 -0
- package/dist/utils/agent-manager.js +133 -0
- package/dist/utils/body.d.ts +11 -0
- package/dist/utils/body.d.ts.map +1 -0
- package/dist/utils/body.js +136 -0
- package/dist/utils/cert.d.ts +12 -0
- package/dist/utils/cert.d.ts.map +1 -0
- package/dist/utils/cert.js +32 -0
- package/dist/utils/concurrency.d.ts +21 -0
- package/dist/utils/concurrency.d.ts.map +1 -0
- package/dist/utils/concurrency.js +116 -0
- package/dist/utils/dns.d.ts +7 -0
- package/dist/utils/dns.d.ts.map +1 -0
- package/dist/utils/dns.js +26 -0
- package/dist/utils/doh.d.ts +3 -0
- package/dist/utils/doh.d.ts.map +1 -0
- package/dist/utils/doh.js +35 -0
- package/dist/utils/header-parser.d.ts +81 -0
- package/dist/utils/header-parser.d.ts.map +1 -0
- package/dist/utils/header-parser.js +457 -0
- package/dist/utils/html-cleaner.d.ts +2 -0
- package/dist/utils/html-cleaner.d.ts.map +1 -0
- package/dist/utils/html-cleaner.js +21 -0
- package/dist/utils/logger.d.ts +33 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +160 -0
- package/dist/utils/progress.d.ts +4 -0
- package/dist/utils/progress.d.ts.map +1 -0
- package/dist/utils/progress.js +49 -0
- package/dist/utils/request-pool.d.ts +23 -0
- package/dist/utils/request-pool.d.ts.map +1 -0
- package/dist/utils/request-pool.js +100 -0
- package/dist/utils/sse.d.ts +8 -0
- package/dist/utils/sse.d.ts.map +1 -0
- package/dist/utils/sse.js +62 -0
- package/dist/utils/streaming.d.ts +18 -0
- package/dist/utils/streaming.d.ts.map +1 -0
- package/dist/utils/streaming.js +83 -0
- package/dist/utils/task-pool.d.ts +38 -0
- package/dist/utils/task-pool.js +104 -0
- package/dist/utils/try-fn.d.ts +4 -0
- package/dist/utils/try-fn.d.ts.map +1 -0
- package/dist/utils/try-fn.js +53 -0
- package/dist/utils/upload.d.ts +10 -0
- package/dist/utils/upload.d.ts.map +1 -0
- package/dist/utils/upload.js +45 -0
- package/dist/utils/user-agent.d.ts +45 -0
- package/dist/utils/user-agent.d.ts.map +1 -0
- package/dist/utils/user-agent.js +100 -0
- package/dist/utils/whois.d.ts +15 -0
- package/dist/utils/whois.d.ts.map +1 -0
- package/dist/utils/whois.js +159 -0
- package/dist/websocket/client.d.ts +38 -0
- package/dist/websocket/client.d.ts.map +1 -0
- package/dist/websocket/client.js +184 -0
- package/package.json +100 -0
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
export type Method = 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'HEAD' | 'OPTIONS' | 'TRACE' | 'CONNECT' | 'PURGE' | 'PROPFIND' | 'PROPPATCH' | 'MKCOL' | 'COPY' | 'MOVE' | 'LOCK' | 'UNLOCK' | 'LINK' | 'UNLINK';
|
|
2
|
+
export interface ProgressEvent {
|
|
3
|
+
loaded: number;
|
|
4
|
+
total?: number;
|
|
5
|
+
percent?: number;
|
|
6
|
+
rate?: number;
|
|
7
|
+
estimated?: number;
|
|
8
|
+
}
|
|
9
|
+
export type ProgressCallback = (progress: ProgressEvent) => void;
|
|
10
|
+
export interface RequestOptions {
|
|
11
|
+
method?: Method;
|
|
12
|
+
headers?: HeadersInit;
|
|
13
|
+
body?: BodyInit | null;
|
|
14
|
+
params?: Record<string, string | number>;
|
|
15
|
+
signal?: AbortSignal;
|
|
16
|
+
throwHttpErrors?: boolean;
|
|
17
|
+
timeout?: number;
|
|
18
|
+
onUploadProgress?: ProgressCallback;
|
|
19
|
+
onDownloadProgress?: ProgressCallback;
|
|
20
|
+
}
|
|
21
|
+
export interface ReckerRequest {
|
|
22
|
+
url: string;
|
|
23
|
+
method: Method;
|
|
24
|
+
headers: Headers;
|
|
25
|
+
body: BodyInit | null;
|
|
26
|
+
signal?: AbortSignal;
|
|
27
|
+
throwHttpErrors?: boolean;
|
|
28
|
+
onUploadProgress?: ProgressCallback;
|
|
29
|
+
onDownloadProgress?: ProgressCallback;
|
|
30
|
+
withHeader(name: string, value: string): ReckerRequest;
|
|
31
|
+
withBody(body: BodyInit): ReckerRequest;
|
|
32
|
+
_hooks?: {
|
|
33
|
+
onDnsLookup?: (info: any) => void;
|
|
34
|
+
onTcpConnect?: (info: any) => void;
|
|
35
|
+
onTlsHandshake?: (info: any) => void;
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
export interface Timings {
|
|
39
|
+
queuing?: number;
|
|
40
|
+
dns?: number;
|
|
41
|
+
tls?: number;
|
|
42
|
+
tcp?: number;
|
|
43
|
+
firstByte?: number;
|
|
44
|
+
content?: number;
|
|
45
|
+
total?: number;
|
|
46
|
+
}
|
|
47
|
+
export interface ConcurrencyConfig {
|
|
48
|
+
max?: number;
|
|
49
|
+
requestsPerInterval?: number;
|
|
50
|
+
interval?: number;
|
|
51
|
+
runner?: {
|
|
52
|
+
concurrency?: number;
|
|
53
|
+
retries?: number;
|
|
54
|
+
retryDelay?: number;
|
|
55
|
+
};
|
|
56
|
+
agent?: {
|
|
57
|
+
connections?: number | 'auto';
|
|
58
|
+
pipelining?: number;
|
|
59
|
+
keepAlive?: boolean;
|
|
60
|
+
keepAliveTimeout?: number;
|
|
61
|
+
keepAliveMaxTimeout?: number;
|
|
62
|
+
connectTimeout?: number;
|
|
63
|
+
perDomainPooling?: boolean;
|
|
64
|
+
};
|
|
65
|
+
http2?: {
|
|
66
|
+
maxConcurrentStreams?: number | 'auto';
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
export interface HTTP2Settings {
|
|
70
|
+
headerTableSize?: number;
|
|
71
|
+
enablePush?: number;
|
|
72
|
+
initialWindowSize?: number;
|
|
73
|
+
maxFrameSize?: number;
|
|
74
|
+
maxConcurrentStreams?: number;
|
|
75
|
+
maxHeaderListSize?: number;
|
|
76
|
+
enableConnectProtocol?: number;
|
|
77
|
+
}
|
|
78
|
+
export interface ConnectionInfo {
|
|
79
|
+
protocol?: string;
|
|
80
|
+
cipher?: string;
|
|
81
|
+
remoteAddress?: string;
|
|
82
|
+
remotePort?: number;
|
|
83
|
+
localAddress?: string;
|
|
84
|
+
localPort?: number;
|
|
85
|
+
http2?: {
|
|
86
|
+
streamId?: number;
|
|
87
|
+
streamWeight?: number;
|
|
88
|
+
streamDependency?: number;
|
|
89
|
+
serverPush?: boolean;
|
|
90
|
+
settingsReceived?: boolean;
|
|
91
|
+
maxConcurrentStreams?: number;
|
|
92
|
+
currentStreams?: number;
|
|
93
|
+
pendingStreams?: number;
|
|
94
|
+
localWindowSize?: number;
|
|
95
|
+
remoteWindowSize?: number;
|
|
96
|
+
localSettings?: HTTP2Settings;
|
|
97
|
+
remoteSettings?: HTTP2Settings;
|
|
98
|
+
};
|
|
99
|
+
http3?: {
|
|
100
|
+
quicVersion?: string;
|
|
101
|
+
zeroRTT?: boolean;
|
|
102
|
+
maxStreams?: number;
|
|
103
|
+
handshakeConfirmed?: boolean;
|
|
104
|
+
};
|
|
105
|
+
reused?: boolean;
|
|
106
|
+
rtt?: number;
|
|
107
|
+
}
|
|
108
|
+
export interface SSEEvent {
|
|
109
|
+
id?: string;
|
|
110
|
+
event?: string;
|
|
111
|
+
data: string;
|
|
112
|
+
retry?: number;
|
|
113
|
+
}
|
|
114
|
+
export interface PaginationConfig {
|
|
115
|
+
pageParam?: string;
|
|
116
|
+
limitParam?: string;
|
|
117
|
+
offsetParam?: string;
|
|
118
|
+
resultsPath?: string;
|
|
119
|
+
nextCursorPath?: string;
|
|
120
|
+
}
|
|
121
|
+
export interface PageResult<T = any> {
|
|
122
|
+
data: T;
|
|
123
|
+
response: ReckerResponse;
|
|
124
|
+
pageNumber: number;
|
|
125
|
+
}
|
|
126
|
+
export type NextFunction = (req: ReckerRequest) => Promise<ReckerResponse>;
|
|
127
|
+
export type Middleware = (req: ReckerRequest, next: NextFunction) => Promise<ReckerResponse>;
|
|
128
|
+
export interface Transport {
|
|
129
|
+
dispatch(req: ReckerRequest): Promise<ReckerResponse>;
|
|
130
|
+
}
|
|
131
|
+
export interface CacheEntry {
|
|
132
|
+
status: number;
|
|
133
|
+
statusText: string;
|
|
134
|
+
headers: Record<string, string>;
|
|
135
|
+
body: string;
|
|
136
|
+
timestamp: number;
|
|
137
|
+
}
|
|
138
|
+
export interface CacheStorage {
|
|
139
|
+
get(key: string): Promise<CacheEntry | undefined | null>;
|
|
140
|
+
set(key: string, value: CacheEntry, ttl: number): Promise<void>;
|
|
141
|
+
delete(key: string): Promise<void>;
|
|
142
|
+
}
|
|
143
|
+
export type CacheStrategy = 'cache-first' | 'network-only' | 'network-first' | 'stale-while-revalidate';
|
|
144
|
+
export type Plugin = (client: any) => void;
|
|
145
|
+
export type HookFunction<T = void> = (req: ReckerRequest, extra?: any) => T | Promise<T>;
|
|
146
|
+
export interface Hooks {
|
|
147
|
+
beforeRequest?: Array<(req: ReckerRequest) => ReckerRequest | void | Promise<ReckerRequest | void>>;
|
|
148
|
+
afterResponse?: Array<(req: ReckerRequest, res: ReckerResponse) => ReckerResponse | void | Promise<ReckerResponse | void>>;
|
|
149
|
+
onError?: Array<(error: Error, req: ReckerRequest) => ReckerResponse | void | Promise<ReckerResponse | void>>;
|
|
150
|
+
onRetry?: Array<(error: Error, attempt: number, delay: number, req: ReckerRequest) => void | Promise<void>>;
|
|
151
|
+
onUrlResolved?: Array<(template: string, url: string, params: Record<string, string | number>, req: ReckerRequest) => void | Promise<void>>;
|
|
152
|
+
onDnsLookup?: Array<(info: {
|
|
153
|
+
domain: string;
|
|
154
|
+
duration: number;
|
|
155
|
+
ip?: string;
|
|
156
|
+
}, req: ReckerRequest) => void>;
|
|
157
|
+
onTcpConnect?: Array<(info: {
|
|
158
|
+
remoteAddress: string;
|
|
159
|
+
duration: number;
|
|
160
|
+
}, req: ReckerRequest) => void>;
|
|
161
|
+
onTlsHandshake?: Array<(info: {
|
|
162
|
+
protocol: string;
|
|
163
|
+
cipher: string;
|
|
164
|
+
duration: number;
|
|
165
|
+
}, req: ReckerRequest) => void>;
|
|
166
|
+
onSocketAssigned?: Array<(info: {
|
|
167
|
+
protocol: string;
|
|
168
|
+
reused: boolean;
|
|
169
|
+
}, req: ReckerRequest) => void>;
|
|
170
|
+
onRequestSent?: Array<(req: ReckerRequest) => void>;
|
|
171
|
+
onResponseStart?: Array<(info: {
|
|
172
|
+
status: number;
|
|
173
|
+
headers: Headers;
|
|
174
|
+
}, req: ReckerRequest) => void>;
|
|
175
|
+
}
|
|
176
|
+
export interface ProxyOptions {
|
|
177
|
+
url: string;
|
|
178
|
+
auth?: {
|
|
179
|
+
username: string;
|
|
180
|
+
password: string;
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
export interface DNSOptions {
|
|
184
|
+
override?: Record<string, string>;
|
|
185
|
+
servers?: string[];
|
|
186
|
+
timeout?: number;
|
|
187
|
+
preferIPv4?: boolean;
|
|
188
|
+
}
|
|
189
|
+
export interface XSRFOptions {
|
|
190
|
+
cookieName?: string;
|
|
191
|
+
headerName?: string;
|
|
192
|
+
}
|
|
193
|
+
export interface CompressionOptions {
|
|
194
|
+
algorithm?: 'gzip' | 'deflate' | 'br';
|
|
195
|
+
threshold?: number;
|
|
196
|
+
force?: boolean;
|
|
197
|
+
methods?: string[];
|
|
198
|
+
}
|
|
199
|
+
export interface HTTP2Options {
|
|
200
|
+
enabled?: boolean;
|
|
201
|
+
maxConcurrentStreams?: number;
|
|
202
|
+
pipelining?: number;
|
|
203
|
+
}
|
|
204
|
+
export interface AgentOptions {
|
|
205
|
+
connections?: number;
|
|
206
|
+
pipelining?: number;
|
|
207
|
+
keepAlive?: boolean;
|
|
208
|
+
keepAliveTimeout?: number;
|
|
209
|
+
keepAliveMaxTimeout?: number;
|
|
210
|
+
connectTimeout?: number;
|
|
211
|
+
perDomainPooling?: boolean;
|
|
212
|
+
}
|
|
213
|
+
export interface ClientOptions {
|
|
214
|
+
baseUrl?: string;
|
|
215
|
+
headers?: HeadersInit;
|
|
216
|
+
middlewares?: Middleware[];
|
|
217
|
+
hooks?: Hooks;
|
|
218
|
+
plugins?: Plugin[];
|
|
219
|
+
transport?: Transport;
|
|
220
|
+
defaults?: {
|
|
221
|
+
params?: Record<string, string | number>;
|
|
222
|
+
};
|
|
223
|
+
pagination?: PaginationConfig;
|
|
224
|
+
debug?: boolean;
|
|
225
|
+
proxy?: ProxyOptions | string;
|
|
226
|
+
dns?: DNSOptions;
|
|
227
|
+
xsrf?: boolean | XSRFOptions;
|
|
228
|
+
compression?: boolean | CompressionOptions;
|
|
229
|
+
http2?: boolean | HTTP2Options;
|
|
230
|
+
concurrency?: number | ConcurrencyConfig;
|
|
231
|
+
}
|
|
232
|
+
export interface ReckerResponse<T = unknown> {
|
|
233
|
+
status: number;
|
|
234
|
+
statusText: string;
|
|
235
|
+
headers: Headers;
|
|
236
|
+
ok: boolean;
|
|
237
|
+
url: string;
|
|
238
|
+
timings?: Timings;
|
|
239
|
+
connection?: ConnectionInfo;
|
|
240
|
+
json<R = T>(): Promise<R>;
|
|
241
|
+
text(): Promise<string>;
|
|
242
|
+
cleanText(): Promise<string>;
|
|
243
|
+
blob(): Promise<Blob>;
|
|
244
|
+
read(): ReadableStream<Uint8Array> | null;
|
|
245
|
+
clone(): ReckerResponse<T>;
|
|
246
|
+
sse(): AsyncGenerator<SSEEvent>;
|
|
247
|
+
download(): AsyncGenerator<ProgressEvent>;
|
|
248
|
+
raw: Response;
|
|
249
|
+
[Symbol.asyncIterator](): AsyncGenerator<Uint8Array>;
|
|
250
|
+
}
|
|
251
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,MAAM,GAEd,KAAK,GACL,MAAM,GACN,KAAK,GACL,QAAQ,GACR,OAAO,GACP,MAAM,GACN,SAAS,GACT,OAAO,GACP,SAAS,GAET,OAAO,GAEP,UAAU,GACV,WAAW,GACX,OAAO,GACP,MAAM,GACN,MAAM,GACN,MAAM,GACN,QAAQ,GAER,MAAM,GACN,QAAQ,CAAC;AAEb,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,gBAAgB,GAAG,CAAC,QAAQ,EAAE,aAAa,KAAK,IAAI,CAAC;AAEjE,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,IAAI,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACzC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;CACvC;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,QAAQ,GAAG,IAAI,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,kBAAkB,CAAC,EAAE,gBAAgB,CAAC;IAGtC,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,aAAa,CAAC;IACvD,QAAQ,CAAC,IAAI,EAAE,QAAQ,GAAG,aAAa,CAAC;IAGxC,MAAM,CAAC,EAAE;QACL,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,CAAC;QAClC,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,CAAC;QACnC,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,CAAC;KACxC,CAAC;CACH;AAED,MAAM,WAAW,OAAO;IACtB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAMD,MAAM,WAAW,iBAAiB;IAmBhC,GAAG,CAAC,EAAE,MAAM,CAAC;IAOb,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAM7B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAMlB,MAAM,CAAC,EAAE;QAEP,WAAW,CAAC,EAAE,MAAM,CAAC;QAErB,OAAO,CAAC,EAAE,MAAM,CAAC;QAEjB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IAMF,KAAK,CAAC,EAAE;QAEN,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QAE9B,UAAU,CAAC,EAAE,MAAM,CAAC;QAEpB,SAAS,CAAC,EAAE,OAAO,CAAC;QAEpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAE1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAE7B,cAAc,CAAC,EAAE,MAAM,CAAC;QAExB,gBAAgB,CAAC,EAAE,OAAO,CAAC;KAC5B,CAAC;IAMF,KAAK,CAAC,EAAE;QAEN,oBAAoB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;KACxC,CAAC;CACH;AAED,MAAM,WAAW,aAAa;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED,MAAM,WAAW,cAAc;IAE7B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,KAAK,CAAC,EAAE;QACN,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAC1B,aAAa,CAAC,EAAE,aAAa,CAAC;QAC9B,cAAc,CAAC,EAAE,aAAa,CAAC;KAChC,CAAC;IAGF,KAAK,CAAC,EAAE;QACN,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,kBAAkB,CAAC,EAAE,OAAO,CAAC;KAC9B,CAAC;IAGF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAGD,MAAM,WAAW,QAAQ;IACvB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU,CAAC,CAAC,GAAG,GAAG;IAC/B,IAAI,EAAE,CAAC,CAAC;IACR,QAAQ,EAAE,cAAc,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,aAAa,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;AAC3E,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,aAAa,EAAE,IAAI,EAAE,YAAY,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;AAE7F,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CACvD;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC;IACzD,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChE,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACpC;AAED,MAAM,MAAM,aAAa,GAAG,aAAa,GAAG,cAAc,GAAG,eAAe,GAAG,wBAAwB,CAAC;AAExG,MAAM,MAAM,MAAM,GAAG,CAAC,MAAM,EAAE,GAAG,KAAK,IAAI,CAAC;AAE3C,MAAM,MAAM,YAAY,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,aAAa,EAAE,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;AAEzF,MAAM,WAAW,KAAK;IACpB,aAAa,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,EAAE,aAAa,KAAK,aAAa,GAAG,IAAI,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC;IACpG,aAAa,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,cAAc,KAAK,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC,CAAC;IAC3H,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,KAAK,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC,CAAC;IAK9G,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IAK5G,aAAa,CAAC,EAAE,KAAK,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IAG5I,WAAW,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC,CAAC;IAC3G,YAAY,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC,CAAC;IACtG,cAAc,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC,CAAC;IAMnH,gBAAgB,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC,CAAC;IAKpG,aAAa,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC,CAAC;IAMpD,eAAe,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,EAAE,GAAG,EAAE,aAAa,KAAK,IAAI,CAAC,CAAC;CACnG;AAED,MAAM,WAAW,YAAY;IAI3B,GAAG,EAAE,MAAM,CAAC;IAIZ,IAAI,CAAC,EAAE;QACL,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAED,MAAM,WAAW,UAAU;IAezB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAalC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAMnB,OAAO,CAAC,EAAE,MAAM,CAAC;IAMjB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAK1B,UAAU,CAAC,EAAE,MAAM,CAAC;IAKpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IAKjC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAAC;IAKtC,SAAS,CAAC,EAAE,MAAM,CAAC;IAKnB,KAAK,CAAC,EAAE,OAAO,CAAC;IAKhB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAM3B,OAAO,CAAC,EAAE,OAAO,CAAC;IAOlB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAQ9B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,YAAY;IAK3B,WAAW,CAAC,EAAE,MAAM,CAAC;IAMrB,UAAU,CAAC,EAAE,MAAM,CAAC;IAMpB,SAAS,CAAC,EAAE,OAAO,CAAC;IAMpB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAM1B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAM7B,cAAc,CAAC,EAAE,MAAM,CAAC;IAOxB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;IAC3B,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,QAAQ,CAAC,EAAE;QACT,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;KAC1C,CAAC;IACF,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,KAAK,CAAC,EAAE,OAAO,CAAC;IAUhB,KAAK,CAAC,EAAE,YAAY,GAAG,MAAM,CAAC;IAoB9B,GAAG,CAAC,EAAE,UAAU,CAAC;IAWjB,IAAI,CAAC,EAAE,OAAO,GAAG,WAAW,CAAC;IAW7B,WAAW,CAAC,EAAE,OAAO,GAAG,kBAAkB,CAAC;IAW3C,KAAK,CAAC,EAAE,OAAO,GAAG,YAAY,CAAC;IAgC/B,WAAW,CAAC,EAAE,MAAM,GAAG,iBAAiB,CAAC;CAC1C;AAED,MAAM,WAAW,cAAc,CAAC,CAAC,GAAG,OAAO;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,EAAE,EAAE,OAAO,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IAGZ,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,UAAU,CAAC,EAAE,cAAc,CAAC;IAG5B,IAAI,CAAC,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IACxB,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAGtB,IAAI,IAAI,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAE1C,KAAK,IAAI,cAAc,CAAC,CAAC,CAAC,CAAC;IAC3B,GAAG,IAAI,cAAc,CAAC,QAAQ,CAAC,CAAC;IAChC,QAAQ,IAAI,cAAc,CAAC,aAAa,CAAC,CAAC;IAC1C,GAAG,EAAE,QAAQ,CAAC;IAGd,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,UAAU,CAAC,CAAC;CACtD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { Agent } from 'undici';
|
|
2
|
+
import type { AgentOptions } from '../types/index.js';
|
|
3
|
+
export interface AgentStats {
|
|
4
|
+
agentCount: number;
|
|
5
|
+
domains: string[];
|
|
6
|
+
totalConnections: number;
|
|
7
|
+
}
|
|
8
|
+
export declare class AgentManager {
|
|
9
|
+
private globalAgent?;
|
|
10
|
+
private domainAgents;
|
|
11
|
+
private options;
|
|
12
|
+
constructor(options?: AgentOptions);
|
|
13
|
+
getGlobalAgent(): Agent;
|
|
14
|
+
getAgentForDomain(domain: string, options?: Partial<AgentOptions>): Agent;
|
|
15
|
+
getAgentForUrl(url: string): Agent;
|
|
16
|
+
createBatchAgent(concurrency: number, requestCount: number, options?: Partial<AgentOptions>): Agent;
|
|
17
|
+
private createAgent;
|
|
18
|
+
getStats(): AgentStats;
|
|
19
|
+
closeDomainAgent(domain: string): Promise<void>;
|
|
20
|
+
closeAll(): Promise<void>;
|
|
21
|
+
destroy(): Promise<void>;
|
|
22
|
+
}
|
|
23
|
+
export declare function createAgent(options?: AgentOptions): Agent;
|
|
24
|
+
export declare function extractDomain(url: string): string | null;
|
|
25
|
+
export declare function analyzeBatchDomains(urls: string[]): {
|
|
26
|
+
strategy: 'single' | 'multi';
|
|
27
|
+
domains: Set<string>;
|
|
28
|
+
};
|
|
29
|
+
//# sourceMappingURL=agent-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent-manager.d.ts","sourceRoot":"","sources":["../../src/utils/agent-manager.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC/B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEtD,MAAM,WAAW,UAAU;IAEzB,UAAU,EAAE,MAAM,CAAC;IAGnB,OAAO,EAAE,MAAM,EAAE,CAAC;IAGlB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAOD,qBAAa,YAAY;IACvB,OAAO,CAAC,WAAW,CAAC,CAAQ;IAC5B,OAAO,CAAC,YAAY,CAAqB;IACzC,OAAO,CAAC,OAAO,CAAyB;gBAE5B,OAAO,GAAE,YAAiB;IAgBtC,cAAc,IAAI,KAAK;IAavB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,KAAK;IAkBzE,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,KAAK;IAmBlC,gBAAgB,CACd,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,EACpB,OAAO,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAC9B,KAAK;IAmCR,OAAO,CAAC,WAAW;IAgBnB,QAAQ,IAAI,UAAU;IAWhB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAW/C,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAoBzB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAgB/B;AAOD,wBAAgB,WAAW,CAAC,OAAO,GAAE,YAAiB,GAAG,KAAK,CAG7D;AAKD,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAOxD;AASD,wBAAgB,mBAAmB,CACjC,IAAI,EAAE,MAAM,EAAE,GACb;IAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;IAAC,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;CAAE,CAcxD"}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { Agent } from 'undici';
|
|
2
|
+
export class AgentManager {
|
|
3
|
+
globalAgent;
|
|
4
|
+
domainAgents;
|
|
5
|
+
options;
|
|
6
|
+
constructor(options = {}) {
|
|
7
|
+
this.domainAgents = new Map();
|
|
8
|
+
this.options = {
|
|
9
|
+
connections: options.connections ?? 10,
|
|
10
|
+
pipelining: options.pipelining ?? 1,
|
|
11
|
+
keepAlive: options.keepAlive ?? true,
|
|
12
|
+
keepAliveTimeout: options.keepAliveTimeout ?? 4000,
|
|
13
|
+
keepAliveMaxTimeout: options.keepAliveMaxTimeout ?? 600000,
|
|
14
|
+
connectTimeout: options.connectTimeout ?? 10000,
|
|
15
|
+
perDomainPooling: options.perDomainPooling ?? true,
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
getGlobalAgent() {
|
|
19
|
+
if (!this.globalAgent) {
|
|
20
|
+
this.globalAgent = this.createAgent(this.options);
|
|
21
|
+
}
|
|
22
|
+
return this.globalAgent;
|
|
23
|
+
}
|
|
24
|
+
getAgentForDomain(domain, options) {
|
|
25
|
+
if (!this.options.perDomainPooling) {
|
|
26
|
+
return this.getGlobalAgent();
|
|
27
|
+
}
|
|
28
|
+
let agent = this.domainAgents.get(domain);
|
|
29
|
+
if (!agent) {
|
|
30
|
+
const agentOptions = { ...this.options, ...options };
|
|
31
|
+
agent = this.createAgent(agentOptions);
|
|
32
|
+
this.domainAgents.set(domain, agent);
|
|
33
|
+
}
|
|
34
|
+
return agent;
|
|
35
|
+
}
|
|
36
|
+
getAgentForUrl(url) {
|
|
37
|
+
try {
|
|
38
|
+
const parsedUrl = new URL(url);
|
|
39
|
+
return this.getAgentForDomain(parsedUrl.hostname);
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return this.getGlobalAgent();
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
createBatchAgent(concurrency, requestCount, options) {
|
|
46
|
+
const smartConnections = Math.max(1, Math.min(Math.ceil(concurrency / 2), 50, requestCount));
|
|
47
|
+
const smartPipelining = requestCount > 20 && concurrency > 5
|
|
48
|
+
? 2
|
|
49
|
+
: 1;
|
|
50
|
+
const batchOptions = {
|
|
51
|
+
...this.options,
|
|
52
|
+
connections: options?.connections ?? smartConnections,
|
|
53
|
+
pipelining: options?.pipelining ?? smartPipelining,
|
|
54
|
+
...options,
|
|
55
|
+
};
|
|
56
|
+
return this.createAgent(batchOptions);
|
|
57
|
+
}
|
|
58
|
+
createAgent(options) {
|
|
59
|
+
return new Agent({
|
|
60
|
+
connections: options.connections,
|
|
61
|
+
pipelining: options.pipelining,
|
|
62
|
+
connect: {
|
|
63
|
+
timeout: options.connectTimeout,
|
|
64
|
+
keepAlive: options.keepAlive,
|
|
65
|
+
keepAliveInitialDelay: options.keepAliveTimeout ?? 1000,
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
getStats() {
|
|
70
|
+
return {
|
|
71
|
+
agentCount: 1 + this.domainAgents.size,
|
|
72
|
+
domains: Array.from(this.domainAgents.keys()),
|
|
73
|
+
totalConnections: (1 + this.domainAgents.size) * this.options.connections,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
async closeDomainAgent(domain) {
|
|
77
|
+
const agent = this.domainAgents.get(domain);
|
|
78
|
+
if (agent) {
|
|
79
|
+
await agent.close();
|
|
80
|
+
this.domainAgents.delete(domain);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
async closeAll() {
|
|
84
|
+
const closePromises = [];
|
|
85
|
+
if (this.globalAgent) {
|
|
86
|
+
closePromises.push(this.globalAgent.close());
|
|
87
|
+
this.globalAgent = undefined;
|
|
88
|
+
}
|
|
89
|
+
for (const agent of this.domainAgents.values()) {
|
|
90
|
+
closePromises.push(agent.close());
|
|
91
|
+
}
|
|
92
|
+
this.domainAgents.clear();
|
|
93
|
+
await Promise.all(closePromises);
|
|
94
|
+
}
|
|
95
|
+
async destroy() {
|
|
96
|
+
const destroyPromises = [];
|
|
97
|
+
if (this.globalAgent) {
|
|
98
|
+
destroyPromises.push(this.globalAgent.destroy());
|
|
99
|
+
this.globalAgent = undefined;
|
|
100
|
+
}
|
|
101
|
+
for (const agent of this.domainAgents.values()) {
|
|
102
|
+
destroyPromises.push(agent.destroy());
|
|
103
|
+
}
|
|
104
|
+
this.domainAgents.clear();
|
|
105
|
+
await Promise.all(destroyPromises);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
export function createAgent(options = {}) {
|
|
109
|
+
const manager = new AgentManager(options);
|
|
110
|
+
return manager.getGlobalAgent();
|
|
111
|
+
}
|
|
112
|
+
export function extractDomain(url) {
|
|
113
|
+
try {
|
|
114
|
+
const parsed = new URL(url);
|
|
115
|
+
return parsed.hostname;
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
export function analyzeBatchDomains(urls) {
|
|
122
|
+
const domains = new Set();
|
|
123
|
+
for (const url of urls) {
|
|
124
|
+
const domain = extractDomain(url);
|
|
125
|
+
if (domain) {
|
|
126
|
+
domains.add(domain);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return {
|
|
130
|
+
strategy: domains.size <= 1 ? 'single' : 'multi',
|
|
131
|
+
domains,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export type BodyInput = string | Blob | ArrayBuffer | FormData | URLSearchParams | ReadableStream | Record<string, any> | null | undefined;
|
|
2
|
+
export interface ProcessedBody {
|
|
3
|
+
body: any;
|
|
4
|
+
contentType?: string;
|
|
5
|
+
}
|
|
6
|
+
export declare function isPlainObject(value: any): boolean;
|
|
7
|
+
export declare function processBody(body: BodyInput): ProcessedBody;
|
|
8
|
+
export declare function createFormData(data: Record<string, any>): FormData;
|
|
9
|
+
export declare function createMultipart(data: Record<string, any>): FormData;
|
|
10
|
+
export declare function isFileUpload(body: any): boolean;
|
|
11
|
+
//# sourceMappingURL=body.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"body.d.ts","sourceRoot":"","sources":["../../src/utils/body.ts"],"names":[],"mappings":"AAIA,MAAM,MAAM,SAAS,GACjB,MAAM,GACN,IAAI,GACJ,WAAW,GACX,QAAQ,GACR,eAAe,GACf,cAAc,GACd,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GACnB,IAAI,GACJ,SAAS,CAAC;AAEd,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,GAAG,CAAC;IACV,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAKD,wBAAgB,aAAa,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CA4BjD;AAKD,wBAAgB,WAAW,CAAC,IAAI,EAAE,SAAS,GAAG,aAAa,CAoE1D;AAMD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,QAAQ,CAqClE;AAYD,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,QAAQ,CAEnE;AAeD,wBAAgB,YAAY,CAAC,IAAI,EAAE,GAAG,GAAG,OAAO,CA8B/C"}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
export function isPlainObject(value) {
|
|
2
|
+
if (typeof value !== 'object' || value === null) {
|
|
3
|
+
return false;
|
|
4
|
+
}
|
|
5
|
+
if (value instanceof Blob ||
|
|
6
|
+
value instanceof FormData ||
|
|
7
|
+
value instanceof URLSearchParams ||
|
|
8
|
+
value instanceof ArrayBuffer ||
|
|
9
|
+
value instanceof ReadableStream) {
|
|
10
|
+
return false;
|
|
11
|
+
}
|
|
12
|
+
if (typeof File !== 'undefined') {
|
|
13
|
+
try {
|
|
14
|
+
if (value instanceof File) {
|
|
15
|
+
return false;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return true;
|
|
22
|
+
}
|
|
23
|
+
export function processBody(body) {
|
|
24
|
+
if (body === null || body === undefined) {
|
|
25
|
+
return { body: undefined };
|
|
26
|
+
}
|
|
27
|
+
if (typeof body === 'string') {
|
|
28
|
+
return {
|
|
29
|
+
body,
|
|
30
|
+
contentType: 'text/plain; charset=utf-8'
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
if (body instanceof FormData) {
|
|
34
|
+
return {
|
|
35
|
+
body,
|
|
36
|
+
contentType: undefined
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
if (body instanceof URLSearchParams) {
|
|
40
|
+
return {
|
|
41
|
+
body,
|
|
42
|
+
contentType: 'application/x-www-form-urlencoded'
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
if (body instanceof Blob || isFile(body)) {
|
|
46
|
+
return {
|
|
47
|
+
body,
|
|
48
|
+
contentType: body.type || 'application/octet-stream'
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
if (body instanceof ArrayBuffer) {
|
|
52
|
+
return {
|
|
53
|
+
body,
|
|
54
|
+
contentType: 'application/octet-stream'
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
if (body instanceof ReadableStream) {
|
|
58
|
+
return {
|
|
59
|
+
body,
|
|
60
|
+
contentType: 'application/octet-stream'
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
if (isPlainObject(body) || Array.isArray(body)) {
|
|
64
|
+
return {
|
|
65
|
+
body: JSON.stringify(body),
|
|
66
|
+
contentType: 'application/json'
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
return {
|
|
70
|
+
body: String(body),
|
|
71
|
+
contentType: 'text/plain; charset=utf-8'
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
export function createFormData(data) {
|
|
75
|
+
const formData = new FormData();
|
|
76
|
+
for (const [key, value] of Object.entries(data)) {
|
|
77
|
+
if (value === null || value === undefined) {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
if (Array.isArray(value)) {
|
|
81
|
+
value.forEach((item) => {
|
|
82
|
+
if (item instanceof Blob || isFile(item)) {
|
|
83
|
+
formData.append(key, item);
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
formData.append(key, String(item));
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
if (value instanceof Blob || isFile(value)) {
|
|
92
|
+
formData.append(key, value);
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
if (typeof value === 'object') {
|
|
96
|
+
formData.append(key, JSON.stringify(value));
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
formData.append(key, String(value));
|
|
100
|
+
}
|
|
101
|
+
return formData;
|
|
102
|
+
}
|
|
103
|
+
export function createMultipart(data) {
|
|
104
|
+
return createFormData(data);
|
|
105
|
+
}
|
|
106
|
+
function isFile(value) {
|
|
107
|
+
if (typeof File === 'undefined') {
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
return value instanceof File;
|
|
111
|
+
}
|
|
112
|
+
export function isFileUpload(body) {
|
|
113
|
+
if (!body)
|
|
114
|
+
return false;
|
|
115
|
+
if (body instanceof Blob || isFile(body)) {
|
|
116
|
+
return true;
|
|
117
|
+
}
|
|
118
|
+
if (body instanceof FormData) {
|
|
119
|
+
for (const value of body.values()) {
|
|
120
|
+
if (value instanceof Blob || isFile(value)) {
|
|
121
|
+
return true;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
if (isPlainObject(body)) {
|
|
126
|
+
for (const value of Object.values(body)) {
|
|
127
|
+
if (value instanceof Blob || isFile(value)) {
|
|
128
|
+
return true;
|
|
129
|
+
}
|
|
130
|
+
if (Array.isArray(value) && value.some(v => v instanceof Blob || isFile(v))) {
|
|
131
|
+
return true;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return false;
|
|
136
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Agent } from 'undici';
|
|
2
|
+
export interface CertificateOptions {
|
|
3
|
+
cert?: string | Buffer;
|
|
4
|
+
key?: string | Buffer;
|
|
5
|
+
ca?: string | Buffer;
|
|
6
|
+
rejectUnauthorized?: boolean;
|
|
7
|
+
}
|
|
8
|
+
export declare function createCertAgent(options: CertificateOptions): Agent;
|
|
9
|
+
export declare function getCertInfo(response: any): {
|
|
10
|
+
authorized: boolean;
|
|
11
|
+
};
|
|
12
|
+
//# sourceMappingURL=cert.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cert.d.ts","sourceRoot":"","sources":["../../src/utils/cert.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAc,MAAM,QAAQ,CAAC;AAG3C,MAAM,WAAW,kBAAkB;IAEjC,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAEvB,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAEtB,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAErB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAsCD,wBAAgB,eAAe,CAAC,OAAO,EAAE,kBAAkB,GAAG,KAAK,CASlE;AAQD,wBAAgB,WAAW,CAAC,QAAQ,EAAE,GAAG;;EAKxC"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { Agent } from 'undici';
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
function loadCert(input) {
|
|
4
|
+
if (!input)
|
|
5
|
+
return undefined;
|
|
6
|
+
if (Buffer.isBuffer(input))
|
|
7
|
+
return input;
|
|
8
|
+
if (input.includes('-----BEGIN')) {
|
|
9
|
+
return input;
|
|
10
|
+
}
|
|
11
|
+
try {
|
|
12
|
+
return readFileSync(input);
|
|
13
|
+
}
|
|
14
|
+
catch (err) {
|
|
15
|
+
return input;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
export function createCertAgent(options) {
|
|
19
|
+
return new Agent({
|
|
20
|
+
connect: {
|
|
21
|
+
cert: loadCert(options.cert),
|
|
22
|
+
key: loadCert(options.key),
|
|
23
|
+
ca: loadCert(options.ca),
|
|
24
|
+
rejectUnauthorized: options.rejectUnauthorized
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
export function getCertInfo(response) {
|
|
29
|
+
return {
|
|
30
|
+
authorized: true,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { AgentOptions, ClientOptions } from '../types/index.js';
|
|
2
|
+
export interface NormalizedConcurrencyConfig {
|
|
3
|
+
max: number;
|
|
4
|
+
requestsPerInterval: number;
|
|
5
|
+
interval: number;
|
|
6
|
+
runner: {
|
|
7
|
+
concurrency: number;
|
|
8
|
+
retries?: number;
|
|
9
|
+
retryDelay?: number;
|
|
10
|
+
};
|
|
11
|
+
agent: AgentOptions & {
|
|
12
|
+
connections: number;
|
|
13
|
+
};
|
|
14
|
+
http2: {
|
|
15
|
+
maxConcurrentStreams: number;
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
export declare function normalizeConcurrency(options: Pick<ClientOptions, 'concurrency' | 'http2'>): NormalizedConcurrencyConfig;
|
|
19
|
+
export declare function calculateOptimalConnections(maxConcurrent: number, http2Enabled: boolean, http2MaxStreams?: number, pipelining?: number): number;
|
|
20
|
+
export declare function createBatchConfig(baseConcurrency: NormalizedConcurrencyConfig, batchConcurrency: number, requestCount: number): NormalizedConcurrencyConfig;
|
|
21
|
+
//# sourceMappingURL=concurrency.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"concurrency.d.ts","sourceRoot":"","sources":["../../src/utils/concurrency.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAEV,YAAY,EAEZ,aAAa,EACd,MAAM,mBAAmB,CAAC;AAK3B,MAAM,WAAW,2BAA2B;IAC1C,GAAG,EAAE,MAAM,CAAC;IACZ,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE;QACN,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,KAAK,EAAE,YAAY,GAAG;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9C,KAAK,EAAE;QACL,oBAAoB,EAAE,MAAM,CAAC;KAC9B,CAAC;CACH;AA+BD,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,IAAI,CAAC,aAAa,EAAE,aAAa,GAAG,OAAO,CAAC,GACpD,2BAA2B,CAuE7B;AAiED,wBAAgB,2BAA2B,CACzC,aAAa,EAAE,MAAM,EACrB,YAAY,EAAE,OAAO,EACrB,eAAe,CAAC,EAAE,MAAM,EACxB,UAAU,GAAE,MAAU,GACrB,MAAM,CAaR;AA4BD,wBAAgB,iBAAiB,CAC/B,eAAe,EAAE,2BAA2B,EAC5C,gBAAgB,EAAE,MAAM,EACxB,YAAY,EAAE,MAAM,GACnB,2BAA2B,CA6B7B"}
|