recker 1.0.26 → 1.0.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/dist/browser/browser/cache.d.ts +40 -0
  2. package/dist/browser/browser/cache.js +199 -0
  3. package/dist/browser/browser/crypto.d.ts +24 -0
  4. package/dist/browser/browser/crypto.js +80 -0
  5. package/dist/browser/browser/index.d.ts +31 -0
  6. package/dist/browser/browser/index.js +31 -0
  7. package/dist/browser/browser/recker.d.ts +26 -0
  8. package/dist/browser/browser/recker.js +61 -0
  9. package/dist/browser/cache/basic-file-storage.d.ts +12 -0
  10. package/dist/browser/cache/basic-file-storage.js +50 -0
  11. package/dist/browser/cache/memory-limits.d.ts +20 -0
  12. package/dist/browser/cache/memory-limits.js +96 -0
  13. package/dist/browser/cache/memory-storage.d.ts +132 -0
  14. package/dist/browser/cache/memory-storage.js +454 -0
  15. package/dist/browser/cache.d.ts +40 -0
  16. package/dist/browser/cache.js +199 -0
  17. package/dist/browser/constants/http-status.d.ts +73 -0
  18. package/dist/browser/constants/http-status.js +156 -0
  19. package/dist/browser/cookies/memory-cookie-jar.d.ts +30 -0
  20. package/dist/browser/cookies/memory-cookie-jar.js +210 -0
  21. package/dist/browser/core/client.d.ts +118 -0
  22. package/dist/browser/core/client.js +667 -0
  23. package/dist/browser/core/errors.d.ts +142 -0
  24. package/dist/browser/core/errors.js +308 -0
  25. package/dist/browser/core/index.d.ts +5 -0
  26. package/dist/browser/core/index.js +5 -0
  27. package/dist/browser/core/request-promise.d.ts +23 -0
  28. package/dist/browser/core/request-promise.js +82 -0
  29. package/dist/browser/core/request.d.ts +20 -0
  30. package/dist/browser/core/request.js +76 -0
  31. package/dist/browser/core/response.d.ts +34 -0
  32. package/dist/browser/core/response.js +178 -0
  33. package/dist/browser/crypto.d.ts +24 -0
  34. package/dist/browser/crypto.js +80 -0
  35. package/dist/browser/index.d.ts +31 -0
  36. package/dist/browser/index.js +31 -0
  37. package/dist/browser/plugins/auth/api-key.d.ts +8 -0
  38. package/dist/browser/plugins/auth/api-key.js +27 -0
  39. package/dist/browser/plugins/auth/auth0.d.ts +33 -0
  40. package/dist/browser/plugins/auth/auth0.js +94 -0
  41. package/dist/browser/plugins/auth/aws-sigv4.d.ts +10 -0
  42. package/dist/browser/plugins/auth/aws-sigv4.js +88 -0
  43. package/dist/browser/plugins/auth/azure-ad.d.ts +48 -0
  44. package/dist/browser/plugins/auth/azure-ad.js +152 -0
  45. package/dist/browser/plugins/auth/basic.d.ts +7 -0
  46. package/dist/browser/plugins/auth/basic.js +13 -0
  47. package/dist/browser/plugins/auth/bearer.d.ts +8 -0
  48. package/dist/browser/plugins/auth/bearer.js +17 -0
  49. package/dist/browser/plugins/auth/cognito.d.ts +45 -0
  50. package/dist/browser/plugins/auth/cognito.js +208 -0
  51. package/dist/browser/plugins/auth/digest.d.ts +8 -0
  52. package/dist/browser/plugins/auth/digest.js +100 -0
  53. package/dist/browser/plugins/auth/firebase.d.ts +32 -0
  54. package/dist/browser/plugins/auth/firebase.js +195 -0
  55. package/dist/browser/plugins/auth/github-app.d.ts +36 -0
  56. package/dist/browser/plugins/auth/github-app.js +170 -0
  57. package/dist/browser/plugins/auth/google-service-account.d.ts +49 -0
  58. package/dist/browser/plugins/auth/google-service-account.js +172 -0
  59. package/dist/browser/plugins/auth/index.d.ts +15 -0
  60. package/dist/browser/plugins/auth/index.js +15 -0
  61. package/dist/browser/plugins/auth/mtls.d.ts +37 -0
  62. package/dist/browser/plugins/auth/mtls.js +140 -0
  63. package/dist/browser/plugins/auth/oauth2.d.ts +8 -0
  64. package/dist/browser/plugins/auth/oauth2.js +26 -0
  65. package/dist/browser/plugins/auth/oidc.d.ts +55 -0
  66. package/dist/browser/plugins/auth/oidc.js +222 -0
  67. package/dist/browser/plugins/auth/okta.d.ts +47 -0
  68. package/dist/browser/plugins/auth/okta.js +157 -0
  69. package/dist/browser/plugins/auth.d.ts +1 -0
  70. package/dist/browser/plugins/auth.js +1 -0
  71. package/dist/browser/plugins/cache.d.ts +15 -0
  72. package/dist/browser/plugins/cache.js +486 -0
  73. package/dist/browser/plugins/circuit-breaker.d.ts +13 -0
  74. package/dist/browser/plugins/circuit-breaker.js +100 -0
  75. package/dist/browser/plugins/compression.d.ts +4 -0
  76. package/dist/browser/plugins/compression.js +130 -0
  77. package/dist/browser/plugins/cookie-jar.d.ts +5 -0
  78. package/dist/browser/plugins/cookie-jar.js +72 -0
  79. package/dist/browser/plugins/dedup.d.ts +5 -0
  80. package/dist/browser/plugins/dedup.js +35 -0
  81. package/dist/browser/plugins/graphql.d.ts +13 -0
  82. package/dist/browser/plugins/graphql.js +58 -0
  83. package/dist/browser/plugins/grpc-web.d.ts +79 -0
  84. package/dist/browser/plugins/grpc-web.js +261 -0
  85. package/dist/browser/plugins/hls.d.ts +105 -0
  86. package/dist/browser/plugins/hls.js +395 -0
  87. package/dist/browser/plugins/jsonrpc.d.ts +75 -0
  88. package/dist/browser/plugins/jsonrpc.js +143 -0
  89. package/dist/browser/plugins/logger.d.ts +13 -0
  90. package/dist/browser/plugins/logger.js +108 -0
  91. package/dist/browser/plugins/odata.d.ts +181 -0
  92. package/dist/browser/plugins/odata.js +564 -0
  93. package/dist/browser/plugins/pagination.d.ts +16 -0
  94. package/dist/browser/plugins/pagination.js +105 -0
  95. package/dist/browser/plugins/rate-limit.d.ts +15 -0
  96. package/dist/browser/plugins/rate-limit.js +162 -0
  97. package/dist/browser/plugins/retry.d.ts +14 -0
  98. package/dist/browser/plugins/retry.js +116 -0
  99. package/dist/browser/plugins/scrape.d.ts +21 -0
  100. package/dist/browser/plugins/scrape.js +82 -0
  101. package/dist/browser/plugins/server-timing.d.ts +7 -0
  102. package/dist/browser/plugins/server-timing.js +24 -0
  103. package/dist/browser/plugins/soap.d.ts +72 -0
  104. package/dist/browser/plugins/soap.js +347 -0
  105. package/dist/browser/plugins/xml.d.ts +9 -0
  106. package/dist/browser/plugins/xml.js +194 -0
  107. package/dist/browser/plugins/xsrf.d.ts +9 -0
  108. package/dist/browser/plugins/xsrf.js +48 -0
  109. package/dist/browser/recker.d.ts +26 -0
  110. package/dist/browser/recker.js +61 -0
  111. package/dist/browser/runner/request-runner.d.ts +46 -0
  112. package/dist/browser/runner/request-runner.js +89 -0
  113. package/dist/browser/scrape/document.d.ts +44 -0
  114. package/dist/browser/scrape/document.js +210 -0
  115. package/dist/browser/scrape/element.d.ts +49 -0
  116. package/dist/browser/scrape/element.js +176 -0
  117. package/dist/browser/scrape/extractors.d.ts +16 -0
  118. package/dist/browser/scrape/extractors.js +356 -0
  119. package/dist/browser/scrape/types.d.ts +107 -0
  120. package/dist/browser/scrape/types.js +1 -0
  121. package/dist/browser/transport/fetch.d.ts +11 -0
  122. package/dist/browser/transport/fetch.js +143 -0
  123. package/dist/browser/transport/undici.d.ts +38 -0
  124. package/dist/browser/transport/undici.js +897 -0
  125. package/dist/browser/types/ai.d.ts +267 -0
  126. package/dist/browser/types/ai.js +1 -0
  127. package/dist/browser/types/index.d.ts +351 -0
  128. package/dist/browser/types/index.js +1 -0
  129. package/dist/browser/types/logger.d.ts +16 -0
  130. package/dist/browser/types/logger.js +66 -0
  131. package/dist/browser/types/udp.d.ts +138 -0
  132. package/dist/browser/types/udp.js +1 -0
  133. package/dist/browser/utils/agent-manager.d.ts +29 -0
  134. package/dist/browser/utils/agent-manager.js +160 -0
  135. package/dist/browser/utils/body.d.ts +10 -0
  136. package/dist/browser/utils/body.js +148 -0
  137. package/dist/browser/utils/charset.d.ts +15 -0
  138. package/dist/browser/utils/charset.js +169 -0
  139. package/dist/browser/utils/concurrency.d.ts +20 -0
  140. package/dist/browser/utils/concurrency.js +120 -0
  141. package/dist/browser/utils/dns.d.ts +6 -0
  142. package/dist/browser/utils/dns.js +26 -0
  143. package/dist/browser/utils/header-parser.d.ts +94 -0
  144. package/dist/browser/utils/header-parser.js +617 -0
  145. package/dist/browser/utils/html-cleaner.d.ts +1 -0
  146. package/dist/browser/utils/html-cleaner.js +21 -0
  147. package/dist/browser/utils/link-header.d.ts +69 -0
  148. package/dist/browser/utils/link-header.js +190 -0
  149. package/dist/browser/utils/optional-require.d.ts +19 -0
  150. package/dist/browser/utils/optional-require.js +105 -0
  151. package/dist/browser/utils/progress.d.ts +8 -0
  152. package/dist/browser/utils/progress.js +82 -0
  153. package/dist/browser/utils/request-pool.d.ts +22 -0
  154. package/dist/browser/utils/request-pool.js +101 -0
  155. package/dist/browser/utils/sse.d.ts +7 -0
  156. package/dist/browser/utils/sse.js +67 -0
  157. package/dist/browser/utils/streaming.d.ts +17 -0
  158. package/dist/browser/utils/streaming.js +84 -0
  159. package/dist/browser/utils/try-fn.d.ts +3 -0
  160. package/dist/browser/utils/try-fn.js +59 -0
  161. package/dist/browser/utils/user-agent.d.ts +44 -0
  162. package/dist/browser/utils/user-agent.js +100 -0
  163. package/dist/browser/utils/whois.d.ts +32 -0
  164. package/dist/browser/utils/whois.js +246 -0
  165. package/dist/browser/websocket/client.d.ts +65 -0
  166. package/dist/browser/websocket/client.js +313 -0
  167. package/dist/cli/index.d.ts +1 -0
  168. package/dist/cli/index.js +1 -0
  169. package/dist/transport/fetch.d.ts +7 -1
  170. package/dist/transport/fetch.js +58 -76
  171. package/package.json +34 -2
@@ -0,0 +1,897 @@
1
+ import { request as undiciRequest, errors as undiciErrors, ProxyAgent, Agent, Client } from 'undici';
2
+ import { HttpResponse } from '../core/response.js';
3
+ import { NetworkError, TimeoutError, MaxSizeExceededError } from '../core/errors.js';
4
+ import { performance } from 'perf_hooks';
5
+ import { AsyncLocalStorage } from 'async_hooks';
6
+ import { channel } from 'node:diagnostics_channel';
7
+ import { createLookupFunction } from '../utils/dns.js';
8
+ import { AgentManager } from '../utils/agent-manager.js';
9
+ import { createProgressStream } from '../utils/progress.js';
10
+ import { nodeToWebStream } from '../utils/streaming.js';
11
+ const undiciRequestChannel = channel('undici:request:create');
12
+ const undiciBodySentChannel = channel('undici:request:bodySent');
13
+ const undiciHeadersChannel = channel('undici:request:headers');
14
+ const undiciConnectChannel = channel('undici:client:connect');
15
+ const requestStorage = new AsyncLocalStorage();
16
+ undiciRequestChannel.subscribe((message) => {
17
+ const store = requestStorage.getStore();
18
+ if (store) {
19
+ store.requestStartTime = performance.now();
20
+ store.timings = { queuing: 0, dns: 0, tcp: 0, tls: 0, firstByte: 0, content: 0, total: 0 };
21
+ store.connection = {};
22
+ }
23
+ });
24
+ undiciBodySentChannel.subscribe((message) => {
25
+ const store = requestStorage.getStore();
26
+ if (store?.hooks && store.hooks.onRequestSent) {
27
+ store.hooks.onRequestSent();
28
+ }
29
+ });
30
+ undiciHeadersChannel.subscribe((message) => {
31
+ const payload = message;
32
+ const store = requestStorage.getStore();
33
+ if (store && payload && payload.timing) {
34
+ const { timing } = payload;
35
+ store.timings.queuing = timing.queuing;
36
+ store.timings.dns = timing.dns;
37
+ store.timings.tcp = timing.tcp;
38
+ store.timings.tls = timing.tls;
39
+ store.timings.firstByte = timing.response;
40
+ store.timings.content = timing.body;
41
+ store.timings.total = timing.ended;
42
+ if (store.hooks) {
43
+ if (store.hooks.onDnsLookup && timing.dns > 0) {
44
+ store.hooks.onDnsLookup({ domain: payload.request.origin, duration: timing.dns });
45
+ }
46
+ if (store.hooks.onTcpConnect && timing.tcp > 0) {
47
+ store.hooks.onTcpConnect({ remoteAddress: '', duration: timing.tcp });
48
+ }
49
+ if (store.hooks.onTlsHandshake && timing.tls > 0) {
50
+ store.hooks.onTlsHandshake({ protocol: '', cipher: '', duration: timing.tls });
51
+ }
52
+ if (store.hooks.onResponseStart) {
53
+ const headers = new Headers();
54
+ for (let i = 0; i < payload.response.headers.length; i += 2) {
55
+ headers.append(payload.response.headers[i], payload.response.headers[i + 1]);
56
+ }
57
+ store.hooks.onResponseStart({ status: payload.response.statusCode, headers });
58
+ }
59
+ }
60
+ }
61
+ });
62
+ undiciConnectChannel.subscribe((message) => {
63
+ const payload = message;
64
+ const store = requestStorage.getStore();
65
+ if (store && payload && payload.socket) {
66
+ const { socket } = payload;
67
+ store.connection.remoteAddress = socket.remoteAddress;
68
+ store.connection.remotePort = socket.remotePort;
69
+ store.connection.localAddress = socket.localAddress;
70
+ store.connection.localPort = socket.localPort;
71
+ const protocol = socket.alpnProtocol || (socket.tlsSocket ? socket.tlsSocket.getProtocol() : undefined);
72
+ store.connection.protocol = protocol;
73
+ store.connection.cipher = socket.tlsSocket ? socket.tlsSocket.getCipher()?.name : undefined;
74
+ if (store.hooks) {
75
+ if (store.hooks.onTcpConnect) {
76
+ store.hooks.onTcpConnect({ remoteAddress: socket.remoteAddress, duration: 0 });
77
+ }
78
+ if (store.hooks.onTlsHandshake && socket.tlsSocket) {
79
+ store.hooks.onTlsHandshake({
80
+ protocol: protocol || 'unknown',
81
+ cipher: socket.tlsSocket.getCipher()?.name || 'unknown',
82
+ duration: 0
83
+ });
84
+ }
85
+ }
86
+ if (protocol === 'h2') {
87
+ const http2Session = socket.session || socket;
88
+ const http2State = http2Session.state;
89
+ const remoteSettings = http2Session.remoteSettings;
90
+ const localSettings = http2Session.localSettings;
91
+ store.connection.http2 = {
92
+ streamId: socket.streamId,
93
+ streamWeight: socket.weight,
94
+ streamDependency: socket.dependency,
95
+ serverPush: Boolean(socket.serverPush),
96
+ settingsReceived: Boolean(remoteSettings),
97
+ maxConcurrentStreams: remoteSettings?.maxConcurrentStreams,
98
+ currentStreams: http2State?.streamCount,
99
+ pendingStreams: http2State?.pendingStreamCount,
100
+ localWindowSize: http2State?.localWindowSize ?? http2State?.effectiveLocalWindowSize,
101
+ remoteWindowSize: http2State?.effectiveRecvDataLength,
102
+ localSettings: localSettings ? { ...localSettings } : undefined,
103
+ remoteSettings: remoteSettings ? { ...remoteSettings } : undefined
104
+ };
105
+ }
106
+ if (protocol === 'h3' || protocol?.startsWith('h3-')) {
107
+ const quicStats = socket.stats;
108
+ const handshakeConfirmed = socket.handshakeConfirmed;
109
+ store.connection.http3 = {
110
+ quicVersion: protocol,
111
+ zeroRTT: socket.zeroRTT || false,
112
+ maxStreams: socket.maxStreams,
113
+ handshakeConfirmed: typeof handshakeConfirmed === 'boolean' ? handshakeConfirmed : undefined
114
+ };
115
+ if (quicStats && typeof quicStats.rtt === 'number') {
116
+ store.connection.rtt = quicStats.rtt;
117
+ }
118
+ }
119
+ store.connection.reused = Boolean(socket.reused);
120
+ }
121
+ });
122
+ function mapTimeoutOptions(requestTimeout, transportDefaults) {
123
+ return {
124
+ connectTimeout: requestTimeout?.connect ??
125
+ requestTimeout?.secureConnect ??
126
+ transportDefaults?.connectTimeout,
127
+ headersTimeout: requestTimeout?.response ??
128
+ transportDefaults?.headersTimeout,
129
+ bodyTimeout: requestTimeout?.send ??
130
+ transportDefaults?.bodyTimeout,
131
+ totalTimeout: requestTimeout?.request
132
+ };
133
+ }
134
+ export class UndiciTransport {
135
+ static requestCounter = 0;
136
+ baseUrl;
137
+ options;
138
+ proxyAgent;
139
+ dnsAgent;
140
+ agentManager;
141
+ proxyBypassList;
142
+ tlsOptions;
143
+ socketClient;
144
+ observability;
145
+ constructor(baseUrl, options = {}) {
146
+ this.baseUrl = baseUrl || '';
147
+ this.options = options;
148
+ this.tlsOptions = options.tls;
149
+ this.observability = options.observability !== false;
150
+ if (options.proxy) {
151
+ const proxyConfig = typeof options.proxy === 'string'
152
+ ? { url: options.proxy }
153
+ : options.proxy;
154
+ const proxyUrl = new URL(proxyConfig.url);
155
+ const proxyType = proxyConfig.type || detectProxyType(proxyUrl.protocol);
156
+ if (proxyType?.startsWith('socks')) {
157
+ throw new NetworkError(`SOCKS proxy (${proxyType}) is not supported. Use an HTTP/HTTPS proxy or a SOCKS-to-HTTP bridge.`, 'ERR_UNSUPPORTED_PROXY_TYPE');
158
+ }
159
+ const proxyAuth = proxyConfig.auth
160
+ ? `${proxyConfig.auth.username}:${proxyConfig.auth.password}`
161
+ : undefined;
162
+ const finalProxyUrl = proxyAuth
163
+ ? proxyConfig.url.replace('://', `://${proxyAuth}@`)
164
+ : proxyConfig.url;
165
+ this.proxyBypassList = proxyConfig.bypass;
166
+ const proxyAgentOptions = {
167
+ uri: finalProxyUrl,
168
+ headers: proxyConfig.headers,
169
+ token: proxyConfig.token,
170
+ proxyTunnel: proxyConfig.tunnel,
171
+ requestTls: mapTlsOptions(proxyConfig.requestTls ?? options.tls),
172
+ proxyTls: mapTlsOptions(proxyConfig.proxyTls),
173
+ };
174
+ if (proxyConfig.connectTimeout) {
175
+ proxyAgentOptions.connectTimeout = proxyConfig.connectTimeout;
176
+ }
177
+ if (proxyConfig.http2) {
178
+ if (!proxyAgentOptions.requestTls) {
179
+ proxyAgentOptions.requestTls = {};
180
+ }
181
+ proxyAgentOptions.requestTls.ALPNProtocols = ['h2', 'http/1.1'];
182
+ }
183
+ this.proxyAgent = new ProxyAgent(proxyAgentOptions);
184
+ }
185
+ this.agentManager = options.agent;
186
+ if (options.dns && !this.agentManager) {
187
+ const lookupFn = createLookupFunction(options.dns);
188
+ this.dnsAgent = new Agent({
189
+ connect: {
190
+ lookup: lookupFn,
191
+ },
192
+ });
193
+ }
194
+ if (!this.agentManager) {
195
+ this.agentManager = new AgentManager({
196
+ connections: options.connections,
197
+ pipelining: options.pipelining,
198
+ keepAlive: options.keepAlive,
199
+ keepAliveTimeout: options.keepAliveTimeout,
200
+ keepAliveMaxTimeout: options.keepAliveMaxTimeout,
201
+ connectTimeout: options.connectTimeout,
202
+ perDomainPooling: options.perDomainPooling,
203
+ localAddress: options.localAddress,
204
+ });
205
+ }
206
+ if (options.socketPath && baseUrl) {
207
+ this.socketClient = new Client(baseUrl, {
208
+ socketPath: options.socketPath
209
+ });
210
+ }
211
+ }
212
+ async dispatch(req) {
213
+ const headers = Object.fromEntries(req.headers);
214
+ const contentLengthHeader = headers['content-length'];
215
+ const uploadTotal = contentLengthHeader ? parseInt(contentLengthHeader, 10) : undefined;
216
+ let currentUrl;
217
+ if (this.baseUrl) {
218
+ const path = req.url.startsWith(this.baseUrl) ? req.url.substring(this.baseUrl.length) : req.url;
219
+ currentUrl = new URL(path, this.baseUrl).toString();
220
+ }
221
+ else {
222
+ currentUrl = req.url;
223
+ }
224
+ const handleRedirectsManually = Boolean(req.beforeRedirect);
225
+ const maxRedirects = req.maxRedirects ?? 20;
226
+ const followRedirects = req.followRedirects !== false;
227
+ const timeouts = mapTimeoutOptions(req.timeout, {
228
+ connectTimeout: this.options.connectTimeout,
229
+ headersTimeout: this.options.headersTimeout,
230
+ bodyTimeout: this.options.bodyTimeout
231
+ });
232
+ let timeoutController;
233
+ let timeoutId;
234
+ if (!this.observability) {
235
+ return this.dispatchFast(req, headers, currentUrl, timeouts, handleRedirectsManually, maxRedirects, followRedirects, uploadTotal);
236
+ }
237
+ const requestContext = {
238
+ timings: {},
239
+ connection: {},
240
+ requestStartTime: 0,
241
+ requestCorrelationId: `r${++UndiciTransport.requestCounter}`,
242
+ hooks: req._hooks
243
+ };
244
+ return requestStorage.run(requestContext, async () => {
245
+ try {
246
+ const startTime = performance.now();
247
+ if (requestContext.requestStartTime === 0) {
248
+ requestContext.requestStartTime = startTime;
249
+ }
250
+ let redirectCount = 0;
251
+ let currentMethod = req.method;
252
+ let currentBody = req.body;
253
+ let currentHeaders = { ...headers };
254
+ let effectiveSignal = req.signal;
255
+ if (timeouts.totalTimeout) {
256
+ timeoutController = new AbortController();
257
+ if (req.signal) {
258
+ const originalSignal = req.signal;
259
+ effectiveSignal = timeoutController.signal;
260
+ const onOriginalAbort = () => {
261
+ timeoutController.abort();
262
+ };
263
+ if (originalSignal.aborted) {
264
+ timeoutController.abort();
265
+ }
266
+ else {
267
+ originalSignal.addEventListener('abort', onOriginalAbort, { once: true });
268
+ }
269
+ }
270
+ else {
271
+ effectiveSignal = timeoutController.signal;
272
+ }
273
+ timeoutId = setTimeout(() => {
274
+ timeoutController.abort();
275
+ }, timeouts.totalTimeout);
276
+ }
277
+ while (true) {
278
+ const dispatcher = this.socketClient || determineDispatcher({
279
+ explicit: req._dispatcher,
280
+ proxyAgent: this.proxyAgent,
281
+ agentManager: this.agentManager,
282
+ dnsAgent: this.dnsAgent,
283
+ url: currentUrl,
284
+ bypass: this.proxyBypassList
285
+ });
286
+ const bodyWithProgress = redirectCount === 0
287
+ ? wrapUploadBody(currentBody, req.onUploadProgress, uploadTotal)
288
+ : currentBody;
289
+ let finalBody = bodyWithProgress;
290
+ if (finalBody instanceof FormData) {
291
+ const tempResponse = new Response(finalBody);
292
+ finalBody = tempResponse.body;
293
+ tempResponse.headers.forEach((value, key) => {
294
+ if (key.toLowerCase() === 'content-type') {
295
+ currentHeaders[key] = value;
296
+ delete currentHeaders['Content-Type'];
297
+ }
298
+ else if (!currentHeaders[key]) {
299
+ currentHeaders[key] = value;
300
+ }
301
+ });
302
+ }
303
+ const undiciOptions = {
304
+ method: currentMethod,
305
+ headers: currentHeaders,
306
+ body: finalBody,
307
+ signal: effectiveSignal,
308
+ dispatcher: dispatcher,
309
+ connectTimeout: timeouts.connectTimeout,
310
+ headersTimeout: timeouts.headersTimeout,
311
+ bodyTimeout: timeouts.bodyTimeout,
312
+ maxRedirections: 0,
313
+ };
314
+ if (finalBody && (finalBody instanceof ReadableStream ||
315
+ (typeof finalBody.pipe === 'function') ||
316
+ finalBody[Symbol.asyncIterator])) {
317
+ undiciOptions.duplex = 'half';
318
+ }
319
+ const tlsOptions = mapTlsOptions(this.tlsOptions);
320
+ if (tlsOptions) {
321
+ undiciOptions.tls = tlsOptions;
322
+ }
323
+ const http2Enabled = req.http2 !== undefined
324
+ ? req.http2
325
+ : this.options.http2?.enabled;
326
+ if (http2Enabled) {
327
+ undiciOptions.allowH2 = true;
328
+ }
329
+ else if (req.http2 === false) {
330
+ undiciOptions.allowH2 = false;
331
+ }
332
+ if (http2Enabled && this.options.http2) {
333
+ if (this.options.http2.maxConcurrentStreams !== undefined) {
334
+ undiciOptions.maxConcurrentStreams = this.options.http2.maxConcurrentStreams;
335
+ }
336
+ if (this.options.http2.pipelining !== undefined) {
337
+ undiciOptions.pipelining = this.options.http2.pipelining;
338
+ }
339
+ }
340
+ let undiciResponse;
341
+ if (this.socketClient) {
342
+ const urlPath = new URL(currentUrl).pathname + new URL(currentUrl).search;
343
+ undiciResponse = await this.socketClient.request({
344
+ path: urlPath || '/',
345
+ method: currentMethod,
346
+ headers: currentHeaders,
347
+ body: bodyWithProgress,
348
+ signal: req.signal,
349
+ });
350
+ }
351
+ else {
352
+ undiciResponse = await undiciRequest(currentUrl, undiciOptions);
353
+ }
354
+ const statusCode = undiciResponse.statusCode;
355
+ const isRedirect = statusCode >= 300 && statusCode < 400;
356
+ if (isRedirect && followRedirects && redirectCount < maxRedirects) {
357
+ const locationHeader = undiciResponse.headers['location'];
358
+ const location = Array.isArray(locationHeader) ? locationHeader[0] : locationHeader;
359
+ if (location) {
360
+ const nextUrl = new URL(location, currentUrl).toString();
361
+ if (handleRedirectsManually) {
362
+ const responseHeaders = new Headers();
363
+ for (const [key, value] of Object.entries(undiciResponse.headers)) {
364
+ if (value !== undefined) {
365
+ if (Array.isArray(value)) {
366
+ value.forEach(v => responseHeaders.append(key, v));
367
+ }
368
+ else {
369
+ responseHeaders.set(key, value);
370
+ }
371
+ }
372
+ }
373
+ const redirectInfo = {
374
+ from: currentUrl,
375
+ to: nextUrl,
376
+ status: statusCode,
377
+ headers: responseHeaders,
378
+ };
379
+ const hookResult = await req.beforeRedirect(redirectInfo);
380
+ if (hookResult === false) {
381
+ const finalResponse = req.onDownloadProgress
382
+ ? wrapDownloadResponse(undiciResponse, req.onDownloadProgress)
383
+ : undiciResponse;
384
+ return new HttpResponse(finalResponse, {
385
+ timings: requestContext.timings,
386
+ connection: requestContext.connection
387
+ });
388
+ }
389
+ if (typeof hookResult === 'string') {
390
+ currentUrl = hookResult;
391
+ }
392
+ else {
393
+ currentUrl = nextUrl;
394
+ }
395
+ }
396
+ else {
397
+ currentUrl = nextUrl;
398
+ }
399
+ if (statusCode === 303 || ((statusCode === 301 || statusCode === 302) && currentMethod !== 'GET' && currentMethod !== 'HEAD')) {
400
+ currentMethod = 'GET';
401
+ currentBody = null;
402
+ delete currentHeaders['content-type'];
403
+ delete currentHeaders['content-length'];
404
+ delete currentHeaders['Content-Type'];
405
+ delete currentHeaders['Content-Length'];
406
+ }
407
+ await undiciResponse.body.arrayBuffer().catch(() => { });
408
+ redirectCount++;
409
+ continue;
410
+ }
411
+ }
412
+ const finalResponse = req.onDownloadProgress
413
+ ? wrapDownloadResponse(undiciResponse, req.onDownloadProgress)
414
+ : undiciResponse;
415
+ const ttfb = performance.now() - startTime;
416
+ const totalTime = performance.now() - requestContext.requestStartTime;
417
+ if (!requestContext.timings.firstByte) {
418
+ requestContext.timings.firstByte = ttfb;
419
+ }
420
+ if (!requestContext.timings.total) {
421
+ requestContext.timings.total = totalTime;
422
+ }
423
+ return new HttpResponse(finalResponse, {
424
+ timings: requestContext.timings,
425
+ connection: requestContext.connection
426
+ });
427
+ }
428
+ }
429
+ catch (error) {
430
+ if (error instanceof undiciErrors.ConnectTimeoutError || error.code === 'UND_ERR_CONNECT_TIMEOUT') {
431
+ throw new TimeoutError(req, {
432
+ phase: 'connect',
433
+ timeout: timeouts.connectTimeout
434
+ });
435
+ }
436
+ if (error instanceof undiciErrors.HeadersTimeoutError || error.code === 'UND_ERR_HEADERS_TIMEOUT') {
437
+ throw new TimeoutError(req, {
438
+ phase: 'response',
439
+ timeout: timeouts.headersTimeout
440
+ });
441
+ }
442
+ if (error instanceof undiciErrors.BodyTimeoutError || error.code === 'UND_ERR_BODY_TIMEOUT') {
443
+ throw new TimeoutError(req, {
444
+ phase: 'send',
445
+ timeout: timeouts.bodyTimeout
446
+ });
447
+ }
448
+ if (error.name === 'AbortError' || error.code === 'ABORT_ERR') {
449
+ throw new TimeoutError(req, {
450
+ phase: 'request',
451
+ timeout: timeouts.totalTimeout
452
+ });
453
+ }
454
+ const code = error.code || error?.cause?.code;
455
+ if (code === 'UND_ERR_HEADERS_OVERFLOW') {
456
+ throw new MaxSizeExceededError(16 * 1024, undefined, req);
457
+ }
458
+ throw new NetworkError(error.message, code, req);
459
+ }
460
+ finally {
461
+ if (timeoutId) {
462
+ clearTimeout(timeoutId);
463
+ }
464
+ }
465
+ });
466
+ }
467
+ async dispatchFast(req, headers, currentUrl, timeouts, handleRedirectsManually, maxRedirects, followRedirects, uploadTotal) {
468
+ let timeoutController;
469
+ let timeoutId;
470
+ try {
471
+ let redirectCount = 0;
472
+ let currentMethod = req.method;
473
+ let currentBody = req.body;
474
+ let currentHeaders = { ...headers };
475
+ let effectiveSignal = req.signal;
476
+ if (timeouts.totalTimeout) {
477
+ timeoutController = new AbortController();
478
+ if (req.signal) {
479
+ const originalSignal = req.signal;
480
+ effectiveSignal = timeoutController.signal;
481
+ const onOriginalAbort = () => {
482
+ timeoutController.abort();
483
+ };
484
+ if (originalSignal.aborted) {
485
+ timeoutController.abort();
486
+ }
487
+ else {
488
+ originalSignal.addEventListener('abort', onOriginalAbort, { once: true });
489
+ }
490
+ }
491
+ else {
492
+ effectiveSignal = timeoutController.signal;
493
+ }
494
+ timeoutId = setTimeout(() => {
495
+ timeoutController.abort();
496
+ }, timeouts.totalTimeout);
497
+ }
498
+ while (true) {
499
+ const dispatcher = this.socketClient || determineDispatcher({
500
+ explicit: req._dispatcher,
501
+ proxyAgent: this.proxyAgent,
502
+ agentManager: this.agentManager,
503
+ dnsAgent: this.dnsAgent,
504
+ url: currentUrl,
505
+ bypass: this.proxyBypassList
506
+ });
507
+ const bodyWithProgress = redirectCount === 0
508
+ ? wrapUploadBody(currentBody, req.onUploadProgress, uploadTotal)
509
+ : currentBody;
510
+ let finalBody = bodyWithProgress;
511
+ if (finalBody instanceof FormData) {
512
+ const tempResponse = new Response(finalBody);
513
+ finalBody = tempResponse.body;
514
+ tempResponse.headers.forEach((value, key) => {
515
+ if (key.toLowerCase() === 'content-type') {
516
+ currentHeaders[key] = value;
517
+ delete currentHeaders['Content-Type'];
518
+ }
519
+ else if (!currentHeaders[key]) {
520
+ currentHeaders[key] = value;
521
+ }
522
+ });
523
+ }
524
+ const undiciOptions = {
525
+ method: currentMethod,
526
+ headers: currentHeaders,
527
+ body: finalBody,
528
+ signal: effectiveSignal,
529
+ dispatcher: dispatcher,
530
+ connectTimeout: timeouts.connectTimeout,
531
+ headersTimeout: timeouts.headersTimeout,
532
+ bodyTimeout: timeouts.bodyTimeout,
533
+ maxRedirections: 0,
534
+ };
535
+ if (finalBody && (finalBody instanceof ReadableStream ||
536
+ (typeof finalBody.pipe === 'function') ||
537
+ finalBody[Symbol.asyncIterator])) {
538
+ undiciOptions.duplex = 'half';
539
+ }
540
+ const tlsOptions = mapTlsOptions(this.tlsOptions);
541
+ if (tlsOptions) {
542
+ undiciOptions.tls = tlsOptions;
543
+ }
544
+ const http2Enabled = req.http2 !== undefined
545
+ ? req.http2
546
+ : this.options.http2?.enabled;
547
+ if (http2Enabled) {
548
+ undiciOptions.allowH2 = true;
549
+ }
550
+ else if (req.http2 === false) {
551
+ undiciOptions.allowH2 = false;
552
+ }
553
+ if (http2Enabled && this.options.http2) {
554
+ if (this.options.http2.maxConcurrentStreams !== undefined) {
555
+ undiciOptions.maxConcurrentStreams = this.options.http2.maxConcurrentStreams;
556
+ }
557
+ if (this.options.http2.pipelining !== undefined) {
558
+ undiciOptions.pipelining = this.options.http2.pipelining;
559
+ }
560
+ }
561
+ let undiciResponse;
562
+ if (this.socketClient) {
563
+ const urlPath = new URL(currentUrl).pathname + new URL(currentUrl).search;
564
+ undiciResponse = await this.socketClient.request({
565
+ path: urlPath || '/',
566
+ method: currentMethod,
567
+ headers: currentHeaders,
568
+ body: bodyWithProgress,
569
+ signal: req.signal,
570
+ });
571
+ }
572
+ else {
573
+ undiciResponse = await undiciRequest(currentUrl, undiciOptions);
574
+ }
575
+ const statusCode = undiciResponse.statusCode;
576
+ const isRedirect = statusCode >= 300 && statusCode < 400;
577
+ if (isRedirect && followRedirects && redirectCount < maxRedirects) {
578
+ const locationHeader = undiciResponse.headers['location'];
579
+ const location = Array.isArray(locationHeader) ? locationHeader[0] : locationHeader;
580
+ if (location) {
581
+ const nextUrl = new URL(location, currentUrl).toString();
582
+ if (handleRedirectsManually) {
583
+ const responseHeaders = new Headers();
584
+ for (const [key, value] of Object.entries(undiciResponse.headers)) {
585
+ if (value !== undefined) {
586
+ if (Array.isArray(value)) {
587
+ value.forEach(v => responseHeaders.append(key, v));
588
+ }
589
+ else {
590
+ responseHeaders.set(key, value);
591
+ }
592
+ }
593
+ }
594
+ const redirectInfo = {
595
+ from: currentUrl,
596
+ to: nextUrl,
597
+ status: statusCode,
598
+ headers: responseHeaders,
599
+ };
600
+ const hookResult = await req.beforeRedirect(redirectInfo);
601
+ if (hookResult === false) {
602
+ const finalResponse = req.onDownloadProgress
603
+ ? wrapDownloadResponse(undiciResponse, req.onDownloadProgress)
604
+ : undiciResponse;
605
+ return new HttpResponse(finalResponse, {
606
+ timings: {},
607
+ connection: {}
608
+ });
609
+ }
610
+ if (typeof hookResult === 'string') {
611
+ currentUrl = hookResult;
612
+ }
613
+ else {
614
+ currentUrl = nextUrl;
615
+ }
616
+ }
617
+ else {
618
+ currentUrl = nextUrl;
619
+ }
620
+ if (statusCode === 303 || ((statusCode === 301 || statusCode === 302) && currentMethod !== 'GET' && currentMethod !== 'HEAD')) {
621
+ currentMethod = 'GET';
622
+ currentBody = null;
623
+ delete currentHeaders['content-type'];
624
+ delete currentHeaders['content-length'];
625
+ delete currentHeaders['Content-Type'];
626
+ delete currentHeaders['Content-Length'];
627
+ }
628
+ await undiciResponse.body.arrayBuffer().catch(() => { });
629
+ redirectCount++;
630
+ continue;
631
+ }
632
+ }
633
+ const finalResponse = req.onDownloadProgress
634
+ ? wrapDownloadResponse(undiciResponse, req.onDownloadProgress)
635
+ : undiciResponse;
636
+ return new HttpResponse(finalResponse, {
637
+ timings: {},
638
+ connection: {}
639
+ });
640
+ }
641
+ }
642
+ catch (error) {
643
+ if (error instanceof undiciErrors.ConnectTimeoutError || error.code === 'UND_ERR_CONNECT_TIMEOUT') {
644
+ throw new TimeoutError(req, {
645
+ phase: 'connect',
646
+ timeout: timeouts.connectTimeout
647
+ });
648
+ }
649
+ if (error instanceof undiciErrors.HeadersTimeoutError || error.code === 'UND_ERR_HEADERS_TIMEOUT') {
650
+ throw new TimeoutError(req, {
651
+ phase: 'response',
652
+ timeout: timeouts.headersTimeout
653
+ });
654
+ }
655
+ if (error instanceof undiciErrors.BodyTimeoutError || error.code === 'UND_ERR_BODY_TIMEOUT') {
656
+ throw new TimeoutError(req, {
657
+ phase: 'send',
658
+ timeout: timeouts.bodyTimeout
659
+ });
660
+ }
661
+ if (error.name === 'AbortError' || error.code === 'ABORT_ERR') {
662
+ throw new TimeoutError(req, {
663
+ phase: 'request',
664
+ timeout: timeouts.totalTimeout
665
+ });
666
+ }
667
+ const code = error.code || error?.cause?.code;
668
+ if (code === 'UND_ERR_HEADERS_OVERFLOW') {
669
+ throw new MaxSizeExceededError(16 * 1024, undefined, req);
670
+ }
671
+ throw new NetworkError(error.message, code, req);
672
+ }
673
+ finally {
674
+ if (timeoutId) {
675
+ clearTimeout(timeoutId);
676
+ }
677
+ }
678
+ }
679
+ }
680
+ function mapTlsOptions(options) {
681
+ if (!options)
682
+ return undefined;
683
+ const tls = {};
684
+ if (options.minVersion)
685
+ tls.minVersion = options.minVersion;
686
+ if (options.maxVersion)
687
+ tls.maxVersion = options.maxVersion;
688
+ if (options.ciphers)
689
+ tls.ciphers = options.ciphers;
690
+ if (options.honorCipherOrder !== undefined)
691
+ tls.honorCipherOrder = options.honorCipherOrder;
692
+ if (options.ca)
693
+ tls.ca = options.ca;
694
+ if (options.cert)
695
+ tls.cert = options.cert;
696
+ if (options.key)
697
+ tls.key = options.key;
698
+ if (options.passphrase)
699
+ tls.passphrase = options.passphrase;
700
+ if (options.rejectUnauthorized !== undefined)
701
+ tls.rejectUnauthorized = options.rejectUnauthorized;
702
+ if (options.alpnProtocols)
703
+ tls.ALPNProtocols = options.alpnProtocols;
704
+ if (options.sessionTimeout !== undefined)
705
+ tls.sessionTimeout = options.sessionTimeout;
706
+ if (options.sessionIdContext)
707
+ tls.sessionIdContext = options.sessionIdContext;
708
+ if (options.servername !== undefined) {
709
+ tls.servername = options.servername === false ? '' : options.servername;
710
+ }
711
+ return tls;
712
+ }
713
+ function determineDispatcher(params) {
714
+ if (params.explicit)
715
+ return params.explicit;
716
+ const { proxyAgent, agentManager, dnsAgent, url, bypass } = params;
717
+ const bypassProxy = shouldBypassProxy(url, bypass);
718
+ if (proxyAgent && !bypassProxy)
719
+ return proxyAgent;
720
+ if (agentManager)
721
+ return agentManager.getAgentForUrl(url);
722
+ if (dnsAgent)
723
+ return dnsAgent;
724
+ return undefined;
725
+ }
726
+ function detectProxyType(protocol) {
727
+ const p = protocol.toLowerCase().replace(':', '');
728
+ switch (p) {
729
+ case 'http':
730
+ return 'http';
731
+ case 'https':
732
+ return 'https';
733
+ case 'socks4':
734
+ return 'socks4';
735
+ case 'socks4a':
736
+ return 'socks4a';
737
+ case 'socks5':
738
+ case 'socks':
739
+ return 'socks5';
740
+ default:
741
+ return undefined;
742
+ }
743
+ }
744
+ function matchesCIDR(ip, cidr) {
745
+ const [range, bits] = cidr.split('/');
746
+ if (!bits)
747
+ return ip === range;
748
+ const mask = parseInt(bits, 10);
749
+ if (isNaN(mask))
750
+ return false;
751
+ const ipParts = ip.split('.').map(Number);
752
+ const rangeParts = range.split('.').map(Number);
753
+ if (ipParts.length !== 4 || rangeParts.length !== 4)
754
+ return false;
755
+ if (ipParts.some(isNaN) || rangeParts.some(isNaN))
756
+ return false;
757
+ const ipNum = (ipParts[0] << 24) | (ipParts[1] << 16) | (ipParts[2] << 8) | ipParts[3];
758
+ const rangeNum = (rangeParts[0] << 24) | (rangeParts[1] << 16) | (rangeParts[2] << 8) | rangeParts[3];
759
+ const maskNum = ~((1 << (32 - mask)) - 1);
760
+ return (ipNum & maskNum) === (rangeNum & maskNum);
761
+ }
762
+ function shouldBypassProxy(url, bypass) {
763
+ if (!bypass || bypass.length === 0)
764
+ return false;
765
+ let hostname = '';
766
+ let port = '';
767
+ try {
768
+ const parsed = new URL(url);
769
+ hostname = parsed.hostname;
770
+ port = parsed.port;
771
+ }
772
+ catch {
773
+ return false;
774
+ }
775
+ for (const rule of bypass) {
776
+ if (rule === '*')
777
+ return true;
778
+ if (rule.includes('/')) {
779
+ if (matchesCIDR(hostname, rule))
780
+ return true;
781
+ continue;
782
+ }
783
+ if (rule.includes(':') && !rule.includes('/')) {
784
+ const [hostRule, portRule] = rule.split(':');
785
+ if (hostname === hostRule && (!portRule || port === portRule))
786
+ return true;
787
+ continue;
788
+ }
789
+ if (rule.startsWith('*.')) {
790
+ const suffix = rule.slice(1);
791
+ if (hostname.endsWith(suffix))
792
+ return true;
793
+ continue;
794
+ }
795
+ if (rule.startsWith('.')) {
796
+ if (hostname.endsWith(rule))
797
+ return true;
798
+ continue;
799
+ }
800
+ if (hostname === rule) {
801
+ return true;
802
+ }
803
+ }
804
+ return false;
805
+ }
806
+ function parseContentLength(headers) {
807
+ if (!headers)
808
+ return undefined;
809
+ if (typeof headers.get === 'function') {
810
+ const raw = headers.get('content-length');
811
+ return raw ? parseInt(raw, 10) : undefined;
812
+ }
813
+ const raw = headers['content-length'] ?? headers['Content-Length'];
814
+ if (raw === undefined)
815
+ return undefined;
816
+ const parsed = parseInt(Array.isArray(raw) ? raw[0] : raw, 10);
817
+ return Number.isFinite(parsed) ? parsed : undefined;
818
+ }
819
+ function wrapDownloadResponse(response, onProgress) {
820
+ if (!onProgress)
821
+ return response;
822
+ if (typeof Response !== 'undefined' && response instanceof Response) {
823
+ if (!response.body)
824
+ return response;
825
+ const total = parseContentLength(response.headers);
826
+ const body = createProgressStream(response.body, onProgress, {
827
+ total,
828
+ direction: 'download'
829
+ });
830
+ return new Response(body, {
831
+ status: response.status,
832
+ statusText: response.statusText,
833
+ headers: response.headers
834
+ });
835
+ }
836
+ const total = parseContentLength(response.headers);
837
+ const nodeBody = response.body;
838
+ if (!nodeBody)
839
+ return response;
840
+ const webBody = createProgressStream(nodeToWebStream(nodeBody), onProgress, {
841
+ total,
842
+ direction: 'download'
843
+ });
844
+ return new Response(webBody, {
845
+ status: response.statusCode,
846
+ statusText: String(response.statusCode),
847
+ headers: response.headers
848
+ });
849
+ }
850
+ function wrapUploadBody(body, onProgress, total) {
851
+ if (body instanceof FormData) {
852
+ return body;
853
+ }
854
+ if (!onProgress || !body)
855
+ return body;
856
+ if (typeof ReadableStream !== 'undefined' && body instanceof ReadableStream) {
857
+ return createProgressStream(body, onProgress, { total, direction: 'upload' });
858
+ }
859
+ if (typeof Blob !== 'undefined' && body instanceof Blob) {
860
+ return createProgressStream(body.stream(), onProgress, { total: body.size, direction: 'upload' });
861
+ }
862
+ if (isNodeReadable(body)) {
863
+ const webStream = nodeToWebStream(body);
864
+ return createProgressStream(webStream, onProgress, { total, direction: 'upload' });
865
+ }
866
+ if (body instanceof ArrayBuffer || ArrayBuffer.isView(body)) {
867
+ const view = body instanceof ArrayBuffer
868
+ ? new Uint8Array(body)
869
+ : new Uint8Array(body.buffer, body.byteOffset, body.byteLength);
870
+ return bufferToProgressStream(view, onProgress);
871
+ }
872
+ if (typeof body === 'string') {
873
+ const encoder = new TextEncoder();
874
+ const view = encoder.encode(body);
875
+ return bufferToProgressStream(view, onProgress);
876
+ }
877
+ return body;
878
+ }
879
+ function bufferToProgressStream(buffer, onProgress, chunkSize = 64 * 1024) {
880
+ let offset = 0;
881
+ const total = buffer.byteLength;
882
+ const stream = new ReadableStream({
883
+ pull(controller) {
884
+ if (offset >= total) {
885
+ controller.close();
886
+ return;
887
+ }
888
+ const end = Math.min(offset + chunkSize, total);
889
+ controller.enqueue(buffer.slice(offset, end));
890
+ offset = end;
891
+ }
892
+ });
893
+ return createProgressStream(stream, onProgress, { total, direction: 'upload' });
894
+ }
895
+ function isNodeReadable(obj) {
896
+ return obj && typeof obj.pipe === 'function' && typeof obj.on === 'function';
897
+ }