urllib 4.3.1 → 4.5.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +0 -2
  2. package/dist/commonjs/FetchOpaqueInterceptor.d.ts +8 -0
  3. package/dist/commonjs/FetchOpaqueInterceptor.js +18 -0
  4. package/dist/commonjs/FormData.d.ts +4 -0
  5. package/dist/commonjs/FormData.js +38 -0
  6. package/dist/commonjs/HttpAgent.d.ts +3 -3
  7. package/dist/commonjs/HttpAgent.js +6 -5
  8. package/dist/commonjs/HttpClient.d.ts +31 -0
  9. package/dist/commonjs/HttpClient.js +62 -98
  10. package/dist/commonjs/Request.d.ts +5 -0
  11. package/dist/commonjs/diagnosticsChannel.js +11 -4
  12. package/dist/commonjs/fetch.d.ts +24 -0
  13. package/dist/commonjs/fetch.js +221 -0
  14. package/dist/commonjs/index.d.ts +2 -1
  15. package/dist/commonjs/index.js +7 -2
  16. package/dist/commonjs/utils.d.ts +4 -0
  17. package/dist/commonjs/utils.js +57 -1
  18. package/dist/esm/FetchOpaqueInterceptor.d.ts +8 -0
  19. package/dist/esm/FetchOpaqueInterceptor.js +12 -0
  20. package/dist/esm/FormData.d.ts +4 -0
  21. package/dist/esm/FormData.js +31 -0
  22. package/dist/esm/HttpAgent.d.ts +3 -3
  23. package/dist/esm/HttpAgent.js +6 -5
  24. package/dist/esm/HttpClient.d.ts +31 -0
  25. package/dist/esm/HttpClient.js +59 -95
  26. package/dist/esm/Request.d.ts +5 -0
  27. package/dist/esm/diagnosticsChannel.js +11 -4
  28. package/dist/esm/fetch.d.ts +24 -0
  29. package/dist/esm/fetch.js +214 -0
  30. package/dist/esm/index.d.ts +2 -1
  31. package/dist/esm/index.js +3 -2
  32. package/dist/esm/utils.d.ts +4 -0
  33. package/dist/esm/utils.js +52 -1
  34. package/dist/package.json +1 -1
  35. package/package.json +7 -5
  36. package/src/FetchOpaqueInterceptor.ts +41 -0
  37. package/src/FormData.ts +32 -0
  38. package/src/HttpAgent.ts +9 -7
  39. package/src/HttpClient.ts +92 -100
  40. package/src/Request.ts +6 -0
  41. package/src/diagnosticsChannel.ts +13 -3
  42. package/src/fetch.ts +263 -0
  43. package/src/index.ts +3 -0
  44. package/src/utils.ts +54 -0
package/src/HttpClient.ts CHANGED
@@ -9,7 +9,6 @@ import {
9
9
  gunzipSync,
10
10
  brotliDecompressSync,
11
11
  } from 'node:zlib';
12
- import { Blob } from 'node:buffer';
13
12
  import { Readable, pipeline } from 'node:stream';
14
13
  import { pipeline as pipelinePromise } from 'node:stream/promises';
15
14
  import { basename } from 'node:path';
@@ -19,13 +18,13 @@ import { performance } from 'node:perf_hooks';
19
18
  import querystring from 'node:querystring';
20
19
  import { setTimeout as sleep } from 'node:timers/promises';
21
20
  import {
22
- FormData,
23
21
  request as undiciRequest,
24
22
  Dispatcher,
25
23
  Agent,
26
24
  getGlobalDispatcher,
27
25
  Pool,
28
26
  } from 'undici';
27
+ import { FormData } from './FormData.js';
29
28
  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
30
29
  // @ts-ignore
31
30
  import undiciSymbols from 'undici/lib/core/symbols.js';
@@ -37,7 +36,7 @@ import { HttpAgent, CheckAddressFunction } from './HttpAgent.js';
37
36
  import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
38
37
  import { RequestURL, RequestOptions, HttpMethod, RequestMeta } from './Request.js';
39
38
  import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response.js';
40
- import { parseJSON, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
39
+ import { parseJSON, digestAuthHeader, globalId, performanceTime, isReadable, updateSocketInfo } from './utils.js';
41
40
  import symbols from './symbols.js';
42
41
  import { initDiagnosticsChannel } from './diagnosticsChannel.js';
43
42
  import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
@@ -47,7 +46,31 @@ type UndiciRequestOption = Exists<Parameters<typeof undiciRequest>[1]>;
47
46
  type PropertyShouldBe<T, K extends keyof T, V> = Omit<T, K> & { [P in K]: V };
48
47
  type IUndiciRequestOption = PropertyShouldBe<UndiciRequestOption, 'headers', IncomingHttpHeaders>;
49
48
 
50
- const PROTO_RE = /^https?:\/\//i;
49
+ export const PROTO_RE = /^https?:\/\//i;
50
+
51
+ export interface UndiciTimingInfo {
52
+ startTime: number;
53
+ redirectStartTime: number;
54
+ redirectEndTime: number;
55
+ postRedirectStartTime: number;
56
+ finalServiceWorkerStartTime: number;
57
+ finalNetworkResponseStartTime: number;
58
+ finalNetworkRequestStartTime: number;
59
+ endTime: number;
60
+ encodedBodySize: number;
61
+ decodedBodySize: number;
62
+ finalConnectionTimingInfo: {
63
+ domainLookupStartTime: number;
64
+ domainLookupEndTime: number;
65
+ connectionStartTime: number;
66
+ connectionEndTime: number;
67
+ secureConnectionStartTime: number;
68
+ // ALPNNegotiatedProtocol: undefined
69
+ };
70
+ }
71
+
72
+ // keep typo compatibility
73
+ export interface UnidiciTimingInfo extends UndiciTimingInfo {}
51
74
 
52
75
  function noop() {
53
76
  // noop
@@ -92,28 +115,6 @@ export type ClientOptions = {
92
115
  },
93
116
  };
94
117
 
95
- // https://github.com/octet-stream/form-data
96
- class BlobFromStream {
97
- #stream;
98
- #type;
99
- constructor(stream: Readable, type: string) {
100
- this.#stream = stream;
101
- this.#type = type;
102
- }
103
-
104
- stream() {
105
- return this.#stream;
106
- }
107
-
108
- get type(): string {
109
- return this.#type;
110
- }
111
-
112
- get [Symbol.toStringTag]() {
113
- return 'Blob';
114
- }
115
- }
116
-
117
118
  export const VERSION = 'VERSION';
118
119
  // 'node-urllib/4.0.0 Node.js/18.19.0 (darwin; x64)'
119
120
  export const HEADER_USER_AGENT =
@@ -135,11 +136,15 @@ export type RequestContext = {
135
136
  retries: number;
136
137
  socketErrorRetries: number;
137
138
  requestStartTime?: number;
139
+ redirects: number;
140
+ history: string[];
138
141
  };
139
142
 
140
- const channels = {
143
+ export const channels = {
141
144
  request: diagnosticsChannel.channel('urllib:request'),
142
145
  response: diagnosticsChannel.channel('urllib:response'),
146
+ fetchRequest: diagnosticsChannel.channel('urllib:fetch:request'),
147
+ fetchResponse: diagnosticsChannel.channel('urllib:fetch:response'),
143
148
  };
144
149
 
145
150
  export type RequestDiagnosticsMessage = {
@@ -167,6 +172,15 @@ export interface PoolStat {
167
172
  size: number;
168
173
  }
169
174
 
175
+ // https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections
176
+ const RedirectStatusCodes = [
177
+ 301, // Moved Permanently
178
+ 302, // Found
179
+ 303, // See Other
180
+ 307, // Temporary Redirect
181
+ 308, // Permanent Redirect
182
+ ];
183
+
170
184
  export class HttpClient extends EventEmitter {
171
185
  #defaultArgs?: RequestOptions;
172
186
  #dispatcher?: Dispatcher;
@@ -271,11 +285,14 @@ export class HttpClient extends EventEmitter {
271
285
  requestContext = {
272
286
  retries: 0,
273
287
  socketErrorRetries: 0,
288
+ redirects: 0,
289
+ history: [],
274
290
  ...requestContext,
275
291
  };
276
292
  if (!requestContext.requestStartTime) {
277
293
  requestContext.requestStartTime = performance.now();
278
294
  }
295
+ requestContext.history.push(requestUrl.href);
279
296
  const requestStartTime = requestContext.requestStartTime;
280
297
 
281
298
  // https://developer.chrome.com/docs/devtools/network/reference/?utm_source=devtools#timing-explanation
@@ -335,7 +352,7 @@ export class HttpClient extends EventEmitter {
335
352
  aborted: false,
336
353
  rt: 0,
337
354
  keepAliveSocket: true,
338
- requestUrls: [],
355
+ requestUrls: requestContext.history,
339
356
  timing,
340
357
  socket: socketInfo,
341
358
  retries: requestContext.retries,
@@ -396,10 +413,13 @@ export class HttpClient extends EventEmitter {
396
413
  isStreamingRequest = true;
397
414
  }
398
415
 
416
+ let maxRedirects = args.maxRedirects ?? 10;
417
+
399
418
  try {
400
419
  const requestOptions: IUndiciRequestOption = {
401
420
  method,
402
- maxRedirections: args.maxRedirects ?? 10,
421
+ // disable undici auto redirect handler
422
+ maxRedirections: 0,
403
423
  headersTimeout,
404
424
  headers,
405
425
  bodyTimeout,
@@ -414,7 +434,7 @@ export class HttpClient extends EventEmitter {
414
434
  requestOptions.reset = args.reset;
415
435
  }
416
436
  if (args.followRedirect === false) {
417
- requestOptions.maxRedirections = 0;
437
+ maxRedirects = 0;
418
438
  }
419
439
 
420
440
  const isGETOrHEAD = requestOptions.method === 'GET' || requestOptions.method === 'HEAD';
@@ -464,21 +484,28 @@ export class HttpClient extends EventEmitter {
464
484
  }
465
485
  }
466
486
  for (const [ index, [ field, file, customFileName ]] of uploadFiles.entries()) {
487
+ let fileName = '';
488
+ let value: any;
467
489
  if (typeof file === 'string') {
468
- // FIXME: support non-ascii filename
469
- // const fileName = encodeURIComponent(basename(file));
470
- // formData.append(field, await fileFromPath(file, `utf-8''${fileName}`, { type: mime.lookup(fileName) || '' }));
471
- const fileName = basename(file);
472
- const fileReadable = createReadStream(file);
473
- formData.append(field, new BlobFromStream(fileReadable, mime.lookup(fileName) || ''), fileName);
490
+ fileName = basename(file);
491
+ value = createReadStream(file);
474
492
  } else if (Buffer.isBuffer(file)) {
475
- formData.append(field, new Blob([ file ]), customFileName || `bufferfile${index}`);
493
+ fileName = customFileName || `bufferfile${index}`;
494
+ value = file;
476
495
  } else if (file instanceof Readable || isReadable(file as any)) {
477
- const fileName = getFileName(file) || customFileName || `streamfile${index}`;
478
- formData.append(field, new BlobFromStream(file, mime.lookup(fileName) || ''), fileName);
496
+ fileName = getFileName(file) || customFileName || `streamfile${index}`;
479
497
  isStreamingRequest = true;
498
+ value = file;
480
499
  }
500
+ const mimeType = mime.lookup(fileName) || '';
501
+ formData.append(field, value, {
502
+ filename: fileName,
503
+ contentType: mimeType,
504
+ });
505
+ debug('formData append field: %s, mimeType: %s, fileName: %s',
506
+ field, mimeType, fileName);
481
507
  }
508
+ Object.assign(headers, formData.getHeaders());
482
509
  requestOptions.body = formData;
483
510
  } else if (args.content) {
484
511
  if (!isGETOrHEAD) {
@@ -535,8 +562,8 @@ export class HttpClient extends EventEmitter {
535
562
  args.socketErrorRetry = 0;
536
563
  }
537
564
 
538
- debug('Request#%d %s %s, headers: %j, headersTimeout: %s, bodyTimeout: %s, isStreamingRequest: %s',
539
- requestId, requestOptions.method, requestUrl.href, headers, headersTimeout, bodyTimeout, isStreamingRequest);
565
+ debug('Request#%d %s %s, headers: %j, headersTimeout: %s, bodyTimeout: %s, isStreamingRequest: %s, maxRedirections: %s, redirects: %s',
566
+ requestId, requestOptions.method, requestUrl.href, headers, headersTimeout, bodyTimeout, isStreamingRequest, maxRedirects, requestContext.redirects);
540
567
  requestOptions.headers = headers;
541
568
  channels.request.publish({
542
569
  request: reqMeta,
@@ -567,18 +594,6 @@ export class HttpClient extends EventEmitter {
567
594
  response = await undiciRequest(requestUrl, requestOptions as UndiciRequestOption);
568
595
  }
569
596
  }
570
-
571
- const context = response.context as { history: URL[] };
572
- let lastUrl = '';
573
- if (context?.history) {
574
- for (const urlObject of context?.history) {
575
- res.requestUrls.push(urlObject.href);
576
- lastUrl = urlObject.href;
577
- }
578
- } else {
579
- res.requestUrls.push(requestUrl.href);
580
- lastUrl = requestUrl.href;
581
- }
582
597
  const contentEncoding = response.headers['content-encoding'];
583
598
  const isCompressedContent = contentEncoding === 'gzip' || contentEncoding === 'br';
584
599
 
@@ -589,6 +604,19 @@ export class HttpClient extends EventEmitter {
589
604
  res.size = parseInt(res.headers['content-length']);
590
605
  }
591
606
 
607
+ // https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections
608
+ if (RedirectStatusCodes.includes(res.statusCode) && maxRedirects > 0 && requestContext.redirects < maxRedirects && !isStreamingRequest) {
609
+ if (res.headers.location) {
610
+ requestContext.redirects++;
611
+ const nextUrl = new URL(res.headers.location, requestUrl.href);
612
+ // Ensure the response is consumed
613
+ await response.body.arrayBuffer();
614
+ debug('Request#%d got response, status: %s, headers: %j, timing: %j, redirect to %s',
615
+ requestId, res.status, res.headers, res.timing, nextUrl.href);
616
+ return await this.#requestInternal(nextUrl.href, options, requestContext);
617
+ }
618
+ }
619
+
592
620
  let data: any = null;
593
621
  if (args.dataType === 'stream') {
594
622
  // only auto decompress on request args.compressed = true
@@ -631,7 +659,7 @@ export class HttpClient extends EventEmitter {
631
659
  }
632
660
  res.rt = performanceTime(requestStartTime);
633
661
  // get real socket info from internalOpaque
634
- this.#updateSocketInfo(socketInfo, internalOpaque);
662
+ updateSocketInfo(socketInfo, internalOpaque);
635
663
 
636
664
  const clientResponse: HttpClientResponse = {
637
665
  opaque: originalOpaque,
@@ -640,12 +668,15 @@ export class HttpClient extends EventEmitter {
640
668
  statusCode: res.status,
641
669
  statusText: res.statusText,
642
670
  headers: res.headers,
643
- url: lastUrl,
644
- redirected: res.requestUrls.length > 1,
671
+ url: requestUrl.href,
672
+ redirected: requestContext.history.length > 1,
645
673
  requestUrls: res.requestUrls,
646
674
  res,
647
675
  };
648
676
 
677
+ debug('Request#%d got response, status: %s, headers: %j, timing: %j',
678
+ requestId, res.status, res.headers, res.timing);
679
+
649
680
  if (args.retry > 0 && requestContext.retries < args.retry) {
650
681
  const isRetry = args.isRetry ?? defaultIsRetry;
651
682
  if (isRetry(clientResponse)) {
@@ -657,8 +688,6 @@ export class HttpClient extends EventEmitter {
657
688
  }
658
689
  }
659
690
 
660
- debug('Request#%d got response, status: %s, headers: %j, timing: %j',
661
- requestId, res.status, res.headers, res.timing);
662
691
  channels.response.publish({
663
692
  request: reqMeta,
664
693
  response: res,
@@ -678,7 +707,8 @@ export class HttpClient extends EventEmitter {
678
707
 
679
708
  return clientResponse;
680
709
  } catch (rawError: any) {
681
- debug('Request#%d throw error: %s', requestId, rawError);
710
+ debug('Request#%d throw error: %s, socketErrorRetry: %s, socketErrorRetries: %s',
711
+ requestId, rawError, args.socketErrorRetry, requestContext.socketErrorRetries);
682
712
  let err = rawError;
683
713
  if (err.name === 'HeadersTimeoutError') {
684
714
  err = new HttpClientRequestTimeoutError(headersTimeout, { cause: err });
@@ -690,6 +720,8 @@ export class HttpClient extends EventEmitter {
690
720
  // auto retry on socket error, https://github.com/node-modules/urllib/issues/454
691
721
  if (args.socketErrorRetry > 0 && requestContext.socketErrorRetries < args.socketErrorRetry) {
692
722
  requestContext.socketErrorRetries++;
723
+ debug('Request#%d retry on socket error, socketErrorRetries: %d',
724
+ requestId, requestContext.socketErrorRetries);
693
725
  return await this.#requestInternal(url, options, requestContext);
694
726
  }
695
727
  }
@@ -702,12 +734,8 @@ export class HttpClient extends EventEmitter {
702
734
  err._rawSocket = err.socket;
703
735
  }
704
736
  err.socket = socketInfo;
705
- // make sure requestUrls not empty
706
- if (res.requestUrls.length === 0) {
707
- res.requestUrls.push(requestUrl.href);
708
- }
709
737
  res.rt = performanceTime(requestStartTime);
710
- this.#updateSocketInfo(socketInfo, internalOpaque, rawError);
738
+ updateSocketInfo(socketInfo, internalOpaque, rawError);
711
739
 
712
740
  channels.response.publish({
713
741
  request: reqMeta,
@@ -729,40 +757,4 @@ export class HttpClient extends EventEmitter {
729
757
  throw err;
730
758
  }
731
759
  }
732
-
733
- #updateSocketInfo(socketInfo: SocketInfo, internalOpaque: any, err?: any) {
734
- const socket = internalOpaque[symbols.kRequestSocket] ?? err?.[symbols.kErrorSocket];
735
- if (socket) {
736
- socketInfo.id = socket[symbols.kSocketId];
737
- socketInfo.handledRequests = socket[symbols.kHandledRequests];
738
- socketInfo.handledResponses = socket[symbols.kHandledResponses];
739
- if (socket[symbols.kSocketLocalAddress]) {
740
- socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
741
- socketInfo.localPort = socket[symbols.kSocketLocalPort];
742
- }
743
- if (socket.remoteAddress) {
744
- socketInfo.remoteAddress = socket.remoteAddress;
745
- socketInfo.remotePort = socket.remotePort;
746
- socketInfo.remoteFamily = socket.remoteFamily;
747
- }
748
- socketInfo.bytesRead = socket.bytesRead;
749
- socketInfo.bytesWritten = socket.bytesWritten;
750
- if (socket[symbols.kSocketConnectErrorTime]) {
751
- socketInfo.connectErrorTime = socket[symbols.kSocketConnectErrorTime];
752
- if (Array.isArray(socket.autoSelectFamilyAttemptedAddresses)) {
753
- socketInfo.attemptedRemoteAddresses = socket.autoSelectFamilyAttemptedAddresses;
754
- }
755
- socketInfo.connectProtocol = socket[symbols.kSocketConnectProtocol];
756
- socketInfo.connectHost = socket[symbols.kSocketConnectHost];
757
- socketInfo.connectPort = socket[symbols.kSocketConnectPort];
758
- }
759
- if (socket[symbols.kSocketConnectedTime]) {
760
- socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
761
- }
762
- if (socket[symbols.kSocketRequestEndTime]) {
763
- socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
764
- }
765
- socket[symbols.kSocketRequestEndTime] = new Date();
766
- }
767
- }
768
760
  }
package/src/Request.ts CHANGED
@@ -3,6 +3,7 @@ import type { EventEmitter } from 'node:events';
3
3
  import type { Dispatcher } from 'undici';
4
4
  import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
5
5
  import type { HttpClientResponse } from './Response.js';
6
+ import { Request } from 'undici';
6
7
 
7
8
  export type HttpMethod = Dispatcher.HttpMethod;
8
9
 
@@ -161,3 +162,8 @@ export type RequestMeta = {
161
162
  ctx?: unknown;
162
163
  retries: number;
163
164
  };
165
+
166
+ export type FetchMeta = {
167
+ requestId: number;
168
+ request: Request,
169
+ };
@@ -143,7 +143,10 @@ export function initDiagnosticsChannel() {
143
143
  subscribe('undici:client:sendHeaders', (message, name) => {
144
144
  const { request, socket } = message as DiagnosticsChannel.ClientSendHeadersMessage & { socket: SocketExtend };
145
145
  const opaque = getRequestOpaque(request, kHandler);
146
- if (!opaque || !opaque[symbols.kRequestId]) return;
146
+ if (!opaque || !opaque[symbols.kRequestId]) {
147
+ debug('[%s] opaque not found', name);
148
+ return;
149
+ }
147
150
 
148
151
  (socket[symbols.kHandledRequests] as number)++;
149
152
  // attach socket to opaque
@@ -165,7 +168,10 @@ export function initDiagnosticsChannel() {
165
168
  subscribe('undici:request:bodySent', (message, name) => {
166
169
  const { request } = message as DiagnosticsChannel.RequestBodySentMessage;
167
170
  const opaque = getRequestOpaque(request, kHandler);
168
- if (!opaque || !opaque[symbols.kRequestId]) return;
171
+ if (!opaque || !opaque[symbols.kRequestId]) {
172
+ debug('[%s] opaque not found', name);
173
+ return;
174
+ }
169
175
 
170
176
  debug('[%s] Request#%d send body', name, opaque[symbols.kRequestId]);
171
177
  if (!opaque[symbols.kEnableRequestTiming]) return;
@@ -176,7 +182,10 @@ export function initDiagnosticsChannel() {
176
182
  subscribe('undici:request:headers', (message, name) => {
177
183
  const { request, response } = message as DiagnosticsChannel.RequestHeadersMessage;
178
184
  const opaque = getRequestOpaque(request, kHandler);
179
- if (!opaque || !opaque[symbols.kRequestId]) return;
185
+ if (!opaque || !opaque[symbols.kRequestId]) {
186
+ debug('[%s] opaque not found', name);
187
+ return;
188
+ }
180
189
 
181
190
  // get socket from opaque
182
191
  const socket = opaque[symbols.kRequestSocket];
@@ -199,6 +208,7 @@ export function initDiagnosticsChannel() {
199
208
  const { request } = message as DiagnosticsChannel.RequestTrailersMessage;
200
209
  const opaque = getRequestOpaque(request, kHandler);
201
210
  if (!opaque || !opaque[symbols.kRequestId]) {
211
+ debug('[%s] opaque not found', name);
202
212
  return;
203
213
  }
204
214
 
package/src/fetch.ts ADDED
@@ -0,0 +1,263 @@
1
+ import { AsyncLocalStorage } from 'node:async_hooks';
2
+ import {
3
+ fetch as UndiciFetch,
4
+ RequestInfo,
5
+ RequestInit,
6
+ Request,
7
+ Response,
8
+ Agent,
9
+ getGlobalDispatcher,
10
+ Pool,
11
+ } from 'undici';
12
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
13
+ // @ts-ignore
14
+ import undiciSymbols from 'undici/lib/core/symbols.js';
15
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
16
+ // @ts-ignore
17
+ import { getResponseState } from 'undici/lib/web/fetch/response.js';
18
+ import {
19
+ channels,
20
+ ClientOptions,
21
+ PoolStat,
22
+ RequestDiagnosticsMessage,
23
+ ResponseDiagnosticsMessage,
24
+ UndiciTimingInfo,
25
+ } from './HttpClient.js';
26
+ import {
27
+ HttpAgent,
28
+ HttpAgentOptions,
29
+ } from './HttpAgent.js';
30
+ import { initDiagnosticsChannel } from './diagnosticsChannel.js';
31
+ import { convertHeader, globalId, performanceTime, updateSocketInfo } from './utils.js';
32
+ import symbols from './symbols.js';
33
+ import {
34
+ FetchMeta,
35
+ HttpMethod,
36
+ RequestMeta,
37
+ } from './Request.js';
38
+ import { FetchOpaque, fetchOpaqueInterceptor } from './FetchOpaqueInterceptor.js';
39
+ import { RawResponseWithMeta, SocketInfo } from './Response.js';
40
+ import { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
41
+
42
+ export interface UrllibRequestInit extends RequestInit {
43
+ // default is true
44
+ timing?: boolean;
45
+ }
46
+
47
+ export type FetchDiagnosticsMessage = {
48
+ fetch: FetchMeta;
49
+ };
50
+
51
+ export type FetchResponseDiagnosticsMessage = {
52
+ fetch: FetchMeta;
53
+ timingInfo?: UndiciTimingInfo;
54
+ response?: Response;
55
+ error?: Error;
56
+ };
57
+
58
+ export class FetchFactory {
59
+ static #dispatcher: Agent;
60
+ static #opaqueLocalStorage = new AsyncLocalStorage<FetchOpaque>();
61
+
62
+ static getDispatcher() {
63
+ return FetchFactory.#dispatcher ?? getGlobalDispatcher();
64
+ }
65
+
66
+ static setDispatcher(dispatcher: Agent) {
67
+ FetchFactory.#dispatcher = dispatcher;
68
+ }
69
+
70
+ static setClientOptions(clientOptions: ClientOptions) {
71
+ let dispatcherOption: Agent.Options = {
72
+ interceptors: {
73
+ Agent: [
74
+ fetchOpaqueInterceptor({
75
+ opaqueLocalStorage: FetchFactory.#opaqueLocalStorage,
76
+ }),
77
+ ],
78
+ Client: [],
79
+ },
80
+ };
81
+ let dispatcherClazz: new (options: Agent.Options) => Agent = Agent;
82
+ if (clientOptions?.lookup || clientOptions?.checkAddress) {
83
+ dispatcherOption = {
84
+ ...dispatcherOption,
85
+ lookup: clientOptions.lookup,
86
+ checkAddress: clientOptions.checkAddress,
87
+ connect: clientOptions.connect,
88
+ allowH2: clientOptions.allowH2,
89
+ } as HttpAgentOptions;
90
+ dispatcherClazz = HttpAgent as unknown as new (options: Agent.Options) => Agent;
91
+ } else if (clientOptions?.connect) {
92
+ dispatcherOption = {
93
+ ...dispatcherOption,
94
+ connect: clientOptions.connect,
95
+ allowH2: clientOptions.allowH2,
96
+ } as HttpAgentOptions;
97
+ dispatcherClazz = Agent;
98
+ } else if (clientOptions?.allowH2) {
99
+ // Support HTTP2
100
+ dispatcherOption = {
101
+ ...dispatcherOption,
102
+ allowH2: clientOptions.allowH2,
103
+ } as HttpAgentOptions;
104
+ dispatcherClazz = Agent;
105
+ }
106
+ FetchFactory.#dispatcher = new dispatcherClazz(dispatcherOption);
107
+ initDiagnosticsChannel();
108
+ }
109
+
110
+ static getDispatcherPoolStats() {
111
+ const agent = FetchFactory.getDispatcher();
112
+ // origin => Pool Instance
113
+ const clients: Map<string, WeakRef<Pool>> | undefined = Reflect.get(agent, undiciSymbols.kClients);
114
+ const poolStatsMap: Record<string, PoolStat> = {};
115
+ if (!clients) {
116
+ return poolStatsMap;
117
+ }
118
+ for (const [ key, ref ] of clients) {
119
+ const pool = typeof ref.deref === 'function' ? ref.deref() : ref as unknown as Pool;
120
+ const stats = pool?.stats;
121
+ if (!stats) continue;
122
+ poolStatsMap[key] = {
123
+ connected: stats.connected,
124
+ free: stats.free,
125
+ pending: stats.pending,
126
+ queued: stats.queued,
127
+ running: stats.running,
128
+ size: stats.size,
129
+ } satisfies PoolStat;
130
+ }
131
+ return poolStatsMap;
132
+ }
133
+
134
+ static async fetch(input: RequestInfo, init?: UrllibRequestInit): Promise<Response> {
135
+ const requestStartTime = performance.now();
136
+ init = init ?? {};
137
+ init.dispatcher = init.dispatcher ?? FetchFactory.#dispatcher;
138
+ const request = new Request(input, init);
139
+ const requestId = globalId('HttpClientRequest');
140
+ // https://developer.chrome.com/docs/devtools/network/reference/?utm_source=devtools#timing-explanation
141
+ const timing = {
142
+ // socket assigned
143
+ queuing: 0,
144
+ // dns lookup time
145
+ // dnslookup: 0,
146
+ // socket connected
147
+ connected: 0,
148
+ // request headers sent
149
+ requestHeadersSent: 0,
150
+ // request sent, including headers and body
151
+ requestSent: 0,
152
+ // Time to first byte (TTFB), the response headers have been received
153
+ waiting: 0,
154
+ // the response body and trailers have been received
155
+ contentDownload: 0,
156
+ };
157
+
158
+ // using opaque to diagnostics channel, binding request and socket
159
+ const internalOpaque = {
160
+ [symbols.kRequestId]: requestId,
161
+ [symbols.kRequestStartTime]: requestStartTime,
162
+ [symbols.kEnableRequestTiming]: !!(init.timing ?? true),
163
+ [symbols.kRequestTiming]: timing,
164
+ // [symbols.kRequestOriginalOpaque]: originalOpaque,
165
+ } as FetchOpaque;
166
+ const reqMeta: RequestMeta = {
167
+ requestId,
168
+ url: request.url,
169
+ args: {
170
+ method: request.method as HttpMethod,
171
+ type: request.method as HttpMethod,
172
+ data: request.body,
173
+ headers: convertHeader(request.headers),
174
+ },
175
+ retries: 0,
176
+ };
177
+ const fetchMeta: FetchMeta = {
178
+ requestId,
179
+ request,
180
+ };
181
+ const socketInfo: SocketInfo = {
182
+ id: 0,
183
+ localAddress: '',
184
+ localPort: 0,
185
+ remoteAddress: '',
186
+ remotePort: 0,
187
+ remoteFamily: '',
188
+ bytesWritten: 0,
189
+ bytesRead: 0,
190
+ handledRequests: 0,
191
+ handledResponses: 0,
192
+ };
193
+ channels.request.publish({
194
+ request: reqMeta,
195
+ } as RequestDiagnosticsMessage);
196
+ channels.fetchRequest.publish({
197
+ fetch: fetchMeta,
198
+ } as FetchDiagnosticsMessage);
199
+
200
+ let res: Response;
201
+ // keep urllib createCallbackResponse style
202
+ const resHeaders: IncomingHttpHeaders = {};
203
+ const urllibResponse = {
204
+ status: -1,
205
+ statusCode: -1,
206
+ statusText: '',
207
+ statusMessage: '',
208
+ headers: resHeaders,
209
+ size: 0,
210
+ aborted: false,
211
+ rt: 0,
212
+ keepAliveSocket: true,
213
+ requestUrls: [
214
+ request.url,
215
+ ],
216
+ timing,
217
+ socket: socketInfo,
218
+ retries: 0,
219
+ socketErrorRetries: 0,
220
+ } as any as RawResponseWithMeta;
221
+ try {
222
+ await FetchFactory.#opaqueLocalStorage.run(internalOpaque, async () => {
223
+ res = await UndiciFetch(input, init);
224
+ });
225
+ } catch (e: any) {
226
+ channels.response.publish({
227
+ fetch: fetchMeta,
228
+ error: e,
229
+ } as FetchResponseDiagnosticsMessage);
230
+ channels.fetchResponse.publish({
231
+ request: reqMeta,
232
+ response: urllibResponse,
233
+ error: e,
234
+ } as ResponseDiagnosticsMessage);
235
+ throw e;
236
+ }
237
+
238
+ // get undici internal response
239
+ const state = getResponseState(res!);
240
+ updateSocketInfo(socketInfo, internalOpaque /* , rawError */);
241
+
242
+ urllibResponse.headers = convertHeader(res!.headers);
243
+ urllibResponse.status = urllibResponse.statusCode = res!.status;
244
+ urllibResponse!.statusMessage = res!.statusText;
245
+ if (urllibResponse.headers['content-length']) {
246
+ urllibResponse.size = parseInt(urllibResponse.headers['content-length']);
247
+ }
248
+ urllibResponse.rt = performanceTime(requestStartTime);
249
+
250
+ channels.fetchResponse.publish({
251
+ fetch: fetchMeta,
252
+ timingInfo: state.timingInfo,
253
+ response: res!,
254
+ } as FetchResponseDiagnosticsMessage);
255
+ channels.response.publish({
256
+ request: reqMeta,
257
+ response: urllibResponse,
258
+ } as ResponseDiagnosticsMessage);
259
+ return res!;
260
+ }
261
+ }
262
+
263
+ export const fetch = FetchFactory.fetch;