urllib 4.3.1 → 4.5.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -2
- package/dist/commonjs/FetchOpaqueInterceptor.d.ts +8 -0
- package/dist/commonjs/FetchOpaqueInterceptor.js +18 -0
- package/dist/commonjs/FormData.d.ts +4 -0
- package/dist/commonjs/FormData.js +38 -0
- package/dist/commonjs/HttpAgent.d.ts +3 -3
- package/dist/commonjs/HttpAgent.js +6 -5
- package/dist/commonjs/HttpClient.d.ts +31 -0
- package/dist/commonjs/HttpClient.js +62 -98
- package/dist/commonjs/Request.d.ts +5 -0
- package/dist/commonjs/diagnosticsChannel.js +11 -4
- package/dist/commonjs/fetch.d.ts +24 -0
- package/dist/commonjs/fetch.js +221 -0
- package/dist/commonjs/index.d.ts +2 -1
- package/dist/commonjs/index.js +7 -2
- package/dist/commonjs/utils.d.ts +4 -0
- package/dist/commonjs/utils.js +57 -1
- package/dist/esm/FetchOpaqueInterceptor.d.ts +8 -0
- package/dist/esm/FetchOpaqueInterceptor.js +12 -0
- package/dist/esm/FormData.d.ts +4 -0
- package/dist/esm/FormData.js +31 -0
- package/dist/esm/HttpAgent.d.ts +3 -3
- package/dist/esm/HttpAgent.js +6 -5
- package/dist/esm/HttpClient.d.ts +31 -0
- package/dist/esm/HttpClient.js +59 -95
- package/dist/esm/Request.d.ts +5 -0
- package/dist/esm/diagnosticsChannel.js +11 -4
- package/dist/esm/fetch.d.ts +24 -0
- package/dist/esm/fetch.js +214 -0
- package/dist/esm/index.d.ts +2 -1
- package/dist/esm/index.js +3 -2
- package/dist/esm/utils.d.ts +4 -0
- package/dist/esm/utils.js +52 -1
- package/dist/package.json +1 -1
- package/package.json +7 -5
- package/src/FetchOpaqueInterceptor.ts +41 -0
- package/src/FormData.ts +32 -0
- package/src/HttpAgent.ts +9 -7
- package/src/HttpClient.ts +92 -100
- package/src/Request.ts +6 -0
- package/src/diagnosticsChannel.ts +13 -3
- package/src/fetch.ts +263 -0
- package/src/index.ts +3 -0
- package/src/utils.ts +54 -0
package/src/HttpClient.ts
CHANGED
@@ -9,7 +9,6 @@ import {
|
|
9
9
|
gunzipSync,
|
10
10
|
brotliDecompressSync,
|
11
11
|
} from 'node:zlib';
|
12
|
-
import { Blob } from 'node:buffer';
|
13
12
|
import { Readable, pipeline } from 'node:stream';
|
14
13
|
import { pipeline as pipelinePromise } from 'node:stream/promises';
|
15
14
|
import { basename } from 'node:path';
|
@@ -19,13 +18,13 @@ import { performance } from 'node:perf_hooks';
|
|
19
18
|
import querystring from 'node:querystring';
|
20
19
|
import { setTimeout as sleep } from 'node:timers/promises';
|
21
20
|
import {
|
22
|
-
FormData,
|
23
21
|
request as undiciRequest,
|
24
22
|
Dispatcher,
|
25
23
|
Agent,
|
26
24
|
getGlobalDispatcher,
|
27
25
|
Pool,
|
28
26
|
} from 'undici';
|
27
|
+
import { FormData } from './FormData.js';
|
29
28
|
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
30
29
|
// @ts-ignore
|
31
30
|
import undiciSymbols from 'undici/lib/core/symbols.js';
|
@@ -37,7 +36,7 @@ import { HttpAgent, CheckAddressFunction } from './HttpAgent.js';
|
|
37
36
|
import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
|
38
37
|
import { RequestURL, RequestOptions, HttpMethod, RequestMeta } from './Request.js';
|
39
38
|
import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response.js';
|
40
|
-
import { parseJSON, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
|
39
|
+
import { parseJSON, digestAuthHeader, globalId, performanceTime, isReadable, updateSocketInfo } from './utils.js';
|
41
40
|
import symbols from './symbols.js';
|
42
41
|
import { initDiagnosticsChannel } from './diagnosticsChannel.js';
|
43
42
|
import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
|
@@ -47,7 +46,31 @@ type UndiciRequestOption = Exists<Parameters<typeof undiciRequest>[1]>;
|
|
47
46
|
type PropertyShouldBe<T, K extends keyof T, V> = Omit<T, K> & { [P in K]: V };
|
48
47
|
type IUndiciRequestOption = PropertyShouldBe<UndiciRequestOption, 'headers', IncomingHttpHeaders>;
|
49
48
|
|
50
|
-
const PROTO_RE = /^https?:\/\//i;
|
49
|
+
export const PROTO_RE = /^https?:\/\//i;
|
50
|
+
|
51
|
+
export interface UndiciTimingInfo {
|
52
|
+
startTime: number;
|
53
|
+
redirectStartTime: number;
|
54
|
+
redirectEndTime: number;
|
55
|
+
postRedirectStartTime: number;
|
56
|
+
finalServiceWorkerStartTime: number;
|
57
|
+
finalNetworkResponseStartTime: number;
|
58
|
+
finalNetworkRequestStartTime: number;
|
59
|
+
endTime: number;
|
60
|
+
encodedBodySize: number;
|
61
|
+
decodedBodySize: number;
|
62
|
+
finalConnectionTimingInfo: {
|
63
|
+
domainLookupStartTime: number;
|
64
|
+
domainLookupEndTime: number;
|
65
|
+
connectionStartTime: number;
|
66
|
+
connectionEndTime: number;
|
67
|
+
secureConnectionStartTime: number;
|
68
|
+
// ALPNNegotiatedProtocol: undefined
|
69
|
+
};
|
70
|
+
}
|
71
|
+
|
72
|
+
// keep typo compatibility
|
73
|
+
export interface UnidiciTimingInfo extends UndiciTimingInfo {}
|
51
74
|
|
52
75
|
function noop() {
|
53
76
|
// noop
|
@@ -92,28 +115,6 @@ export type ClientOptions = {
|
|
92
115
|
},
|
93
116
|
};
|
94
117
|
|
95
|
-
// https://github.com/octet-stream/form-data
|
96
|
-
class BlobFromStream {
|
97
|
-
#stream;
|
98
|
-
#type;
|
99
|
-
constructor(stream: Readable, type: string) {
|
100
|
-
this.#stream = stream;
|
101
|
-
this.#type = type;
|
102
|
-
}
|
103
|
-
|
104
|
-
stream() {
|
105
|
-
return this.#stream;
|
106
|
-
}
|
107
|
-
|
108
|
-
get type(): string {
|
109
|
-
return this.#type;
|
110
|
-
}
|
111
|
-
|
112
|
-
get [Symbol.toStringTag]() {
|
113
|
-
return 'Blob';
|
114
|
-
}
|
115
|
-
}
|
116
|
-
|
117
118
|
export const VERSION = 'VERSION';
|
118
119
|
// 'node-urllib/4.0.0 Node.js/18.19.0 (darwin; x64)'
|
119
120
|
export const HEADER_USER_AGENT =
|
@@ -135,11 +136,15 @@ export type RequestContext = {
|
|
135
136
|
retries: number;
|
136
137
|
socketErrorRetries: number;
|
137
138
|
requestStartTime?: number;
|
139
|
+
redirects: number;
|
140
|
+
history: string[];
|
138
141
|
};
|
139
142
|
|
140
|
-
const channels = {
|
143
|
+
export const channels = {
|
141
144
|
request: diagnosticsChannel.channel('urllib:request'),
|
142
145
|
response: diagnosticsChannel.channel('urllib:response'),
|
146
|
+
fetchRequest: diagnosticsChannel.channel('urllib:fetch:request'),
|
147
|
+
fetchResponse: diagnosticsChannel.channel('urllib:fetch:response'),
|
143
148
|
};
|
144
149
|
|
145
150
|
export type RequestDiagnosticsMessage = {
|
@@ -167,6 +172,15 @@ export interface PoolStat {
|
|
167
172
|
size: number;
|
168
173
|
}
|
169
174
|
|
175
|
+
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections
|
176
|
+
const RedirectStatusCodes = [
|
177
|
+
301, // Moved Permanently
|
178
|
+
302, // Found
|
179
|
+
303, // See Other
|
180
|
+
307, // Temporary Redirect
|
181
|
+
308, // Permanent Redirect
|
182
|
+
];
|
183
|
+
|
170
184
|
export class HttpClient extends EventEmitter {
|
171
185
|
#defaultArgs?: RequestOptions;
|
172
186
|
#dispatcher?: Dispatcher;
|
@@ -271,11 +285,14 @@ export class HttpClient extends EventEmitter {
|
|
271
285
|
requestContext = {
|
272
286
|
retries: 0,
|
273
287
|
socketErrorRetries: 0,
|
288
|
+
redirects: 0,
|
289
|
+
history: [],
|
274
290
|
...requestContext,
|
275
291
|
};
|
276
292
|
if (!requestContext.requestStartTime) {
|
277
293
|
requestContext.requestStartTime = performance.now();
|
278
294
|
}
|
295
|
+
requestContext.history.push(requestUrl.href);
|
279
296
|
const requestStartTime = requestContext.requestStartTime;
|
280
297
|
|
281
298
|
// https://developer.chrome.com/docs/devtools/network/reference/?utm_source=devtools#timing-explanation
|
@@ -335,7 +352,7 @@ export class HttpClient extends EventEmitter {
|
|
335
352
|
aborted: false,
|
336
353
|
rt: 0,
|
337
354
|
keepAliveSocket: true,
|
338
|
-
requestUrls:
|
355
|
+
requestUrls: requestContext.history,
|
339
356
|
timing,
|
340
357
|
socket: socketInfo,
|
341
358
|
retries: requestContext.retries,
|
@@ -396,10 +413,13 @@ export class HttpClient extends EventEmitter {
|
|
396
413
|
isStreamingRequest = true;
|
397
414
|
}
|
398
415
|
|
416
|
+
let maxRedirects = args.maxRedirects ?? 10;
|
417
|
+
|
399
418
|
try {
|
400
419
|
const requestOptions: IUndiciRequestOption = {
|
401
420
|
method,
|
402
|
-
|
421
|
+
// disable undici auto redirect handler
|
422
|
+
maxRedirections: 0,
|
403
423
|
headersTimeout,
|
404
424
|
headers,
|
405
425
|
bodyTimeout,
|
@@ -414,7 +434,7 @@ export class HttpClient extends EventEmitter {
|
|
414
434
|
requestOptions.reset = args.reset;
|
415
435
|
}
|
416
436
|
if (args.followRedirect === false) {
|
417
|
-
|
437
|
+
maxRedirects = 0;
|
418
438
|
}
|
419
439
|
|
420
440
|
const isGETOrHEAD = requestOptions.method === 'GET' || requestOptions.method === 'HEAD';
|
@@ -464,21 +484,28 @@ export class HttpClient extends EventEmitter {
|
|
464
484
|
}
|
465
485
|
}
|
466
486
|
for (const [ index, [ field, file, customFileName ]] of uploadFiles.entries()) {
|
487
|
+
let fileName = '';
|
488
|
+
let value: any;
|
467
489
|
if (typeof file === 'string') {
|
468
|
-
|
469
|
-
|
470
|
-
// formData.append(field, await fileFromPath(file, `utf-8''${fileName}`, { type: mime.lookup(fileName) || '' }));
|
471
|
-
const fileName = basename(file);
|
472
|
-
const fileReadable = createReadStream(file);
|
473
|
-
formData.append(field, new BlobFromStream(fileReadable, mime.lookup(fileName) || ''), fileName);
|
490
|
+
fileName = basename(file);
|
491
|
+
value = createReadStream(file);
|
474
492
|
} else if (Buffer.isBuffer(file)) {
|
475
|
-
|
493
|
+
fileName = customFileName || `bufferfile${index}`;
|
494
|
+
value = file;
|
476
495
|
} else if (file instanceof Readable || isReadable(file as any)) {
|
477
|
-
|
478
|
-
formData.append(field, new BlobFromStream(file, mime.lookup(fileName) || ''), fileName);
|
496
|
+
fileName = getFileName(file) || customFileName || `streamfile${index}`;
|
479
497
|
isStreamingRequest = true;
|
498
|
+
value = file;
|
480
499
|
}
|
500
|
+
const mimeType = mime.lookup(fileName) || '';
|
501
|
+
formData.append(field, value, {
|
502
|
+
filename: fileName,
|
503
|
+
contentType: mimeType,
|
504
|
+
});
|
505
|
+
debug('formData append field: %s, mimeType: %s, fileName: %s',
|
506
|
+
field, mimeType, fileName);
|
481
507
|
}
|
508
|
+
Object.assign(headers, formData.getHeaders());
|
482
509
|
requestOptions.body = formData;
|
483
510
|
} else if (args.content) {
|
484
511
|
if (!isGETOrHEAD) {
|
@@ -535,8 +562,8 @@ export class HttpClient extends EventEmitter {
|
|
535
562
|
args.socketErrorRetry = 0;
|
536
563
|
}
|
537
564
|
|
538
|
-
debug('Request#%d %s %s, headers: %j, headersTimeout: %s, bodyTimeout: %s, isStreamingRequest: %s',
|
539
|
-
requestId, requestOptions.method, requestUrl.href, headers, headersTimeout, bodyTimeout, isStreamingRequest);
|
565
|
+
debug('Request#%d %s %s, headers: %j, headersTimeout: %s, bodyTimeout: %s, isStreamingRequest: %s, maxRedirections: %s, redirects: %s',
|
566
|
+
requestId, requestOptions.method, requestUrl.href, headers, headersTimeout, bodyTimeout, isStreamingRequest, maxRedirects, requestContext.redirects);
|
540
567
|
requestOptions.headers = headers;
|
541
568
|
channels.request.publish({
|
542
569
|
request: reqMeta,
|
@@ -567,18 +594,6 @@ export class HttpClient extends EventEmitter {
|
|
567
594
|
response = await undiciRequest(requestUrl, requestOptions as UndiciRequestOption);
|
568
595
|
}
|
569
596
|
}
|
570
|
-
|
571
|
-
const context = response.context as { history: URL[] };
|
572
|
-
let lastUrl = '';
|
573
|
-
if (context?.history) {
|
574
|
-
for (const urlObject of context?.history) {
|
575
|
-
res.requestUrls.push(urlObject.href);
|
576
|
-
lastUrl = urlObject.href;
|
577
|
-
}
|
578
|
-
} else {
|
579
|
-
res.requestUrls.push(requestUrl.href);
|
580
|
-
lastUrl = requestUrl.href;
|
581
|
-
}
|
582
597
|
const contentEncoding = response.headers['content-encoding'];
|
583
598
|
const isCompressedContent = contentEncoding === 'gzip' || contentEncoding === 'br';
|
584
599
|
|
@@ -589,6 +604,19 @@ export class HttpClient extends EventEmitter {
|
|
589
604
|
res.size = parseInt(res.headers['content-length']);
|
590
605
|
}
|
591
606
|
|
607
|
+
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Redirections
|
608
|
+
if (RedirectStatusCodes.includes(res.statusCode) && maxRedirects > 0 && requestContext.redirects < maxRedirects && !isStreamingRequest) {
|
609
|
+
if (res.headers.location) {
|
610
|
+
requestContext.redirects++;
|
611
|
+
const nextUrl = new URL(res.headers.location, requestUrl.href);
|
612
|
+
// Ensure the response is consumed
|
613
|
+
await response.body.arrayBuffer();
|
614
|
+
debug('Request#%d got response, status: %s, headers: %j, timing: %j, redirect to %s',
|
615
|
+
requestId, res.status, res.headers, res.timing, nextUrl.href);
|
616
|
+
return await this.#requestInternal(nextUrl.href, options, requestContext);
|
617
|
+
}
|
618
|
+
}
|
619
|
+
|
592
620
|
let data: any = null;
|
593
621
|
if (args.dataType === 'stream') {
|
594
622
|
// only auto decompress on request args.compressed = true
|
@@ -631,7 +659,7 @@ export class HttpClient extends EventEmitter {
|
|
631
659
|
}
|
632
660
|
res.rt = performanceTime(requestStartTime);
|
633
661
|
// get real socket info from internalOpaque
|
634
|
-
|
662
|
+
updateSocketInfo(socketInfo, internalOpaque);
|
635
663
|
|
636
664
|
const clientResponse: HttpClientResponse = {
|
637
665
|
opaque: originalOpaque,
|
@@ -640,12 +668,15 @@ export class HttpClient extends EventEmitter {
|
|
640
668
|
statusCode: res.status,
|
641
669
|
statusText: res.statusText,
|
642
670
|
headers: res.headers,
|
643
|
-
url:
|
644
|
-
redirected:
|
671
|
+
url: requestUrl.href,
|
672
|
+
redirected: requestContext.history.length > 1,
|
645
673
|
requestUrls: res.requestUrls,
|
646
674
|
res,
|
647
675
|
};
|
648
676
|
|
677
|
+
debug('Request#%d got response, status: %s, headers: %j, timing: %j',
|
678
|
+
requestId, res.status, res.headers, res.timing);
|
679
|
+
|
649
680
|
if (args.retry > 0 && requestContext.retries < args.retry) {
|
650
681
|
const isRetry = args.isRetry ?? defaultIsRetry;
|
651
682
|
if (isRetry(clientResponse)) {
|
@@ -657,8 +688,6 @@ export class HttpClient extends EventEmitter {
|
|
657
688
|
}
|
658
689
|
}
|
659
690
|
|
660
|
-
debug('Request#%d got response, status: %s, headers: %j, timing: %j',
|
661
|
-
requestId, res.status, res.headers, res.timing);
|
662
691
|
channels.response.publish({
|
663
692
|
request: reqMeta,
|
664
693
|
response: res,
|
@@ -678,7 +707,8 @@ export class HttpClient extends EventEmitter {
|
|
678
707
|
|
679
708
|
return clientResponse;
|
680
709
|
} catch (rawError: any) {
|
681
|
-
debug('Request#%d throw error: %s
|
710
|
+
debug('Request#%d throw error: %s, socketErrorRetry: %s, socketErrorRetries: %s',
|
711
|
+
requestId, rawError, args.socketErrorRetry, requestContext.socketErrorRetries);
|
682
712
|
let err = rawError;
|
683
713
|
if (err.name === 'HeadersTimeoutError') {
|
684
714
|
err = new HttpClientRequestTimeoutError(headersTimeout, { cause: err });
|
@@ -690,6 +720,8 @@ export class HttpClient extends EventEmitter {
|
|
690
720
|
// auto retry on socket error, https://github.com/node-modules/urllib/issues/454
|
691
721
|
if (args.socketErrorRetry > 0 && requestContext.socketErrorRetries < args.socketErrorRetry) {
|
692
722
|
requestContext.socketErrorRetries++;
|
723
|
+
debug('Request#%d retry on socket error, socketErrorRetries: %d',
|
724
|
+
requestId, requestContext.socketErrorRetries);
|
693
725
|
return await this.#requestInternal(url, options, requestContext);
|
694
726
|
}
|
695
727
|
}
|
@@ -702,12 +734,8 @@ export class HttpClient extends EventEmitter {
|
|
702
734
|
err._rawSocket = err.socket;
|
703
735
|
}
|
704
736
|
err.socket = socketInfo;
|
705
|
-
// make sure requestUrls not empty
|
706
|
-
if (res.requestUrls.length === 0) {
|
707
|
-
res.requestUrls.push(requestUrl.href);
|
708
|
-
}
|
709
737
|
res.rt = performanceTime(requestStartTime);
|
710
|
-
|
738
|
+
updateSocketInfo(socketInfo, internalOpaque, rawError);
|
711
739
|
|
712
740
|
channels.response.publish({
|
713
741
|
request: reqMeta,
|
@@ -729,40 +757,4 @@ export class HttpClient extends EventEmitter {
|
|
729
757
|
throw err;
|
730
758
|
}
|
731
759
|
}
|
732
|
-
|
733
|
-
#updateSocketInfo(socketInfo: SocketInfo, internalOpaque: any, err?: any) {
|
734
|
-
const socket = internalOpaque[symbols.kRequestSocket] ?? err?.[symbols.kErrorSocket];
|
735
|
-
if (socket) {
|
736
|
-
socketInfo.id = socket[symbols.kSocketId];
|
737
|
-
socketInfo.handledRequests = socket[symbols.kHandledRequests];
|
738
|
-
socketInfo.handledResponses = socket[symbols.kHandledResponses];
|
739
|
-
if (socket[symbols.kSocketLocalAddress]) {
|
740
|
-
socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
|
741
|
-
socketInfo.localPort = socket[symbols.kSocketLocalPort];
|
742
|
-
}
|
743
|
-
if (socket.remoteAddress) {
|
744
|
-
socketInfo.remoteAddress = socket.remoteAddress;
|
745
|
-
socketInfo.remotePort = socket.remotePort;
|
746
|
-
socketInfo.remoteFamily = socket.remoteFamily;
|
747
|
-
}
|
748
|
-
socketInfo.bytesRead = socket.bytesRead;
|
749
|
-
socketInfo.bytesWritten = socket.bytesWritten;
|
750
|
-
if (socket[symbols.kSocketConnectErrorTime]) {
|
751
|
-
socketInfo.connectErrorTime = socket[symbols.kSocketConnectErrorTime];
|
752
|
-
if (Array.isArray(socket.autoSelectFamilyAttemptedAddresses)) {
|
753
|
-
socketInfo.attemptedRemoteAddresses = socket.autoSelectFamilyAttemptedAddresses;
|
754
|
-
}
|
755
|
-
socketInfo.connectProtocol = socket[symbols.kSocketConnectProtocol];
|
756
|
-
socketInfo.connectHost = socket[symbols.kSocketConnectHost];
|
757
|
-
socketInfo.connectPort = socket[symbols.kSocketConnectPort];
|
758
|
-
}
|
759
|
-
if (socket[symbols.kSocketConnectedTime]) {
|
760
|
-
socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
|
761
|
-
}
|
762
|
-
if (socket[symbols.kSocketRequestEndTime]) {
|
763
|
-
socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
|
764
|
-
}
|
765
|
-
socket[symbols.kSocketRequestEndTime] = new Date();
|
766
|
-
}
|
767
|
-
}
|
768
760
|
}
|
package/src/Request.ts
CHANGED
@@ -3,6 +3,7 @@ import type { EventEmitter } from 'node:events';
|
|
3
3
|
import type { Dispatcher } from 'undici';
|
4
4
|
import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
|
5
5
|
import type { HttpClientResponse } from './Response.js';
|
6
|
+
import { Request } from 'undici';
|
6
7
|
|
7
8
|
export type HttpMethod = Dispatcher.HttpMethod;
|
8
9
|
|
@@ -161,3 +162,8 @@ export type RequestMeta = {
|
|
161
162
|
ctx?: unknown;
|
162
163
|
retries: number;
|
163
164
|
};
|
165
|
+
|
166
|
+
export type FetchMeta = {
|
167
|
+
requestId: number;
|
168
|
+
request: Request,
|
169
|
+
};
|
@@ -143,7 +143,10 @@ export function initDiagnosticsChannel() {
|
|
143
143
|
subscribe('undici:client:sendHeaders', (message, name) => {
|
144
144
|
const { request, socket } = message as DiagnosticsChannel.ClientSendHeadersMessage & { socket: SocketExtend };
|
145
145
|
const opaque = getRequestOpaque(request, kHandler);
|
146
|
-
if (!opaque || !opaque[symbols.kRequestId])
|
146
|
+
if (!opaque || !opaque[symbols.kRequestId]) {
|
147
|
+
debug('[%s] opaque not found', name);
|
148
|
+
return;
|
149
|
+
}
|
147
150
|
|
148
151
|
(socket[symbols.kHandledRequests] as number)++;
|
149
152
|
// attach socket to opaque
|
@@ -165,7 +168,10 @@ export function initDiagnosticsChannel() {
|
|
165
168
|
subscribe('undici:request:bodySent', (message, name) => {
|
166
169
|
const { request } = message as DiagnosticsChannel.RequestBodySentMessage;
|
167
170
|
const opaque = getRequestOpaque(request, kHandler);
|
168
|
-
if (!opaque || !opaque[symbols.kRequestId])
|
171
|
+
if (!opaque || !opaque[symbols.kRequestId]) {
|
172
|
+
debug('[%s] opaque not found', name);
|
173
|
+
return;
|
174
|
+
}
|
169
175
|
|
170
176
|
debug('[%s] Request#%d send body', name, opaque[symbols.kRequestId]);
|
171
177
|
if (!opaque[symbols.kEnableRequestTiming]) return;
|
@@ -176,7 +182,10 @@ export function initDiagnosticsChannel() {
|
|
176
182
|
subscribe('undici:request:headers', (message, name) => {
|
177
183
|
const { request, response } = message as DiagnosticsChannel.RequestHeadersMessage;
|
178
184
|
const opaque = getRequestOpaque(request, kHandler);
|
179
|
-
if (!opaque || !opaque[symbols.kRequestId])
|
185
|
+
if (!opaque || !opaque[symbols.kRequestId]) {
|
186
|
+
debug('[%s] opaque not found', name);
|
187
|
+
return;
|
188
|
+
}
|
180
189
|
|
181
190
|
// get socket from opaque
|
182
191
|
const socket = opaque[symbols.kRequestSocket];
|
@@ -199,6 +208,7 @@ export function initDiagnosticsChannel() {
|
|
199
208
|
const { request } = message as DiagnosticsChannel.RequestTrailersMessage;
|
200
209
|
const opaque = getRequestOpaque(request, kHandler);
|
201
210
|
if (!opaque || !opaque[symbols.kRequestId]) {
|
211
|
+
debug('[%s] opaque not found', name);
|
202
212
|
return;
|
203
213
|
}
|
204
214
|
|
package/src/fetch.ts
ADDED
@@ -0,0 +1,263 @@
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
2
|
+
import {
|
3
|
+
fetch as UndiciFetch,
|
4
|
+
RequestInfo,
|
5
|
+
RequestInit,
|
6
|
+
Request,
|
7
|
+
Response,
|
8
|
+
Agent,
|
9
|
+
getGlobalDispatcher,
|
10
|
+
Pool,
|
11
|
+
} from 'undici';
|
12
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
13
|
+
// @ts-ignore
|
14
|
+
import undiciSymbols from 'undici/lib/core/symbols.js';
|
15
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
16
|
+
// @ts-ignore
|
17
|
+
import { getResponseState } from 'undici/lib/web/fetch/response.js';
|
18
|
+
import {
|
19
|
+
channels,
|
20
|
+
ClientOptions,
|
21
|
+
PoolStat,
|
22
|
+
RequestDiagnosticsMessage,
|
23
|
+
ResponseDiagnosticsMessage,
|
24
|
+
UndiciTimingInfo,
|
25
|
+
} from './HttpClient.js';
|
26
|
+
import {
|
27
|
+
HttpAgent,
|
28
|
+
HttpAgentOptions,
|
29
|
+
} from './HttpAgent.js';
|
30
|
+
import { initDiagnosticsChannel } from './diagnosticsChannel.js';
|
31
|
+
import { convertHeader, globalId, performanceTime, updateSocketInfo } from './utils.js';
|
32
|
+
import symbols from './symbols.js';
|
33
|
+
import {
|
34
|
+
FetchMeta,
|
35
|
+
HttpMethod,
|
36
|
+
RequestMeta,
|
37
|
+
} from './Request.js';
|
38
|
+
import { FetchOpaque, fetchOpaqueInterceptor } from './FetchOpaqueInterceptor.js';
|
39
|
+
import { RawResponseWithMeta, SocketInfo } from './Response.js';
|
40
|
+
import { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
|
41
|
+
|
42
|
+
export interface UrllibRequestInit extends RequestInit {
|
43
|
+
// default is true
|
44
|
+
timing?: boolean;
|
45
|
+
}
|
46
|
+
|
47
|
+
export type FetchDiagnosticsMessage = {
|
48
|
+
fetch: FetchMeta;
|
49
|
+
};
|
50
|
+
|
51
|
+
export type FetchResponseDiagnosticsMessage = {
|
52
|
+
fetch: FetchMeta;
|
53
|
+
timingInfo?: UndiciTimingInfo;
|
54
|
+
response?: Response;
|
55
|
+
error?: Error;
|
56
|
+
};
|
57
|
+
|
58
|
+
export class FetchFactory {
|
59
|
+
static #dispatcher: Agent;
|
60
|
+
static #opaqueLocalStorage = new AsyncLocalStorage<FetchOpaque>();
|
61
|
+
|
62
|
+
static getDispatcher() {
|
63
|
+
return FetchFactory.#dispatcher ?? getGlobalDispatcher();
|
64
|
+
}
|
65
|
+
|
66
|
+
static setDispatcher(dispatcher: Agent) {
|
67
|
+
FetchFactory.#dispatcher = dispatcher;
|
68
|
+
}
|
69
|
+
|
70
|
+
static setClientOptions(clientOptions: ClientOptions) {
|
71
|
+
let dispatcherOption: Agent.Options = {
|
72
|
+
interceptors: {
|
73
|
+
Agent: [
|
74
|
+
fetchOpaqueInterceptor({
|
75
|
+
opaqueLocalStorage: FetchFactory.#opaqueLocalStorage,
|
76
|
+
}),
|
77
|
+
],
|
78
|
+
Client: [],
|
79
|
+
},
|
80
|
+
};
|
81
|
+
let dispatcherClazz: new (options: Agent.Options) => Agent = Agent;
|
82
|
+
if (clientOptions?.lookup || clientOptions?.checkAddress) {
|
83
|
+
dispatcherOption = {
|
84
|
+
...dispatcherOption,
|
85
|
+
lookup: clientOptions.lookup,
|
86
|
+
checkAddress: clientOptions.checkAddress,
|
87
|
+
connect: clientOptions.connect,
|
88
|
+
allowH2: clientOptions.allowH2,
|
89
|
+
} as HttpAgentOptions;
|
90
|
+
dispatcherClazz = HttpAgent as unknown as new (options: Agent.Options) => Agent;
|
91
|
+
} else if (clientOptions?.connect) {
|
92
|
+
dispatcherOption = {
|
93
|
+
...dispatcherOption,
|
94
|
+
connect: clientOptions.connect,
|
95
|
+
allowH2: clientOptions.allowH2,
|
96
|
+
} as HttpAgentOptions;
|
97
|
+
dispatcherClazz = Agent;
|
98
|
+
} else if (clientOptions?.allowH2) {
|
99
|
+
// Support HTTP2
|
100
|
+
dispatcherOption = {
|
101
|
+
...dispatcherOption,
|
102
|
+
allowH2: clientOptions.allowH2,
|
103
|
+
} as HttpAgentOptions;
|
104
|
+
dispatcherClazz = Agent;
|
105
|
+
}
|
106
|
+
FetchFactory.#dispatcher = new dispatcherClazz(dispatcherOption);
|
107
|
+
initDiagnosticsChannel();
|
108
|
+
}
|
109
|
+
|
110
|
+
static getDispatcherPoolStats() {
|
111
|
+
const agent = FetchFactory.getDispatcher();
|
112
|
+
// origin => Pool Instance
|
113
|
+
const clients: Map<string, WeakRef<Pool>> | undefined = Reflect.get(agent, undiciSymbols.kClients);
|
114
|
+
const poolStatsMap: Record<string, PoolStat> = {};
|
115
|
+
if (!clients) {
|
116
|
+
return poolStatsMap;
|
117
|
+
}
|
118
|
+
for (const [ key, ref ] of clients) {
|
119
|
+
const pool = typeof ref.deref === 'function' ? ref.deref() : ref as unknown as Pool;
|
120
|
+
const stats = pool?.stats;
|
121
|
+
if (!stats) continue;
|
122
|
+
poolStatsMap[key] = {
|
123
|
+
connected: stats.connected,
|
124
|
+
free: stats.free,
|
125
|
+
pending: stats.pending,
|
126
|
+
queued: stats.queued,
|
127
|
+
running: stats.running,
|
128
|
+
size: stats.size,
|
129
|
+
} satisfies PoolStat;
|
130
|
+
}
|
131
|
+
return poolStatsMap;
|
132
|
+
}
|
133
|
+
|
134
|
+
static async fetch(input: RequestInfo, init?: UrllibRequestInit): Promise<Response> {
|
135
|
+
const requestStartTime = performance.now();
|
136
|
+
init = init ?? {};
|
137
|
+
init.dispatcher = init.dispatcher ?? FetchFactory.#dispatcher;
|
138
|
+
const request = new Request(input, init);
|
139
|
+
const requestId = globalId('HttpClientRequest');
|
140
|
+
// https://developer.chrome.com/docs/devtools/network/reference/?utm_source=devtools#timing-explanation
|
141
|
+
const timing = {
|
142
|
+
// socket assigned
|
143
|
+
queuing: 0,
|
144
|
+
// dns lookup time
|
145
|
+
// dnslookup: 0,
|
146
|
+
// socket connected
|
147
|
+
connected: 0,
|
148
|
+
// request headers sent
|
149
|
+
requestHeadersSent: 0,
|
150
|
+
// request sent, including headers and body
|
151
|
+
requestSent: 0,
|
152
|
+
// Time to first byte (TTFB), the response headers have been received
|
153
|
+
waiting: 0,
|
154
|
+
// the response body and trailers have been received
|
155
|
+
contentDownload: 0,
|
156
|
+
};
|
157
|
+
|
158
|
+
// using opaque to diagnostics channel, binding request and socket
|
159
|
+
const internalOpaque = {
|
160
|
+
[symbols.kRequestId]: requestId,
|
161
|
+
[symbols.kRequestStartTime]: requestStartTime,
|
162
|
+
[symbols.kEnableRequestTiming]: !!(init.timing ?? true),
|
163
|
+
[symbols.kRequestTiming]: timing,
|
164
|
+
// [symbols.kRequestOriginalOpaque]: originalOpaque,
|
165
|
+
} as FetchOpaque;
|
166
|
+
const reqMeta: RequestMeta = {
|
167
|
+
requestId,
|
168
|
+
url: request.url,
|
169
|
+
args: {
|
170
|
+
method: request.method as HttpMethod,
|
171
|
+
type: request.method as HttpMethod,
|
172
|
+
data: request.body,
|
173
|
+
headers: convertHeader(request.headers),
|
174
|
+
},
|
175
|
+
retries: 0,
|
176
|
+
};
|
177
|
+
const fetchMeta: FetchMeta = {
|
178
|
+
requestId,
|
179
|
+
request,
|
180
|
+
};
|
181
|
+
const socketInfo: SocketInfo = {
|
182
|
+
id: 0,
|
183
|
+
localAddress: '',
|
184
|
+
localPort: 0,
|
185
|
+
remoteAddress: '',
|
186
|
+
remotePort: 0,
|
187
|
+
remoteFamily: '',
|
188
|
+
bytesWritten: 0,
|
189
|
+
bytesRead: 0,
|
190
|
+
handledRequests: 0,
|
191
|
+
handledResponses: 0,
|
192
|
+
};
|
193
|
+
channels.request.publish({
|
194
|
+
request: reqMeta,
|
195
|
+
} as RequestDiagnosticsMessage);
|
196
|
+
channels.fetchRequest.publish({
|
197
|
+
fetch: fetchMeta,
|
198
|
+
} as FetchDiagnosticsMessage);
|
199
|
+
|
200
|
+
let res: Response;
|
201
|
+
// keep urllib createCallbackResponse style
|
202
|
+
const resHeaders: IncomingHttpHeaders = {};
|
203
|
+
const urllibResponse = {
|
204
|
+
status: -1,
|
205
|
+
statusCode: -1,
|
206
|
+
statusText: '',
|
207
|
+
statusMessage: '',
|
208
|
+
headers: resHeaders,
|
209
|
+
size: 0,
|
210
|
+
aborted: false,
|
211
|
+
rt: 0,
|
212
|
+
keepAliveSocket: true,
|
213
|
+
requestUrls: [
|
214
|
+
request.url,
|
215
|
+
],
|
216
|
+
timing,
|
217
|
+
socket: socketInfo,
|
218
|
+
retries: 0,
|
219
|
+
socketErrorRetries: 0,
|
220
|
+
} as any as RawResponseWithMeta;
|
221
|
+
try {
|
222
|
+
await FetchFactory.#opaqueLocalStorage.run(internalOpaque, async () => {
|
223
|
+
res = await UndiciFetch(input, init);
|
224
|
+
});
|
225
|
+
} catch (e: any) {
|
226
|
+
channels.response.publish({
|
227
|
+
fetch: fetchMeta,
|
228
|
+
error: e,
|
229
|
+
} as FetchResponseDiagnosticsMessage);
|
230
|
+
channels.fetchResponse.publish({
|
231
|
+
request: reqMeta,
|
232
|
+
response: urllibResponse,
|
233
|
+
error: e,
|
234
|
+
} as ResponseDiagnosticsMessage);
|
235
|
+
throw e;
|
236
|
+
}
|
237
|
+
|
238
|
+
// get undici internal response
|
239
|
+
const state = getResponseState(res!);
|
240
|
+
updateSocketInfo(socketInfo, internalOpaque /* , rawError */);
|
241
|
+
|
242
|
+
urllibResponse.headers = convertHeader(res!.headers);
|
243
|
+
urllibResponse.status = urllibResponse.statusCode = res!.status;
|
244
|
+
urllibResponse!.statusMessage = res!.statusText;
|
245
|
+
if (urllibResponse.headers['content-length']) {
|
246
|
+
urllibResponse.size = parseInt(urllibResponse.headers['content-length']);
|
247
|
+
}
|
248
|
+
urllibResponse.rt = performanceTime(requestStartTime);
|
249
|
+
|
250
|
+
channels.fetchResponse.publish({
|
251
|
+
fetch: fetchMeta,
|
252
|
+
timingInfo: state.timingInfo,
|
253
|
+
response: res!,
|
254
|
+
} as FetchResponseDiagnosticsMessage);
|
255
|
+
channels.response.publish({
|
256
|
+
request: reqMeta,
|
257
|
+
response: urllibResponse,
|
258
|
+
} as ResponseDiagnosticsMessage);
|
259
|
+
return res!;
|
260
|
+
}
|
261
|
+
}
|
262
|
+
|
263
|
+
export const fetch = FetchFactory.fetch;
|