urllib 4.8.2 → 4.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -42
- package/dist/commonjs/BaseAgent.d.ts +2 -2
- package/dist/commonjs/BaseAgent.js +1 -1
- package/dist/commonjs/FetchOpaqueInterceptor.d.ts +4 -0
- package/dist/commonjs/FetchOpaqueInterceptor.js +1 -1
- package/dist/commonjs/FormData.js +2 -2
- package/dist/commonjs/HttpAgent.d.ts +3 -2
- package/dist/commonjs/HttpAgent.js +1 -1
- package/dist/commonjs/HttpClient.d.ts +22 -22
- package/dist/commonjs/HttpClient.js +46 -25
- package/dist/commonjs/HttpClientError.d.ts +1 -1
- package/dist/commonjs/IncomingHttpHeaders.d.ts +1 -1
- package/dist/commonjs/Request.d.ts +5 -5
- package/dist/commonjs/diagnosticsChannel.js +3 -4
- package/dist/commonjs/fetch.d.ts +6 -5
- package/dist/commonjs/fetch.js +6 -8
- package/dist/commonjs/index.d.ts +14 -11
- package/dist/commonjs/index.js +3 -2
- package/dist/commonjs/symbols.d.ts +2 -2
- package/dist/commonjs/symbols.js +3 -2
- package/dist/commonjs/utils.d.ts +3 -3
- package/dist/commonjs/utils.js +17 -11
- package/dist/esm/BaseAgent.d.ts +2 -2
- package/dist/esm/BaseAgent.js +2 -2
- package/dist/esm/FetchOpaqueInterceptor.d.ts +4 -0
- package/dist/esm/FetchOpaqueInterceptor.js +1 -1
- package/dist/esm/FormData.js +2 -2
- package/dist/esm/HttpAgent.d.ts +3 -2
- package/dist/esm/HttpAgent.js +1 -1
- package/dist/esm/HttpClient.d.ts +22 -22
- package/dist/esm/HttpClient.js +47 -26
- package/dist/esm/HttpClientError.d.ts +1 -1
- package/dist/esm/IncomingHttpHeaders.d.ts +1 -1
- package/dist/esm/Request.d.ts +5 -5
- package/dist/esm/diagnosticsChannel.js +3 -4
- package/dist/esm/fetch.d.ts +6 -5
- package/dist/esm/fetch.js +7 -9
- package/dist/esm/index.d.ts +14 -11
- package/dist/esm/index.js +4 -3
- package/dist/esm/symbols.d.ts +2 -2
- package/dist/esm/symbols.js +3 -2
- package/dist/esm/utils.d.ts +3 -3
- package/dist/esm/utils.js +17 -11
- package/dist/package.json +1 -1
- package/package.json +92 -73
- package/src/BaseAgent.ts +4 -5
- package/src/FetchOpaqueInterceptor.ts +1 -0
- package/src/FormData.ts +3 -2
- package/src/HttpAgent.ts +9 -9
- package/src/HttpClient.ts +150 -88
- package/src/HttpClientError.ts +1 -1
- package/src/IncomingHttpHeaders.ts +2 -1
- package/src/Request.ts +15 -7
- package/src/Response.ts +1 -0
- package/src/diagnosticsChannel.ts +55 -21
- package/src/fetch.ts +36 -44
- package/src/formstream.d.ts +5 -1
- package/src/index.ts +38 -24
- package/src/symbols.ts +24 -1
- package/src/utils.ts +34 -26
package/src/HttpClient.ts
CHANGED
|
@@ -1,53 +1,46 @@
|
|
|
1
1
|
import diagnosticsChannel from 'node:diagnostics_channel';
|
|
2
|
+
import type { Channel } from 'node:diagnostics_channel';
|
|
2
3
|
import { EventEmitter } from 'node:events';
|
|
3
|
-
import {
|
|
4
|
+
import { createReadStream } from 'node:fs';
|
|
4
5
|
import { STATUS_CODES } from 'node:http';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
createGunzip,
|
|
8
|
-
createBrotliDecompress,
|
|
9
|
-
gunzipSync,
|
|
10
|
-
brotliDecompressSync,
|
|
11
|
-
} from 'node:zlib';
|
|
12
|
-
import { Readable, pipeline } from 'node:stream';
|
|
13
|
-
import { pipeline as pipelinePromise } from 'node:stream/promises';
|
|
6
|
+
import type { LookupFunction } from 'node:net';
|
|
14
7
|
import { basename } from 'node:path';
|
|
15
|
-
import { createReadStream } from 'node:fs';
|
|
16
|
-
import { format as urlFormat } from 'node:url';
|
|
17
8
|
import { performance } from 'node:perf_hooks';
|
|
18
9
|
import querystring from 'node:querystring';
|
|
10
|
+
import { Readable, pipeline } from 'node:stream';
|
|
11
|
+
import { pipeline as pipelinePromise } from 'node:stream/promises';
|
|
19
12
|
import { setTimeout as sleep } from 'node:timers/promises';
|
|
20
|
-
import {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
13
|
+
import { format as urlFormat } from 'node:url';
|
|
14
|
+
import { debuglog } from 'node:util';
|
|
15
|
+
import { createGunzip, createBrotliDecompress, gunzipSync, brotliDecompressSync } from 'node:zlib';
|
|
16
|
+
|
|
17
|
+
// Compatible with old style formstream
|
|
18
|
+
import FormStream from 'formstream';
|
|
19
|
+
import mime from 'mime-types';
|
|
20
|
+
import qs from 'qs';
|
|
21
|
+
import { request as undiciRequest, Dispatcher, Agent, getGlobalDispatcher, Pool } from 'undici';
|
|
27
22
|
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
28
23
|
// @ts-ignore
|
|
29
24
|
import undiciSymbols from 'undici/lib/core/symbols.js';
|
|
30
|
-
|
|
31
|
-
import
|
|
32
|
-
|
|
33
|
-
import FormStream from 'formstream';
|
|
25
|
+
|
|
26
|
+
import { initDiagnosticsChannel } from './diagnosticsChannel.js';
|
|
27
|
+
import type { FetchOpaque } from './FetchOpaqueInterceptor.js';
|
|
34
28
|
import { FormData } from './FormData.js';
|
|
35
|
-
import { HttpAgent
|
|
29
|
+
import { HttpAgent } from './HttpAgent.js';
|
|
30
|
+
import type { CheckAddressFunction } from './HttpAgent.js';
|
|
31
|
+
import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
|
|
36
32
|
import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
|
|
37
|
-
import { RequestURL, RequestOptions, HttpMethod, RequestMeta } from './Request.js';
|
|
38
|
-
import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response.js';
|
|
39
|
-
import { parseJSON, digestAuthHeader, globalId, performanceTime, isReadable, updateSocketInfo } from './utils.js';
|
|
33
|
+
import type { RequestURL, RequestOptions, HttpMethod, RequestMeta } from './Request.js';
|
|
34
|
+
import type { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response.js';
|
|
40
35
|
import symbols from './symbols.js';
|
|
41
|
-
import {
|
|
42
|
-
import { HttpClientConnectTimeoutError, HttpClientRequestTimeoutError } from './HttpClientError.js';
|
|
43
|
-
import { FetchOpaque } from './FetchOpaqueInterceptor.js';
|
|
36
|
+
import { parseJSON, digestAuthHeader, globalId, performanceTime, isReadable, updateSocketInfo } from './utils.js';
|
|
44
37
|
|
|
45
38
|
type Exists<T> = T extends undefined ? never : T;
|
|
46
39
|
type UndiciRequestOption = Exists<Parameters<typeof undiciRequest>[1]>;
|
|
47
40
|
type PropertyShouldBe<T, K extends keyof T, V> = Omit<T, K> & { [P in K]: V };
|
|
48
41
|
type IUndiciRequestOption = PropertyShouldBe<UndiciRequestOption, 'headers', IncomingHttpHeaders>;
|
|
49
42
|
|
|
50
|
-
export const PROTO_RE = /^https?:\/\//i;
|
|
43
|
+
export const PROTO_RE: RegExp = /^https?:\/\//i;
|
|
51
44
|
|
|
52
45
|
export interface UndiciTimingInfo {
|
|
53
46
|
startTime: number;
|
|
@@ -73,7 +66,7 @@ export interface UndiciTimingInfo {
|
|
|
73
66
|
// keep typo compatibility
|
|
74
67
|
export interface UnidiciTimingInfo extends UndiciTimingInfo {}
|
|
75
68
|
|
|
76
|
-
function noop() {
|
|
69
|
+
function noop(): void {
|
|
77
70
|
// noop
|
|
78
71
|
}
|
|
79
72
|
|
|
@@ -88,23 +81,23 @@ export type ClientOptions = {
|
|
|
88
81
|
*/
|
|
89
82
|
lookup?: LookupFunction;
|
|
90
83
|
/**
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
84
|
+
* check request address to protect from SSRF and similar attacks.
|
|
85
|
+
* It receive two arguments(ip and family) and should return true or false to identified the address is legal or not.
|
|
86
|
+
* It rely on lookup and have the same version requirement.
|
|
87
|
+
*/
|
|
95
88
|
checkAddress?: CheckAddressFunction;
|
|
96
89
|
connect?: {
|
|
97
90
|
key?: string | Buffer;
|
|
98
91
|
/**
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
92
|
+
* A string or Buffer containing the certificate key of the client in PEM format.
|
|
93
|
+
* Notes: This is necessary only if using the client certificate authentication
|
|
94
|
+
*/
|
|
102
95
|
cert?: string | Buffer;
|
|
103
96
|
/**
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
97
|
+
* If `true`, the server certificate is verified against the list of supplied CAs.
|
|
98
|
+
* An 'error' event is emitted if verification fails.
|
|
99
|
+
* Default: `true`
|
|
100
|
+
*/
|
|
108
101
|
rejectUnauthorized?: boolean;
|
|
109
102
|
/**
|
|
110
103
|
* socketPath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
|
|
@@ -114,15 +107,14 @@ export type ClientOptions = {
|
|
|
114
107
|
* connect timeout, default is 10000ms
|
|
115
108
|
*/
|
|
116
109
|
timeout?: number;
|
|
117
|
-
}
|
|
110
|
+
};
|
|
118
111
|
};
|
|
119
112
|
|
|
120
|
-
export const VERSION = 'VERSION';
|
|
113
|
+
export const VERSION: string = 'VERSION';
|
|
121
114
|
// 'node-urllib/4.0.0 Node.js/18.19.0 (darwin; x64)'
|
|
122
|
-
export const HEADER_USER_AGENT =
|
|
123
|
-
`node-urllib/${VERSION} Node.js/${process.version.substring(1)} (${process.platform}; ${process.arch})`;
|
|
115
|
+
export const HEADER_USER_AGENT: string = `node-urllib/${VERSION} Node.js/${process.version.substring(1)} (${process.platform}; ${process.arch})`;
|
|
124
116
|
|
|
125
|
-
function getFileName(stream: Readable) {
|
|
117
|
+
function getFileName(stream: Readable): string {
|
|
126
118
|
const filePath: string = (stream as any).path;
|
|
127
119
|
if (filePath) {
|
|
128
120
|
return basename(filePath);
|
|
@@ -130,7 +122,7 @@ function getFileName(stream: Readable) {
|
|
|
130
122
|
return '';
|
|
131
123
|
}
|
|
132
124
|
|
|
133
|
-
function defaultIsRetry(response: HttpClientResponse) {
|
|
125
|
+
function defaultIsRetry(response: HttpClientResponse): boolean {
|
|
134
126
|
return response.status >= 500;
|
|
135
127
|
}
|
|
136
128
|
|
|
@@ -142,7 +134,12 @@ export type RequestContext = {
|
|
|
142
134
|
history: string[];
|
|
143
135
|
};
|
|
144
136
|
|
|
145
|
-
export const channels
|
|
137
|
+
export const channels: {
|
|
138
|
+
request: Channel;
|
|
139
|
+
response: Channel;
|
|
140
|
+
fetchRequest: Channel;
|
|
141
|
+
fetchResponse: Channel;
|
|
142
|
+
} = {
|
|
146
143
|
request: diagnosticsChannel.channel('urllib:request'),
|
|
147
144
|
response: diagnosticsChannel.channel('urllib:response'),
|
|
148
145
|
fetchRequest: diagnosticsChannel.channel('urllib:fetch:request'),
|
|
@@ -187,6 +184,12 @@ const RedirectStatusCodes = [
|
|
|
187
184
|
308, // Permanent Redirect
|
|
188
185
|
];
|
|
189
186
|
|
|
187
|
+
// Credential-bearing headers that must not be forwarded across an origin
|
|
188
|
+
// boundary when following a redirect, to avoid leaking them to a third party.
|
|
189
|
+
// Matches the WHATWG Fetch spec and undici's RedirectHandler.
|
|
190
|
+
// https://fetch.spec.whatwg.org/#http-redirect-fetch
|
|
191
|
+
const CrossOriginSensitiveHeaders = new Set(['authorization', 'cookie', 'proxy-authorization']);
|
|
192
|
+
|
|
190
193
|
export class HttpClient extends EventEmitter {
|
|
191
194
|
#defaultArgs?: RequestOptions;
|
|
192
195
|
#dispatcher?: Dispatcher;
|
|
@@ -215,15 +218,15 @@ export class HttpClient extends EventEmitter {
|
|
|
215
218
|
initDiagnosticsChannel();
|
|
216
219
|
}
|
|
217
220
|
|
|
218
|
-
getDispatcher() {
|
|
221
|
+
getDispatcher(): Dispatcher {
|
|
219
222
|
return this.#dispatcher ?? getGlobalDispatcher();
|
|
220
223
|
}
|
|
221
224
|
|
|
222
|
-
setDispatcher(dispatcher: Dispatcher) {
|
|
225
|
+
setDispatcher(dispatcher: Dispatcher): void {
|
|
223
226
|
this.#dispatcher = dispatcher;
|
|
224
227
|
}
|
|
225
228
|
|
|
226
|
-
getDispatcherPoolStats() {
|
|
229
|
+
getDispatcherPoolStats(): Record<string, PoolStat> {
|
|
227
230
|
const agent = this.getDispatcher();
|
|
228
231
|
// origin => Pool Instance
|
|
229
232
|
const clients: Map<string, WeakRef<Pool>> | undefined = Reflect.get(agent, undiciSymbols.kClients);
|
|
@@ -231,8 +234,8 @@ export class HttpClient extends EventEmitter {
|
|
|
231
234
|
if (!clients) {
|
|
232
235
|
return poolStatsMap;
|
|
233
236
|
}
|
|
234
|
-
for (const [
|
|
235
|
-
const pool = (typeof ref.deref === 'function' ? ref.deref() : ref) as unknown as
|
|
237
|
+
for (const [key, ref] of clients) {
|
|
238
|
+
const pool = (typeof ref.deref === 'function' ? ref.deref() : ref) as unknown as Pool & { dispatcher: Pool };
|
|
236
239
|
// NOTE: pool become to { dispatcher: Pool } in undici@v7
|
|
237
240
|
const stats = pool?.stats ?? pool?.dispatcher?.stats;
|
|
238
241
|
if (!stats) continue;
|
|
@@ -249,16 +252,20 @@ export class HttpClient extends EventEmitter {
|
|
|
249
252
|
return poolStatsMap;
|
|
250
253
|
}
|
|
251
254
|
|
|
252
|
-
async request<T = any>(url: RequestURL, options?: RequestOptions) {
|
|
255
|
+
async request<T = any>(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse<T>> {
|
|
253
256
|
return await this.#requestInternal<T>(url, options);
|
|
254
257
|
}
|
|
255
258
|
|
|
256
259
|
// alias to request, keep compatible with urllib@2 HttpClient.curl
|
|
257
|
-
async curl<T = any>(url: RequestURL, options?: RequestOptions) {
|
|
260
|
+
async curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<HttpClientResponse<T>> {
|
|
258
261
|
return await this.request<T>(url, options);
|
|
259
262
|
}
|
|
260
263
|
|
|
261
|
-
async #requestInternal<T>(
|
|
264
|
+
async #requestInternal<T>(
|
|
265
|
+
url: RequestURL,
|
|
266
|
+
options?: RequestOptions,
|
|
267
|
+
requestContext?: RequestContext,
|
|
268
|
+
): Promise<HttpClientResponse<T>> {
|
|
262
269
|
const requestId = globalId('HttpClientRequest');
|
|
263
270
|
let requestUrl: URL;
|
|
264
271
|
if (typeof url === 'string') {
|
|
@@ -472,20 +479,20 @@ export class HttpClient extends EventEmitter {
|
|
|
472
479
|
const formData = new FormData();
|
|
473
480
|
const uploadFiles: [string, string | Readable | Buffer, string?][] = [];
|
|
474
481
|
if (Array.isArray(args.files)) {
|
|
475
|
-
for (const [
|
|
482
|
+
for (const [index, file] of args.files.entries()) {
|
|
476
483
|
const field = index === 0 ? 'file' : `file${index}`;
|
|
477
|
-
uploadFiles.push([
|
|
484
|
+
uploadFiles.push([field, file]);
|
|
478
485
|
}
|
|
479
486
|
} else if (args.files instanceof Readable || isReadable(args.files as any)) {
|
|
480
|
-
uploadFiles.push([
|
|
487
|
+
uploadFiles.push(['file', args.files as Readable]);
|
|
481
488
|
} else if (typeof args.files === 'string' || Buffer.isBuffer(args.files)) {
|
|
482
|
-
uploadFiles.push([
|
|
489
|
+
uploadFiles.push(['file', args.files]);
|
|
483
490
|
} else if (typeof args.files === 'object') {
|
|
484
491
|
const files = args.files as Record<string, string | Readable | Buffer>;
|
|
485
492
|
for (const field in files) {
|
|
486
493
|
// set custom fileName
|
|
487
494
|
const file = files[field];
|
|
488
|
-
uploadFiles.push([
|
|
495
|
+
uploadFiles.push([field, file, field]);
|
|
489
496
|
}
|
|
490
497
|
}
|
|
491
498
|
// set normal fields first
|
|
@@ -494,7 +501,7 @@ export class HttpClient extends EventEmitter {
|
|
|
494
501
|
formData.append(field, args.data[field]);
|
|
495
502
|
}
|
|
496
503
|
}
|
|
497
|
-
for (const [
|
|
504
|
+
for (const [index, [field, file, customFileName]] of uploadFiles.entries()) {
|
|
498
505
|
let fileName = '';
|
|
499
506
|
let value: any;
|
|
500
507
|
if (typeof file === 'string') {
|
|
@@ -513,8 +520,7 @@ export class HttpClient extends EventEmitter {
|
|
|
513
520
|
filename: fileName,
|
|
514
521
|
contentType: mimeType,
|
|
515
522
|
});
|
|
516
|
-
debug('formData append field: %s, mimeType: %s, fileName: %s',
|
|
517
|
-
field, mimeType, fileName);
|
|
523
|
+
debug('formData append field: %s, mimeType: %s, fileName: %s', field, mimeType, fileName);
|
|
518
524
|
}
|
|
519
525
|
Object.assign(headers, formData.getHeaders());
|
|
520
526
|
requestOptions.body = formData;
|
|
@@ -530,9 +536,8 @@ export class HttpClient extends EventEmitter {
|
|
|
530
536
|
isStreamingRequest = isReadable(args.content);
|
|
531
537
|
}
|
|
532
538
|
} else if (args.data) {
|
|
533
|
-
const isStringOrBufferOrReadable =
|
|
534
|
-
|| Buffer.isBuffer(args.data)
|
|
535
|
-
|| isReadable(args.data);
|
|
539
|
+
const isStringOrBufferOrReadable =
|
|
540
|
+
typeof args.data === 'string' || Buffer.isBuffer(args.data) || isReadable(args.data);
|
|
536
541
|
if (isGETOrHEAD) {
|
|
537
542
|
if (!isStringOrBufferOrReadable) {
|
|
538
543
|
let query: string;
|
|
@@ -550,9 +555,11 @@ export class HttpClient extends EventEmitter {
|
|
|
550
555
|
requestOptions.body = args.data;
|
|
551
556
|
isStreamingRequest = isReadable(args.data);
|
|
552
557
|
} else {
|
|
553
|
-
if (
|
|
554
|
-
|
|
555
|
-
|
|
558
|
+
if (
|
|
559
|
+
args.contentType === 'json' ||
|
|
560
|
+
args.contentType === 'application/json' ||
|
|
561
|
+
headers['content-type']?.startsWith('application/json')
|
|
562
|
+
) {
|
|
556
563
|
requestOptions.body = JSON.stringify(args.data);
|
|
557
564
|
if (!headers['content-type']) {
|
|
558
565
|
headers['content-type'] = 'application/json';
|
|
@@ -578,8 +585,19 @@ export class HttpClient extends EventEmitter {
|
|
|
578
585
|
args.socketErrorRetry = 0;
|
|
579
586
|
}
|
|
580
587
|
|
|
581
|
-
debug(
|
|
582
|
-
|
|
588
|
+
debug(
|
|
589
|
+
'Request#%d %s %s, headers: %j, headersTimeout: %s, bodyTimeout: %s, isStreamingRequest: %s, isStreamingResponse: %s, maxRedirections: %s, redirects: %s',
|
|
590
|
+
requestId,
|
|
591
|
+
requestOptions.method,
|
|
592
|
+
requestUrl.href,
|
|
593
|
+
headers,
|
|
594
|
+
headersTimeout,
|
|
595
|
+
bodyTimeout,
|
|
596
|
+
isStreamingRequest,
|
|
597
|
+
isStreamingResponse,
|
|
598
|
+
maxRedirects,
|
|
599
|
+
requestContext.redirects,
|
|
600
|
+
);
|
|
583
601
|
requestOptions.headers = headers;
|
|
584
602
|
channels.request.publish({
|
|
585
603
|
request: reqMeta,
|
|
@@ -589,17 +607,25 @@ export class HttpClient extends EventEmitter {
|
|
|
589
607
|
}
|
|
590
608
|
|
|
591
609
|
let response = await undiciRequest(requestUrl, requestOptions as UndiciRequestOption);
|
|
592
|
-
if (
|
|
593
|
-
|
|
610
|
+
if (
|
|
611
|
+
response.statusCode === 401 &&
|
|
612
|
+
(response.headers['www-authenticate'] || response.headers['x-www-authenticate']) &&
|
|
613
|
+
!requestOptions.headers.authorization &&
|
|
614
|
+
args.digestAuth
|
|
615
|
+
) {
|
|
594
616
|
// handle digest auth
|
|
595
617
|
const authenticateHeaders = response.headers['www-authenticate'] ?? response.headers['x-www-authenticate'];
|
|
596
618
|
const authenticate = Array.isArray(authenticateHeaders)
|
|
597
|
-
? authenticateHeaders.find(authHeader => authHeader.startsWith('Digest '))
|
|
619
|
+
? authenticateHeaders.find((authHeader) => authHeader.startsWith('Digest '))
|
|
598
620
|
: authenticateHeaders;
|
|
599
621
|
if (authenticate && authenticate.startsWith('Digest ')) {
|
|
600
622
|
debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', requestId, requestUrl.href, authenticate);
|
|
601
|
-
requestOptions.headers.authorization = digestAuthHeader(
|
|
602
|
-
|
|
623
|
+
requestOptions.headers.authorization = digestAuthHeader(
|
|
624
|
+
requestOptions.method!,
|
|
625
|
+
`${requestUrl.pathname}${requestUrl.search}`,
|
|
626
|
+
authenticate,
|
|
627
|
+
args.digestAuth,
|
|
628
|
+
);
|
|
603
629
|
debug('Request#%d %s: auth with digest header: %s', requestId, url, requestOptions.headers.authorization);
|
|
604
630
|
if (Array.isArray(response.headers['set-cookie'])) {
|
|
605
631
|
// FIXME: merge exists cookie header
|
|
@@ -627,9 +653,31 @@ export class HttpClient extends EventEmitter {
|
|
|
627
653
|
const nextUrl = new URL(res.headers.location, requestUrl.href);
|
|
628
654
|
// Ensure the response is consumed
|
|
629
655
|
await response.body.arrayBuffer();
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
656
|
+
let redirectOptions = options;
|
|
657
|
+
// Do not forward credential-bearing headers to a different origin.
|
|
658
|
+
if (nextUrl.origin !== requestUrl.origin) {
|
|
659
|
+
const cleanedHeaders: IncomingHttpHeaders = {};
|
|
660
|
+
if (options?.headers) {
|
|
661
|
+
for (const name in options.headers) {
|
|
662
|
+
if (!CrossOriginSensitiveHeaders.has(name.toLowerCase())) {
|
|
663
|
+
cleanedHeaders[name] = options.headers[name];
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
// Clone so the caller's options object is never mutated, and drop
|
|
668
|
+
// credentials so they are not re-applied on the new origin,
|
|
669
|
+
// including Basic/digest auth inherited from the client's defaultArgs.
|
|
670
|
+
redirectOptions = { ...options, headers: cleanedHeaders, auth: undefined, digestAuth: undefined };
|
|
671
|
+
}
|
|
672
|
+
debug(
|
|
673
|
+
'Request#%d got response, status: %s, headers: %j, timing: %j, redirect to %s',
|
|
674
|
+
requestId,
|
|
675
|
+
res.status,
|
|
676
|
+
res.headers,
|
|
677
|
+
res.timing,
|
|
678
|
+
nextUrl.href,
|
|
679
|
+
);
|
|
680
|
+
return await this.#requestInternal(nextUrl.href, redirectOptions, requestContext);
|
|
633
681
|
}
|
|
634
682
|
}
|
|
635
683
|
|
|
@@ -690,8 +738,14 @@ export class HttpClient extends EventEmitter {
|
|
|
690
738
|
res,
|
|
691
739
|
};
|
|
692
740
|
|
|
693
|
-
debug(
|
|
694
|
-
|
|
741
|
+
debug(
|
|
742
|
+
'Request#%d got response, status: %s, headers: %j, timing: %j, socket: %j',
|
|
743
|
+
requestId,
|
|
744
|
+
res.status,
|
|
745
|
+
res.headers,
|
|
746
|
+
res.timing,
|
|
747
|
+
res.socket,
|
|
748
|
+
);
|
|
695
749
|
|
|
696
750
|
if (args.retry > 0 && requestContext.retries < args.retry) {
|
|
697
751
|
const isRetry = args.isRetry ?? defaultIsRetry;
|
|
@@ -723,8 +777,13 @@ export class HttpClient extends EventEmitter {
|
|
|
723
777
|
|
|
724
778
|
return clientResponse;
|
|
725
779
|
} catch (rawError: any) {
|
|
726
|
-
debug(
|
|
727
|
-
|
|
780
|
+
debug(
|
|
781
|
+
'Request#%d throw error: %s, socketErrorRetry: %s, socketErrorRetries: %s',
|
|
782
|
+
requestId,
|
|
783
|
+
rawError,
|
|
784
|
+
args.socketErrorRetry,
|
|
785
|
+
requestContext.socketErrorRetries,
|
|
786
|
+
);
|
|
728
787
|
let err = rawError;
|
|
729
788
|
if (err.name === 'HeadersTimeoutError') {
|
|
730
789
|
err = new HttpClientRequestTimeoutError(headersTimeout, { cause: err });
|
|
@@ -738,8 +797,11 @@ export class HttpClient extends EventEmitter {
|
|
|
738
797
|
// auto retry on socket error, https://github.com/node-modules/urllib/issues/454
|
|
739
798
|
if (args.socketErrorRetry > 0 && requestContext.socketErrorRetries < args.socketErrorRetry) {
|
|
740
799
|
requestContext.socketErrorRetries++;
|
|
741
|
-
debug(
|
|
742
|
-
|
|
800
|
+
debug(
|
|
801
|
+
'Request#%d retry on socket error, socketErrorRetries: %d',
|
|
802
|
+
requestId,
|
|
803
|
+
requestContext.socketErrorRetries,
|
|
804
|
+
);
|
|
743
805
|
return await this.#requestInternal(url, options, requestContext);
|
|
744
806
|
}
|
|
745
807
|
}
|
package/src/HttpClientError.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import type { Except } from 'type-fest';
|
|
2
1
|
import type { IncomingHttpHeaders as HTTPIncomingHttpHeaders } from 'node:http';
|
|
3
2
|
|
|
3
|
+
import type { Except } from 'type-fest';
|
|
4
|
+
|
|
4
5
|
// fix set-cookie type define https://github.com/nodejs/undici/pull/1893
|
|
5
6
|
export interface IncomingHttpHeaders extends Except<HTTPIncomingHttpHeaders, 'set-cookie'> {
|
|
6
7
|
'set-cookie'?: string | string[];
|
package/src/Request.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import type { Readable, Writable } from 'node:stream';
|
|
2
1
|
import type { EventEmitter } from 'node:events';
|
|
2
|
+
import type { Readable, Writable } from 'node:stream';
|
|
3
|
+
|
|
3
4
|
import type { Dispatcher } from 'undici';
|
|
5
|
+
import { Request } from 'undici';
|
|
6
|
+
|
|
4
7
|
import type { IncomingHttpHeaders } from './IncomingHttpHeaders.js';
|
|
5
8
|
import type { HttpClientResponse } from './Response.js';
|
|
6
|
-
import { Request } from 'undici';
|
|
7
9
|
|
|
8
10
|
export type HttpMethod = Dispatcher.HttpMethod;
|
|
9
11
|
|
|
@@ -36,10 +38,16 @@ export type RequestOptions = {
|
|
|
36
38
|
*/
|
|
37
39
|
writeStream?: Writable;
|
|
38
40
|
/**
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
files?:
|
|
41
|
+
* The files will send with multipart/form-data format, base on formstream.
|
|
42
|
+
* If method not set, will use POST method by default.
|
|
43
|
+
*/
|
|
44
|
+
files?:
|
|
45
|
+
| Array<Readable | Buffer | string>
|
|
46
|
+
| Record<string, Readable | Buffer | string>
|
|
47
|
+
| Readable
|
|
48
|
+
| Buffer
|
|
49
|
+
| string
|
|
50
|
+
| object;
|
|
43
51
|
/** Type of request data, could be 'json'. If it's 'json', will auto set Content-Type: 'application/json' header. */
|
|
44
52
|
contentType?: string;
|
|
45
53
|
/**
|
|
@@ -165,5 +173,5 @@ export type RequestMeta = {
|
|
|
165
173
|
|
|
166
174
|
export type FetchMeta = {
|
|
167
175
|
requestId: number;
|
|
168
|
-
request: Request
|
|
176
|
+
request: Request;
|
|
169
177
|
};
|
package/src/Response.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import diagnosticsChannel from 'node:diagnostics_channel';
|
|
2
|
+
import { Socket } from 'node:net';
|
|
2
3
|
import { performance } from 'node:perf_hooks';
|
|
3
4
|
import { debuglog } from 'node:util';
|
|
4
|
-
|
|
5
|
-
import { DiagnosticsChannel } from 'undici';
|
|
5
|
+
|
|
6
|
+
import type { DiagnosticsChannel } from 'undici';
|
|
7
|
+
|
|
6
8
|
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
7
9
|
// @ts-ignore
|
|
8
10
|
import symbols from './symbols.js';
|
|
@@ -54,7 +56,7 @@ function formatSocket(socket: SocketExtend) {
|
|
|
54
56
|
|
|
55
57
|
// make sure error contains socket info
|
|
56
58
|
const destroySocket = Socket.prototype.destroy;
|
|
57
|
-
Socket.prototype.destroy = function(err?: any) {
|
|
59
|
+
Socket.prototype.destroy = function (err?: any) {
|
|
58
60
|
if (err) {
|
|
59
61
|
Object.defineProperty(err, symbols.kErrorSocket, {
|
|
60
62
|
// don't show on console log
|
|
@@ -80,7 +82,7 @@ function getRequestOpaque(request: DiagnosticsChannel.Request, kHandler?: symbol
|
|
|
80
82
|
return handler?.opts?.opaque ?? handler?.opaque;
|
|
81
83
|
}
|
|
82
84
|
|
|
83
|
-
export function initDiagnosticsChannel() {
|
|
85
|
+
export function initDiagnosticsChannel(): void {
|
|
84
86
|
// make sure init global DiagnosticsChannel once
|
|
85
87
|
if (initedDiagnosticsChannel) return;
|
|
86
88
|
initedDiagnosticsChannel = true;
|
|
@@ -104,14 +106,24 @@ export function initDiagnosticsChannel() {
|
|
|
104
106
|
if (!opaque || !opaque[symbols.kRequestId]) return;
|
|
105
107
|
|
|
106
108
|
Reflect.set(request, symbols.kRequestInternalOpaque, opaque);
|
|
107
|
-
debug(
|
|
108
|
-
|
|
109
|
+
debug(
|
|
110
|
+
'[%s] Request#%d %s %s, path: %s, headers: %j',
|
|
111
|
+
name,
|
|
112
|
+
opaque[symbols.kRequestId],
|
|
113
|
+
request.method,
|
|
114
|
+
request.origin,
|
|
115
|
+
request.path,
|
|
116
|
+
request.headers,
|
|
117
|
+
);
|
|
109
118
|
if (!opaque[symbols.kEnableRequestTiming]) return;
|
|
110
119
|
opaque[symbols.kRequestTiming].queuing = performanceTime(opaque[symbols.kRequestStartTime]);
|
|
111
120
|
});
|
|
112
121
|
|
|
113
122
|
subscribe('undici:client:connectError', (message, name) => {
|
|
114
|
-
const { error, connectParams, socket } = message as DiagnosticsChannel.ClientConnectErrorMessage & {
|
|
123
|
+
const { error, connectParams, socket } = message as DiagnosticsChannel.ClientConnectErrorMessage & {
|
|
124
|
+
error: any;
|
|
125
|
+
socket: SocketExtend;
|
|
126
|
+
};
|
|
115
127
|
let sock = socket;
|
|
116
128
|
if (!sock && error[symbols.kErrorSocket]) {
|
|
117
129
|
sock = error[symbols.kErrorSocket];
|
|
@@ -129,11 +141,16 @@ export function initDiagnosticsChannel() {
|
|
|
129
141
|
sock[symbols.kSocketConnectProtocol] = connectParams.protocol;
|
|
130
142
|
sock[symbols.kSocketConnectHost] = connectParams.host;
|
|
131
143
|
sock[symbols.kSocketConnectPort] = connectParams.port;
|
|
132
|
-
debug(
|
|
133
|
-
|
|
144
|
+
debug(
|
|
145
|
+
'[%s] Socket#%d connectError, connectParams: %j, error: %s, (sock: %j)',
|
|
146
|
+
name,
|
|
147
|
+
sock[symbols.kSocketId],
|
|
148
|
+
connectParams,
|
|
149
|
+
(error as Error).message,
|
|
150
|
+
formatSocket(sock),
|
|
151
|
+
);
|
|
134
152
|
} else {
|
|
135
|
-
debug('[%s] connectError, connectParams: %j, error: %o',
|
|
136
|
-
name, connectParams, error);
|
|
153
|
+
debug('[%s] connectError, connectParams: %j, error: %o', name, connectParams, error);
|
|
137
154
|
}
|
|
138
155
|
});
|
|
139
156
|
|
|
@@ -166,17 +183,24 @@ export function initDiagnosticsChannel() {
|
|
|
166
183
|
(socket[symbols.kHandledRequests] as number)++;
|
|
167
184
|
// attach socket to opaque
|
|
168
185
|
opaque[symbols.kRequestSocket] = socket;
|
|
169
|
-
debug(
|
|
170
|
-
|
|
171
|
-
|
|
186
|
+
debug(
|
|
187
|
+
'[%s] Request#%d send headers on Socket#%d (handled %d requests, sock: %j)',
|
|
188
|
+
name,
|
|
189
|
+
opaque[symbols.kRequestId],
|
|
190
|
+
socket[symbols.kSocketId],
|
|
191
|
+
socket[symbols.kHandledRequests],
|
|
192
|
+
formatSocket(socket),
|
|
193
|
+
);
|
|
172
194
|
|
|
173
195
|
if (!opaque[symbols.kEnableRequestTiming]) return;
|
|
174
196
|
opaque[symbols.kRequestTiming].requestHeadersSent = performanceTime(opaque[symbols.kRequestStartTime]);
|
|
175
197
|
// first socket need to calculate the connected time
|
|
176
198
|
if (socket[symbols.kHandledRequests] === 1) {
|
|
177
199
|
// kSocketStartTime - kRequestStartTime = connected time
|
|
178
|
-
opaque[symbols.kRequestTiming].connected =
|
|
179
|
-
|
|
200
|
+
opaque[symbols.kRequestTiming].connected = performanceTime(
|
|
201
|
+
opaque[symbols.kRequestStartTime],
|
|
202
|
+
socket[symbols.kSocketStartTime] as number,
|
|
203
|
+
);
|
|
180
204
|
}
|
|
181
205
|
});
|
|
182
206
|
|
|
@@ -206,12 +230,22 @@ export function initDiagnosticsChannel() {
|
|
|
206
230
|
const socket = opaque[symbols.kRequestSocket];
|
|
207
231
|
if (socket) {
|
|
208
232
|
socket[symbols.kHandledResponses]++;
|
|
209
|
-
debug(
|
|
210
|
-
|
|
211
|
-
|
|
233
|
+
debug(
|
|
234
|
+
'[%s] Request#%d get %s response headers on Socket#%d (handled %d responses, sock: %j)',
|
|
235
|
+
name,
|
|
236
|
+
opaque[symbols.kRequestId],
|
|
237
|
+
response.statusCode,
|
|
238
|
+
socket[symbols.kSocketId],
|
|
239
|
+
socket[symbols.kHandledResponses],
|
|
240
|
+
formatSocket(socket),
|
|
241
|
+
);
|
|
212
242
|
} else {
|
|
213
|
-
debug(
|
|
214
|
-
|
|
243
|
+
debug(
|
|
244
|
+
'[%s] Request#%d get %s response headers on Unknown Socket',
|
|
245
|
+
name,
|
|
246
|
+
opaque[symbols.kRequestId],
|
|
247
|
+
response.statusCode,
|
|
248
|
+
);
|
|
215
249
|
}
|
|
216
250
|
|
|
217
251
|
if (!opaque[symbols.kEnableRequestTiming]) return;
|