urllib 3.17.2 → 3.18.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{src/cjs → dist/commonjs}/HttpAgent.d.ts +1 -3
- package/{src/esm → dist/commonjs}/HttpClient.d.ts +4 -4
- package/{src/cjs → dist/commonjs}/HttpClient.js +56 -42
- package/{src/cjs → dist/commonjs}/Request.d.ts +7 -2
- package/{src/cjs → dist/commonjs}/diagnosticsChannel.js +39 -39
- package/{src/esm → dist/commonjs}/index.d.ts +6 -6
- package/{src/cjs → dist/commonjs}/index.js +8 -8
- package/dist/commonjs/package.json +1 -0
- package/{src/cjs → dist/commonjs}/symbols.d.ts +1 -1
- package/{src/cjs → dist/commonjs}/symbols.js +1 -1
- package/{src/cjs → dist/commonjs}/utils.d.ts +1 -1
- package/{src → dist}/esm/HttpAgent.d.ts +2 -4
- package/{src/cjs → dist/esm}/HttpClient.d.ts +7 -7
- package/{src → dist}/esm/HttpClient.js +25 -11
- package/{src → dist}/esm/Request.d.ts +11 -6
- package/{src → dist}/esm/Response.d.ts +2 -2
- package/{src/cjs → dist/esm}/index.d.ts +6 -6
- package/dist/esm/package.json +1 -0
- package/{src → dist}/esm/symbols.d.ts +1 -1
- package/{src → dist}/esm/symbols.js +1 -1
- package/{src → dist}/esm/utils.d.ts +1 -1
- package/package.json +27 -14
- package/src/HttpAgent.ts +2 -2
- package/src/HttpClient.ts +28 -16
- package/src/Request.ts +7 -2
- package/src/diagnosticsChannel.ts +2 -2
- package/src/index.ts +5 -5
- package/src/symbols.ts +1 -1
- package/src/utils.ts +1 -1
- package/src/cjs/package.json +0 -3
- package/src/esm/package.json +0 -3
- /package/{src/cjs → dist/commonjs}/HttpAgent.js +0 -0
- /package/{src/cjs → dist/commonjs}/Request.js +0 -0
- /package/{src/cjs → dist/commonjs}/Response.d.ts +0 -0
- /package/{src/cjs → dist/commonjs}/Response.js +0 -0
- /package/{src/cjs → dist/commonjs}/diagnosticsChannel.d.ts +0 -0
- /package/{src/cjs → dist/commonjs}/utils.js +0 -0
- /package/{src → dist}/esm/HttpAgent.js +0 -0
- /package/{src → dist}/esm/Request.js +0 -0
- /package/{src → dist}/esm/Response.js +0 -0
- /package/{src → dist}/esm/diagnosticsChannel.d.ts +0 -0
- /package/{src → dist}/esm/diagnosticsChannel.js +0 -0
- /package/{src → dist}/esm/index.js +0 -0
- /package/{src → dist}/esm/utils.js +0 -0
@@ -1,8 +1,6 @@
|
|
1
1
|
/// <reference types="node" />
|
2
2
|
import { LookupFunction } from 'node:net';
|
3
|
-
import { Agent } from 'undici';
|
4
|
-
import type Dispatcher from 'undici/types/dispatcher';
|
5
|
-
import type buildConnector from 'undici/types/connector';
|
3
|
+
import { Agent, Dispatcher, buildConnector } from 'undici';
|
6
4
|
export type CheckAddressFunction = (ip: string, family: number | string) => boolean;
|
7
5
|
export type HttpAgentOptions = {
|
8
6
|
lookup?: LookupFunction;
|
@@ -3,9 +3,9 @@
|
|
3
3
|
/// <reference types="node" />
|
4
4
|
import { EventEmitter } from 'node:events';
|
5
5
|
import { LookupFunction } from 'node:net';
|
6
|
-
import { CheckAddressFunction } from './HttpAgent';
|
7
|
-
import { RequestURL, RequestOptions, RequestMeta } from './Request';
|
8
|
-
import { RawResponseWithMeta, HttpClientResponse } from './Response';
|
6
|
+
import { CheckAddressFunction } from './HttpAgent.js';
|
7
|
+
import { RequestURL, RequestOptions, RequestMeta } from './Request.js';
|
8
|
+
import { RawResponseWithMeta, HttpClientResponse } from './Response.js';
|
9
9
|
export type ClientOptions = {
|
10
10
|
defaultArgs?: RequestOptions;
|
11
11
|
/**
|
@@ -31,7 +31,7 @@ export type ClientOptions = {
|
|
31
31
|
*/
|
32
32
|
rejectUnauthorized?: boolean;
|
33
33
|
/**
|
34
|
-
*
|
34
|
+
* socketPath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
|
35
35
|
*/
|
36
36
|
socketPath?: string | null;
|
37
37
|
};
|
@@ -21,13 +21,14 @@ const formdata_node_1 = require("formdata-node");
|
|
21
21
|
const form_data_encoder_1 = require("form-data-encoder");
|
22
22
|
const default_user_agent_1 = __importDefault(require("default-user-agent"));
|
23
23
|
const mime_types_1 = __importDefault(require("mime-types"));
|
24
|
+
const qs_1 = __importDefault(require("qs"));
|
24
25
|
const pump_1 = __importDefault(require("pump"));
|
25
26
|
// Compatible with old style formstream
|
26
27
|
const formstream_1 = __importDefault(require("formstream"));
|
27
|
-
const
|
28
|
-
const
|
29
|
-
const
|
30
|
-
const
|
28
|
+
const HttpAgent_js_1 = require("./HttpAgent.js");
|
29
|
+
const utils_js_1 = require("./utils.js");
|
30
|
+
const symbols_js_1 = __importDefault(require("./symbols.js"));
|
31
|
+
const diagnosticsChannel_js_1 = require("./diagnosticsChannel.js");
|
31
32
|
const PROTO_RE = /^https?:\/\//i;
|
32
33
|
const FormData = undici_1.FormData ?? formdata_node_1.FormData;
|
33
34
|
// impl promise pipeline on Node.js 14
|
@@ -72,7 +73,7 @@ class HttpClientRequestTimeoutError extends Error {
|
|
72
73
|
Error.captureStackTrace(this, this.constructor);
|
73
74
|
}
|
74
75
|
}
|
75
|
-
exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.
|
76
|
+
exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.18.1-beta.0');
|
76
77
|
function getFileName(stream) {
|
77
78
|
const filePath = stream.path;
|
78
79
|
if (filePath) {
|
@@ -94,13 +95,13 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
94
95
|
super();
|
95
96
|
this.#defaultArgs = clientOptions?.defaultArgs;
|
96
97
|
if (clientOptions?.lookup || clientOptions?.checkAddress || clientOptions?.connect) {
|
97
|
-
this.#dispatcher = new
|
98
|
+
this.#dispatcher = new HttpAgent_js_1.HttpAgent({
|
98
99
|
lookup: clientOptions.lookup,
|
99
100
|
checkAddress: clientOptions.checkAddress,
|
100
101
|
connect: clientOptions.connect,
|
101
102
|
});
|
102
103
|
}
|
103
|
-
(0,
|
104
|
+
(0, diagnosticsChannel_js_1.initDiagnosticsChannel)();
|
104
105
|
}
|
105
106
|
async request(url, options) {
|
106
107
|
return await this.#requestInternal(url, options);
|
@@ -110,7 +111,7 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
110
111
|
return await this.request(url, options);
|
111
112
|
}
|
112
113
|
async #requestInternal(url, options, requestContext) {
|
113
|
-
const requestId = (0,
|
114
|
+
const requestId = (0, utils_js_1.globalId)('HttpClientRequest');
|
114
115
|
let requestUrl;
|
115
116
|
if (typeof url === 'string') {
|
116
117
|
if (!PROTO_RE.test(url)) {
|
@@ -167,14 +168,14 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
167
168
|
// the response body and trailers have been received
|
168
169
|
contentDownload: 0,
|
169
170
|
};
|
170
|
-
const
|
171
|
+
const originalOpaque = args.opaque;
|
171
172
|
// using opaque to diagnostics channel, binding request and socket
|
172
173
|
const internalOpaque = {
|
173
|
-
[
|
174
|
-
[
|
175
|
-
[
|
176
|
-
[
|
177
|
-
[
|
174
|
+
[symbols_js_1.default.kRequestId]: requestId,
|
175
|
+
[symbols_js_1.default.kRequestStartTime]: requestStartTime,
|
176
|
+
[symbols_js_1.default.kEnableRequestTiming]: !!args.timing,
|
177
|
+
[symbols_js_1.default.kRequestTiming]: timing,
|
178
|
+
[symbols_js_1.default.kRequestOriginalOpaque]: originalOpaque,
|
178
179
|
};
|
179
180
|
const reqMeta = {
|
180
181
|
requestId,
|
@@ -290,7 +291,7 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
290
291
|
if (args.stream && !args.content) {
|
291
292
|
// convert old style stream to new stream
|
292
293
|
// https://nodejs.org/dist/latest-v18.x/docs/api/stream.html#readablewrapstream
|
293
|
-
if ((0,
|
294
|
+
if ((0, utils_js_1.isReadable)(args.stream) && !(args.stream instanceof node_stream_1.Readable)) {
|
294
295
|
debug('Request#%d convert old style stream to Readable', requestId);
|
295
296
|
args.stream = new node_stream_1.Readable().wrap(args.stream);
|
296
297
|
isStreamingRequest = true;
|
@@ -314,7 +315,7 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
314
315
|
uploadFiles.push([field, file]);
|
315
316
|
}
|
316
317
|
}
|
317
|
-
else if (args.files instanceof node_stream_1.Readable || (0,
|
318
|
+
else if (args.files instanceof node_stream_1.Readable || (0, utils_js_1.isReadable)(args.files)) {
|
318
319
|
uploadFiles.push(['file', args.files]);
|
319
320
|
}
|
320
321
|
else if (typeof args.files === 'string' || Buffer.isBuffer(args.files)) {
|
@@ -343,7 +344,7 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
343
344
|
else if (Buffer.isBuffer(file)) {
|
344
345
|
formData.append(field, new node_buffer_1.Blob([file]), `bufferfile${index}`);
|
345
346
|
}
|
346
|
-
else if (file instanceof node_stream_1.Readable || (0,
|
347
|
+
else if (file instanceof node_stream_1.Readable || (0, utils_js_1.isReadable)(file)) {
|
347
348
|
const fileName = getFileName(file) || `streamfile${index}`;
|
348
349
|
formData.append(field, new BlobFromStream(file, mime_types_1.default.lookup(fileName) || ''), fileName);
|
349
350
|
isStreamingRequest = true;
|
@@ -372,27 +373,35 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
372
373
|
else if (typeof args.content === 'string' && !headers['content-type']) {
|
373
374
|
headers['content-type'] = 'text/plain;charset=UTF-8';
|
374
375
|
}
|
375
|
-
isStreamingRequest = (0,
|
376
|
+
isStreamingRequest = (0, utils_js_1.isReadable)(args.content);
|
376
377
|
}
|
377
378
|
}
|
378
379
|
else if (args.data) {
|
379
380
|
const isStringOrBufferOrReadable = typeof args.data === 'string'
|
380
381
|
|| Buffer.isBuffer(args.data)
|
381
|
-
|| (0,
|
382
|
+
|| (0, utils_js_1.isReadable)(args.data);
|
382
383
|
if (isGETOrHEAD) {
|
383
384
|
if (!isStringOrBufferOrReadable) {
|
384
|
-
|
385
|
-
const
|
386
|
-
|
387
|
-
|
388
|
-
requestUrl.
|
385
|
+
if (args.nestedQuerystring) {
|
386
|
+
const querystring = qs_1.default.stringify(args.data);
|
387
|
+
// reset the requestUrl
|
388
|
+
const href = requestUrl.href;
|
389
|
+
requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + querystring);
|
390
|
+
}
|
391
|
+
else {
|
392
|
+
for (const field in args.data) {
|
393
|
+
const fieldValue = args.data[field];
|
394
|
+
if (fieldValue === undefined)
|
395
|
+
continue;
|
396
|
+
requestUrl.searchParams.append(field, fieldValue);
|
397
|
+
}
|
389
398
|
}
|
390
399
|
}
|
391
400
|
}
|
392
401
|
else {
|
393
402
|
if (isStringOrBufferOrReadable) {
|
394
403
|
requestOptions.body = args.data;
|
395
|
-
isStreamingRequest = (0,
|
404
|
+
isStreamingRequest = (0, utils_js_1.isReadable)(args.data);
|
396
405
|
}
|
397
406
|
else {
|
398
407
|
if (args.contentType === 'json'
|
@@ -405,7 +414,12 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
405
414
|
}
|
406
415
|
else {
|
407
416
|
headers['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
|
408
|
-
|
417
|
+
if (args.nestedQuerystring) {
|
418
|
+
requestOptions.body = qs_1.default.stringify(args.data);
|
419
|
+
}
|
420
|
+
else {
|
421
|
+
requestOptions.body = new URLSearchParams(args.data).toString();
|
422
|
+
}
|
409
423
|
}
|
410
424
|
}
|
411
425
|
}
|
@@ -432,7 +446,7 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
432
446
|
: authenticateHeaders;
|
433
447
|
if (authenticate && authenticate.startsWith('Digest ')) {
|
434
448
|
debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', requestId, requestUrl.href, authenticate);
|
435
|
-
requestOptions.headers.authorization = (0,
|
449
|
+
requestOptions.headers.authorization = (0, utils_js_1.digestAuthHeader)(requestOptions.method, `${requestUrl.pathname}${requestUrl.search}`, authenticate, args.digestAuth);
|
436
450
|
debug('Request#%d %s: auth with digest header: %s', requestId, url, requestOptions.headers.authorization);
|
437
451
|
if (Array.isArray(response.headers['set-cookie'])) {
|
438
452
|
// FIXME: merge exists cookie header
|
@@ -507,15 +521,15 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
507
521
|
data = null;
|
508
522
|
}
|
509
523
|
else {
|
510
|
-
data = (0,
|
524
|
+
data = (0, utils_js_1.parseJSON)(data.toString(), args.fixJSONCtlChars);
|
511
525
|
}
|
512
526
|
}
|
513
527
|
}
|
514
|
-
res.rt = (0,
|
528
|
+
res.rt = (0, utils_js_1.performanceTime)(requestStartTime);
|
515
529
|
// get real socket info from internalOpaque
|
516
530
|
this.#updateSocketInfo(socketInfo, internalOpaque);
|
517
531
|
const clientResponse = {
|
518
|
-
opaque:
|
532
|
+
opaque: originalOpaque,
|
519
533
|
data,
|
520
534
|
status: res.status,
|
521
535
|
statusCode: res.status,
|
@@ -530,7 +544,7 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
530
544
|
const isRetry = args.isRetry ?? defaultIsRetry;
|
531
545
|
if (isRetry(clientResponse)) {
|
532
546
|
if (args.retryDelay) {
|
533
|
-
await (0,
|
547
|
+
await (0, utils_js_1.sleep)(args.retryDelay);
|
534
548
|
}
|
535
549
|
requestContext.retries++;
|
536
550
|
return await this.#requestInternal(url, options, requestContext);
|
@@ -570,7 +584,7 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
570
584
|
return await this.#requestInternal(url, options, requestContext);
|
571
585
|
}
|
572
586
|
}
|
573
|
-
err.opaque =
|
587
|
+
err.opaque = originalOpaque;
|
574
588
|
err.status = res.status;
|
575
589
|
err.headers = res.headers;
|
576
590
|
err.res = res;
|
@@ -583,7 +597,7 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
583
597
|
if (res.requestUrls.length === 0) {
|
584
598
|
res.requestUrls.push(requestUrl.href);
|
585
599
|
}
|
586
|
-
res.rt = (0,
|
600
|
+
res.rt = (0, utils_js_1.performanceTime)(requestStartTime);
|
587
601
|
this.#updateSocketInfo(socketInfo, internalOpaque);
|
588
602
|
channels.response.publish({
|
589
603
|
request: reqMeta,
|
@@ -606,21 +620,21 @@ class HttpClient extends node_events_1.EventEmitter {
|
|
606
620
|
}
|
607
621
|
}
|
608
622
|
#updateSocketInfo(socketInfo, internalOpaque) {
|
609
|
-
const socket = internalOpaque[
|
623
|
+
const socket = internalOpaque[symbols_js_1.default.kRequestSocket];
|
610
624
|
if (socket) {
|
611
|
-
socketInfo.id = socket[
|
612
|
-
socketInfo.handledRequests = socket[
|
613
|
-
socketInfo.handledResponses = socket[
|
614
|
-
socketInfo.localAddress = socket[
|
615
|
-
socketInfo.localPort = socket[
|
625
|
+
socketInfo.id = socket[symbols_js_1.default.kSocketId];
|
626
|
+
socketInfo.handledRequests = socket[symbols_js_1.default.kHandledRequests];
|
627
|
+
socketInfo.handledResponses = socket[symbols_js_1.default.kHandledResponses];
|
628
|
+
socketInfo.localAddress = socket[symbols_js_1.default.kSocketLocalAddress];
|
629
|
+
socketInfo.localPort = socket[symbols_js_1.default.kSocketLocalPort];
|
616
630
|
socketInfo.remoteAddress = socket.remoteAddress;
|
617
631
|
socketInfo.remotePort = socket.remotePort;
|
618
632
|
socketInfo.remoteFamily = socket.remoteFamily;
|
619
633
|
socketInfo.bytesRead = socket.bytesRead;
|
620
634
|
socketInfo.bytesWritten = socket.bytesWritten;
|
621
|
-
socketInfo.connectedTime = socket[
|
622
|
-
socketInfo.lastRequestEndTime = socket[
|
623
|
-
socket[
|
635
|
+
socketInfo.connectedTime = socket[symbols_js_1.default.kSocketConnectedTime];
|
636
|
+
socketInfo.lastRequestEndTime = socket[symbols_js_1.default.kSocketRequestEndTime];
|
637
|
+
socket[symbols_js_1.default.kSocketRequestEndTime] = new Date();
|
624
638
|
}
|
625
639
|
}
|
626
640
|
}
|
@@ -4,8 +4,8 @@
|
|
4
4
|
/// <reference types="node" />
|
5
5
|
import { Readable, Writable } from 'node:stream';
|
6
6
|
import type { IncomingHttpHeaders } from 'node:http';
|
7
|
-
import type Dispatcher from 'undici
|
8
|
-
import type { HttpClientResponse } from './Response';
|
7
|
+
import type { Dispatcher } from 'undici';
|
8
|
+
import type { HttpClientResponse } from './Response.js';
|
9
9
|
export type HttpMethod = Dispatcher.HttpMethod;
|
10
10
|
export type RequestURL = string | URL;
|
11
11
|
export type FixJSONCtlCharsHandler = (data: string) => string;
|
@@ -44,6 +44,11 @@ export type RequestOptions = {
|
|
44
44
|
* Default is 'buffer'.
|
45
45
|
*/
|
46
46
|
dataType?: 'text' | 'html' | 'json' | 'buffer' | 'stream';
|
47
|
+
/**
|
48
|
+
* urllib default use URLSearchParams to stringify form data which don't support nested object,
|
49
|
+
* will use qs instead of URLSearchParams to support nested object by set this option to true.
|
50
|
+
*/
|
51
|
+
nestedQuerystring?: boolean;
|
47
52
|
/**
|
48
53
|
* @deprecated
|
49
54
|
* Only for d.ts keep compatible with urllib@2, don't use it anymore.
|
@@ -7,8 +7,8 @@ exports.initDiagnosticsChannel = void 0;
|
|
7
7
|
const node_diagnostics_channel_1 = __importDefault(require("node:diagnostics_channel"));
|
8
8
|
const node_perf_hooks_1 = require("node:perf_hooks");
|
9
9
|
const node_util_1 = require("node:util");
|
10
|
-
const
|
11
|
-
const
|
10
|
+
const symbols_js_1 = __importDefault(require("./symbols.js"));
|
11
|
+
const utils_js_1 = require("./utils.js");
|
12
12
|
const debug = (0, node_util_1.debuglog)('urllib:DiagnosticsChannel');
|
13
13
|
let initedDiagnosticsChannel = false;
|
14
14
|
// https://undici.nodejs.org/#/docs/api/DiagnosticsChannel
|
@@ -34,8 +34,8 @@ function formatSocket(socket) {
|
|
34
34
|
if (!socket)
|
35
35
|
return socket;
|
36
36
|
return {
|
37
|
-
localAddress: socket[
|
38
|
-
localPort: socket[
|
37
|
+
localAddress: socket[symbols_js_1.default.kSocketLocalAddress],
|
38
|
+
localPort: socket[symbols_js_1.default.kSocketLocalPort],
|
39
39
|
remoteAddress: socket.remoteAddress,
|
40
40
|
remotePort: socket.remotePort,
|
41
41
|
};
|
@@ -61,27 +61,27 @@ function initDiagnosticsChannel() {
|
|
61
61
|
}
|
62
62
|
const opaque = request[kHandler]?.opts?.opaque;
|
63
63
|
// ignore non HttpClient Request
|
64
|
-
if (!opaque || !opaque[
|
64
|
+
if (!opaque || !opaque[symbols_js_1.default.kRequestId])
|
65
65
|
return;
|
66
|
-
debug('[%s] Request#%d %s %s, path: %s, headers: %o', name, opaque[
|
67
|
-
if (!opaque[
|
66
|
+
debug('[%s] Request#%d %s %s, path: %s, headers: %o', name, opaque[symbols_js_1.default.kRequestId], request.method, request.origin, request.path, request.headers);
|
67
|
+
if (!opaque[symbols_js_1.default.kEnableRequestTiming])
|
68
68
|
return;
|
69
|
-
opaque[
|
69
|
+
opaque[symbols_js_1.default.kRequestTiming].queuing = (0, utils_js_1.performanceTime)(opaque[symbols_js_1.default.kRequestStartTime]);
|
70
70
|
});
|
71
71
|
// diagnosticsChannel.channel('undici:client:beforeConnect')
|
72
72
|
// diagnosticsChannel.channel('undici:client:connectError')
|
73
73
|
// This message is published after a connection is established.
|
74
74
|
subscribe('undici:client:connected', (message, name) => {
|
75
75
|
const { socket } = message;
|
76
|
-
socket[
|
77
|
-
socket[
|
78
|
-
socket[
|
79
|
-
socket[
|
80
|
-
socket[
|
76
|
+
socket[symbols_js_1.default.kSocketId] = (0, utils_js_1.globalId)('UndiciSocket');
|
77
|
+
socket[symbols_js_1.default.kSocketStartTime] = node_perf_hooks_1.performance.now();
|
78
|
+
socket[symbols_js_1.default.kSocketConnectedTime] = new Date();
|
79
|
+
socket[symbols_js_1.default.kHandledRequests] = 0;
|
80
|
+
socket[symbols_js_1.default.kHandledResponses] = 0;
|
81
81
|
// copy local address to symbol, avoid them be reset after request error throw
|
82
|
-
socket[
|
83
|
-
socket[
|
84
|
-
debug('[%s] Socket#%d connected (sock: %o)', name, socket[
|
82
|
+
socket[symbols_js_1.default.kSocketLocalAddress] = socket.localAddress;
|
83
|
+
socket[symbols_js_1.default.kSocketLocalPort] = socket.localPort;
|
84
|
+
debug('[%s] Socket#%d connected (sock: %o)', name, socket[symbols_js_1.default.kSocketId], formatSocket(socket));
|
85
85
|
});
|
86
86
|
// This message is published right before the first byte of the request is written to the socket.
|
87
87
|
subscribe('undici:client:sendHeaders', (message, name) => {
|
@@ -89,20 +89,20 @@ function initDiagnosticsChannel() {
|
|
89
89
|
if (!kHandler)
|
90
90
|
return;
|
91
91
|
const opaque = request[kHandler]?.opts?.opaque;
|
92
|
-
if (!opaque || !opaque[
|
92
|
+
if (!opaque || !opaque[symbols_js_1.default.kRequestId])
|
93
93
|
return;
|
94
|
-
socket[
|
94
|
+
socket[symbols_js_1.default.kHandledRequests]++;
|
95
95
|
// attach socket to opaque
|
96
|
-
opaque[
|
97
|
-
debug('[%s] Request#%d send headers on Socket#%d (handled %d requests, sock: %o)', name, opaque[
|
98
|
-
if (!opaque[
|
96
|
+
opaque[symbols_js_1.default.kRequestSocket] = socket;
|
97
|
+
debug('[%s] Request#%d send headers on Socket#%d (handled %d requests, sock: %o)', name, opaque[symbols_js_1.default.kRequestId], socket[symbols_js_1.default.kSocketId], socket[symbols_js_1.default.kHandledRequests], formatSocket(socket));
|
98
|
+
if (!opaque[symbols_js_1.default.kEnableRequestTiming])
|
99
99
|
return;
|
100
|
-
opaque[
|
100
|
+
opaque[symbols_js_1.default.kRequestTiming].requestHeadersSent = (0, utils_js_1.performanceTime)(opaque[symbols_js_1.default.kRequestStartTime]);
|
101
101
|
// first socket need to caculate the connected time
|
102
|
-
if (socket[
|
102
|
+
if (socket[symbols_js_1.default.kHandledRequests] === 1) {
|
103
103
|
// kSocketStartTime - kRequestStartTime = connected time
|
104
|
-
opaque[
|
105
|
-
(0,
|
104
|
+
opaque[symbols_js_1.default.kRequestTiming].connected =
|
105
|
+
(0, utils_js_1.performanceTime)(opaque[symbols_js_1.default.kRequestStartTime], socket[symbols_js_1.default.kSocketStartTime]);
|
106
106
|
}
|
107
107
|
});
|
108
108
|
subscribe('undici:request:bodySent', (message, name) => {
|
@@ -110,12 +110,12 @@ function initDiagnosticsChannel() {
|
|
110
110
|
if (!kHandler)
|
111
111
|
return;
|
112
112
|
const opaque = request[kHandler]?.opts?.opaque;
|
113
|
-
if (!opaque || !opaque[
|
113
|
+
if (!opaque || !opaque[symbols_js_1.default.kRequestId])
|
114
114
|
return;
|
115
|
-
debug('[%s] Request#%d send body', name, opaque[
|
116
|
-
if (!opaque[
|
115
|
+
debug('[%s] Request#%d send body', name, opaque[symbols_js_1.default.kRequestId]);
|
116
|
+
if (!opaque[symbols_js_1.default.kEnableRequestTiming])
|
117
117
|
return;
|
118
|
-
opaque[
|
118
|
+
opaque[symbols_js_1.default.kRequestTiming].requestSent = (0, utils_js_1.performanceTime)(opaque[symbols_js_1.default.kRequestStartTime]);
|
119
119
|
});
|
120
120
|
// This message is published after the response headers have been received, i.e. the response has been completed.
|
121
121
|
subscribe('undici:request:headers', (message, name) => {
|
@@ -123,15 +123,15 @@ function initDiagnosticsChannel() {
|
|
123
123
|
if (!kHandler)
|
124
124
|
return;
|
125
125
|
const opaque = request[kHandler]?.opts?.opaque;
|
126
|
-
if (!opaque || !opaque[
|
126
|
+
if (!opaque || !opaque[symbols_js_1.default.kRequestId])
|
127
127
|
return;
|
128
128
|
// get socket from opaque
|
129
|
-
const socket = opaque[
|
130
|
-
socket[
|
131
|
-
debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses, sock: %o)', name, opaque[
|
132
|
-
if (!opaque[
|
129
|
+
const socket = opaque[symbols_js_1.default.kRequestSocket];
|
130
|
+
socket[symbols_js_1.default.kHandledResponses]++;
|
131
|
+
debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses, sock: %o)', name, opaque[symbols_js_1.default.kRequestId], response.statusCode, socket[symbols_js_1.default.kSocketId], socket[symbols_js_1.default.kHandledResponses], formatSocket(socket));
|
132
|
+
if (!opaque[symbols_js_1.default.kEnableRequestTiming])
|
133
133
|
return;
|
134
|
-
opaque[
|
134
|
+
opaque[symbols_js_1.default.kRequestTiming].waiting = (0, utils_js_1.performanceTime)(opaque[symbols_js_1.default.kRequestStartTime]);
|
135
135
|
});
|
136
136
|
// This message is published after the response body and trailers have been received, i.e. the response has been completed.
|
137
137
|
subscribe('undici:request:trailers', (message, name) => {
|
@@ -139,12 +139,12 @@ function initDiagnosticsChannel() {
|
|
139
139
|
if (!kHandler)
|
140
140
|
return;
|
141
141
|
const opaque = request[kHandler]?.opts?.opaque;
|
142
|
-
if (!opaque || !opaque[
|
142
|
+
if (!opaque || !opaque[symbols_js_1.default.kRequestId])
|
143
143
|
return;
|
144
|
-
debug('[%s] Request#%d get response body and trailers', name, opaque[
|
145
|
-
if (!opaque[
|
144
|
+
debug('[%s] Request#%d get response body and trailers', name, opaque[symbols_js_1.default.kRequestId]);
|
145
|
+
if (!opaque[symbols_js_1.default.kEnableRequestTiming])
|
146
146
|
return;
|
147
|
-
opaque[
|
147
|
+
opaque[symbols_js_1.default.kRequestTiming].contentDownload = (0, utils_js_1.performanceTime)(opaque[symbols_js_1.default.kRequestStartTime]);
|
148
148
|
});
|
149
149
|
// This message is published if the request is going to error, but it has not errored yet.
|
150
150
|
// subscribe('undici:request:error', (message, name) => {
|
@@ -1,10 +1,10 @@
|
|
1
|
-
import { RequestOptions, RequestURL } from './Request';
|
2
|
-
export declare function request<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response").HttpClientResponse<T>>;
|
3
|
-
export declare function curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response").HttpClientResponse<T>>;
|
1
|
+
import { RequestOptions, RequestURL } from './Request.js';
|
2
|
+
export declare function request<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
|
3
|
+
export declare function curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
|
4
4
|
export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
|
5
|
-
export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, RequestDiagnosticsMessage, ResponseDiagnosticsMessage, } from './HttpClient';
|
6
|
-
export { RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod, FixJSONCtlCharsHandler, FixJSONCtlChars, } from './Request';
|
7
|
-
export { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse } from './Response';
|
5
|
+
export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, RequestDiagnosticsMessage, ResponseDiagnosticsMessage, } from './HttpClient.js';
|
6
|
+
export { RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod, FixJSONCtlCharsHandler, FixJSONCtlChars, } from './Request.js';
|
7
|
+
export { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse } from './Response.js';
|
8
8
|
declare const _default: {
|
9
9
|
request: typeof request;
|
10
10
|
curl: typeof curl;
|
@@ -5,14 +5,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
6
6
|
exports.USER_AGENT = exports.HttpClient2 = exports.HttpClient = exports.getGlobalDispatcher = exports.setGlobalDispatcher = exports.Dispatcher = exports.Agent = exports.ProxyAgent = exports.MockAgent = exports.curl = exports.request = void 0;
|
7
7
|
const ylru_1 = __importDefault(require("ylru"));
|
8
|
-
const
|
8
|
+
const HttpClient_js_1 = require("./HttpClient.js");
|
9
9
|
let httpclient;
|
10
10
|
const domainSocketHttpclients = new ylru_1.default(50);
|
11
11
|
async function request(url, options) {
|
12
12
|
if (options?.socketPath) {
|
13
13
|
let domainSocketHttpclient = domainSocketHttpclients.get(options.socketPath);
|
14
14
|
if (!domainSocketHttpclient) {
|
15
|
-
domainSocketHttpclient = new
|
15
|
+
domainSocketHttpclient = new HttpClient_js_1.HttpClient({
|
16
16
|
connect: { socketPath: options.socketPath },
|
17
17
|
});
|
18
18
|
domainSocketHttpclients.set(options.socketPath, domainSocketHttpclient);
|
@@ -20,7 +20,7 @@ async function request(url, options) {
|
|
20
20
|
return await domainSocketHttpclient.request(url, options);
|
21
21
|
}
|
22
22
|
if (!httpclient) {
|
23
|
-
httpclient = new
|
23
|
+
httpclient = new HttpClient_js_1.HttpClient({});
|
24
24
|
}
|
25
25
|
return await httpclient.request(url, options);
|
26
26
|
}
|
@@ -42,12 +42,12 @@ Object.defineProperty(exports, "Dispatcher", { enumerable: true, get: function (
|
|
42
42
|
Object.defineProperty(exports, "setGlobalDispatcher", { enumerable: true, get: function () { return undici_1.setGlobalDispatcher; } });
|
43
43
|
Object.defineProperty(exports, "getGlobalDispatcher", { enumerable: true, get: function () { return undici_1.getGlobalDispatcher; } });
|
44
44
|
// HttpClient2 is keep compatible with urlib@2 HttpClient2
|
45
|
-
var
|
46
|
-
Object.defineProperty(exports, "HttpClient", { enumerable: true, get: function () { return
|
47
|
-
Object.defineProperty(exports, "HttpClient2", { enumerable: true, get: function () { return
|
48
|
-
Object.defineProperty(exports, "USER_AGENT", { enumerable: true, get: function () { return
|
45
|
+
var HttpClient_js_2 = require("./HttpClient.js");
|
46
|
+
Object.defineProperty(exports, "HttpClient", { enumerable: true, get: function () { return HttpClient_js_2.HttpClient; } });
|
47
|
+
Object.defineProperty(exports, "HttpClient2", { enumerable: true, get: function () { return HttpClient_js_2.HttpClient; } });
|
48
|
+
Object.defineProperty(exports, "USER_AGENT", { enumerable: true, get: function () { return HttpClient_js_2.HEADER_USER_AGENT; } });
|
49
49
|
exports.default = {
|
50
50
|
request,
|
51
51
|
curl,
|
52
|
-
USER_AGENT:
|
52
|
+
USER_AGENT: HttpClient_js_1.HEADER_USER_AGENT,
|
53
53
|
};
|
@@ -0,0 +1 @@
|
|
1
|
+
{"type":"commonjs"}
|
@@ -14,5 +14,5 @@ exports.default = {
|
|
14
14
|
kRequestStartTime: Symbol('request start time'),
|
15
15
|
kEnableRequestTiming: Symbol('enable request timing or not'),
|
16
16
|
kRequestTiming: Symbol('request timing'),
|
17
|
-
|
17
|
+
kRequestOriginalOpaque: Symbol('request original opaque'),
|
18
18
|
};
|
@@ -1,4 +1,4 @@
|
|
1
|
-
import { FixJSONCtlChars } from './Request';
|
1
|
+
import type { FixJSONCtlChars } from './Request.js';
|
2
2
|
export declare function parseJSON(data: string, fixJSONCtlChars?: FixJSONCtlChars): string;
|
3
3
|
export declare function sleep(ms: number): Promise<void>;
|
4
4
|
export declare function digestAuthHeader(method: string, uri: string, wwwAuthenticate: string, userpass: string): string;
|
@@ -1,8 +1,6 @@
|
|
1
|
-
/// <reference types="node" />
|
1
|
+
/// <reference types="node" resolution-mode="require"/>
|
2
2
|
import { LookupFunction } from 'node:net';
|
3
|
-
import { Agent } from 'undici';
|
4
|
-
import type Dispatcher from 'undici/types/dispatcher';
|
5
|
-
import type buildConnector from 'undici/types/connector';
|
3
|
+
import { Agent, Dispatcher, buildConnector } from 'undici';
|
6
4
|
export type CheckAddressFunction = (ip: string, family: number | string) => boolean;
|
7
5
|
export type HttpAgentOptions = {
|
8
6
|
lookup?: LookupFunction;
|
@@ -1,11 +1,11 @@
|
|
1
|
-
/// <reference types="node" />
|
2
|
-
/// <reference types="node" />
|
3
|
-
/// <reference types="node" />
|
1
|
+
/// <reference types="node" resolution-mode="require"/>
|
2
|
+
/// <reference types="node" resolution-mode="require"/>
|
3
|
+
/// <reference types="node" resolution-mode="require"/>
|
4
4
|
import { EventEmitter } from 'node:events';
|
5
5
|
import { LookupFunction } from 'node:net';
|
6
|
-
import { CheckAddressFunction } from './HttpAgent';
|
7
|
-
import { RequestURL, RequestOptions, RequestMeta } from './Request';
|
8
|
-
import { RawResponseWithMeta, HttpClientResponse } from './Response';
|
6
|
+
import { CheckAddressFunction } from './HttpAgent.js';
|
7
|
+
import { RequestURL, RequestOptions, RequestMeta } from './Request.js';
|
8
|
+
import { RawResponseWithMeta, HttpClientResponse } from './Response.js';
|
9
9
|
export type ClientOptions = {
|
10
10
|
defaultArgs?: RequestOptions;
|
11
11
|
/**
|
@@ -31,7 +31,7 @@ export type ClientOptions = {
|
|
31
31
|
*/
|
32
32
|
rejectUnauthorized?: boolean;
|
33
33
|
/**
|
34
|
-
*
|
34
|
+
* socketPath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
|
35
35
|
*/
|
36
36
|
socketPath?: string | null;
|
37
37
|
};
|
@@ -15,6 +15,7 @@ import { FormData as FormDataNode } from 'formdata-node';
|
|
15
15
|
import { FormDataEncoder } from 'form-data-encoder';
|
16
16
|
import createUserAgent from 'default-user-agent';
|
17
17
|
import mime from 'mime-types';
|
18
|
+
import qs from 'qs';
|
18
19
|
import pump from 'pump';
|
19
20
|
// Compatible with old style formstream
|
20
21
|
import FormStream from 'formstream';
|
@@ -66,7 +67,7 @@ class HttpClientRequestTimeoutError extends Error {
|
|
66
67
|
Error.captureStackTrace(this, this.constructor);
|
67
68
|
}
|
68
69
|
}
|
69
|
-
export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.
|
70
|
+
export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.18.1-beta.0');
|
70
71
|
function getFileName(stream) {
|
71
72
|
const filePath = stream.path;
|
72
73
|
if (filePath) {
|
@@ -161,14 +162,14 @@ export class HttpClient extends EventEmitter {
|
|
161
162
|
// the response body and trailers have been received
|
162
163
|
contentDownload: 0,
|
163
164
|
};
|
164
|
-
const
|
165
|
+
const originalOpaque = args.opaque;
|
165
166
|
// using opaque to diagnostics channel, binding request and socket
|
166
167
|
const internalOpaque = {
|
167
168
|
[symbols.kRequestId]: requestId,
|
168
169
|
[symbols.kRequestStartTime]: requestStartTime,
|
169
170
|
[symbols.kEnableRequestTiming]: !!args.timing,
|
170
171
|
[symbols.kRequestTiming]: timing,
|
171
|
-
[symbols.
|
172
|
+
[symbols.kRequestOriginalOpaque]: originalOpaque,
|
172
173
|
};
|
173
174
|
const reqMeta = {
|
174
175
|
requestId,
|
@@ -375,11 +376,19 @@ export class HttpClient extends EventEmitter {
|
|
375
376
|
|| isReadable(args.data);
|
376
377
|
if (isGETOrHEAD) {
|
377
378
|
if (!isStringOrBufferOrReadable) {
|
378
|
-
|
379
|
-
const
|
380
|
-
|
381
|
-
|
382
|
-
requestUrl.
|
379
|
+
if (args.nestedQuerystring) {
|
380
|
+
const querystring = qs.stringify(args.data);
|
381
|
+
// reset the requestUrl
|
382
|
+
const href = requestUrl.href;
|
383
|
+
requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + querystring);
|
384
|
+
}
|
385
|
+
else {
|
386
|
+
for (const field in args.data) {
|
387
|
+
const fieldValue = args.data[field];
|
388
|
+
if (fieldValue === undefined)
|
389
|
+
continue;
|
390
|
+
requestUrl.searchParams.append(field, fieldValue);
|
391
|
+
}
|
383
392
|
}
|
384
393
|
}
|
385
394
|
}
|
@@ -399,7 +408,12 @@ export class HttpClient extends EventEmitter {
|
|
399
408
|
}
|
400
409
|
else {
|
401
410
|
headers['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
|
402
|
-
|
411
|
+
if (args.nestedQuerystring) {
|
412
|
+
requestOptions.body = qs.stringify(args.data);
|
413
|
+
}
|
414
|
+
else {
|
415
|
+
requestOptions.body = new URLSearchParams(args.data).toString();
|
416
|
+
}
|
403
417
|
}
|
404
418
|
}
|
405
419
|
}
|
@@ -509,7 +523,7 @@ export class HttpClient extends EventEmitter {
|
|
509
523
|
// get real socket info from internalOpaque
|
510
524
|
this.#updateSocketInfo(socketInfo, internalOpaque);
|
511
525
|
const clientResponse = {
|
512
|
-
opaque:
|
526
|
+
opaque: originalOpaque,
|
513
527
|
data,
|
514
528
|
status: res.status,
|
515
529
|
statusCode: res.status,
|
@@ -564,7 +578,7 @@ export class HttpClient extends EventEmitter {
|
|
564
578
|
return await this.#requestInternal(url, options, requestContext);
|
565
579
|
}
|
566
580
|
}
|
567
|
-
err.opaque =
|
581
|
+
err.opaque = originalOpaque;
|
568
582
|
err.status = res.status;
|
569
583
|
err.headers = res.headers;
|
570
584
|
err.res = res;
|
@@ -1,11 +1,11 @@
|
|
1
|
-
/// <reference types="node" />
|
2
|
-
/// <reference types="node" />
|
3
|
-
/// <reference types="node" />
|
4
|
-
/// <reference types="node" />
|
1
|
+
/// <reference types="node" resolution-mode="require"/>
|
2
|
+
/// <reference types="node" resolution-mode="require"/>
|
3
|
+
/// <reference types="node" resolution-mode="require"/>
|
4
|
+
/// <reference types="node" resolution-mode="require"/>
|
5
5
|
import { Readable, Writable } from 'node:stream';
|
6
6
|
import type { IncomingHttpHeaders } from 'node:http';
|
7
|
-
import type Dispatcher from 'undici
|
8
|
-
import type { HttpClientResponse } from './Response';
|
7
|
+
import type { Dispatcher } from 'undici';
|
8
|
+
import type { HttpClientResponse } from './Response.js';
|
9
9
|
export type HttpMethod = Dispatcher.HttpMethod;
|
10
10
|
export type RequestURL = string | URL;
|
11
11
|
export type FixJSONCtlCharsHandler = (data: string) => string;
|
@@ -44,6 +44,11 @@ export type RequestOptions = {
|
|
44
44
|
* Default is 'buffer'.
|
45
45
|
*/
|
46
46
|
dataType?: 'text' | 'html' | 'json' | 'buffer' | 'stream';
|
47
|
+
/**
|
48
|
+
* urllib default use URLSearchParams to stringify form data which don't support nested object,
|
49
|
+
* will use qs instead of URLSearchParams to support nested object by set this option to true.
|
50
|
+
*/
|
51
|
+
nestedQuerystring?: boolean;
|
47
52
|
/**
|
48
53
|
* @deprecated
|
49
54
|
* Only for d.ts keep compatible with urllib@2, don't use it anymore.
|
@@ -1,5 +1,5 @@
|
|
1
|
-
/// <reference types="node" />
|
2
|
-
/// <reference types="node" />
|
1
|
+
/// <reference types="node" resolution-mode="require"/>
|
2
|
+
/// <reference types="node" resolution-mode="require"/>
|
3
3
|
import { Readable } from 'node:stream';
|
4
4
|
import { IncomingHttpHeaders } from 'node:http';
|
5
5
|
export type SocketInfo = {
|
@@ -1,10 +1,10 @@
|
|
1
|
-
import { RequestOptions, RequestURL } from './Request';
|
2
|
-
export declare function request<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response").HttpClientResponse<T>>;
|
3
|
-
export declare function curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response").HttpClientResponse<T>>;
|
1
|
+
import { RequestOptions, RequestURL } from './Request.js';
|
2
|
+
export declare function request<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
|
3
|
+
export declare function curl<T = any>(url: RequestURL, options?: RequestOptions): Promise<import("./Response.js").HttpClientResponse<T>>;
|
4
4
|
export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
|
5
|
-
export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, RequestDiagnosticsMessage, ResponseDiagnosticsMessage, } from './HttpClient';
|
6
|
-
export { RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod, FixJSONCtlCharsHandler, FixJSONCtlChars, } from './Request';
|
7
|
-
export { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse } from './Response';
|
5
|
+
export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, RequestDiagnosticsMessage, ResponseDiagnosticsMessage, } from './HttpClient.js';
|
6
|
+
export { RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod, FixJSONCtlCharsHandler, FixJSONCtlChars, } from './Request.js';
|
7
|
+
export { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse } from './Response.js';
|
8
8
|
declare const _default: {
|
9
9
|
request: typeof request;
|
10
10
|
curl: typeof curl;
|
@@ -0,0 +1 @@
|
|
1
|
+
{"type":"module"}
|
@@ -12,5 +12,5 @@ export default {
|
|
12
12
|
kRequestStartTime: Symbol('request start time'),
|
13
13
|
kEnableRequestTiming: Symbol('enable request timing or not'),
|
14
14
|
kRequestTiming: Symbol('request timing'),
|
15
|
-
|
15
|
+
kRequestOriginalOpaque: Symbol('request original opaque'),
|
16
16
|
};
|
@@ -1,4 +1,4 @@
|
|
1
|
-
import { FixJSONCtlChars } from './Request';
|
1
|
+
import type { FixJSONCtlChars } from './Request.js';
|
2
2
|
export declare function parseJSON(data: string, fixJSONCtlChars?: FixJSONCtlChars): string;
|
3
3
|
export declare function sleep(ms: number): Promise<void>;
|
4
4
|
export declare function digestAuthHeader(method: string, uri: string, wwwAuthenticate: string, userpass: string): string;
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "urllib",
|
3
|
-
"version": "3.
|
3
|
+
"version": "3.18.1-beta.0",
|
4
4
|
"publishConfig": {
|
5
5
|
"tag": "latest"
|
6
6
|
},
|
@@ -19,21 +19,28 @@
|
|
19
19
|
"author": "fengmk2 <fengmk2@gmail.com> (https://github.com/fengmk2)",
|
20
20
|
"homepage": "https://github.com/node-modules/urllib",
|
21
21
|
"type": "module",
|
22
|
+
"tshy": {
|
23
|
+
"exports": {
|
24
|
+
".": "./src/index.ts",
|
25
|
+
"./package.json": "./package.json"
|
26
|
+
}
|
27
|
+
},
|
22
28
|
"exports": {
|
23
29
|
".": {
|
24
30
|
"import": {
|
25
|
-
"types": "./
|
26
|
-
"default": "./
|
31
|
+
"types": "./dist/esm/index.d.ts",
|
32
|
+
"default": "./dist/esm/index.js"
|
27
33
|
},
|
28
34
|
"require": {
|
29
|
-
"types": "./
|
30
|
-
"default": "./
|
35
|
+
"types": "./dist/commonjs/index.d.ts",
|
36
|
+
"default": "./dist/commonjs/index.js"
|
31
37
|
}
|
32
|
-
}
|
38
|
+
},
|
39
|
+
"./package.json": "./package.json"
|
33
40
|
},
|
34
|
-
"
|
35
|
-
"main": "./src/cjs/index.js",
|
41
|
+
"typings": "./dist/commonjs/index.d.ts",
|
36
42
|
"files": [
|
43
|
+
"dist",
|
37
44
|
"src"
|
38
45
|
],
|
39
46
|
"repository": {
|
@@ -43,21 +50,22 @@
|
|
43
50
|
"scripts": {
|
44
51
|
"lint": "eslint src test --ext .ts --cache",
|
45
52
|
"prebuild": "npm run clean",
|
46
|
-
"build": "tsc --version &&
|
53
|
+
"build": "tsc --version && tshy && npm run build:version",
|
47
54
|
"postbuild": "rm -rf src/*.tsbuildinfo",
|
48
|
-
"build:cjs": "tsc -p ./tsconfig.build.cjs.json",
|
49
|
-
"build:esm": "tsc -p ./tsconfig.build.esm.json && node ./scripts/esm_import_fix.js",
|
50
55
|
"build:version": "node ./scripts/replace_urllib_version.js",
|
51
56
|
"build:cjs:test": "cd test/cjs && rm -rf node_modules && npm link ../.. && node index.js",
|
52
57
|
"build:esm:test": "cd test/esm && rm -rf node_modules && npm link ../.. && node index.js",
|
53
|
-
"build:test": "
|
54
|
-
"test
|
58
|
+
"build:mts:test": "cd test/mts && rm -rf node_modules && npm link ../.. && tsc",
|
59
|
+
"build:test": "npm run build && npm run build:cjs:test && npm run build:esm:test && npm run build:mts:test && npm run test-tsc",
|
60
|
+
"test-tsc": "npm run test-tsc:cjs && npm run test-tsc:esm",
|
61
|
+
"test-tsc:cjs": "cd test/fixtures/ts && rm -rf node_modules && npm link ../../.. && npm run build",
|
62
|
+
"test-tsc:esm": "cd test/fixtures/ts-esm && rm -rf node_modules && npm link ../../.. && npm run build",
|
55
63
|
"test": "npm run lint && vitest run",
|
56
64
|
"test-keepalive": "cross-env TEST_KEEPALIVE_COUNT=50 vitest run --test-timeout 180000 keep-alive-header.test.ts",
|
57
65
|
"cov": "vitest run --coverage",
|
58
66
|
"ci": "npm run lint && npm run cov && npm run build:test",
|
59
67
|
"contributor": "git-contributor",
|
60
|
-
"clean": "rm -rf
|
68
|
+
"clean": "rm -rf dist",
|
61
69
|
"prepublishOnly": "npm run build"
|
62
70
|
},
|
63
71
|
"dependencies": {
|
@@ -68,15 +76,19 @@
|
|
68
76
|
"formstream": "^1.1.1",
|
69
77
|
"mime-types": "^2.1.35",
|
70
78
|
"pump": "^3.0.0",
|
79
|
+
"qs": "^6.11.2",
|
71
80
|
"undici": "^5.22.1",
|
72
81
|
"ylru": "^1.3.2"
|
73
82
|
},
|
74
83
|
"devDependencies": {
|
84
|
+
"@tsconfig/node18": "^18.2.1",
|
85
|
+
"@tsconfig/strictest": "^2.0.2",
|
75
86
|
"@types/busboy": "^1.5.0",
|
76
87
|
"@types/default-user-agent": "^1.0.0",
|
77
88
|
"@types/mime-types": "^2.1.1",
|
78
89
|
"@types/node": "^20.2.1",
|
79
90
|
"@types/pump": "^1.1.1",
|
91
|
+
"@types/qs": "^6.9.7",
|
80
92
|
"@types/selfsigned": "^2.0.1",
|
81
93
|
"@types/tar-stream": "^2.2.2",
|
82
94
|
"@vitest/coverage-v8": "^0.32.0",
|
@@ -89,6 +101,7 @@
|
|
89
101
|
"proxy": "^1.0.2",
|
90
102
|
"selfsigned": "^2.0.1",
|
91
103
|
"tar-stream": "^2.2.0",
|
104
|
+
"tshy": "^1.0.0-3",
|
92
105
|
"typescript": "^5.0.4",
|
93
106
|
"vitest": "^0.32.0"
|
94
107
|
},
|
package/src/HttpAgent.ts
CHANGED
@@ -2,9 +2,9 @@ import dns from 'node:dns';
|
|
2
2
|
import { LookupFunction, isIP } from 'node:net';
|
3
3
|
import {
|
4
4
|
Agent,
|
5
|
+
Dispatcher,
|
6
|
+
buildConnector,
|
5
7
|
} from 'undici';
|
6
|
-
import type Dispatcher from 'undici/types/dispatcher';
|
7
|
-
import type buildConnector from 'undici/types/connector';
|
8
8
|
|
9
9
|
export type CheckAddressFunction = (ip: string, family: number | string) => boolean;
|
10
10
|
|
package/src/HttpClient.ts
CHANGED
@@ -26,15 +26,16 @@ import { FormData as FormDataNode } from 'formdata-node';
|
|
26
26
|
import { FormDataEncoder } from 'form-data-encoder';
|
27
27
|
import createUserAgent from 'default-user-agent';
|
28
28
|
import mime from 'mime-types';
|
29
|
+
import qs from 'qs';
|
29
30
|
import pump from 'pump';
|
30
31
|
// Compatible with old style formstream
|
31
32
|
import FormStream from 'formstream';
|
32
|
-
import { HttpAgent, CheckAddressFunction } from './HttpAgent';
|
33
|
-
import { RequestURL, RequestOptions, HttpMethod, RequestMeta } from './Request';
|
34
|
-
import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response';
|
35
|
-
import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime, isReadable } from './utils';
|
36
|
-
import symbols from './symbols';
|
37
|
-
import { initDiagnosticsChannel } from './diagnosticsChannel';
|
33
|
+
import { HttpAgent, CheckAddressFunction } from './HttpAgent.js';
|
34
|
+
import { RequestURL, RequestOptions, HttpMethod, RequestMeta } from './Request.js';
|
35
|
+
import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response.js';
|
36
|
+
import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime, isReadable } from './utils.js';
|
37
|
+
import symbols from './symbols.js';
|
38
|
+
import { initDiagnosticsChannel } from './diagnosticsChannel.js';
|
38
39
|
|
39
40
|
type Exists<T> = T extends undefined ? never : T;
|
40
41
|
type UndiciRequestOption = Exists<Parameters<typeof undiciRequest>[1]>;
|
@@ -87,7 +88,7 @@ export type ClientOptions = {
|
|
87
88
|
rejectUnauthorized?: boolean;
|
88
89
|
|
89
90
|
/**
|
90
|
-
*
|
91
|
+
* socketPath string | null (optional) - Default: null - An IPC endpoint, either Unix domain socket or Windows named pipe
|
91
92
|
*/
|
92
93
|
socketPath?: string | null;
|
93
94
|
},
|
@@ -244,14 +245,14 @@ export class HttpClient extends EventEmitter {
|
|
244
245
|
// the response body and trailers have been received
|
245
246
|
contentDownload: 0,
|
246
247
|
};
|
247
|
-
const
|
248
|
+
const originalOpaque = args.opaque;
|
248
249
|
// using opaque to diagnostics channel, binding request and socket
|
249
250
|
const internalOpaque = {
|
250
251
|
[symbols.kRequestId]: requestId,
|
251
252
|
[symbols.kRequestStartTime]: requestStartTime,
|
252
253
|
[symbols.kEnableRequestTiming]: !!args.timing,
|
253
254
|
[symbols.kRequestTiming]: timing,
|
254
|
-
[symbols.
|
255
|
+
[symbols.kRequestOriginalOpaque]: originalOpaque,
|
255
256
|
};
|
256
257
|
const reqMeta = {
|
257
258
|
requestId,
|
@@ -452,10 +453,17 @@ export class HttpClient extends EventEmitter {
|
|
452
453
|
|| isReadable(args.data);
|
453
454
|
if (isGETOrHEAD) {
|
454
455
|
if (!isStringOrBufferOrReadable) {
|
455
|
-
|
456
|
-
const
|
457
|
-
|
458
|
-
requestUrl.
|
456
|
+
if (args.nestedQuerystring) {
|
457
|
+
const querystring = qs.stringify(args.data);
|
458
|
+
// reset the requestUrl
|
459
|
+
const href = requestUrl.href;
|
460
|
+
requestUrl = new URL(href + (href.includes('?') ? '&' : '?') + querystring);
|
461
|
+
} else {
|
462
|
+
for (const field in args.data) {
|
463
|
+
const fieldValue = args.data[field];
|
464
|
+
if (fieldValue === undefined) continue;
|
465
|
+
requestUrl.searchParams.append(field, fieldValue);
|
466
|
+
}
|
459
467
|
}
|
460
468
|
}
|
461
469
|
} else {
|
@@ -472,7 +480,11 @@ export class HttpClient extends EventEmitter {
|
|
472
480
|
}
|
473
481
|
} else {
|
474
482
|
headers['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8';
|
475
|
-
|
483
|
+
if (args.nestedQuerystring) {
|
484
|
+
requestOptions.body = qs.stringify(args.data);
|
485
|
+
} else {
|
486
|
+
requestOptions.body = new URLSearchParams(args.data).toString();
|
487
|
+
}
|
476
488
|
}
|
477
489
|
}
|
478
490
|
}
|
@@ -582,7 +594,7 @@ export class HttpClient extends EventEmitter {
|
|
582
594
|
this.#updateSocketInfo(socketInfo, internalOpaque);
|
583
595
|
|
584
596
|
const clientResponse: HttpClientResponse = {
|
585
|
-
opaque:
|
597
|
+
opaque: originalOpaque,
|
586
598
|
data,
|
587
599
|
status: res.status,
|
588
600
|
statusCode: res.status,
|
@@ -637,7 +649,7 @@ export class HttpClient extends EventEmitter {
|
|
637
649
|
return await this.#requestInternal(url, options, requestContext);
|
638
650
|
}
|
639
651
|
}
|
640
|
-
err.opaque =
|
652
|
+
err.opaque = originalOpaque;
|
641
653
|
err.status = res.status;
|
642
654
|
err.headers = res.headers;
|
643
655
|
err.res = res;
|
package/src/Request.ts
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
import { Readable, Writable } from 'node:stream';
|
2
2
|
import type { IncomingHttpHeaders } from 'node:http';
|
3
|
-
import type Dispatcher from 'undici
|
3
|
+
import type { Dispatcher } from 'undici';
|
4
4
|
import type {
|
5
5
|
HttpClientResponse,
|
6
|
-
} from './Response';
|
6
|
+
} from './Response.js';
|
7
7
|
|
8
8
|
export type HttpMethod = Dispatcher.HttpMethod;
|
9
9
|
|
@@ -46,6 +46,11 @@ export type RequestOptions = {
|
|
46
46
|
* Default is 'buffer'.
|
47
47
|
*/
|
48
48
|
dataType?: 'text' | 'html' | 'json' | 'buffer' | 'stream';
|
49
|
+
/**
|
50
|
+
* urllib default use URLSearchParams to stringify form data which don't support nested object,
|
51
|
+
* will use qs instead of URLSearchParams to support nested object by set this option to true.
|
52
|
+
*/
|
53
|
+
nestedQuerystring?: boolean;
|
49
54
|
/**
|
50
55
|
* @deprecated
|
51
56
|
* Only for d.ts keep compatible with urllib@2, don't use it anymore.
|
@@ -3,8 +3,8 @@ import { performance } from 'node:perf_hooks';
|
|
3
3
|
import { debuglog } from 'node:util';
|
4
4
|
import { Socket } from 'node:net';
|
5
5
|
import { DiagnosticsChannel } from 'undici';
|
6
|
-
import symbols from './symbols';
|
7
|
-
import { globalId, performanceTime } from './utils';
|
6
|
+
import symbols from './symbols.js';
|
7
|
+
import { globalId, performanceTime } from './utils.js';
|
8
8
|
|
9
9
|
const debug = debuglog('urllib:DiagnosticsChannel');
|
10
10
|
let initedDiagnosticsChannel = false;
|
package/src/index.ts
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
import LRU from 'ylru';
|
2
|
-
import { HttpClient, HEADER_USER_AGENT } from './HttpClient';
|
3
|
-
import { RequestOptions, RequestURL } from './Request';
|
2
|
+
import { HttpClient, HEADER_USER_AGENT } from './HttpClient.js';
|
3
|
+
import { RequestOptions, RequestURL } from './Request.js';
|
4
4
|
|
5
5
|
let httpclient: HttpClient;
|
6
6
|
const domainSocketHttpclients = new LRU(50);
|
@@ -39,14 +39,14 @@ export {
|
|
39
39
|
export {
|
40
40
|
HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT,
|
41
41
|
RequestDiagnosticsMessage, ResponseDiagnosticsMessage,
|
42
|
-
} from './HttpClient';
|
42
|
+
} from './HttpClient.js';
|
43
43
|
// RequestOptions2 is keep compatible with urlib@2 RequestOptions2
|
44
44
|
export {
|
45
45
|
RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod,
|
46
46
|
FixJSONCtlCharsHandler, FixJSONCtlChars,
|
47
|
-
} from './Request';
|
47
|
+
} from './Request.js';
|
48
48
|
|
49
|
-
export { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse } from './Response';
|
49
|
+
export { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse } from './Response.js';
|
50
50
|
|
51
51
|
export default {
|
52
52
|
request,
|
package/src/symbols.ts
CHANGED
@@ -12,5 +12,5 @@ export default {
|
|
12
12
|
kRequestStartTime: Symbol('request start time'),
|
13
13
|
kEnableRequestTiming: Symbol('enable request timing or not'),
|
14
14
|
kRequestTiming: Symbol('request timing'),
|
15
|
-
|
15
|
+
kRequestOriginalOpaque: Symbol('request original opaque'),
|
16
16
|
};
|
package/src/utils.ts
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import { randomBytes, createHash } from 'node:crypto';
|
2
2
|
import { Readable } from 'node:stream';
|
3
3
|
import { performance } from 'node:perf_hooks';
|
4
|
-
import { FixJSONCtlChars } from './Request';
|
4
|
+
import type { FixJSONCtlChars } from './Request.js';
|
5
5
|
|
6
6
|
const JSONCtlCharsMap = {
|
7
7
|
'"': '\\"', // \u0022
|
package/src/cjs/package.json
DELETED
package/src/esm/package.json
DELETED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|