urllib 2.36.0 → 2.37.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/History.md +30 -0
- package/README.md +12 -7
- package/lib/index.d.ts +6 -0
- package/lib/urllib.js +75 -37
- package/package.json +3 -3
package/History.md
CHANGED
@@ -1,4 +1,34 @@
|
|
1
1
|
|
2
|
+
2.37.2 / 2021-06-07
|
3
|
+
==================
|
4
|
+
|
5
|
+
**fixes**
|
6
|
+
* [[`04734a1`](http://github.com/node-modules/urllib/commit/04734a13e146da501f4e38333d5de9f9bbb259d1)] - fix: 🐛 should trigger response event when follow redirect (#361) (Hongcai Deng <<admin@dhchouse.com>>)
|
7
|
+
|
8
|
+
2.37.1 / 2021-04-15
|
9
|
+
==================
|
10
|
+
|
11
|
+
**others**
|
12
|
+
* [[`d50afda`](http://github.com/node-modules/urllib/commit/d50afda91befff75f044a1ad83689f7a459c5d32)] - Update proxy-agent to v4 to resolve vulnerability (#355) (Chad McElligott <<chad.mcelligott@gmail.com>>)
|
13
|
+
|
14
|
+
2.37.0 / 2021-04-02
|
15
|
+
==================
|
16
|
+
|
17
|
+
**features**
|
18
|
+
* [[`2acf75b`](http://github.com/node-modules/urllib/commit/2acf75b75050667f7341e6990078e0330e704af1)] - feat: add unix domain socket file support (#352) (Khaidi Chu <<i@2333.moe>>)
|
19
|
+
|
20
|
+
**others**
|
21
|
+
* [[`f2a42d1`](http://github.com/node-modules/urllib/commit/f2a42d19d2236979c16eea13c229839e5a78b381)] - chore: LICENSE link typo (fengmk2 <<fengmk2@gmail.com>>)
|
22
|
+
* [[`28a0152`](http://github.com/node-modules/urllib/commit/28a0152c0ae5d4470d88b8122724b6228b48b319)] - chore: update contributors (fengmk2 <<fengmk2@gmail.com>>)
|
23
|
+
* [[`5674670`](http://github.com/node-modules/urllib/commit/56746704d61116d99846d94480d223a9e3c42913)] - test: add socket checker test case (#341) (fengmk2 <<fengmk2@gmail.com>>)
|
24
|
+
* [[`d953ec0`](http://github.com/node-modules/urllib/commit/d953ec0e55ec28d553a24c5cf0fcb64715e7c8f9)] - test: add testcase for using retry and writeStream in HttpClient2 (#350) (Haoliang Gao <<sakura9515@gmail.com>>)
|
25
|
+
|
26
|
+
2.36.1 / 2020-06-09
|
27
|
+
==================
|
28
|
+
|
29
|
+
**others**
|
30
|
+
* [[`35eecc0`](http://github.com/node-modules/urllib/commit/35eecc021f55e324b2520f91a032a33517d94f29)] - chore: add type declaration for RequestOptions#keepHeaderCase (#345) (William <<willizm.cn@gmail.com>>)
|
31
|
+
|
2
32
|
2.36.0 / 2020-06-08
|
3
33
|
==================
|
4
34
|
|
package/README.md
CHANGED
@@ -165,6 +165,7 @@ httpclient.request('http://nodejs.org', function (err, body) {
|
|
165
165
|
- ***lookup*** Function - Custom DNS lookup function, default is `dns.lookup`. Require node >= 4.0.0(for http protocol) and node >=8(for https protocol)
|
166
166
|
- ***checkAddress*** Function: optional, check request address to protect from SSRF and similar attacks. It receive tow arguments(`ip` and `family`) and should return true or false to identified the address is legal or not. It rely on `lookup` and have the same version requirement.
|
167
167
|
- ***trace*** Boolean - Enable capture stack include call site of library entrance, default is `false`.
|
168
|
+
- ***socketPath*** String - optional Unix Domain Socket. (Refer to [Node.js Document](https://nodejs.org/dist/latest-v14.x/docs/api/http.html#http_http_request_options_callback))
|
168
169
|
- ***callback(err, data, res)*** Function - Optional callback.
|
169
170
|
- **err** Error - Would be `null` if no error accured.
|
170
171
|
- **data** Buffer | Object - The data responsed. Would be a Buffer if `dataType` is set to `text` or an JSON parsed into Object if it's set to `json`.
|
@@ -374,6 +375,10 @@ options extends from urllib, besides below
|
|
374
375
|
- ***retryDelay*** Number - wait a delay(ms) between retries.
|
375
376
|
- ***isRetry*** Function - determine whether retry, a response object as the first argument. it will retry when status >= 500 by default. Request error is not included.
|
376
377
|
|
378
|
+
#### Warning
|
379
|
+
|
380
|
+
It's not supported by using retry and writeStream, because the retry request can't stop the stream which is consuming.
|
381
|
+
|
377
382
|
## Proxy
|
378
383
|
|
379
384
|
Support both `http` and `https` protocol.
|
@@ -444,20 +449,20 @@ When open the trace, urllib may have poor perfomance, please consider carefully.
|
|
444
449
|
|
445
450
|
## Contributors
|
446
451
|
|
447
|
-
|[<img src="https://avatars0.githubusercontent.com/u/156269?v=4" width="100px;"/><br/><sub><b>fengmk2</b></sub>](https://github.com/fengmk2)<br/>|[<img src="https://avatars3.githubusercontent.com/u/985607?v=4" width="100px;"/><br/><sub><b>dead-horse</b></sub>](https://github.com/dead-horse)<br/>|[<img src="https://avatars2.githubusercontent.com/u/288288?v=4" width="100px;"/><br/><sub><b>xingrz</b></sub>](https://github.com/xingrz)<br/>|[<img src="https://
|
452
|
+
|[<img src="https://avatars0.githubusercontent.com/u/156269?v=4" width="100px;"/><br/><sub><b>fengmk2</b></sub>](https://github.com/fengmk2)<br/>|[<img src="https://avatars3.githubusercontent.com/u/985607?v=4" width="100px;"/><br/><sub><b>dead-horse</b></sub>](https://github.com/dead-horse)<br/>|[<img src="https://avatars2.githubusercontent.com/u/288288?v=4" width="100px;"/><br/><sub><b>xingrz</b></sub>](https://github.com/xingrz)<br/>|[<img src="https://avatars1.githubusercontent.com/u/360661?v=4" width="100px;"/><br/><sub><b>popomore</b></sub>](https://github.com/popomore)<br/>|[<img src="https://avatars3.githubusercontent.com/u/327019?v=4" width="100px;"/><br/><sub><b>JacksonTian</b></sub>](https://github.com/JacksonTian)<br/>|[<img src="https://avatars3.githubusercontent.com/u/543405?v=4" width="100px;"/><br/><sub><b>ibigbug</b></sub>](https://github.com/ibigbug)<br/>|
|
448
453
|
| :---: | :---: | :---: | :---: | :---: | :---: |
|
449
|
-
|[<img src="https://avatars1.githubusercontent.com/u/
|
450
|
-
|[<img src="https://
|
451
|
-
|[<img src="https://
|
452
|
-
[<img src="https://
|
454
|
+
|[<img src="https://avatars1.githubusercontent.com/u/14790466?v=4" width="100px;"/><br/><sub><b>greenkeeperio-bot</b></sub>](https://github.com/greenkeeperio-bot)<br/>|[<img src="https://avatars2.githubusercontent.com/u/2569835?v=4" width="100px;"/><br/><sub><b>haoxins</b></sub>](https://github.com/haoxins)<br/>|[<img src="https://avatars2.githubusercontent.com/u/227713?v=4" width="100px;"/><br/><sub><b>atian25</b></sub>](https://github.com/atian25)<br/>|[<img src="https://avatars3.githubusercontent.com/u/5381764?v=4" width="100px;"/><br/><sub><b>paambaati</b></sub>](https://github.com/paambaati)<br/>|[<img src="https://avatars1.githubusercontent.com/u/1147375?v=4" width="100px;"/><br/><sub><b>alsotang</b></sub>](https://github.com/alsotang)<br/>|[<img src="https://avatars1.githubusercontent.com/u/546535?v=4" width="100px;"/><br/><sub><b>leoner</b></sub>](https://github.com/leoner)<br/>|
|
455
|
+
|[<img src="https://avatars3.githubusercontent.com/u/1747852?v=4" width="100px;"/><br/><sub><b>isayme</b></sub>](https://github.com/isayme)<br/>|[<img src="https://avatars0.githubusercontent.com/u/252317?v=4" width="100px;"/><br/><sub><b>cyjake</b></sub>](https://github.com/cyjake)<br/>|[<img src="https://avatars0.githubusercontent.com/u/5856440?v=4" width="100px;"/><br/><sub><b>whxaxes</b></sub>](https://github.com/whxaxes)<br/>|[<img src="https://avatars1.githubusercontent.com/u/5139554?v=4" width="100px;"/><br/><sub><b>danielwpz</b></sub>](https://github.com/danielwpz)<br/>|[<img src="https://avatars3.githubusercontent.com/u/5127897?v=4" width="100px;"/><br/><sub><b>danielsss</b></sub>](https://github.com/danielsss)<br/>|[<img src="https://avatars2.githubusercontent.com/u/3367820?v=4" width="100px;"/><br/><sub><b>Jeff-Tian</b></sub>](https://github.com/Jeff-Tian)<br/>|
|
456
|
+
|[<img src="https://avatars1.githubusercontent.com/u/32407?v=4" width="100px;"/><br/><sub><b>jedahan</b></sub>](https://github.com/jedahan)<br/>|[<img src="https://avatars3.githubusercontent.com/u/2842176?v=4" width="100px;"/><br/><sub><b>XadillaX</b></sub>](https://github.com/XadillaX)<br/>|[<img src="https://avatars1.githubusercontent.com/u/17075261?v=4" width="100px;"/><br/><sub><b>nick-ng</b></sub>](https://github.com/nick-ng)<br/>|[<img src="https://avatars2.githubusercontent.com/u/1706595?v=4" width="100px;"/><br/><sub><b>rishavsharan</b></sub>](https://github.com/rishavsharan)<br/>|[<img src="https://avatars1.githubusercontent.com/u/1886161?v=4" width="100px;"/><br/><sub><b>willizm</b></sub>](https://github.com/willizm)<br/>|[<img src="https://avatars1.githubusercontent.com/u/7227589?v=4" width="100px;"/><br/><sub><b>davidkhala</b></sub>](https://github.com/davidkhala)<br/>|
|
457
|
+
[<img src="https://avatars0.githubusercontent.com/u/535479?v=4" width="100px;"/><br/><sub><b>aleafs</b></sub>](https://github.com/aleafs)<br/>|[<img src="https://avatars1.githubusercontent.com/u/3689968?v=4" width="100px;"/><br/><sub><b>Amunu</b></sub>](https://github.com/Amunu)<br/>|[<img src="https://avatars3.githubusercontent.com/in/9426?v=4" width="100px;"/><br/><sub><b>azure-pipelines[bot]</b></sub>](https://github.com/apps/azure-pipelines)<br/>|[<img src="https://avatars3.githubusercontent.com/u/929503?v=4" width="100px;"/><br/><sub><b>yuzhigang33</b></sub>](https://github.com/yuzhigang33)<br/>|[<img src="https://avatars1.githubusercontent.com/u/981128?v=4" width="100px;"/><br/><sub><b>fishbar</b></sub>](https://github.com/fishbar)<br/>|[<img src="https://avatars2.githubusercontent.com/u/1207064?v=4" width="100px;"/><br/><sub><b>gxcsoccer</b></sub>](https://github.com/gxcsoccer)<br/>
|
453
458
|
|
454
|
-
This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `
|
459
|
+
This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Sat Jan 16 2021 23:16:33 GMT+0800`.
|
455
460
|
|
456
461
|
<!-- GITCONTRIBUTOR_END -->
|
457
462
|
|
458
463
|
## License
|
459
464
|
|
460
|
-
[MIT](LICENSE
|
465
|
+
[MIT](LICENSE)
|
461
466
|
|
462
467
|
|
463
468
|
[bluebird]: https://github.com/petkaantonov/bluebird
|
package/lib/index.d.ts
CHANGED
@@ -54,6 +54,8 @@ export interface RequestOptions {
|
|
54
54
|
fixJSONCtlChars?: boolean;
|
55
55
|
/** Request headers. */
|
56
56
|
headers?: IncomingHttpHeaders;
|
57
|
+
/** by default will convert header keys to lowercase */
|
58
|
+
keepHeaderCase?: boolean;
|
57
59
|
/**
|
58
60
|
* Request timeout in milliseconds for connecting phase and response receiving phase.
|
59
61
|
* Defaults to exports.
|
@@ -128,6 +130,10 @@ export interface RequestOptions {
|
|
128
130
|
* It rely on lookup and have the same version requirement.
|
129
131
|
*/
|
130
132
|
checkAddress?: (ip: string, family: number | string) => boolean;
|
133
|
+
/**
|
134
|
+
* UNIX domain socket path. (Windows is not supported)
|
135
|
+
*/
|
136
|
+
socketPath?: string;
|
131
137
|
}
|
132
138
|
|
133
139
|
export interface HttpClientResponse<T> {
|
package/lib/urllib.js
CHANGED
@@ -138,6 +138,7 @@ var SOCKET_RESPONSE_COUNT = '_URLLIB_SOCKET_RESPONSE_COUNT';
|
|
138
138
|
* Require node >= 4.0.0 and only work on `http` protocol.
|
139
139
|
* - {Boolean} [enableProxy]: optional, enable proxy request. Default is `false`.
|
140
140
|
* - {String|Object} [proxy]: optional proxy agent uri or options. Default is `null`.
|
141
|
+
* - {String} [socketPath]: optional, unix domain socket file path.
|
141
142
|
* - {Function} checkAddress: optional, check request address to protect from SSRF and similar attacks.
|
142
143
|
* @param {Function} [callback]: callback(error, data, res). If missing callback, will return a promise object.
|
143
144
|
* @return {HttpRequest} req object.
|
@@ -315,6 +316,9 @@ function requestWithCallback(url, args, callback) {
|
|
315
316
|
options.headers[key] = args.headers[name];
|
316
317
|
}
|
317
318
|
}
|
319
|
+
if (args.socketPath) {
|
320
|
+
options.socketPath = args.socketPath;
|
321
|
+
}
|
318
322
|
|
319
323
|
var sslNames = [
|
320
324
|
'pfx',
|
@@ -516,6 +520,7 @@ function requestWithCallback(url, args, callback) {
|
|
516
520
|
}
|
517
521
|
return;
|
518
522
|
}
|
523
|
+
|
519
524
|
var cb = callback;
|
520
525
|
callback = null;
|
521
526
|
var headers = {};
|
@@ -525,49 +530,16 @@ function requestWithCallback(url, args, callback) {
|
|
525
530
|
headers = res.headers;
|
526
531
|
}
|
527
532
|
|
528
|
-
|
529
|
-
|
530
|
-
&& !options.headers.authorization && args.digestAuth) {
|
531
|
-
var authenticate = headers['www-authenticate'];
|
532
|
-
if (authenticate.indexOf('Digest ') >= 0) {
|
533
|
-
debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', reqId, url, authenticate);
|
534
|
-
options.headers.authorization = digestAuthHeader(options.method, options.path, authenticate, args.digestAuth);
|
535
|
-
debug('Request#%d %s: auth with digest header: %s', reqId, url, options.headers.authorization);
|
536
|
-
if (res.headers['set-cookie']) {
|
537
|
-
options.headers.cookie = res.headers['set-cookie'].join(';');
|
538
|
-
}
|
539
|
-
args.headers = options.headers;
|
540
|
-
return exports.requestWithCallback(url, args, cb);
|
541
|
-
}
|
533
|
+
if (handleDigestAuth(res, cb)) {
|
534
|
+
return;
|
542
535
|
}
|
543
536
|
|
544
|
-
var
|
545
|
-
if (timing) {
|
546
|
-
timing.contentDownload = requestUseTime;
|
547
|
-
}
|
537
|
+
var response = createCallbackResponse(data, res);
|
548
538
|
|
549
539
|
debug('[%sms] done, %s bytes HTTP %s %s %s %s, keepAliveSocket: %s, timing: %j, socketHandledRequests: %s, socketHandledResponses: %s',
|
550
|
-
requestUseTime, responseSize, statusCode, options.method, options.host, options.path,
|
540
|
+
response.requestUseTime, responseSize, statusCode, options.method, options.host, options.path,
|
551
541
|
keepAliveSocket, timing, socketHandledRequests, socketHandledResponses);
|
552
542
|
|
553
|
-
var response = {
|
554
|
-
status: statusCode,
|
555
|
-
statusCode: statusCode,
|
556
|
-
statusMessage: statusMessage,
|
557
|
-
headers: headers,
|
558
|
-
size: responseSize,
|
559
|
-
aborted: responseAborted,
|
560
|
-
rt: requestUseTime,
|
561
|
-
keepAliveSocket: keepAliveSocket,
|
562
|
-
data: data,
|
563
|
-
requestUrls: args.requestUrls,
|
564
|
-
timing: timing,
|
565
|
-
remoteAddress: remoteAddress,
|
566
|
-
remotePort: remotePort,
|
567
|
-
socketHandledRequests: socketHandledRequests,
|
568
|
-
socketHandledResponses: socketHandledResponses,
|
569
|
-
};
|
570
|
-
|
571
543
|
if (err) {
|
572
544
|
var agentStatus = '';
|
573
545
|
if (agent && typeof agent.getCurrentStatus === 'function') {
|
@@ -616,6 +588,45 @@ function requestWithCallback(url, args, callback) {
|
|
616
588
|
|
617
589
|
cb(err, data, args.streaming ? res : response);
|
618
590
|
|
591
|
+
emitResponseEvent(err, response);
|
592
|
+
}
|
593
|
+
|
594
|
+
function createAndEmitResponseEvent(data, res) {
|
595
|
+
var response = createCallbackResponse(data, res);
|
596
|
+
emitResponseEvent(null, response);
|
597
|
+
}
|
598
|
+
|
599
|
+
function createCallbackResponse(data, res) {
|
600
|
+
var requestUseTime = Date.now() - requestStartTime;
|
601
|
+
if (timing) {
|
602
|
+
timing.contentDownload = requestUseTime;
|
603
|
+
}
|
604
|
+
|
605
|
+
var headers = {};
|
606
|
+
if (res && res.headers) {
|
607
|
+
headers = res.headers;
|
608
|
+
}
|
609
|
+
|
610
|
+
return {
|
611
|
+
status: statusCode,
|
612
|
+
statusCode: statusCode,
|
613
|
+
statusMessage: statusMessage,
|
614
|
+
headers: headers,
|
615
|
+
size: responseSize,
|
616
|
+
aborted: responseAborted,
|
617
|
+
rt: requestUseTime,
|
618
|
+
keepAliveSocket: keepAliveSocket,
|
619
|
+
data: data,
|
620
|
+
requestUrls: args.requestUrls,
|
621
|
+
timing: timing,
|
622
|
+
remoteAddress: remoteAddress,
|
623
|
+
remotePort: remotePort,
|
624
|
+
socketHandledRequests: socketHandledRequests,
|
625
|
+
socketHandledResponses: socketHandledResponses,
|
626
|
+
};
|
627
|
+
}
|
628
|
+
|
629
|
+
function emitResponseEvent(err, response) {
|
619
630
|
if (args.emitter) {
|
620
631
|
// keep to use the same reqMeta object on request event before
|
621
632
|
reqMeta.url = parsedUrl.href;
|
@@ -633,6 +644,30 @@ function requestWithCallback(url, args, callback) {
|
|
633
644
|
}
|
634
645
|
}
|
635
646
|
|
647
|
+
function handleDigestAuth(res, cb) {
|
648
|
+
var headers = {};
|
649
|
+
if (res && res.headers) {
|
650
|
+
headers = res.headers;
|
651
|
+
}
|
652
|
+
// handle digest auth
|
653
|
+
if (statusCode === 401 && headers['www-authenticate']
|
654
|
+
&& !options.headers.authorization && args.digestAuth) {
|
655
|
+
var authenticate = headers['www-authenticate'];
|
656
|
+
if (authenticate.indexOf('Digest ') >= 0) {
|
657
|
+
debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', reqId, url, authenticate);
|
658
|
+
options.headers.authorization = digestAuthHeader(options.method, options.path, authenticate, args.digestAuth);
|
659
|
+
debug('Request#%d %s: auth with digest header: %s', reqId, url, options.headers.authorization);
|
660
|
+
if (res.headers['set-cookie']) {
|
661
|
+
options.headers.cookie = res.headers['set-cookie'].join(';');
|
662
|
+
}
|
663
|
+
args.headers = options.headers;
|
664
|
+
exports.requestWithCallback(url, args, cb);
|
665
|
+
return true;
|
666
|
+
}
|
667
|
+
}
|
668
|
+
return false;
|
669
|
+
}
|
670
|
+
|
636
671
|
function handleRedirect(res) {
|
637
672
|
var err = null;
|
638
673
|
if (args.followRedirect && statuses.redirect[res.statusCode]) { // handle redirect
|
@@ -736,6 +771,7 @@ function requestWithCallback(url, args, callback) {
|
|
736
771
|
var result = handleRedirect(res);
|
737
772
|
if (result.redirect) {
|
738
773
|
res.resume();
|
774
|
+
createAndEmitResponseEvent(null, res);
|
739
775
|
return;
|
740
776
|
}
|
741
777
|
if (result.error) {
|
@@ -777,6 +813,7 @@ function requestWithCallback(url, args, callback) {
|
|
777
813
|
var result = handleRedirect(res);
|
778
814
|
if (result.redirect) {
|
779
815
|
res.resume();
|
816
|
+
createAndEmitResponseEvent(null, res);
|
780
817
|
return;
|
781
818
|
}
|
782
819
|
if (result.error) {
|
@@ -870,6 +907,7 @@ function requestWithCallback(url, args, callback) {
|
|
870
907
|
return done(result.error, body, res);
|
871
908
|
}
|
872
909
|
if (result.redirect) {
|
910
|
+
createAndEmitResponseEvent(null, res);
|
873
911
|
return;
|
874
912
|
}
|
875
913
|
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "urllib",
|
3
|
-
"version": "2.
|
3
|
+
"version": "2.37.2",
|
4
4
|
"description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more.",
|
5
5
|
"keywords": [
|
6
6
|
"urllib",
|
@@ -42,7 +42,7 @@
|
|
42
42
|
"humanize-ms": "^1.2.0",
|
43
43
|
"iconv-lite": "^0.4.15",
|
44
44
|
"ip": "^1.1.5",
|
45
|
-
"proxy-agent": "^
|
45
|
+
"proxy-agent": "^4.0.1",
|
46
46
|
"pump": "^3.0.0",
|
47
47
|
"qs": "^6.4.0",
|
48
48
|
"statuses": "^1.3.1",
|
@@ -83,7 +83,7 @@
|
|
83
83
|
"os": {
|
84
84
|
"github": "linux, windows, macos"
|
85
85
|
},
|
86
|
-
"version": "
|
86
|
+
"version": "8, 10, 12, 14"
|
87
87
|
},
|
88
88
|
"license": "MIT"
|
89
89
|
}
|