urllib 2.36.1 → 2.37.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/History.md +30 -0
- package/README.md +12 -7
- package/lib/index.d.ts +4 -0
- package/lib/urllib.js +74 -37
- package/package.json +3 -3
package/History.md
CHANGED
@@ -1,4 +1,34 @@
|
|
1
1
|
|
2
|
+
2.37.3 / 2021-07-05
|
3
|
+
==================
|
4
|
+
|
5
|
+
**fixes**
|
6
|
+
* [[`b730a6c`](http://github.com/node-modules/urllib/commit/b730a6cdc0465bea1c08d50bfa3b5e95bd17b31f)] - fix: 🐛 redirect status code (#363) (Hongcai Deng <<admin@dhchouse.com>>)
|
7
|
+
|
8
|
+
2.37.2 / 2021-06-07
|
9
|
+
==================
|
10
|
+
|
11
|
+
**fixes**
|
12
|
+
* [[`04734a1`](http://github.com/node-modules/urllib/commit/04734a13e146da501f4e38333d5de9f9bbb259d1)] - fix: 🐛 should trigger response event when follow redirect (#361) (Hongcai Deng <<admin@dhchouse.com>>)
|
13
|
+
|
14
|
+
2.37.1 / 2021-04-15
|
15
|
+
==================
|
16
|
+
|
17
|
+
**others**
|
18
|
+
* [[`d50afda`](http://github.com/node-modules/urllib/commit/d50afda91befff75f044a1ad83689f7a459c5d32)] - Update proxy-agent to v4 to resolve vulnerability (#355) (Chad McElligott <<chad.mcelligott@gmail.com>>)
|
19
|
+
|
20
|
+
2.37.0 / 2021-04-02
|
21
|
+
==================
|
22
|
+
|
23
|
+
**features**
|
24
|
+
* [[`2acf75b`](http://github.com/node-modules/urllib/commit/2acf75b75050667f7341e6990078e0330e704af1)] - feat: add unix domain socket file support (#352) (Khaidi Chu <<i@2333.moe>>)
|
25
|
+
|
26
|
+
**others**
|
27
|
+
* [[`f2a42d1`](http://github.com/node-modules/urllib/commit/f2a42d19d2236979c16eea13c229839e5a78b381)] - chore: LICENSE link typo (fengmk2 <<fengmk2@gmail.com>>)
|
28
|
+
* [[`28a0152`](http://github.com/node-modules/urllib/commit/28a0152c0ae5d4470d88b8122724b6228b48b319)] - chore: update contributors (fengmk2 <<fengmk2@gmail.com>>)
|
29
|
+
* [[`5674670`](http://github.com/node-modules/urllib/commit/56746704d61116d99846d94480d223a9e3c42913)] - test: add socket checker test case (#341) (fengmk2 <<fengmk2@gmail.com>>)
|
30
|
+
* [[`d953ec0`](http://github.com/node-modules/urllib/commit/d953ec0e55ec28d553a24c5cf0fcb64715e7c8f9)] - test: add testcase for using retry and writeStream in HttpClient2 (#350) (Haoliang Gao <<sakura9515@gmail.com>>)
|
31
|
+
|
2
32
|
2.36.1 / 2020-06-09
|
3
33
|
==================
|
4
34
|
|
package/README.md
CHANGED
@@ -165,6 +165,7 @@ httpclient.request('http://nodejs.org', function (err, body) {
|
|
165
165
|
- ***lookup*** Function - Custom DNS lookup function, default is `dns.lookup`. Require node >= 4.0.0(for http protocol) and node >=8(for https protocol)
|
166
166
|
- ***checkAddress*** Function: optional, check request address to protect from SSRF and similar attacks. It receive tow arguments(`ip` and `family`) and should return true or false to identified the address is legal or not. It rely on `lookup` and have the same version requirement.
|
167
167
|
- ***trace*** Boolean - Enable capture stack include call site of library entrance, default is `false`.
|
168
|
+
- ***socketPath*** String - optional Unix Domain Socket. (Refer to [Node.js Document](https://nodejs.org/dist/latest-v14.x/docs/api/http.html#http_http_request_options_callback))
|
168
169
|
- ***callback(err, data, res)*** Function - Optional callback.
|
169
170
|
- **err** Error - Would be `null` if no error accured.
|
170
171
|
- **data** Buffer | Object - The data responsed. Would be a Buffer if `dataType` is set to `text` or an JSON parsed into Object if it's set to `json`.
|
@@ -374,6 +375,10 @@ options extends from urllib, besides below
|
|
374
375
|
- ***retryDelay*** Number - wait a delay(ms) between retries.
|
375
376
|
- ***isRetry*** Function - determine whether retry, a response object as the first argument. it will retry when status >= 500 by default. Request error is not included.
|
376
377
|
|
378
|
+
#### Warning
|
379
|
+
|
380
|
+
It's not supported by using retry and writeStream, because the retry request can't stop the stream which is consuming.
|
381
|
+
|
377
382
|
## Proxy
|
378
383
|
|
379
384
|
Support both `http` and `https` protocol.
|
@@ -444,20 +449,20 @@ When open the trace, urllib may have poor perfomance, please consider carefully.
|
|
444
449
|
|
445
450
|
## Contributors
|
446
451
|
|
447
|
-
|[<img src="https://avatars0.githubusercontent.com/u/156269?v=4" width="100px;"/><br/><sub><b>fengmk2</b></sub>](https://github.com/fengmk2)<br/>|[<img src="https://avatars3.githubusercontent.com/u/985607?v=4" width="100px;"/><br/><sub><b>dead-horse</b></sub>](https://github.com/dead-horse)<br/>|[<img src="https://avatars2.githubusercontent.com/u/288288?v=4" width="100px;"/><br/><sub><b>xingrz</b></sub>](https://github.com/xingrz)<br/>|[<img src="https://
|
452
|
+
|[<img src="https://avatars0.githubusercontent.com/u/156269?v=4" width="100px;"/><br/><sub><b>fengmk2</b></sub>](https://github.com/fengmk2)<br/>|[<img src="https://avatars3.githubusercontent.com/u/985607?v=4" width="100px;"/><br/><sub><b>dead-horse</b></sub>](https://github.com/dead-horse)<br/>|[<img src="https://avatars2.githubusercontent.com/u/288288?v=4" width="100px;"/><br/><sub><b>xingrz</b></sub>](https://github.com/xingrz)<br/>|[<img src="https://avatars1.githubusercontent.com/u/360661?v=4" width="100px;"/><br/><sub><b>popomore</b></sub>](https://github.com/popomore)<br/>|[<img src="https://avatars3.githubusercontent.com/u/327019?v=4" width="100px;"/><br/><sub><b>JacksonTian</b></sub>](https://github.com/JacksonTian)<br/>|[<img src="https://avatars3.githubusercontent.com/u/543405?v=4" width="100px;"/><br/><sub><b>ibigbug</b></sub>](https://github.com/ibigbug)<br/>|
|
448
453
|
| :---: | :---: | :---: | :---: | :---: | :---: |
|
449
|
-
|[<img src="https://avatars1.githubusercontent.com/u/
|
450
|
-
|[<img src="https://
|
451
|
-
|[<img src="https://
|
452
|
-
[<img src="https://
|
454
|
+
|[<img src="https://avatars1.githubusercontent.com/u/14790466?v=4" width="100px;"/><br/><sub><b>greenkeeperio-bot</b></sub>](https://github.com/greenkeeperio-bot)<br/>|[<img src="https://avatars2.githubusercontent.com/u/2569835?v=4" width="100px;"/><br/><sub><b>haoxins</b></sub>](https://github.com/haoxins)<br/>|[<img src="https://avatars2.githubusercontent.com/u/227713?v=4" width="100px;"/><br/><sub><b>atian25</b></sub>](https://github.com/atian25)<br/>|[<img src="https://avatars3.githubusercontent.com/u/5381764?v=4" width="100px;"/><br/><sub><b>paambaati</b></sub>](https://github.com/paambaati)<br/>|[<img src="https://avatars1.githubusercontent.com/u/1147375?v=4" width="100px;"/><br/><sub><b>alsotang</b></sub>](https://github.com/alsotang)<br/>|[<img src="https://avatars1.githubusercontent.com/u/546535?v=4" width="100px;"/><br/><sub><b>leoner</b></sub>](https://github.com/leoner)<br/>|
|
455
|
+
|[<img src="https://avatars3.githubusercontent.com/u/1747852?v=4" width="100px;"/><br/><sub><b>isayme</b></sub>](https://github.com/isayme)<br/>|[<img src="https://avatars0.githubusercontent.com/u/252317?v=4" width="100px;"/><br/><sub><b>cyjake</b></sub>](https://github.com/cyjake)<br/>|[<img src="https://avatars0.githubusercontent.com/u/5856440?v=4" width="100px;"/><br/><sub><b>whxaxes</b></sub>](https://github.com/whxaxes)<br/>|[<img src="https://avatars1.githubusercontent.com/u/5139554?v=4" width="100px;"/><br/><sub><b>danielwpz</b></sub>](https://github.com/danielwpz)<br/>|[<img src="https://avatars3.githubusercontent.com/u/5127897?v=4" width="100px;"/><br/><sub><b>danielsss</b></sub>](https://github.com/danielsss)<br/>|[<img src="https://avatars2.githubusercontent.com/u/3367820?v=4" width="100px;"/><br/><sub><b>Jeff-Tian</b></sub>](https://github.com/Jeff-Tian)<br/>|
|
456
|
+
|[<img src="https://avatars1.githubusercontent.com/u/32407?v=4" width="100px;"/><br/><sub><b>jedahan</b></sub>](https://github.com/jedahan)<br/>|[<img src="https://avatars3.githubusercontent.com/u/2842176?v=4" width="100px;"/><br/><sub><b>XadillaX</b></sub>](https://github.com/XadillaX)<br/>|[<img src="https://avatars1.githubusercontent.com/u/17075261?v=4" width="100px;"/><br/><sub><b>nick-ng</b></sub>](https://github.com/nick-ng)<br/>|[<img src="https://avatars2.githubusercontent.com/u/1706595?v=4" width="100px;"/><br/><sub><b>rishavsharan</b></sub>](https://github.com/rishavsharan)<br/>|[<img src="https://avatars1.githubusercontent.com/u/1886161?v=4" width="100px;"/><br/><sub><b>willizm</b></sub>](https://github.com/willizm)<br/>|[<img src="https://avatars1.githubusercontent.com/u/7227589?v=4" width="100px;"/><br/><sub><b>davidkhala</b></sub>](https://github.com/davidkhala)<br/>|
|
457
|
+
[<img src="https://avatars0.githubusercontent.com/u/535479?v=4" width="100px;"/><br/><sub><b>aleafs</b></sub>](https://github.com/aleafs)<br/>|[<img src="https://avatars1.githubusercontent.com/u/3689968?v=4" width="100px;"/><br/><sub><b>Amunu</b></sub>](https://github.com/Amunu)<br/>|[<img src="https://avatars3.githubusercontent.com/in/9426?v=4" width="100px;"/><br/><sub><b>azure-pipelines[bot]</b></sub>](https://github.com/apps/azure-pipelines)<br/>|[<img src="https://avatars3.githubusercontent.com/u/929503?v=4" width="100px;"/><br/><sub><b>yuzhigang33</b></sub>](https://github.com/yuzhigang33)<br/>|[<img src="https://avatars1.githubusercontent.com/u/981128?v=4" width="100px;"/><br/><sub><b>fishbar</b></sub>](https://github.com/fishbar)<br/>|[<img src="https://avatars2.githubusercontent.com/u/1207064?v=4" width="100px;"/><br/><sub><b>gxcsoccer</b></sub>](https://github.com/gxcsoccer)<br/>
|
453
458
|
|
454
|
-
This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `
|
459
|
+
This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Sat Jan 16 2021 23:16:33 GMT+0800`.
|
455
460
|
|
456
461
|
<!-- GITCONTRIBUTOR_END -->
|
457
462
|
|
458
463
|
## License
|
459
464
|
|
460
|
-
[MIT](LICENSE
|
465
|
+
[MIT](LICENSE)
|
461
466
|
|
462
467
|
|
463
468
|
[bluebird]: https://github.com/petkaantonov/bluebird
|
package/lib/index.d.ts
CHANGED
@@ -130,6 +130,10 @@ export interface RequestOptions {
|
|
130
130
|
* It rely on lookup and have the same version requirement.
|
131
131
|
*/
|
132
132
|
checkAddress?: (ip: string, family: number | string) => boolean;
|
133
|
+
/**
|
134
|
+
* UNIX domain socket path. (Windows is not supported)
|
135
|
+
*/
|
136
|
+
socketPath?: string;
|
133
137
|
}
|
134
138
|
|
135
139
|
export interface HttpClientResponse<T> {
|
package/lib/urllib.js
CHANGED
@@ -138,6 +138,7 @@ var SOCKET_RESPONSE_COUNT = '_URLLIB_SOCKET_RESPONSE_COUNT';
|
|
138
138
|
* Require node >= 4.0.0 and only work on `http` protocol.
|
139
139
|
* - {Boolean} [enableProxy]: optional, enable proxy request. Default is `false`.
|
140
140
|
* - {String|Object} [proxy]: optional proxy agent uri or options. Default is `null`.
|
141
|
+
* - {String} [socketPath]: optional, unix domain socket file path.
|
141
142
|
* - {Function} checkAddress: optional, check request address to protect from SSRF and similar attacks.
|
142
143
|
* @param {Function} [callback]: callback(error, data, res). If missing callback, will return a promise object.
|
143
144
|
* @return {HttpRequest} req object.
|
@@ -315,6 +316,9 @@ function requestWithCallback(url, args, callback) {
|
|
315
316
|
options.headers[key] = args.headers[name];
|
316
317
|
}
|
317
318
|
}
|
319
|
+
if (args.socketPath) {
|
320
|
+
options.socketPath = args.socketPath;
|
321
|
+
}
|
318
322
|
|
319
323
|
var sslNames = [
|
320
324
|
'pfx',
|
@@ -516,6 +520,7 @@ function requestWithCallback(url, args, callback) {
|
|
516
520
|
}
|
517
521
|
return;
|
518
522
|
}
|
523
|
+
|
519
524
|
var cb = callback;
|
520
525
|
callback = null;
|
521
526
|
var headers = {};
|
@@ -525,49 +530,16 @@ function requestWithCallback(url, args, callback) {
|
|
525
530
|
headers = res.headers;
|
526
531
|
}
|
527
532
|
|
528
|
-
|
529
|
-
|
530
|
-
&& !options.headers.authorization && args.digestAuth) {
|
531
|
-
var authenticate = headers['www-authenticate'];
|
532
|
-
if (authenticate.indexOf('Digest ') >= 0) {
|
533
|
-
debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', reqId, url, authenticate);
|
534
|
-
options.headers.authorization = digestAuthHeader(options.method, options.path, authenticate, args.digestAuth);
|
535
|
-
debug('Request#%d %s: auth with digest header: %s', reqId, url, options.headers.authorization);
|
536
|
-
if (res.headers['set-cookie']) {
|
537
|
-
options.headers.cookie = res.headers['set-cookie'].join(';');
|
538
|
-
}
|
539
|
-
args.headers = options.headers;
|
540
|
-
return exports.requestWithCallback(url, args, cb);
|
541
|
-
}
|
533
|
+
if (handleDigestAuth(res, cb)) {
|
534
|
+
return;
|
542
535
|
}
|
543
536
|
|
544
|
-
var
|
545
|
-
if (timing) {
|
546
|
-
timing.contentDownload = requestUseTime;
|
547
|
-
}
|
537
|
+
var response = createCallbackResponse(data, res);
|
548
538
|
|
549
539
|
debug('[%sms] done, %s bytes HTTP %s %s %s %s, keepAliveSocket: %s, timing: %j, socketHandledRequests: %s, socketHandledResponses: %s',
|
550
|
-
requestUseTime, responseSize, statusCode, options.method, options.host, options.path,
|
540
|
+
response.requestUseTime, responseSize, statusCode, options.method, options.host, options.path,
|
551
541
|
keepAliveSocket, timing, socketHandledRequests, socketHandledResponses);
|
552
542
|
|
553
|
-
var response = {
|
554
|
-
status: statusCode,
|
555
|
-
statusCode: statusCode,
|
556
|
-
statusMessage: statusMessage,
|
557
|
-
headers: headers,
|
558
|
-
size: responseSize,
|
559
|
-
aborted: responseAborted,
|
560
|
-
rt: requestUseTime,
|
561
|
-
keepAliveSocket: keepAliveSocket,
|
562
|
-
data: data,
|
563
|
-
requestUrls: args.requestUrls,
|
564
|
-
timing: timing,
|
565
|
-
remoteAddress: remoteAddress,
|
566
|
-
remotePort: remotePort,
|
567
|
-
socketHandledRequests: socketHandledRequests,
|
568
|
-
socketHandledResponses: socketHandledResponses,
|
569
|
-
};
|
570
|
-
|
571
543
|
if (err) {
|
572
544
|
var agentStatus = '';
|
573
545
|
if (agent && typeof agent.getCurrentStatus === 'function') {
|
@@ -616,6 +588,44 @@ function requestWithCallback(url, args, callback) {
|
|
616
588
|
|
617
589
|
cb(err, data, args.streaming ? res : response);
|
618
590
|
|
591
|
+
emitResponseEvent(err, response);
|
592
|
+
}
|
593
|
+
|
594
|
+
function createAndEmitResponseEvent(data, res) {
|
595
|
+
var response = createCallbackResponse(data, res);
|
596
|
+
emitResponseEvent(null, response);
|
597
|
+
}
|
598
|
+
|
599
|
+
function createCallbackResponse(data, res) {
|
600
|
+
var requestUseTime = Date.now() - requestStartTime;
|
601
|
+
if (timing) {
|
602
|
+
timing.contentDownload = requestUseTime;
|
603
|
+
}
|
604
|
+
|
605
|
+
var headers = res && res.headers || {};
|
606
|
+
var resStatusCode = res && res.statusCode || statusCode;
|
607
|
+
var resStatusMessage = res && res.statusMessage || statusMessage;
|
608
|
+
|
609
|
+
return {
|
610
|
+
status: resStatusCode,
|
611
|
+
statusCode: resStatusCode,
|
612
|
+
statusMessage: resStatusMessage,
|
613
|
+
headers: headers,
|
614
|
+
size: responseSize,
|
615
|
+
aborted: responseAborted,
|
616
|
+
rt: requestUseTime,
|
617
|
+
keepAliveSocket: keepAliveSocket,
|
618
|
+
data: data,
|
619
|
+
requestUrls: args.requestUrls,
|
620
|
+
timing: timing,
|
621
|
+
remoteAddress: remoteAddress,
|
622
|
+
remotePort: remotePort,
|
623
|
+
socketHandledRequests: socketHandledRequests,
|
624
|
+
socketHandledResponses: socketHandledResponses,
|
625
|
+
};
|
626
|
+
}
|
627
|
+
|
628
|
+
function emitResponseEvent(err, response) {
|
619
629
|
if (args.emitter) {
|
620
630
|
// keep to use the same reqMeta object on request event before
|
621
631
|
reqMeta.url = parsedUrl.href;
|
@@ -633,6 +643,30 @@ function requestWithCallback(url, args, callback) {
|
|
633
643
|
}
|
634
644
|
}
|
635
645
|
|
646
|
+
function handleDigestAuth(res, cb) {
|
647
|
+
var headers = {};
|
648
|
+
if (res && res.headers) {
|
649
|
+
headers = res.headers;
|
650
|
+
}
|
651
|
+
// handle digest auth
|
652
|
+
if (statusCode === 401 && headers['www-authenticate']
|
653
|
+
&& !options.headers.authorization && args.digestAuth) {
|
654
|
+
var authenticate = headers['www-authenticate'];
|
655
|
+
if (authenticate.indexOf('Digest ') >= 0) {
|
656
|
+
debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', reqId, url, authenticate);
|
657
|
+
options.headers.authorization = digestAuthHeader(options.method, options.path, authenticate, args.digestAuth);
|
658
|
+
debug('Request#%d %s: auth with digest header: %s', reqId, url, options.headers.authorization);
|
659
|
+
if (res.headers['set-cookie']) {
|
660
|
+
options.headers.cookie = res.headers['set-cookie'].join(';');
|
661
|
+
}
|
662
|
+
args.headers = options.headers;
|
663
|
+
exports.requestWithCallback(url, args, cb);
|
664
|
+
return true;
|
665
|
+
}
|
666
|
+
}
|
667
|
+
return false;
|
668
|
+
}
|
669
|
+
|
636
670
|
function handleRedirect(res) {
|
637
671
|
var err = null;
|
638
672
|
if (args.followRedirect && statuses.redirect[res.statusCode]) { // handle redirect
|
@@ -736,6 +770,7 @@ function requestWithCallback(url, args, callback) {
|
|
736
770
|
var result = handleRedirect(res);
|
737
771
|
if (result.redirect) {
|
738
772
|
res.resume();
|
773
|
+
createAndEmitResponseEvent(null, res);
|
739
774
|
return;
|
740
775
|
}
|
741
776
|
if (result.error) {
|
@@ -777,6 +812,7 @@ function requestWithCallback(url, args, callback) {
|
|
777
812
|
var result = handleRedirect(res);
|
778
813
|
if (result.redirect) {
|
779
814
|
res.resume();
|
815
|
+
createAndEmitResponseEvent(null, res);
|
780
816
|
return;
|
781
817
|
}
|
782
818
|
if (result.error) {
|
@@ -870,6 +906,7 @@ function requestWithCallback(url, args, callback) {
|
|
870
906
|
return done(result.error, body, res);
|
871
907
|
}
|
872
908
|
if (result.redirect) {
|
909
|
+
createAndEmitResponseEvent(null, res);
|
873
910
|
return;
|
874
911
|
}
|
875
912
|
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "urllib",
|
3
|
-
"version": "2.
|
3
|
+
"version": "2.37.3",
|
4
4
|
"description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more.",
|
5
5
|
"keywords": [
|
6
6
|
"urllib",
|
@@ -42,7 +42,7 @@
|
|
42
42
|
"humanize-ms": "^1.2.0",
|
43
43
|
"iconv-lite": "^0.4.15",
|
44
44
|
"ip": "^1.1.5",
|
45
|
-
"proxy-agent": "^
|
45
|
+
"proxy-agent": "^4.0.1",
|
46
46
|
"pump": "^3.0.0",
|
47
47
|
"qs": "^6.4.0",
|
48
48
|
"statuses": "^1.3.1",
|
@@ -83,7 +83,7 @@
|
|
83
83
|
"os": {
|
84
84
|
"github": "linux, windows, macos"
|
85
85
|
},
|
86
|
-
"version": "
|
86
|
+
"version": "8, 10, 12, 14"
|
87
87
|
},
|
88
88
|
"license": "MIT"
|
89
89
|
}
|