urllib 3.0.0-alpha.1 → 3.0.2

package/README.md

@@ -79,58 +79,27 @@ console.log('status: %s, body size: %d, headers: %j', res.statusCode, data.lengt
 - ***options*** Object - Optional
   - ***method*** String - Request method, defaults to `GET`. Could be `GET`, `POST`, `DELETE` or `PUT`. Alias 'type'.
   - ***data*** Object - Data to be sent. Will be stringify automatically.
-  - ***dataAsQueryString*** Boolean - Force convert `data` to query string.
   - ***content*** String | [Buffer](http://nodejs.org/api/buffer.html) - Manually set the content of payload. If set, `data` will be ignored.
   - ***stream*** [stream.Readable](http://nodejs.org/api/stream.html#stream_class_stream_readable) - Stream to be pipe to the remote. If set, `data` and `content` will be ignored.
   - ***writeStream*** [stream.Writable](http://nodejs.org/api/stream.html#stream_class_stream_writable) - A writable stream to be piped by the response stream. Responding data will be write to this stream and `callback` will be called with `data` set `null` after finished writing.
   - ***files*** {Array<ReadStream|Buffer|String> | Object | ReadStream | Buffer | String - The files will send with `multipart/form-data` format, base on `formstream`. If `method` not set, will use `POST` method by default.
-  - ***consumeWriteStream*** [true] - consume the writeStream, invoke the callback after writeStream close.
   - ***contentType*** String - Type of request data. Could be `json` (**Notes**: not use `application/json` here). If it's `json`, will auto set `Content-Type: application/json` header.
-  - ***nestedQuerystring*** Boolean - urllib default use querystring to stringify form data which don't support nested object, will use [qs](https://github.com/ljharb/qs) instead of querystring to support nested object by set this option to true.
   - ***dataType*** String - Type of response data. Could be `text` or `json`. If it's `text`, the `callback`ed `data` would be a String. If it's `json`, the `data` of callback would be a parsed JSON Object and will auto set `Accept: application/json` header. Default `callback`ed `data` would be a `Buffer`.
   - **fixJSONCtlChars** Boolean - Fix the control characters (U+0000 through U+001F) before JSON parse response. Default is `false`.
   - ***headers*** Object - Request headers.
-  - ***keepHeaderCase*** Boolean - by default will convert header keys to lowercase
   - ***timeout*** Number | Array - Request timeout in milliseconds for connecting phase and response receiving phase. Defaults to `exports.TIMEOUT`, both are 5s. You can use `timeout: 5000` to tell urllib use same timeout on two phase or set them seperately such as `timeout: [3000, 5000]`, which will set connecting timeout to 3s and response 5s.
   - ***auth*** String - `username:password` used in HTTP Basic Authorization.
-  - ***digestAuth*** String - `username:password` used in HTTP [Digest Authorization](http://en.wikipedia.org/wiki/Digest_access_authentication).
   - ***agent*** [http.Agent](http://nodejs.org/api/http.html#http_class_http_agent) - HTTP Agent object.
       Set `false` if you does not use agent.
   - ***httpsAgent*** [https.Agent](http://nodejs.org/api/https.html#https_class_https_agent) - HTTPS Agent object.
       Set `false` if you does not use agent.
-  - ***ca*** String | Buffer | Array - An array of strings or Buffers of trusted certificates.
-      If this is omitted several well known "root" CAs will be used, like VeriSign.
-      These are used to authorize connections.
-      **Notes**: This is necessary only if the server uses the self-signed certificate
-  - ***rejectUnauthorized*** Boolean - If true, the server certificate is verified against the list of supplied CAs.
-      An 'error' event is emitted if verification fails. Default: true.
-  - ***pfx*** String | Buffer - A string or Buffer containing the private key,
-      certificate and CA certs of the server in PFX or PKCS12 format.
-  - ***key*** String | Buffer - A string or Buffer containing the private key of the client in PEM format.
-      **Notes**: This is necessary only if using the client certificate authentication
-  - ***cert*** String | Buffer - A string or Buffer containing the certificate key of the client in PEM format.
-      **Notes**: This is necessary only if using the client certificate authentication
-  - ***passphrase*** String - A string of passphrase for the private key or pfx.
-  - ***ciphers*** String - A string describing the ciphers to use or exclude.
-  - ***secureProtocol*** String - The SSL method to use, e.g. SSLv3_method to force SSL version 3.
   - ***followRedirect*** Boolean - follow HTTP 3xx responses as redirects. defaults to false.
   - ***maxRedirects*** Number - The maximum number of redirects to follow, defaults to 10.
   - ***formatRedirectUrl*** Function - Format the redirect url by your self. Default is `url.resolve(from, to)`.
   - ***beforeRequest*** Function - Before request hook, you can change every thing here.
   - ***streaming*** Boolean - let you get the `res` object when request  connected, default `false`. alias `customResponse`
-  - ***gzip*** Boolean - Accept gzip response content and auto decode it, default is `false`.
+  - ***gzip*** Boolean - Accept `gzip, br` response content and auto decode it, default is `false`.
   - ***timing*** Boolean - Enable timing or not, default is `false`.
-  - ***enableProxy*** Boolean - Enable proxy request, default is `false`.
-  - ***proxy*** String | Object - proxy agent uri or options, default is `null`.
-  - ***lookup*** Function - Custom DNS lookup function, default is `dns.lookup`. Require node >= 4.0.0(for http protocol) and node >=8(for https protocol)
-  - ***checkAddress*** Function: optional, check request address to protect from SSRF and similar attacks. It receive tow arguments(`ip` and `family`) and should return true or false to identified the address is legal or not. It rely on `lookup` and have the same version requirement.
-  - ***trace*** Boolean - Enable capture stack include call site of library entrance, default is `false`.
-  - ***socketPath*** String - optional Unix Domain Socket. (Refer to [Node.js Document](https://nodejs.org/dist/latest-v14.x/docs/api/http.html#http_http_request_options_callback))
-
-#### Returns
-
-- **data** Buffer | Object - The data responsed. Would be a Buffer if `dataType` is set to `text` or an JSON parsed into Object if it's set to `json`.
-- **res** [http.IncomingMessage](http://nodejs.org/api/http.html#http_http_incomingmessage) - The response.
 
 #### Options: `options.data`
 
@@ -150,14 +119,14 @@ For `GET` request, `data` will be stringify to query string, e.g. `http://exampl
 
 For others like `POST`, `PATCH` or `PUT` request,
 in defaults, the `data` will be stringify into `application/x-www-form-urlencoded` format
-if `Content-Type` header is not set.
+if `content-type` header is not set.
 
-If `Content-type` is `application/json`, the `data` will be `JSON.stringify` to JSON data format.
+If `content-type` is `application/json`, the `data` will be `JSON.stringify` to JSON data format.
 
 #### Options: `options.content`
 
 `options.content` is useful when you wish to construct the request body by yourself,
-for example making a `Content-Type: application/json` request.
+for example making a `content-type: application/json` request.
 
 Notes that if you want to send a JSON body, you should stringify it yourself:
 
@@ -178,8 +147,8 @@ It would make a HTTP request like:
 
 ```http
 POST / HTTP/1.1
-Host: example.com
-Content-Type: application/json
+host: example.com
+content-type: application/json
 
 {
   "a": "hello",
@@ -193,7 +162,7 @@ This exmaple can use `options.data` with `application/json` content type:
 await request('https://example.com', {
   method: 'POST',
   headers: {
-    'Content-Type': 'application/json'
+    'content-type': 'application/json'
   },
   data: {
     a: 'hello',
@@ -268,7 +237,6 @@ Response is normal object, it contains:
 - `status` or `statusCode`: response status code.
   - `-1` meaning some network error like `ENOTFOUND`
   - `-2` meaning ConnectionTimeoutError
-- `statusMessage`: response status message.
 - `headers`: response http headers, default is `{}`
 - `size`: response size
 - `aborted`: response was aborted or not
@@ -279,75 +247,72 @@ Response is normal object, it contains:
 - `socketHandledRequests`: socket already handled request count
 - `socketHandledResponses`: socket already handled response count
 
-#### Response: `res.aborted`
-
-If the underlaying connection was terminated before `response.end()` was called,
-`res.aborted` should be `true`.
+## Run test with debug log
 
-```js
-import { createServer } from 'http';
-
-createServer((req, res) => {
-  req.resume();
-  req.on('end', () => {
-    res.write('foo haha\n');
-    setTimeout(() => {
-      res.write('foo haha 2');
-      setTimeout(() => {
-        res.socket.end();
-      }, 300);
-    }, 200);
-    return;
-  });
-}).listen(2022);
-
-const { data, res } = await request('http://127.0.0.1:2022/socket.end');
-assert.equal(data.toString(), 'foo haha\nfoo haha 2');
-assert(res.aborted);
+```bash
+NODE_DEBUG=urllib npm test
 ```
 
-### HttpClient2
+## Mocking Request
 
-HttpClient2 base on HttpClient.
+export from [undici](https://undici.nodejs.org/#/docs/best-practices/mocking-request)
 
-#### Options
+```ts
+import { strict as assert } from 'assert';
+import { MockAgent, setGlobalDispatcher, request } from 'urllib';
 
-options extends from urllib, besides below
+const mockAgent = new MockAgent();
+setGlobalDispatcher(mockAgent);
 
-- ***retry*** Number - a retry count, when get an error, it will request again until reach the retry count.
-- ***retryDelay*** Number - wait a delay(ms) between retries.
-- ***isRetry*** Function - determine whether retry, a response object as the first argument. it will retry when status >= 500 by default. Request error is not included.
+const mockPool = mockAgent.get('http://localhost:7001');
 
-#### Warning
+mockPool.intercept({
+  path: '/foo',
+  method: 'POST',
+}).reply(400, {
+  message: 'mock 400 bad request',
+});
 
-It's not supported by using retry and writeStream, because the retry request can't stop the stream which is consuming.
+const response = await request('http://localhost:7001/foo', {
+  method: 'POST',
+  dataType: 'json',
+});
+assert.equal(response.status, 400);
+assert.deepEqual(response.data, { message: 'mock 400 bad request' });
+```
 
-### Trace
+## Benchmarks
 
-If set trace true, error stack will contains full call stack, like
+Fork [undici benchmarks script](https://github.com/fengmk2/undici/blob/urllib-benchmark/benchmarks/benchmark.js)
 
-```
-Error: connect ECONNREFUSED 127.0.0.1:11
-    at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1113:14)
-    --------------------
-    at ~/workspace/urllib/lib/urllib.js:150:13
-    at new Promise (<anonymous>)
-    at Object.request (~/workspace/urllib/lib/urllib.js:149:10)
-    at Context.<anonymous> (~/workspace/urllib/test/urllib_promise.test.js:49:19)
-    ....
-```
+### Connections 1
+
+| Tests               | Samples |        Result | Tolerance | Difference with slowest |
+|---------------------|---------|---------------|-----------|-------------------------|
+| http - no keepalive |     15 | 6.38 req/sec | ± 2.44 % |                      - |
+| http - keepalive    |     10 | 6.77 req/sec | ± 2.35 % |               + 6.13 % |
+| urllib2 - request   |     45 | 40.13 req/sec | ± 2.88 % |             + 528.66 % |
+| urllib3 - request   |     10 | 58.51 req/sec | ± 2.52 % |             + 816.64 % |
+| undici - pipeline   |      5 | 59.12 req/sec | ± 2.47 % |             + 826.18 % |
+| undici - fetch      |     15 | 60.42 req/sec | ± 3.00 % |             + 846.60 % |
+| undici - dispatch   |      5 | 60.58 req/sec | ± 1.39 % |             + 848.99 % |
+| undici - stream     |      5 | 61.30 req/sec | ± 1.31 % |             + 860.39 % |
+| undici - request    |      5 | 61.74 req/sec | ± 2.03 % |             + 867.20 % |
 
-When open the trace, urllib may have poor perfomance, please consider carefully.
+### Connections 50
 
-## TODO
+| Tests               | Samples |           Result | Tolerance | Difference with slowest |
+|---------------------|---------|------------------|-----------|-------------------------|
+| urllib2 - request   |     51 | 1465.40 req/sec | ± 14.40 % |                      - |
+| undici - fetch      |     40 | 3121.10 req/sec | ± 2.82 % |             + 112.99 % |
+| http - no keepalive |     45 | 3355.42 req/sec | ± 2.84 % |             + 128.98 % |
+| http - keepalive    |     51 | 5179.55 req/sec | ± 36.61 % |             + 253.46 % |
+| urllib3 - request   |     30 | 7045.86 req/sec | ± 2.93 % |             + 380.82 % |
+| undici - pipeline   |     50 | 8306.92 req/sec | ± 2.99 % |             + 466.87 % |
+| undici - request    |     51 | 9552.59 req/sec | ± 13.13 % |             + 551.88 % |
+| undici - stream     |     45 | 12523.45 req/sec | ± 2.97 % |             + 754.61 % |
+| undici - dispatch   |     51 | 12970.18 req/sec | ± 3.15 % |             + 785.10 % |
 
-- ❎ Support Proxy
-- ✅ Upload file like form upload
-- ✅ Auto redirect handle
-- ✅ https & self-signed certificate
-- ✅ Connection timeout & Response timeout
-- ✅ Support `Accept-Encoding=gzip` by default
-- ✅ Support [Digest access authentication](http://en.wikipedia.org/wiki/Digest_access_authentication)
 
 <!-- GITCONTRIBUTOR_START -->
 
@@ -356,18 +321,16 @@ When open the trace, urllib may have poor perfomance, please consider carefully.
 |[<img src="https://avatars.githubusercontent.com/u/156269?v=4" width="100px;"/><br/><sub><b>fengmk2</b></sub>](https://github.com/fengmk2)<br/>|[<img src="https://avatars.githubusercontent.com/u/985607?v=4" width="100px;"/><br/><sub><b>dead-horse</b></sub>](https://github.com/dead-horse)<br/>|[<img src="https://avatars.githubusercontent.com/u/288288?v=4" width="100px;"/><br/><sub><b>xingrz</b></sub>](https://github.com/xingrz)<br/>|[<img src="https://avatars.githubusercontent.com/u/360661?v=4" width="100px;"/><br/><sub><b>popomore</b></sub>](https://github.com/popomore)<br/>|[<img src="https://avatars.githubusercontent.com/u/327019?v=4" width="100px;"/><br/><sub><b>JacksonTian</b></sub>](https://github.com/JacksonTian)<br/>|[<img src="https://avatars.githubusercontent.com/u/543405?v=4" width="100px;"/><br/><sub><b>ibigbug</b></sub>](https://github.com/ibigbug)<br/>|
 | :---: | :---: | :---: | :---: | :---: | :---: |
 |[<img src="https://avatars.githubusercontent.com/u/14790466?v=4" width="100px;"/><br/><sub><b>greenkeeperio-bot</b></sub>](https://github.com/greenkeeperio-bot)<br/>|[<img src="https://avatars.githubusercontent.com/u/227713?v=4" width="100px;"/><br/><sub><b>atian25</b></sub>](https://github.com/atian25)<br/>|[<img src="https://avatars.githubusercontent.com/u/5381764?v=4" width="100px;"/><br/><sub><b>paambaati</b></sub>](https://github.com/paambaati)<br/>|[<img src="https://avatars.githubusercontent.com/u/1433247?v=4" width="100px;"/><br/><sub><b>denghongcai</b></sub>](https://github.com/denghongcai)<br/>|[<img src="https://avatars.githubusercontent.com/u/2842176?v=4" width="100px;"/><br/><sub><b>XadillaX</b></sub>](https://github.com/XadillaX)<br/>|[<img src="https://avatars.githubusercontent.com/u/1147375?v=4" width="100px;"/><br/><sub><b>alsotang</b></sub>](https://github.com/alsotang)<br/>|
-|[<img src="https://avatars.githubusercontent.com/u/546535?v=4" width="100px;"/><br/><sub><b>leoner</b></sub>](https://github.com/leoner)<br/>|[<img src="https://avatars.githubusercontent.com/u/19908330?v=4" width="100px;"/><br/><sub><b>hyj1991</b></sub>](https://github.com/hyj1991)<br/>|[<img src="https://avatars.githubusercontent.com/u/1747852?v=4" width="100px;"/><br/><sub><b>isayme</b></sub>](https://github.com/isayme)<br/>|[<img src="https://avatars.githubusercontent.com/u/252317?v=4" width="100px;"/><br/><sub><b>cyjake</b></sub>](https://github.com/cyjake)<br/>|[<img src="https://avatars.githubusercontent.com/u/5856440?v=4" width="100px;"/><br/><sub><b>whxaxes</b></sub>](https://github.com/whxaxes)<br/>|[<img src="https://avatars.githubusercontent.com/u/309219?v=4" width="100px;"/><br/><sub><b>chadxz</b></sub>](https://github.com/chadxz)<br/>|
-|[<img src="https://avatars.githubusercontent.com/u/5139554?v=4" width="100px;"/><br/><sub><b>danielwpz</b></sub>](https://github.com/danielwpz)<br/>|[<img src="https://avatars.githubusercontent.com/u/5127897?v=4" width="100px;"/><br/><sub><b>danielsss</b></sub>](https://github.com/danielsss)<br/>|[<img src="https://avatars.githubusercontent.com/u/3367820?v=4" width="100px;"/><br/><sub><b>Jeff-Tian</b></sub>](https://github.com/Jeff-Tian)<br/>|[<img src="https://avatars.githubusercontent.com/u/32407?v=4" width="100px;"/><br/><sub><b>jedahan</b></sub>](https://github.com/jedahan)<br/>|[<img src="https://avatars.githubusercontent.com/u/17075261?v=4" width="100px;"/><br/><sub><b>nick-ng</b></sub>](https://github.com/nick-ng)<br/>|[<img src="https://avatars.githubusercontent.com/u/1706595?v=4" width="100px;"/><br/><sub><b>rishavsharan</b></sub>](https://github.com/rishavsharan)<br/>|
-|[<img src="https://avatars.githubusercontent.com/u/1886161?v=4" width="100px;"/><br/><sub><b>willizm</b></sub>](https://github.com/willizm)<br/>|[<img src="https://avatars.githubusercontent.com/u/7227589?v=4" width="100px;"/><br/><sub><b>davidkhala</b></sub>](https://github.com/davidkhala)<br/>|[<img src="https://avatars.githubusercontent.com/u/535479?v=4" width="100px;"/><br/><sub><b>aleafs</b></sub>](https://github.com/aleafs)<br/>|[<img src="https://avatars.githubusercontent.com/u/3689968?v=4" width="100px;"/><br/><sub><b>Amunu</b></sub>](https://github.com/Amunu)<br/>|[<img src="https://avatars.githubusercontent.com/in/9426?v=4" width="100px;"/><br/><sub><b>azure-pipelines[bot]</b></sub>](https://github.com/apps/azure-pipelines)<br/>|[<img src="https://avatars.githubusercontent.com/u/1281323?v=4" width="100px;"/><br/><sub><b>changzhiwin</b></sub>](https://github.com/changzhiwin)<br/>|
-|[<img src="https://avatars.githubusercontent.com/u/929503?v=4" width="100px;"/><br/><sub><b>yuzhigang33</b></sub>](https://github.com/yuzhigang33)<br/>|[<img src="https://avatars.githubusercontent.com/u/981128?v=4" width="100px;"/><br/><sub><b>fishbar</b></sub>](https://github.com/fishbar)<br/>|[<img src="https://avatars.githubusercontent.com/u/1207064?v=4" width="100px;"/><br/><sub><b>gxcsoccer</b></sub>](https://github.com/gxcsoccer)<br/>|[<img src="https://avatars.githubusercontent.com/u/6897780?v=4" width="100px;"/><br/><sub><b>killagu</b></sub>](https://github.com/killagu)<br/>|[<img src="https://avatars.githubusercontent.com/u/17476119?v=4" width="100px;"/><br/><sub><b>mars-coder</b></sub>](https://github.com/mars-coder)<br/>|[<img src="https://avatars.githubusercontent.com/u/929179?v=4" width="100px;"/><br/><sub><b>rockdai</b></sub>](https://github.com/rockdai)<br/>|
+|[<img src="https://avatars.githubusercontent.com/u/546535?v=4" width="100px;"/><br/><sub><b>leoner</b></sub>](https://github.com/leoner)<br/>|[<img src="https://avatars.githubusercontent.com/u/19908330?v=4" width="100px;"/><br/><sub><b>hyj1991</b></sub>](https://github.com/hyj1991)<br/>|[<img src="https://avatars.githubusercontent.com/u/1747852?v=4" width="100px;"/><br/><sub><b>isayme</b></sub>](https://github.com/isayme)<br/>|[<img src="https://avatars.githubusercontent.com/u/6897780?v=4" width="100px;"/><br/><sub><b>killagu</b></sub>](https://github.com/killagu)<br/>|[<img src="https://avatars.githubusercontent.com/u/252317?v=4" width="100px;"/><br/><sub><b>cyjake</b></sub>](https://github.com/cyjake)<br/>|[<img src="https://avatars.githubusercontent.com/u/5856440?v=4" width="100px;"/><br/><sub><b>whxaxes</b></sub>](https://github.com/whxaxes)<br/>|
+|[<img src="https://avatars.githubusercontent.com/u/309219?v=4" width="100px;"/><br/><sub><b>chadxz</b></sub>](https://github.com/chadxz)<br/>|[<img src="https://avatars.githubusercontent.com/u/5139554?v=4" width="100px;"/><br/><sub><b>danielwpz</b></sub>](https://github.com/danielwpz)<br/>|[<img src="https://avatars.githubusercontent.com/u/5127897?v=4" width="100px;"/><br/><sub><b>danielsss</b></sub>](https://github.com/danielsss)<br/>|[<img src="https://avatars.githubusercontent.com/u/3367820?v=4" width="100px;"/><br/><sub><b>Jeff-Tian</b></sub>](https://github.com/Jeff-Tian)<br/>|[<img src="https://avatars.githubusercontent.com/u/32407?v=4" width="100px;"/><br/><sub><b>jedahan</b></sub>](https://github.com/jedahan)<br/>|[<img src="https://avatars.githubusercontent.com/u/17075261?v=4" width="100px;"/><br/><sub><b>nick-ng</b></sub>](https://github.com/nick-ng)<br/>|
+|[<img src="https://avatars.githubusercontent.com/u/1706595?v=4" width="100px;"/><br/><sub><b>rishavsharan</b></sub>](https://github.com/rishavsharan)<br/>|[<img src="https://avatars.githubusercontent.com/u/1886161?v=4" width="100px;"/><br/><sub><b>willizm</b></sub>](https://github.com/willizm)<br/>|[<img src="https://avatars.githubusercontent.com/u/7227589?v=4" width="100px;"/><br/><sub><b>davidkhala</b></sub>](https://github.com/davidkhala)<br/>|[<img src="https://avatars.githubusercontent.com/u/535479?v=4" width="100px;"/><br/><sub><b>aleafs</b></sub>](https://github.com/aleafs)<br/>|[<img src="https://avatars.githubusercontent.com/u/3689968?v=4" width="100px;"/><br/><sub><b>Amunu</b></sub>](https://github.com/Amunu)<br/>|[<img src="https://avatars.githubusercontent.com/in/9426?v=4" width="100px;"/><br/><sub><b>azure-pipelines[bot]</b></sub>](https://github.com/apps/azure-pipelines)<br/>|
+|[<img src="https://avatars.githubusercontent.com/u/1281323?v=4" width="100px;"/><br/><sub><b>changzhiwin</b></sub>](https://github.com/changzhiwin)<br/>|[<img src="https://avatars.githubusercontent.com/u/929503?v=4" width="100px;"/><br/><sub><b>yuzhigang33</b></sub>](https://github.com/yuzhigang33)<br/>|[<img src="https://avatars.githubusercontent.com/u/981128?v=4" width="100px;"/><br/><sub><b>fishbar</b></sub>](https://github.com/fishbar)<br/>|[<img src="https://avatars.githubusercontent.com/u/1207064?v=4" width="100px;"/><br/><sub><b>gxcsoccer</b></sub>](https://github.com/gxcsoccer)<br/>|[<img src="https://avatars.githubusercontent.com/u/17476119?v=4" width="100px;"/><br/><sub><b>mars-coder</b></sub>](https://github.com/mars-coder)<br/>|[<img src="https://avatars.githubusercontent.com/u/929179?v=4" width="100px;"/><br/><sub><b>rockdai</b></sub>](https://github.com/rockdai)<br/>|
 [<img src="https://avatars.githubusercontent.com/u/2196373?v=4" width="100px;"/><br/><sub><b>dickeylth</b></sub>](https://github.com/dickeylth)<br/>|[<img src="https://avatars.githubusercontent.com/u/13050025?v=4" width="100px;"/><br/><sub><b>aladdin-add</b></sub>](https://github.com/aladdin-add)<br/>
 
-This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Fri Mar 25 2022 00:05:23 GMT+0800`.
+This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Tue Jul 05 2022 16:17:31 GMT+0800`.
 
 <!-- GITCONTRIBUTOR_END -->
 
 ## License
 
 [MIT](LICENSE)
-
-[bluebird]: https://github.com/petkaantonov/bluebird

package/package.json

@@ -1,6 +1,9 @@
 {
   "name": "urllib",
-  "version": "3.0.0-alpha.1",
+  "version": "3.0.2",
+  "publishConfig": {
+    "tag": "latest"
+  },
   "description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more. Base undici fetch API.",
   "keywords": [
     "urllib",
@@ -54,15 +57,19 @@
   "dependencies": {
     "default-user-agent": "^1.0.0",
     "digest-header": "^0.0.1",
-    "ip": "^1.1.5",
+    "form-data-encoder": "^1.7.2",
+    "formdata-node": "^4.3.3",
     "mime-types": "^2.1.35",
-    "undici": "^5.4.0"
+    "pump": "^3.0.0",
+    "undici": "^5.6.0"
   },
   "devDependencies": {
     "@types/busboy": "^1.5.0",
     "@types/default-user-agent": "^1.0.0",
     "@types/jest": "28",
     "@types/mime-types": "^2.1.1",
+    "@types/pump": "^1.1.1",
+    "@types/selfsigned": "^2.0.1",
     "busboy": "^1.6.0",
     "coffee": "5",
     "egg-ci": "2",
@@ -72,19 +79,16 @@
     "iconv-lite": "^0.6.3",
     "jest": "28",
     "jest-summary-reporter": "^0.0.2",
-    "p-event": "^4.2.0",
+    "selfsigned": "^2.0.1",
     "ts-jest": "28",
     "tslib": "^2.4.0",
     "typescript": "4"
   },
   "engines": {
-    "node": ">= 16.5.0"
+    "node": ">= 14.0.0"
   },
   "ci": {
-    "version": "16, 18"
-  },
-  "publishConfig": {
-    "tag": "next"
+    "version": "14, 16, 18"
   },
   "license": "MIT"
 }

package/src/HttpAgent.ts

@@ -0,0 +1,72 @@
+import dns from 'dns';
+import { LookupFunction, isIP } from 'net';
+import {
+  Agent,
+} from 'undici';
+import { DispatchHandlers } from 'undici/types/dispatcher';
+import { BuildOptions } from 'undici/types/connector';
+
+export type CheckAddressFunction = (ip: string, family: number | string) => boolean;
+
+export type HttpAgentOptions = {
+  lookup?: LookupFunction;
+  checkAddress?: CheckAddressFunction;
+  connect?: BuildOptions,
+};
+
+class IllegalAddressError extends Error {
+  hostname: string;
+  ip: string;
+  family: number;
+
+  constructor(hostname: string, ip: string, family: number) {
+    const message = 'illegal address';
+    super(message);
+    this.name = this.constructor.name;
+    this.hostname = hostname;
+    this.ip = ip;
+    this.family = family;
+    Error.captureStackTrace(this, this.constructor);
+  }
+}
+
+export class HttpAgent extends Agent {
+  #checkAddress?: CheckAddressFunction;
+
+  constructor(options: HttpAgentOptions) {
+    /* eslint node/prefer-promises/dns: off*/
+    const _lookup = options.lookup ?? dns.lookup;
+    const lookup: LookupFunction = (hostname, dnsOptions, callback) => {
+      _lookup(hostname, dnsOptions, (err, address, family) => {
+        if (err) return callback(err, address, family);
+        if (options.checkAddress && !options.checkAddress(address, family)) {
+          err = new IllegalAddressError(hostname, address, family);
+        }
+        callback(err, address, family);
+      });
+    };
+    super({
+      connect: { ...options.connect, lookup },
+    });
+    this.#checkAddress = options.checkAddress;
+  }
+
+  dispatch(options: Agent.DispatchOptions, handler: DispatchHandlers): boolean {
+    if (this.#checkAddress && options.origin) {
+      const originUrl = typeof options.origin === 'string' ? new URL(options.origin) : options.origin;
+      let hostname = originUrl.hostname;
+      // [2001:db8:2de::e13] => 2001:db8:2de::e13
+      if (hostname.startsWith('[') && hostname.endsWith(']')) {
+        hostname = hostname.substring(1, hostname.length - 1);
+      }
+      const family = isIP(hostname);
+      if (family === 4 || family === 6) {
+        // if request hostname is ip, custom lookup won't excute
+        if (!this.#checkAddress(hostname, family)) {
+          throw new IllegalAddressError(hostname, hostname, family);
+        }
+      }
+    }
+    return super.dispatch(options, handler);
+  }
+}