urllib 2.37.1 → 2.38.0

package/History.md

@@ -1,4 +1,32 @@
 
+2.38.0 / 2021-11-24
+==================
+
+**others**
+  * [[`6d19f99`](http://github.com/node-modules/urllib/commit/6d19f99f6490df67832d1d2a9c39743c0de1cbd5)] - tsd: add timing interface (#375) (弘树@阿里 <<dickeylth@users.noreply.github.com>>)
+  * [[`9c9c65b`](http://github.com/node-modules/urllib/commit/9c9c65b0e8344c1a30c157628a675440b8a616c1)] - test: check TypeScript type definitions (#373) (fengmk2 <<fengmk2@gmail.com>>)
+
+2.37.4 / 2021-09-07
+==================
+
+**fixes**
+  * [[`1e6622a`](http://github.com/node-modules/urllib/commit/1e6622a571b3e6f72c5f187a224bddedcbc382cf)] - fix: upgrade proxy-agent to fix the security vulnerability. (#368) (hyj1991 <<yeekwanvong@gmail.com>>)
+
+**others**
+  * [[`ae27498`](http://github.com/node-modules/urllib/commit/ae2749811e40262da8bf2641599b066344fa6b05)] - docs: notes contentType's value (#349) (changzhi <<changzhiwin@gmail.com>>)
+
+2.37.3 / 2021-07-05
+==================
+
+**fixes**
+  * [[`b730a6c`](http://github.com/node-modules/urllib/commit/b730a6cdc0465bea1c08d50bfa3b5e95bd17b31f)] - fix: 🐛 redirect status code (#363) (Hongcai Deng <<admin@dhchouse.com>>)
+
+2.37.2 / 2021-06-07
+==================
+
+**fixes**
+  * [[`04734a1`](http://github.com/node-modules/urllib/commit/04734a13e146da501f4e38333d5de9f9bbb259d1)] - fix: 🐛 should trigger response event when follow redirect (#361) (Hongcai Deng <<admin@dhchouse.com>>)
+
 2.37.1 / 2021-04-15
 ==================
 

package/README.md

@@ -1,21 +1,15 @@
 # urllib
 
 [![NPM version][npm-image]][npm-url]
-[![build status][travis-image]][travis-url]
-[![Build Status](https://dev.azure.com/eggjs/egg/_apis/build/status/node-modules.urllib)](https://dev.azure.com/eggjs/egg/_build/latest?definitionId=7)
+[![Node.js CI](https://github.com/node-modules/urllib/actions/workflows/nodejs.yml/badge.svg)](https://github.com/node-modules/urllib/actions/workflows/nodejs.yml)
 [![Test coverage][codecov-image]][codecov-url]
-[![David deps][david-image]][david-url]
 [![Known Vulnerabilities][snyk-image]][snyk-url]
 [![npm download][download-image]][download-url]
 
 [npm-image]: https://img.shields.io/npm/v/urllib.svg?style=flat-square
 [npm-url]: https://npmjs.org/package/urllib
-[travis-image]: https://img.shields.io/travis/node-modules/urllib.svg?style=flat-square
-[travis-url]: https://travis-ci.org/node-modules/urllib
 [codecov-image]: https://codecov.io/gh/node-modules/urllib/branch/master/graph/badge.svg
 [codecov-url]: https://codecov.io/gh/node-modules/urllib
-[david-image]: https://img.shields.io/david/node-modules/urllib.svg?style=flat-square
-[david-url]: https://david-dm.org/node-modules/urllib
 [snyk-image]: https://snyk.io/test/npm/urllib/badge.svg?style=flat-square
 [snyk-url]: https://snyk.io/test/npm/urllib
 [download-image]: https://img.shields.io/npm/dm/urllib.svg?style=flat-square
@@ -125,7 +119,7 @@ httpclient.request('http://nodejs.org', function (err, body) {
     - ***writeStream*** [stream.Writable](http://nodejs.org/api/stream.html#stream_class_stream_writable) - A writable stream to be piped by the response stream. Responding data will be write to this stream and `callback` will be called with `data` set `null` after finished writing.
     - ***files*** {Array<ReadStream|Buffer|String> | Object | ReadStream | Buffer | String - The files will send with `multipart/form-data` format, base on `formstream`. If `method` not set, will use `POST` method by default.
     - ***consumeWriteStream*** [true] - consume the writeStream, invoke the callback after writeStream close.
-    - ***contentType*** String - Type of request data. Could be `json`. If it's `json`, will auto set `Content-Type: application/json` header.
+    - ***contentType*** String - Type of request data. Could be `json` (**Notes**: not use `application/json` here). If it's `json`, will auto set `Content-Type: application/json` header.
     - ***nestedQuerystring*** Boolean - urllib default use querystring to stringify form data which don't support nested object, will use [qs](https://github.com/ljharb/qs) instead of querystring to support nested object by set this option to true.
     - ***dataType*** String - Type of response data. Could be `text` or `json`. If it's `text`, the `callback`ed `data` would be a String. If it's `json`, the `data` of callback would be a parsed JSON Object and will auto set `Accept: application/json` header. Default `callback`ed `data` would be a `Buffer`.
     - **fixJSONCtlChars** Boolean - Fix the control characters (U+0000 through U+001F) before JSON parse response. Default is `false`.

package/lib/index.d.ts

@@ -140,7 +140,19 @@ export interface HttpClientResponse<T> {
   data: T;
   status: number;
   headers:  OutgoingHttpHeaders;
-  res: http.IncomingMessage;
+  res: http.IncomingMessage & {
+    /**
+     * https://eggjs.org/en/core/httpclient.html#timing-boolean
+     */
+    timing?: {
+      queuing: number;
+      dnslookup: number;
+      connected: number;
+      requestSent: number;
+      waiting: number;
+      contentDownload: number;
+    }
+  };
 }
 
 

package/lib/index.test-d.ts

@@ -0,0 +1,10 @@
+import { expectType } from 'tsd';
+import { curl } from '..';
+
+// curl
+expectType<Buffer>((await curl<Buffer>('http://a.com')).data);
+// RequestOptions
+expectType<Buffer>((await curl<Buffer>('http://a.com', {})).data);
+expectType<string>((await curl<string>('http://a.com', {
+  method: 'HEAD',
+})).data);

package/lib/urllib.js

@@ -520,6 +520,7 @@ function requestWithCallback(url, args, callback) {
       }
       return;
     }
+
     var cb = callback;
     callback = null;
     var headers = {};
@@ -529,49 +530,16 @@ function requestWithCallback(url, args, callback) {
       headers = res.headers;
     }
 
-    // handle digest auth
-    if (statusCode === 401 && headers['www-authenticate']
-        && !options.headers.authorization && args.digestAuth) {
-      var authenticate = headers['www-authenticate'];
-      if (authenticate.indexOf('Digest ') >= 0) {
-        debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', reqId, url, authenticate);
-        options.headers.authorization = digestAuthHeader(options.method, options.path, authenticate, args.digestAuth);
-        debug('Request#%d %s: auth with digest header: %s', reqId, url, options.headers.authorization);
-        if (res.headers['set-cookie']) {
-          options.headers.cookie = res.headers['set-cookie'].join(';');
-        }
-        args.headers = options.headers;
-        return exports.requestWithCallback(url, args, cb);
-      }
+    if (handleDigestAuth(res, cb)) {
+      return;
     }
 
-    var requestUseTime = Date.now() - requestStartTime;
-    if (timing) {
-      timing.contentDownload = requestUseTime;
-    }
+    var response = createCallbackResponse(data, res);
 
     debug('[%sms] done, %s bytes HTTP %s %s %s %s, keepAliveSocket: %s, timing: %j, socketHandledRequests: %s, socketHandledResponses: %s',
-      requestUseTime, responseSize, statusCode, options.method, options.host, options.path,
+      response.requestUseTime, responseSize, statusCode, options.method, options.host, options.path,
       keepAliveSocket, timing, socketHandledRequests, socketHandledResponses);
 
-    var response = {
-      status: statusCode,
-      statusCode: statusCode,
-      statusMessage: statusMessage,
-      headers: headers,
-      size: responseSize,
-      aborted: responseAborted,
-      rt: requestUseTime,
-      keepAliveSocket: keepAliveSocket,
-      data: data,
-      requestUrls: args.requestUrls,
-      timing: timing,
-      remoteAddress: remoteAddress,
-      remotePort: remotePort,
-      socketHandledRequests: socketHandledRequests,
-      socketHandledResponses: socketHandledResponses,
-    };
-
     if (err) {
       var agentStatus = '';
       if (agent && typeof agent.getCurrentStatus === 'function') {
@@ -620,6 +588,44 @@ function requestWithCallback(url, args, callback) {
 
     cb(err, data, args.streaming ? res : response);
 
+    emitResponseEvent(err, response);
+  }
+
+  function createAndEmitResponseEvent(data, res) {
+    var response = createCallbackResponse(data, res);
+    emitResponseEvent(null, response);
+  }
+
+  function createCallbackResponse(data, res) {
+    var requestUseTime = Date.now() - requestStartTime;
+    if (timing) {
+      timing.contentDownload = requestUseTime;
+    }
+
+    var headers = res && res.headers || {};
+    var resStatusCode = res && res.statusCode || statusCode;
+    var resStatusMessage = res && res.statusMessage || statusMessage;
+
+    return {
+      status: resStatusCode,
+      statusCode: resStatusCode,
+      statusMessage: resStatusMessage,
+      headers: headers,
+      size: responseSize,
+      aborted: responseAborted,
+      rt: requestUseTime,
+      keepAliveSocket: keepAliveSocket,
+      data: data,
+      requestUrls: args.requestUrls,
+      timing: timing,
+      remoteAddress: remoteAddress,
+      remotePort: remotePort,
+      socketHandledRequests: socketHandledRequests,
+      socketHandledResponses: socketHandledResponses,
+    };
+  }
+
+  function emitResponseEvent(err, response) {
     if (args.emitter) {
       // keep to use the same reqMeta object on request event before
       reqMeta.url = parsedUrl.href;
@@ -637,6 +643,30 @@ function requestWithCallback(url, args, callback) {
     }
   }
 
+  function handleDigestAuth(res, cb) {
+    var headers = {};
+    if (res && res.headers) {
+      headers = res.headers;
+    }
+    // handle digest auth
+    if (statusCode === 401 && headers['www-authenticate']
+        && !options.headers.authorization && args.digestAuth) {
+      var authenticate = headers['www-authenticate'];
+      if (authenticate.indexOf('Digest ') >= 0) {
+        debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', reqId, url, authenticate);
+        options.headers.authorization = digestAuthHeader(options.method, options.path, authenticate, args.digestAuth);
+        debug('Request#%d %s: auth with digest header: %s', reqId, url, options.headers.authorization);
+        if (res.headers['set-cookie']) {
+          options.headers.cookie = res.headers['set-cookie'].join(';');
+        }
+        args.headers = options.headers;
+        exports.requestWithCallback(url, args, cb);
+        return true;
+      }
+    }
+    return false;
+  }
+
   function handleRedirect(res) {
     var err = null;
     if (args.followRedirect && statuses.redirect[res.statusCode]) {  // handle redirect
@@ -740,6 +770,7 @@ function requestWithCallback(url, args, callback) {
       var result = handleRedirect(res);
       if (result.redirect) {
         res.resume();
+        createAndEmitResponseEvent(null, res);
         return;
       }
       if (result.error) {
@@ -781,6 +812,7 @@ function requestWithCallback(url, args, callback) {
       var result = handleRedirect(res);
       if (result.redirect) {
         res.resume();
+        createAndEmitResponseEvent(null, res);
         return;
       }
       if (result.error) {
@@ -874,6 +906,7 @@ function requestWithCallback(url, args, callback) {
         return done(result.error, body, res);
       }
       if (result.redirect) {
+        createAndEmitResponseEvent(null, res);
         return;
       }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "urllib",
-  "version": "2.37.1",
+  "version": "2.38.0",
   "description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more.",
   "keywords": [
     "urllib",
@@ -23,10 +23,11 @@
     "url": "git://github.com/node-modules/urllib.git"
   },
   "scripts": {
+    "tsd": "node test/tsd.js",
     "test-local": "mocha -t 30000 -r intelli-espower-loader test/*.test.js",
     "test": "npm run lint && npm run test-local",
     "test-cov": "istanbul cover node_modules/mocha/bin/_mocha -- -t 30000 -r intelli-espower-loader test/*.test.js",
-    "ci": "npm run lint && npm run test-cov",
+    "ci": "npm run lint && npm run tsd && npm run test-cov",
     "lint": "jshint .",
     "autod": "autod -w --prefix '^' -t test -e examples",
     "contributor": "git-contributor"
@@ -42,7 +43,7 @@
     "humanize-ms": "^1.2.0",
     "iconv-lite": "^0.4.15",
     "ip": "^1.1.5",
-    "proxy-agent": "^4.0.1",
+    "proxy-agent": "^5.0.0",
     "pump": "^3.0.0",
     "qs": "^6.4.0",
     "statuses": "^1.3.1",
@@ -73,7 +74,8 @@
     "spy": "^1.0.0",
     "tar": "^4.4.8",
     "through2": "^2.0.3",
-    "typescript": "^3.2.2"
+    "tsd": "^0.18.0",
+    "typescript": "^4.4.4"
   },
   "engines": {
     "node": ">= 0.10.0"
@@ -83,7 +85,7 @@
     "os": {
       "github": "linux, windows, macos"
     },
-    "version": "8, 10, 12, 14"
+    "version": "8, 10, 12, 14, 16"
   },
   "license": "MIT"
 }