urllib 2.37.0 → 2.37.4

package/History.md

@@ -1,4 +1,31 @@
 
+2.37.4 / 2021-09-07
+==================
+
+**fixes**
+  * [[`1e6622a`](http://github.com/node-modules/urllib/commit/1e6622a571b3e6f72c5f187a224bddedcbc382cf)] - fix: upgrade proxy-agent to fix the security vulnerability. (#368) (hyj1991 <<yeekwanvong@gmail.com>>)
+
+**others**
+  * [[`ae27498`](http://github.com/node-modules/urllib/commit/ae2749811e40262da8bf2641599b066344fa6b05)] - docs: notes contentType's value (#349) (changzhi <<changzhiwin@gmail.com>>)
+
+2.37.3 / 2021-07-05
+==================
+
+**fixes**
+  * [[`b730a6c`](http://github.com/node-modules/urllib/commit/b730a6cdc0465bea1c08d50bfa3b5e95bd17b31f)] - fix: 🐛 redirect status code (#363) (Hongcai Deng <<admin@dhchouse.com>>)
+
+2.37.2 / 2021-06-07
+==================
+
+**fixes**
+  * [[`04734a1`](http://github.com/node-modules/urllib/commit/04734a13e146da501f4e38333d5de9f9bbb259d1)] - fix: 🐛 should trigger response event when follow redirect (#361) (Hongcai Deng <<admin@dhchouse.com>>)
+
+2.37.1 / 2021-04-15
+==================
+
+**others**
+  * [[`d50afda`](http://github.com/node-modules/urllib/commit/d50afda91befff75f044a1ad83689f7a459c5d32)] - Update proxy-agent to v4 to resolve vulnerability (#355) (Chad McElligott <<chad.mcelligott@gmail.com>>)
+
 2.37.0 / 2021-04-02
 ==================
 

package/README.md

@@ -125,7 +125,7 @@ httpclient.request('http://nodejs.org', function (err, body) {
     - ***writeStream*** [stream.Writable](http://nodejs.org/api/stream.html#stream_class_stream_writable) - A writable stream to be piped by the response stream. Responding data will be write to this stream and `callback` will be called with `data` set `null` after finished writing.
     - ***files*** {Array<ReadStream|Buffer|String> | Object | ReadStream | Buffer | String - The files will send with `multipart/form-data` format, base on `formstream`. If `method` not set, will use `POST` method by default.
     - ***consumeWriteStream*** [true] - consume the writeStream, invoke the callback after writeStream close.
-    - ***contentType*** String - Type of request data. Could be `json`. If it's `json`, will auto set `Content-Type: application/json` header.
+    - ***contentType*** String - Type of request data. Could be `json` (**Notes**: not use `application/json` here). If it's `json`, will auto set `Content-Type: application/json` header.
     - ***nestedQuerystring*** Boolean - urllib default use querystring to stringify form data which don't support nested object, will use [qs](https://github.com/ljharb/qs) instead of querystring to support nested object by set this option to true.
     - ***dataType*** String - Type of response data. Could be `text` or `json`. If it's `text`, the `callback`ed `data` would be a String. If it's `json`, the `data` of callback would be a parsed JSON Object and will auto set `Accept: application/json` header. Default `callback`ed `data` would be a `Buffer`.
     - **fixJSONCtlChars** Boolean - Fix the control characters (U+0000 through U+001F) before JSON parse response. Default is `false`.

package/lib/urllib.js

@@ -520,6 +520,7 @@ function requestWithCallback(url, args, callback) {
       }
       return;
     }
+
     var cb = callback;
     callback = null;
     var headers = {};
@@ -529,49 +530,16 @@ function requestWithCallback(url, args, callback) {
       headers = res.headers;
     }
 
-    // handle digest auth
-    if (statusCode === 401 && headers['www-authenticate']
-        && !options.headers.authorization && args.digestAuth) {
-      var authenticate = headers['www-authenticate'];
-      if (authenticate.indexOf('Digest ') >= 0) {
-        debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', reqId, url, authenticate);
-        options.headers.authorization = digestAuthHeader(options.method, options.path, authenticate, args.digestAuth);
-        debug('Request#%d %s: auth with digest header: %s', reqId, url, options.headers.authorization);
-        if (res.headers['set-cookie']) {
-          options.headers.cookie = res.headers['set-cookie'].join(';');
-        }
-        args.headers = options.headers;
-        return exports.requestWithCallback(url, args, cb);
-      }
+    if (handleDigestAuth(res, cb)) {
+      return;
     }
 
-    var requestUseTime = Date.now() - requestStartTime;
-    if (timing) {
-      timing.contentDownload = requestUseTime;
-    }
+    var response = createCallbackResponse(data, res);
 
     debug('[%sms] done, %s bytes HTTP %s %s %s %s, keepAliveSocket: %s, timing: %j, socketHandledRequests: %s, socketHandledResponses: %s',
-      requestUseTime, responseSize, statusCode, options.method, options.host, options.path,
+      response.requestUseTime, responseSize, statusCode, options.method, options.host, options.path,
       keepAliveSocket, timing, socketHandledRequests, socketHandledResponses);
 
-    var response = {
-      status: statusCode,
-      statusCode: statusCode,
-      statusMessage: statusMessage,
-      headers: headers,
-      size: responseSize,
-      aborted: responseAborted,
-      rt: requestUseTime,
-      keepAliveSocket: keepAliveSocket,
-      data: data,
-      requestUrls: args.requestUrls,
-      timing: timing,
-      remoteAddress: remoteAddress,
-      remotePort: remotePort,
-      socketHandledRequests: socketHandledRequests,
-      socketHandledResponses: socketHandledResponses,
-    };
-
     if (err) {
       var agentStatus = '';
       if (agent && typeof agent.getCurrentStatus === 'function') {
@@ -620,6 +588,44 @@ function requestWithCallback(url, args, callback) {
 
     cb(err, data, args.streaming ? res : response);
 
+    emitResponseEvent(err, response);
+  }
+
+  function createAndEmitResponseEvent(data, res) {
+    var response = createCallbackResponse(data, res);
+    emitResponseEvent(null, response);
+  }
+
+  function createCallbackResponse(data, res) {
+    var requestUseTime = Date.now() - requestStartTime;
+    if (timing) {
+      timing.contentDownload = requestUseTime;
+    }
+
+    var headers = res && res.headers || {};
+    var resStatusCode = res && res.statusCode || statusCode;
+    var resStatusMessage = res && res.statusMessage || statusMessage;
+
+    return {
+      status: resStatusCode,
+      statusCode: resStatusCode,
+      statusMessage: resStatusMessage,
+      headers: headers,
+      size: responseSize,
+      aborted: responseAborted,
+      rt: requestUseTime,
+      keepAliveSocket: keepAliveSocket,
+      data: data,
+      requestUrls: args.requestUrls,
+      timing: timing,
+      remoteAddress: remoteAddress,
+      remotePort: remotePort,
+      socketHandledRequests: socketHandledRequests,
+      socketHandledResponses: socketHandledResponses,
+    };
+  }
+
+  function emitResponseEvent(err, response) {
     if (args.emitter) {
       // keep to use the same reqMeta object on request event before
       reqMeta.url = parsedUrl.href;
@@ -637,6 +643,30 @@ function requestWithCallback(url, args, callback) {
     }
   }
 
+  function handleDigestAuth(res, cb) {
+    var headers = {};
+    if (res && res.headers) {
+      headers = res.headers;
+    }
+    // handle digest auth
+    if (statusCode === 401 && headers['www-authenticate']
+        && !options.headers.authorization && args.digestAuth) {
+      var authenticate = headers['www-authenticate'];
+      if (authenticate.indexOf('Digest ') >= 0) {
+        debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', reqId, url, authenticate);
+        options.headers.authorization = digestAuthHeader(options.method, options.path, authenticate, args.digestAuth);
+        debug('Request#%d %s: auth with digest header: %s', reqId, url, options.headers.authorization);
+        if (res.headers['set-cookie']) {
+          options.headers.cookie = res.headers['set-cookie'].join(';');
+        }
+        args.headers = options.headers;
+        exports.requestWithCallback(url, args, cb);
+        return true;
+      }
+    }
+    return false;
+  }
+
   function handleRedirect(res) {
     var err = null;
     if (args.followRedirect && statuses.redirect[res.statusCode]) {  // handle redirect
@@ -740,6 +770,7 @@ function requestWithCallback(url, args, callback) {
       var result = handleRedirect(res);
       if (result.redirect) {
         res.resume();
+        createAndEmitResponseEvent(null, res);
         return;
       }
       if (result.error) {
@@ -781,6 +812,7 @@ function requestWithCallback(url, args, callback) {
       var result = handleRedirect(res);
       if (result.redirect) {
         res.resume();
+        createAndEmitResponseEvent(null, res);
         return;
       }
       if (result.error) {
@@ -874,6 +906,7 @@ function requestWithCallback(url, args, callback) {
         return done(result.error, body, res);
       }
       if (result.redirect) {
+        createAndEmitResponseEvent(null, res);
         return;
       }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "urllib",
-  "version": "2.37.0",
+  "version": "2.37.4",
   "description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more.",
   "keywords": [
     "urllib",
@@ -42,7 +42,7 @@
     "humanize-ms": "^1.2.0",
     "iconv-lite": "^0.4.15",
     "ip": "^1.1.5",
-    "proxy-agent": "^3.1.0",
+    "proxy-agent": "^5.0.0",
     "pump": "^3.0.0",
     "qs": "^6.4.0",
     "statuses": "^1.3.1",