urllib 2.34.1 → 2.36.1

package/History.md

@@ -1,4 +1,35 @@
 
+2.36.1 / 2020-06-09
+==================
+
+**others**
+  * [[`35eecc0`](http://github.com/node-modules/urllib/commit/35eecc021f55e324b2520f91a032a33517d94f29)] - chore: add type declaration for RequestOptions#keepHeaderCase (#345) (William <<willizm.cn@gmail.com>>)
+
+2.36.0 / 2020-06-08
+==================
+
+**fixes**
+  * [[`15c4170`](http://github.com/node-modules/urllib/commit/15c4170beab4404b0576caa01e199f445c476b7d)] - fix: allow to keep origin header key case (#344) (TZ | 天猪 <<atian25@qq.com>>)
+
+**others**
+  * [[`31c03b8`](http://github.com/node-modules/urllib/commit/31c03b853ae5db00c7223c0a1a545dfcf0b07aa5)] - chore: Update and rename LICENSE.txt to LICENSE (#343) (Yuxiang LIU <<david-khala@hotmail.com>>)
+
+2.35.0 / 2020-05-15
+==================
+
+**features**
+  * [[`47c21bd`](http://github.com/node-modules/urllib/commit/47c21bd93648080589bdc6528d9e9cf3e0489951)] - feat: change header to lowercase (#337) (TZ | 天猪 <<atian25@qq.com>>)
+
+**fixes**
+  * [[`8f2ca64`](http://github.com/node-modules/urllib/commit/8f2ca648608995d140d0ca3873ef728bdbd4ee41)] - fix: need to handle response data on close event (#340) (fengmk2 <<fengmk2@gmail.com>>)
+  * [[`175ad2b`](http://github.com/node-modules/urllib/commit/175ad2b3e17196626c701ef72396dd6932d14c7a)] - fix: res.socket is null in node-v14.x (#339) (hyj1991 <<yeekwanvong@gmail.com>>)
+
+2.34.2 / 2019-12-09
+==================
+
+**fixes**
+  * [[`67d5b1c`](http://github.com/node-modules/urllib/commit/67d5b1c35dc302778aa4b992ea4998fe427e3cc8)] - fix: index.d.ts (#334) (Daniels.Sun <<better.sunjian@gmail.com>>)
+
 2.34.1 / 2019-09-02
 ==================
 

package/{LICENSE.txt → LICENSE}

@@ -1,4 +1,4 @@
-This software is licensed under the MIT License.
+MIT License
 
 Copyright(c) 2011 - 2014 fengmk2 and other contributors.
 Copyright(c) 2015 - present node-modules and other contributors.
@@ -10,13 +10,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -130,6 +130,7 @@ httpclient.request('http://nodejs.org', function (err, body) {
     - ***dataType*** String - Type of response data. Could be `text` or `json`. If it's `text`, the `callback`ed `data` would be a String. If it's `json`, the `data` of callback would be a parsed JSON Object and will auto set `Accept: application/json` header. Default `callback`ed `data` would be a `Buffer`.
     - **fixJSONCtlChars** Boolean - Fix the control characters (U+0000 through U+001F) before JSON parse response. Default is `false`.
     - ***headers*** Object - Request headers.
+    - ***keepHeaderCase*** Boolean - by default will convert header keys to lowercase
     - ***timeout*** Number | Array - Request timeout in milliseconds for connecting phase and response receiving phase. Defaults to `exports.TIMEOUT`, both are 5s. You can use `timeout: 5000` to tell urllib use same timeout on two phase or set them seperately such as `timeout: [3000, 5000]`, which will set connecting timeout to 3s and response 5s.
     - ***auth*** String - `username:password` used in HTTP Basic Authorization.
     - ***digestAuth*** String - `username:password` used in HTTP [Digest Authorization](http://en.wikipedia.org/wiki/Digest_access_authentication).

package/lib/index.d.ts

@@ -31,6 +31,11 @@ export interface RequestOptions {
   writeStream?: Writable;
   /** consume the writeStream, invoke the callback after writeStream close. */
   consumeWriteStream?: boolean;
+  /** 
+    * The files will send with multipart/form-data format, base on formstream.
+    * If method not set, will use POST method by default.
+    */
+  files?: Array<Readable | Buffer | string> | object | Readable | Buffer | string;
   /** Type of request data.Could be json.If it's json, will auto set Content-Type: application/json header. */
   contentType?: string;
   /**
@@ -49,6 +54,8 @@ export interface RequestOptions {
   fixJSONCtlChars?: boolean;
   /** Request headers. */
   headers?: IncomingHttpHeaders;
+  /** by default will convert header keys to lowercase */
+  keepHeaderCase?: boolean;
   /**
    * Request timeout in milliseconds for connecting phase and response receiving phase.
    * Defaults to exports.

package/lib/urllib.js

@@ -105,6 +105,7 @@ var SOCKET_RESPONSE_COUNT = '_URLLIB_SOCKET_RESPONSE_COUNT';
  *       before JSON parse response. Default is `false`.
  *       `fixJSONCtlChars` can be a function, will pass data to the first argument. e.g.: `data = fixJSONCtlChars(data)`
  *   - {Object} [headers]: optional, request headers
+ *   - {Boolean} [keepHeaderCase]: optional, by default will convert header keys to lowercase
  *   - {Number|Array} [timeout]: request timeout(in milliseconds), default is `exports.TIMEOUTS containing connect timeout and response timeout`
  *   - {Agent} [agent]: optional, http agent. Set `false` if you does not use agent.
  *   - {Agent} [httpsAgent]: optional, https agent. Set `false` if you does not use agent.
@@ -196,6 +197,7 @@ exports.requestThunk = function requestThunk(url, args) {
 };
 
 function requestWithCallback(url, args, callback) {
+  var req;
   // requestWithCallback(url, callback)
   if (!url || (typeof url !== 'string' && typeof url !== 'object')) {
     var msg = util.format('expect request url to be a string or a http request options, but got %j', url);
@@ -299,12 +301,18 @@ function requestWithCallback(url, args, callback) {
     // https://github.com/nodejs/node/blob/archived-io.js-v0.12/lib/net.js#L952
     lookup: lookup,
   };
+
+  var originHeaderKeys = {};
   if (args.headers) {
     // only allow enumerable and ownProperty value of args.headers
     var names = utility.getOwnEnumerables(args.headers, true);
     for (var i = 0; i < names.length; i++) {
       var name = names[i];
-      options.headers[name] = args.headers[name];
+      var key = name.toLowerCase();
+      if (key !== name) {
+        originHeaderKeys[key] = name;
+      }
+      options.headers[key] = args.headers[name];
     }
   }
 
@@ -386,7 +394,7 @@ function requestWithCallback(url, args, callback) {
     var formHeaderNames = utility.getOwnEnumerables(formHeaders, true);
     for (var i = 0; i < formHeaderNames.length; i++) {
       var name = formHeaderNames[i];
-      options.headers[name] = formHeaders[name];
+      options.headers[name.toLowerCase()] = formHeaders[name];
     }
     debug('set multipart headers: %j, method: %s', formHeaders, options.method);
     args.stream = form;
@@ -399,7 +407,7 @@ function requestWithCallback(url, args, callback) {
           // read: GET, HEAD, use query string
           body = args.nestedQuerystring ? qs.stringify(body) : querystring.stringify(body);
         } else {
-          var contentType = options.headers['Content-Type'] || options.headers['content-type'];
+          var contentType = options.headers['content-type'];
           // auto add application/x-www-form-urlencoded when using urlencode form request
           if (!contentType) {
             if (args.contentType === 'json') {
@@ -407,7 +415,7 @@ function requestWithCallback(url, args, callback) {
             } else {
               contentType = 'application/x-www-form-urlencoded';
             }
-            options.headers['Content-Type'] = contentType;
+            options.headers['content-type'] = contentType;
           }
 
           if (parseContentType(contentType).type === 'application/json') {
@@ -433,13 +441,15 @@ function requestWithCallback(url, args, callback) {
       if (!Buffer.isBuffer(body)) {
         length = Buffer.byteLength(body);
       }
-      requestSize = options.headers['Content-Length'] = length;
+      requestSize = length;
+
+      options.headers['content-length'] = length.toString();
     }
   }
 
   if (args.dataType === 'json') {
-    if (!options.headers.Accept && !options.headers.accept) {
-      options.headers.Accept = 'application/json';
+    if (!options.headers.accept) {
+      options.headers.accept = 'application/json';
     }
   }
 
@@ -447,6 +457,7 @@ function requestWithCallback(url, args, callback) {
     // you can use this hook to change every thing.
     args.beforeRequest(options);
   }
+
   var connectTimer = null;
   var responseTimer = null;
   var __err = null;
@@ -516,14 +527,14 @@ function requestWithCallback(url, args, callback) {
 
     // handle digest auth
     if (statusCode === 401 && headers['www-authenticate']
-        && !options.headers.Authorization && args.digestAuth) {
+        && !options.headers.authorization && args.digestAuth) {
       var authenticate = headers['www-authenticate'];
       if (authenticate.indexOf('Digest ') >= 0) {
         debug('Request#%d %s: got digest auth header WWW-Authenticate: %s', reqId, url, authenticate);
-        options.headers.Authorization = digestAuthHeader(options.method, options.path, authenticate, args.digestAuth);
-        debug('Request#%d %s: auth with digest header: %s', reqId, url, options.headers.Authorization);
+        options.headers.authorization = digestAuthHeader(options.method, options.path, authenticate, args.digestAuth);
+        debug('Request#%d %s: auth with digest header: %s', reqId, url, options.headers.authorization);
         if (res.headers['set-cookie']) {
-          options.headers.Cookie = res.headers['set-cookie'].join(';');
+          options.headers.cookie = res.headers['set-cookie'].join(';');
         }
         args.headers = options.headers;
         return exports.requestWithCallback(url, args, cb);
@@ -592,10 +603,11 @@ function requestWithCallback(url, args, callback) {
           if (serverSocketTimeout < freeSocketTimeout) {
             // https://github.com/node-modules/agentkeepalive/blob/master/lib/agent.js#L127
             // agentkeepalive@4
+            var socket = res.socket || (req && req.socket);
             if (agent.options && agent.options.freeSocketTimeout) {
-              res.socket.freeSocketTimeout = serverSocketTimeout;
+              socket.freeSocketTimeout = serverSocketTimeout;
             } else {
-              res.socket.freeSocketKeepAliveTimeout = serverSocketTimeout;
+              socket.freeSocketKeepAliveTimeout = serverSocketTimeout;
             }
           }
         }
@@ -637,9 +649,9 @@ function requestWithCallback(url, args, callback) {
         debug('Request#%d %s: `redirected` from %s to %s', reqId, options.path, url, newUrl);
         // make sure timer stop
         cancelResponseTimer();
-        // should clean up headers.Host on `location: http://other-domain/url`
-        if (options.headers.Host && PROTO_RE.test(location)) {
-          options.headers.Host = null;
+        // should clean up headers.host on `location: http://other-domain/url`
+        if (options.headers.host && PROTO_RE.test(location)) {
+          options.headers.host = null;
           args.headers = options.headers;
         }
         // avoid done will be execute in the future change.
@@ -663,23 +675,20 @@ function requestWithCallback(url, args, callback) {
     if (options.headers['user-agent']) {
       delete options.headers['user-agent'];
     }
-    if (options.headers['User-Agent']) {
-      delete options.headers['User-Agent'];
-    }
   } else {
     // need to set user-agent
-    var hasAgentHeader = options.headers['User-Agent'] || options.headers['user-agent'];
+    var hasAgentHeader = options.headers['user-agent'];
     if (!hasAgentHeader) {
-      options.headers['User-Agent'] = USER_AGENT;
+      options.headers['user-agent'] = USER_AGENT;
     }
   }
 
   if (args.gzip) {
     var isAcceptEncodingNull = (args.headers && (args.headers['Accept-Encoding'] === null || args.headers['accept-encoding'] === null));
     if (!isAcceptEncodingNull) {
-      var hasAcceptEncodingHeader = options.headers['Accept-Encoding'] || options.headers['accept-encoding'];
+      var hasAcceptEncodingHeader = options.headers['accept-encoding'];
       if (!hasAcceptEncodingHeader) {
-        options.headers['Accept-Encoding'] = 'gzip, deflate';
+        options.headers['accept-encoding'] = 'gzip, deflate';
       }
     }
   }
@@ -737,11 +746,6 @@ function requestWithCallback(url, args, callback) {
       return done(null, null, res);
     }
 
-    res.on('close', function () {
-      debug('Request#%d %s: `res close` event emit, total size %d, socket handled %s requests and %s responses',
-        reqId, url, responseSize, socketHandledRequests, socketHandledResponses);
-    });
-
     res.on('error', function () {
       debug('Request#%d %s: `res error` event emit, total size %d, socket handled %s requests and %s responses',
         reqId, url, responseSize, socketHandledRequests, socketHandledResponses);
@@ -843,10 +847,18 @@ function requestWithCallback(url, args, callback) {
       chunks.push(chunk);
     });
 
-    res.on('end', function () {
+    var isEmitted = false;
+    function handleResponseCloseAndEnd(event) {
+      debug('Request#%d %s: `res %s` event emit, total size %d, socket handled %s requests and %s responses',
+        reqId, url, event, responseSize, socketHandledRequests, socketHandledResponses);
+      if (isEmitted) {
+        return;
+      }
+      isEmitted = true;
+
       var body = Buffer.concat(chunks, responseSize);
-      debug('Request#%d %s: `res end` event emit, total size %d, _dumped: %s',
-        reqId, url, responseSize, res._dumped);
+      debug('Request#%d %s: _dumped: %s',
+        reqId, url, res._dumped);
 
       if (__err) {
         // req.abort() after `res data` event emit.
@@ -898,6 +910,14 @@ function requestWithCallback(url, args, callback) {
 
         done(err, data, res);
       });
+    }
+
+    // node >= 14 only emit close if req abort
+    res.on('close', function () {
+      handleResponseCloseAndEnd('close');
+    });
+    res.on('end', function () {
+      handleResponseCloseAndEnd('end');
     });
   }
 
@@ -966,10 +986,27 @@ function requestWithCallback(url, args, callback) {
     }
   }
 
-  var req;
   // request headers checker will throw error
   try {
-    req = httplib.request(options, onResponse);
+    var finalOptions = options;
+
+    // restore origin header key
+    if (args.keepHeaderCase) {
+      var originKeys = Object.keys(originHeaderKeys);
+      if (originKeys.length) {
+        var finalHeaders = {};
+        var names = utility.getOwnEnumerables(options.headers, true);
+        for (var i = 0; i < names.length; i++) {
+          var name = names[i];
+          finalHeaders[originHeaderKeys[name] || name] = options.headers[name];
+        }
+
+        finalOptions = Object.assign({}, options);
+        finalOptions.headers = finalHeaders;
+      }
+    }
+
+    req = httplib.request(finalOptions, onResponse);
     if (args.trace) {
       req._callSite = {};
       Error.captureStackTrace(req._callSite, requestWithCallback);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "urllib",
-  "version": "2.34.1",
+  "version": "2.36.1",
   "description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more.",
   "keywords": [
     "urllib",
@@ -58,6 +58,7 @@
     "busboy": "^0.2.14",
     "co": "*",
     "coffee": "1",
+    "egg-ci": "^1.15.0",
     "git-contributor": "^1.0.10",
     "http-proxy": "^1.16.2",
     "intelli-espower-loader": "^1.0.1",
@@ -69,6 +70,7 @@
     "pedding": "^1.1.0",
     "power-assert": "^1.4.2",
     "semver": "5",
+    "spy": "^1.0.0",
     "tar": "^4.4.8",
     "through2": "^2.0.3",
     "typescript": "^3.2.2"
@@ -76,5 +78,12 @@
   "engines": {
     "node": ">= 0.10.0"
   },
+  "ci": {
+    "type": "github",
+    "os": {
+      "github": "linux, windows, macos"
+    },
+    "version": "6, 8, 10, 12, 14"
+  },
   "license": "MIT"
 }