urllib 2.9.0 → 2.11.1

package/History.md

@@ -1,4 +1,26 @@
 
+2.11.1 / 2016-08-04
+==================
+
+  * fix: catch http.request sync error (#199)
+
+2.11.0 / 2016-06-26
+==================
+
+  * deps: upgrade deps from ~ to ^ (#189)
+
+2.10.0 / 2016-06-21
+==================
+
+  * feat: add an options consumeWriteStream (#187)
+  * chore(package): update statuses to version 1.3.0 (#174)
+
+2.9.1 / 2016-05-09
+==================
+
+  * fix: check url before request (#172)
+  * chore(package): update any-promise to version 1.2.0 (#171)
+
 2.9.0 / 2016-04-21
 ==================
 

package/README.md

@@ -5,6 +5,7 @@
 [![appveyor build status][appveyor-image]][appveyor-url]
 [![Test coverage][codecov-image]][codecov-url]
 [![David deps][david-image]][david-url]
+[![Known Vulnerabilities][snyk-image]][snyk-url]
 [![npm download][download-image]][download-url]
 
 [npm-image]: https://img.shields.io/npm/v/urllib.svg?style=flat-square
@@ -13,10 +14,12 @@
 [travis-url]: https://travis-ci.org/node-modules/urllib
 [appveyor-image]: https://ci.appveyor.com/api/projects/status/wpnl7r1llxyruja9/branch/master?svg=true
 [appveyor-url]: https://ci.appveyor.com/project/fengmk2/urllib-54ds2
-[codecov-image]: https://codecov.io/github/node-modules/urllib/coverage.svg?branch=master
-[codecov-url]: https://codecov.io/github/node-modules/urllib?branch=master
+[codecov-image]: https://codecov.io/gh/node-modules/urllib/branch/master/graph/badge.svg
+[codecov-url]: https://codecov.io/gh/node-modules/urllib
 [david-image]: https://img.shields.io/david/node-modules/urllib.svg?style=flat-square
 [david-url]: https://david-dm.org/node-modules/urllib
+[snyk-image]: https://snyk.io/test/npm/urllib/badge.svg?style=flat-square
+[snyk-url]: https://snyk.io/test/npm/urllib
 [download-image]: https://img.shields.io/npm/dm/urllib.svg?style=flat-square
 [download-url]: https://npmjs.org/package/urllib
 
@@ -121,6 +124,7 @@ httpclient.request('http://nodejs.org', function (err, body) {
     - ***content*** String | [Buffer](http://nodejs.org/api/buffer.html) - Manually set the content of payload. If set, `data` will be ignored.
     - ***stream*** [stream.Readable](http://nodejs.org/api/stream.html#stream_class_stream_readable) - Stream to be pipe to the remote. If set, `data` and `content` will be ignored.
     - ***writeStream*** [stream.Writable](http://nodejs.org/api/stream.html#stream_class_stream_writable) - A writable stream to be piped by the response stream. Responding data will be write to this stream and `callback` will be called with `data` set `null` after finished writing.
+    - ***consumeWriteStream*** [true] - consume the writeStream, invoke the callback after writeStream close.
     - ***contentType*** String - Type of request data. Could be `json`. If it's `json`, will auto set `Content-Type: application/json` header.
     - ***dataType*** String - Type of response data. Could be `text` or `json`. If it's `text`, the `callback`ed `data` would be a String. If it's `json`, the `data` of callback would be a parsed JSON Object. Default `callback`ed `data` would be a `Buffer`.
     - **fixJSONCtlChars** Boolean - Fix the control characters (U+0000 through U+001F) before JSON parse response. Default is `false`.

package/lib/urllib.js

@@ -1,21 +1,10 @@
-/**
- * Copyright(c) node-modules and other contributors.
- * MIT Licensed
- *
- * Authors:
- *   fengmk2 <fengmk2@gmail.com> (http://fengmk2.com)
- */
-
-"use strict";
-
-/**
- * Module dependencies.
- */
+'use strict';
 
 var debug = require('debug')('urllib');
 var http = require('http');
 var https = require('https');
 var urlutil = require('url');
+var util = require('util');
 var qs = require('querystring');
 var zlib = require('zlib');
 var ua = require('default-user-agent');
@@ -82,6 +71,7 @@ var PROTO_RE = /^https?:\/\//i;
  *   - {WriteStream} [writeStream]: writable stream to save response data.
  *       If you use this, callback's data should be null.
  *       We will just `pipe(ws, {end: true})`.
+ *   - {consumeWriteStream} [true]: consume the writeStream, invoke the callback after writeStream close.
  *   - {String} [method]: optional, could be GET | POST | DELETE | PUT, default is GET
  *   - {String} [contentType]: optional, request data type, could be `json`, default is undefined
  *   - {String} [dataType]: optional, response data type, could be `text` or `json`, default is buffer
@@ -173,6 +163,11 @@ exports.requestThunk = function (url, args) {
 
 exports.requestWithCallback = function (url, args, callback) {
   // requestWithCallback(url, callback)
+  if (!url || (typeof url !== 'string' && typeof url !== 'object')) {
+    var msg = util.format('expect request url to be a string or a http request options, but got %j', url);
+    throw new Error(msg);
+  }
+
   if (arguments.length === 2 && typeof args === 'function') {
     callback = args;
     args = null;
@@ -485,7 +480,7 @@ exports.requestWithCallback = function (url, args, callback) {
 
   args.requestUrls.push(url);
 
-  var req = httplib.request(options, function (res) {
+  function onResponse(res) {
     debug('Request#%d %s `req response` event emit: status %d, headers: %j',
       reqId, url, res.statusCode, res.headers);
 
@@ -531,7 +526,12 @@ exports.requestWithCallback = function (url, args, callback) {
         writeStream.end();
         return done(result.error, null, res);
       }
-      writeStream.on('close', done.bind(null, null, null, res));
+      // you can set consumeWriteStream false that only wait response end
+      if (args.consumeWriteStream === false) {
+        res.on('end', done.bind(null, null, null, res));
+      } else {
+        writeStream.on('close', done.bind(null, null, null, res));
+      }
       return res.pipe(writeStream);
     }
 
@@ -628,7 +628,15 @@ exports.requestWithCallback = function (url, args, callback) {
         done(err, data, res);
       });
     });
-  });
+  }
+
+  var req;
+  // request headers checker will throw error
+  try {
+    req = httplib.request(options, onResponse);
+  } catch (err) {
+    return done(err);
+  }
 
   var abortRequest = function () {
     debug('Request#%d %s abort, connected: %s', reqId, url, connected);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "urllib",
-  "version": "2.9.0",
+  "version": "2.11.1",
   "description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more.",
   "keywords": [
     "urllib",
@@ -22,37 +22,40 @@
     "url": "git://github.com/node-modules/urllib.git"
   },
   "scripts": {
-    "test": "mocha -R spec -t 30000 -r should -r should-http test/*.test.js",
-    "test-cov": "istanbul cover node_modules/mocha/bin/_mocha -- -t 30000 -r should -r should-http test/*.test.js",
+    "test": "mocha -R spec -t 30000 -r should -r should-http -r intelli-espower-loader test/*.test.js",
+    "test-cov": "istanbul cover node_modules/mocha/bin/_mocha -- -t 30000 -r should -r should-http -r intelli-espower-loader test/*.test.js",
     "ci": "npm run lint && npm run test-cov",
     "lint": "jshint .",
-    "autod": "autod -w --prefix '~' --devprefix '^' -t test -e examples"
+    "autod": "autod -w --prefix '^' -t test -e examples"
   },
   "dependencies": {
-    "any-promise": "~1.1.0",
-    "debug": "~2.2.0",
-    "default-user-agent": "~1.0.0",
-    "digest-header": "~0.0.1",
-    "humanize-ms": "~1.0.1",
-    "iconv-lite": "~0.4.13",
-    "media-typer": "~0.3.0",
-    "statuses": "~1.2.1"
+    "any-promise": "^1.2.0",
+    "debug": "^2.2.0",
+    "default-user-agent": "^1.0.0",
+    "digest-header": "^0.0.1",
+    "humanize-ms": "^1.2.0",
+    "iconv-lite": "^0.4.13",
+    "media-typer": "^0.3.0",
+    "statuses": "^1.3.0"
   },
   "devDependencies": {
-    "agentkeepalive": "^2.1.1",
+    "agentkeepalive": "2",
     "autod": "*",
     "bluebird": "*",
     "co": "*",
     "coffee": "1",
-    "formstream": "^1.0.0",
+    "formstream": "1",
+    "intelli-espower-loader": "^1.0.1",
     "istanbul": "*",
     "jshint": "*",
     "mocha": "*",
-    "pedding": "^1.0.0",
+    "pedding": "1",
+    "power-assert": "^1.4.1",
     "semver": "5",
-    "should": "^8.2.2",
+    "should": "8",
     "should-http": "*",
-    "tar": "^2.2.1"
+    "tar": "2",
+    "through2": "2"
   },
   "engines": {
     "node": ">= 0.10.0"