superagent 4.0.0-alpha.1 → 4.1.0

package/lib/node/http2wrapper.js

@@ -0,0 +1,188 @@
+'use strict';
+
+const http2 = require('http2');
+const Stream = require('stream');
+const util = require('util');
+const net = require('net');
+const tls = require('tls');
+const parse = require('url').parse;
+
+const {
+  HTTP2_HEADER_PATH,
+  HTTP2_HEADER_STATUS,
+  HTTP2_HEADER_METHOD,
+  HTTP2_HEADER_AUTHORITY,
+  HTTP2_HEADER_HOST,
+  HTTP2_HEADER_SET_COOKIE,
+  NGHTTP2_CANCEL,
+} = http2.constants;
+
+
+function setProtocol(protocol) {
+  return {
+    request: function (options) {
+      return new Request(protocol, options);
+    }
+  }
+}
+
+function Request(protocol, options) {
+  Stream.call(this);
+  const defaultPort = protocol === 'https:' ? 443 : 80;
+  const defaultHost = 'localhost'
+  const port = options.port || defaultPort;
+  const host = options.host || defaultHost;
+
+  delete options.port
+  delete options.host
+
+  this.method = options.method;
+  this.path = options.path;
+  this.protocol = protocol;
+  this.host = host;
+
+  delete options.method
+  delete options.path
+
+  const sessionOptions = Object.assign({}, options);
+  if (options.socketPath) {
+    sessionOptions.socketPath = options.socketPath;
+    sessionOptions.createConnection = this.createUnixConnection.bind(this);
+  }
+
+  this._headers = {};
+
+  const session = http2.connect(`${protocol}//${host}:${port}`, sessionOptions);
+  this.setHeader('host', `${host}:${port}`)
+
+  session.on('error', (err) => this.emit('error', err));
+
+  this.session = session;
+}
+
+/**
+ * Inherit from `Stream` (which inherits from `EventEmitter`).
+ */
+util.inherits(Request, Stream);
+
+Request.prototype.createUnixConnection = function (authority, options) {
+  switch (this.protocol) {
+    case 'http:':
+      return net.connect(options.socketPath);
+    case 'https:':
+      options.ALPNProtocols = ['h2'];
+      options.servername = this.host;
+      options.allowHalfOpen = true;
+      return tls.connect(options.socketPath, options);
+    default:
+      throw new Error('Unsupported protocol', this.protocol);
+  }
+}
+
+Request.prototype.setNoDelay = function (bool) {
+  // We can not use setNoDelay with HTTP/2.
+  // Node 10 limits http2session.socket methods to ones safe to use with HTTP/2.
+  // See also https://nodejs.org/api/http2.html#http2_http2session_socket
+}
+
+Request.prototype.getFrame = function () {
+  if (this.frame) {
+    return this.frame;
+  }
+
+  const method = {
+    [HTTP2_HEADER_PATH]: this.path,
+    [HTTP2_HEADER_METHOD]: this.method,
+  }
+
+  let headers = this.mapToHttp2Header(this._headers);
+
+  headers = Object.assign(headers, method);
+
+  const frame = this.session.request(headers);
+  frame.once('response', (headers, flags) => {
+    headers = this.mapToHttpHeader(headers);
+    frame.headers = headers;
+    frame.status = frame.statusCode = headers[HTTP2_HEADER_STATUS];
+    this.emit('response', frame);
+  });
+
+  this._headerSent = true;
+
+  frame.once('drain', () => this.emit('drain'));
+  frame.on('error', (err) => this.emit('error', err));
+  frame.on('close', () => this.session.close());
+
+  this.frame = frame;
+  return frame;
+}
+
+Request.prototype.mapToHttpHeader = function (headers) {
+  const keys = Object.keys(headers);
+  const http2Headers = {};
+  for (var i = 0; i < keys.length; i++) {
+    let key = keys[i];
+    let value = headers[key];
+    key = key.toLowerCase();
+    switch (key) {
+      case HTTP2_HEADER_SET_COOKIE:
+        value = Array.isArray(value) ? value : [value];
+        break;
+      default:
+        break;
+    }
+    http2Headers[key] = value;
+  }
+  return http2Headers;
+}
+
+Request.prototype.mapToHttp2Header = function (headers) {
+  const keys = Object.keys(headers);
+  const http2Headers = {};
+  for (var i = 0; i < keys.length; i++) {
+    let key = keys[i];
+    let value = headers[key];
+    key = key.toLowerCase();
+    switch (key) {
+      case HTTP2_HEADER_HOST:
+        key = HTTP2_HEADER_AUTHORITY;
+        value = /^http\:\/\/|^https\:\/\//.test(value) ? parse(value).host : value;
+        break;
+      default:
+        break;
+    }
+    http2Headers[key] = value;
+  }
+  return http2Headers;
+}
+
+Request.prototype.setHeader = function (name, value) {
+  this._headers[name.toLowerCase()] = value;
+}
+
+Request.prototype.getHeader = function (name) {
+  return this._headers[name.toLowerCase()];
+}
+
+Request.prototype.write = function (data, encoding) {
+  const frame = this.getFrame();
+  return frame.write(data, encoding);
+};
+
+Request.prototype.pipe = function (stream, options) {
+  const frame = this.getFrame();
+  return frame.pipe(stream, options);
+}
+
+Request.prototype.end = function (data) {
+  const frame = this.getFrame();
+  frame.end(data);
+}
+
+Request.prototype.abort = function (data) {
+  const frame = this.getFrame();
+  frame.close(NGHTTP2_CANCEL);
+  this.session.destroy();
+}
+
+exports.setProtocol = setProtocol;

package/lib/node/index.js

@@ -15,7 +15,6 @@ let methods = require('methods');
 const Stream = require('stream');
 const utils = require('../utils');
 const unzip = require('./unzip').unzip;
-const extend = require('extend');
 const mime = require('mime');
 const https = require('https');
 const http = require('http');
@@ -27,6 +26,11 @@ const pkg = require('../../package.json');
 const RequestBase = require('../request-base');
 const CookieJar = require('cookiejar');
 
+let http2;
+try {
+  http2 = require('./http2wrapper');
+} catch (_) {}
+
 function request(method, url) {
   // callback
   if ('function' == typeof url) {
@@ -81,6 +85,7 @@ mime.define({
 exports.protocols = {
   'http:': http,
   'https:': https,
+  'http2:': http2,
 };
 
 /**
@@ -144,6 +149,7 @@ function _initHeaders(req) {
 function Request(method, url) {
   Stream.call(this);
   if ('string' != typeof url) url = format(url);
+  this._enableHttp2 = !!process.env.HTTP2_TEST; // internal only
   this._agent = false;
   this._formData = null;
   this.method = method;
@@ -168,6 +174,43 @@ function Request(method, url) {
 util.inherits(Request, Stream);
 RequestBase(Request.prototype);
 
+/**
+ * Enable or Disable http2.
+ *
+ * Enable http2.
+ *
+ * ``` js
+ * request.get('http://localhost/')
+ *   .http2()
+ *   .end(callback);
+ *
+ * request.get('http://localhost/')
+ *   .http2(true)
+ *   .end(callback);
+ * ```
+ *
+ * Disable http2.
+ *
+ * ``` js
+ * request = request.http2();
+ * request.get('http://localhost/')
+ *   .http2(false)
+ *   .end(callback);
+ * ```
+ *
+ * @param {Boolean} enable
+ * @return {Request} for chaining
+ * @api public
+ */
+
+Request.prototype.http2 = function(bool){
+  if (exports.protocols['http2:'] === undefined) {
+    throw new Error('superagent: this version of Node.js does not support http2');
+  }
+  this._enableHttp2 = bool === undefined ? true : bool;
+  return this;
+}
+
 /**
  * Queue the given `file` as an attachment to the specified `field`,
  * with optional `options` (or filename).
@@ -319,7 +362,7 @@ Request.prototype.query = function(val){
   if ('string' == typeof val) {
     this._query.push(val);
   } else {
-    extend(this.qs, val);
+    Object.assign(this.qs, val);
   }
   return this;
 };
@@ -504,7 +547,7 @@ Request.prototype.auth = function(user, pass, options){
     options = { type: 'basic' };
   }
 
-  const encoder = string => new Buffer(string).toString('base64');
+  const encoder = string => new Buffer.from(string).toString('base64');
 
   return this._auth(user, pass, options, encoder);
 };
@@ -631,6 +674,24 @@ Request.prototype.request = function(){
     url.path = unixParts[2];
   }
 
+  // Override IP address of a hostname
+  if (this._connectOverride) {
+    const hostname = url.hostname;
+    const match = hostname in this._connectOverride ? this._connectOverride[hostname] : this._connectOverride['*'];
+    if (match) {
+      // backup the real host
+      if (!this._header['host']) {
+        this.set('host', url.host);
+      }
+      // wrap [ipv6]
+      url.host = /:/.test(match) ? `[${match}]` : match;
+      if (url.port) {
+        url.host += `:${url.port}`;
+      }
+      url.hostname = match;
+    }
+  }
+
   // options
   options.method = this.method;
   options.port = url.port;
@@ -648,8 +709,12 @@ Request.prototype.request = function(){
     options.servername = this._header['host'].replace(/:[0-9]+$/,'');
   }
 
+  if (this._trustLocalhost && /^(?:localhost|127\.0\.0\.\d+|(0*:)+:0*1)$/.test(url.hostname)) {
+    options.rejectUnauthorized = false;
+  }
+
   // initiate request
-  const mod = exports.protocols[url.protocol];
+  const mod = this._enableHttp2 ? exports.protocols['http2:'].setProtocol(url.protocol) : exports.protocols[url.protocol];
 
   // request
   const req = (this.req = mod.request(options));
@@ -695,10 +760,10 @@ Request.prototype.request = function(){
 
   // add cookies
   if (this.cookies) {
-    if(this.header.hasOwnProperty('cookie')) {
+    if(this._header.hasOwnProperty('cookie')) {
       // merge
       const tmpJar = new CookieJar.CookieJar();
-      tmpJar.setCookies(this.header.cookie.split(';'));
+      tmpJar.setCookies(this._header.cookie.split(';'));
       tmpJar.setCookies(this.cookies.split(';'));
       req.setHeader('Cookie',tmpJar.getCookies(CookieJar.CookieAccessInfo.All).toValueString());
     } else {
@@ -789,6 +854,11 @@ Request.prototype._emitResponse = function(body, files) {
     response.body = body;
   }
   response.files = files;
+  if (this._endCalled) {
+    response.pipe = function() {
+      throw Error("end() has already been called, so it's too late to start piping");
+    }
+  }
   this.emit('response', response);
   return response;
 };
@@ -798,8 +868,8 @@ Request.prototype.end = function(fn) {
   debug('%s %s', this.method, this.url);
 
   if (this._endCalled) {
-    console.warn(
-      'Warning: .end() was called twice. This is not supported in superagent'
+    throw Error(
+      '.end() was called twice. This is not supported in superagent'
     );
   }
   this._endCalled = true;
@@ -807,7 +877,7 @@ Request.prototype.end = function(fn) {
   // store callback
   this._callback = fn || noop;
 
-  return this._end();
+  this._end();
 };
 
 Request.prototype._end = function() {
@@ -815,7 +885,6 @@ Request.prototype._end = function() {
 
   let data = this._data;
   const req = this.req;
-  let buffer = this._buffer;
   const method = this.method;
 
   this._setTimeouts();
@@ -827,7 +896,7 @@ Request.prototype._end = function() {
       let contentType = req.getHeader('Content-Type');
       // Parse out just the content type from the header (ignore the charset)
       if (contentType) contentType = contentType.split(';')[0];
-      let serialize = exports.serialize[contentType];
+      let serialize = this._serializer || exports.serialize[contentType];
       if (!serialize && isJSON(contentType)) {
         serialize = exports.serialize['application/json'];
       }
@@ -857,7 +926,6 @@ Request.prototype._end = function() {
     const type = mime.split('/')[0];
     const multipart = 'multipart' == type;
     const redirect = isRedirect(res.statusCode);
-    let parser = this._parser;
     const responseType = this._responseType;
 
     this.res = res;
@@ -878,10 +946,19 @@ Request.prototype._end = function() {
       unzip(req, res);
     }
 
+    let buffer = this._buffer;
     if (buffer === undefined && mime in exports.buffer){
       buffer = !!exports.buffer[mime];
     }
 
+    let parser = this._parser;
+    if (undefined === buffer) {
+      if (parser) {
+        console.warn("A custom superagent parser has been set, but buffering strategy for the parser hasn't been configured. Call `req.buffer(true or false)` or set `superagent.buffer[mime] = true or false`");
+        buffer = true;
+      }
+    }
+
     if (!parser) {
       if (responseType) {
         parser = exports.parse.image; // It's actually a generic Buffer
@@ -905,6 +982,9 @@ Request.prototype._end = function() {
         buffer = (buffer !== false);
       } else if (buffer) {
         parser = exports.parse.text;
+      } else if (undefined === buffer) {
+        parser = exports.parse.image; // It's actually a generic Buffer
+        buffer = true;
       }
     }
 
@@ -913,6 +993,7 @@ Request.prototype._end = function() {
       buffer = true;
     }
 
+    this._resBuffered = buffer;
     let parserHandlesEnd = false;
     if (buffer) {
       // Protectiona against zip bombs and other nuisance
@@ -1059,8 +1140,6 @@ Request.prototype._end = function() {
   } else {
     req.end(data);
   }
-
-  return this;
 };
 
 /**
@@ -1082,6 +1161,35 @@ Request.prototype._shouldUnzip = res => {
   return /^\s*(?:deflate|gzip)\s*$/.test(res.headers['content-encoding']);
 };
 
+/**
+ * Overrides DNS for selected hostnames. Takes object mapping hostnames to IP addresses.
+ *
+ * When making a request to a URL with a hostname exactly matching a key in the object,
+ * use the given IP address to connect, instead of using DNS to resolve the hostname.
+ *
+ * A special host `*` matches every hostname (keep redirects in mind!)
+ *
+ *      request.connect({
+ *        'test.example.com': '127.0.0.1',
+ *        'ipv6.example.com': '::1',
+ *      })
+ */
+Request.prototype.connect = function(connectOverride) {
+  if ('string' === typeof connectOverride) {
+    this._connectOverride = {'*': connectOverride};
+  } else if ('object' === typeof connectOverride) {
+    this._connectOverride = connectOverride;
+  } else {
+    this._connectOverride = undefined;
+  }
+  return this;
+};
+
+Request.prototype.trustLocalhost = function(toggle) {
+    this._trustLocalhost = toggle === undefined ? true : toggle;
+    return this;
+};
+
 // generate HTTP verb methods
 if (methods.indexOf('del') == -1) {
   // create a copy so we don't cause conflicts with

package/lib/node/response.js

@@ -36,7 +36,7 @@ function Response(req) {
   this.text = res.text;
   this.body = res.body !== undefined ? res.body : {};
   this.files = res.files || {};
-  this.buffered = 'string' == typeof this.text;
+  this.buffered = req._resBuffered;
   this.header = this.headers = res.headers;
   this._setStatusProperties(res.statusCode);
   this._setHeaderProperties(this.header);

package/lib/node/unzip.js

@@ -59,9 +59,9 @@ exports.unzip = (req, res) => {
   const _on = res.on;
   res.on = function(type, fn) {
     if ('data' == type || 'end' == type) {
-      stream.on(type, fn);
+      stream.on(type, fn.bind(res));
     } else if ('error' == type) {
-      stream.on(type, fn);
+      stream.on(type, fn.bind(res));
       _on.call(res, type, fn);
     } else {
       _on.call(res, type, fn);

package/lib/request-base.js

@@ -236,6 +236,14 @@ RequestBase.prototype.then = function then(resolve, reject) {
     }
     this._fullfilledPromise = new Promise((innerResolve, innerReject) => {
       self.on('error', innerReject);
+      self.on('abort', () => {
+        const err = new Error('Aborted');
+        err.code = "ABORTED";
+        err.status = this.status;
+        err.method = this.method;
+        err.url = this.url;
+        innerReject(err);
+      });
       self.end((err, res) => {
         if (err) innerReject(err);
         else innerResolve(res);

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "superagent",
-  "version": "4.0.0-alpha.1",
+  "version": "4.1.0",
   "description": "elegant & feature rich browser / node HTTP with a fluent API",
   "scripts": {
     "prepare": "make all",
-    "test": "make test"
+    "test": "make test",
+    "test-http2": "make test-node-http2"
   },
   "keywords": [
     "http",
@@ -25,15 +26,14 @@
   },
   "dependencies": {
     "component-emitter": "^1.2.0",
-    "cookiejar": "^2.1.0",
-    "debug": "^3.1.0",
-    "extend": "^3.0.0",
-    "form-data": "^2.3.2",
+    "cookiejar": "^2.1.2",
+    "debug": "^4.1.0",
+    "form-data": "^2.3.3",
     "formidable": "^1.2.0",
     "methods": "^1.1.1",
-    "mime": "^2.0.3",
-    "qs": "^6.5.1",
-    "readable-stream": "^2.3.5"
+    "mime": "^2.4.0",
+    "qs": "^6.6.0",
+    "readable-stream": "^3.0.6"
   },
   "devDependencies": {
     "Base64": "^1.0.1",
@@ -42,16 +42,16 @@
     "babelify": "^8.0.0",
     "basic-auth-connect": "^1.0.0",
     "body-parser": "^1.18.2",
-    "browserify": "^16.2.0",
+    "browserify": "^16.2.3",
     "cookie-parser": "^1.4.3",
     "express": "^4.16.3",
     "express-session": "^1.15.6",
-    "marked": "^0.3.19",
+    "marked": "^0.5.2",
     "mocha": "^3.5.3",
-    "multer": "^1.3.0",
+    "multer": "^1.4.1",
     "should": "^13.2.0",
     "should-http": "^0.1.1",
-    "zuul": "^3.11.1"
+    "zuul": "^3.12.0"
   },
   "browser": {
     "./lib/node/index.js": "./lib/client.js",

package/superagent.js

@@ -274,6 +274,8 @@ RequestBase.prototype._retry = function () {
  */
 
 RequestBase.prototype.then = function then(resolve, reject) {
+  var _this = this;
+
   if (!this._fullfilledPromise) {
     var self = this;
     if (this._endCalled) {
@@ -281,6 +283,14 @@ RequestBase.prototype.then = function then(resolve, reject) {
     }
     this._fullfilledPromise = new Promise(function (innerResolve, innerReject) {
       self.on('error', innerReject);
+      self.on('abort', function () {
+        var err = new Error('Aborted');
+        err.code = "ABORTED";
+        err.status = _this.status;
+        err.method = _this.method;
+        err.url = _this.url;
+        innerReject(err);
+      });
       self.end(function (err, res) {
         if (err) innerReject(err);else innerResolve(res);
       });
@@ -1799,7 +1809,7 @@ Request.prototype.end = function (fn) {
   // querystring
   this._finalizeQueryString();
 
-  return this._end();
+  this._end();
 };
 
 Request.prototype._end = function () {
@@ -1901,7 +1911,6 @@ Request.prototype._end = function () {
   // IE11 xhr.send(undefined) sends 'undefined' string as POST payload (instead of nothing)
   // We need null here if data is undefined
   xhr.send(typeof data !== 'undefined' ? data : null);
-  return this;
 };
 
 request.agent = function () {