superagent 3.8.1 → 4.0.0-beta.5

package/lib/node/index.js

@@ -15,7 +15,6 @@ let methods = require('methods');
 const Stream = require('stream');
 const utils = require('../utils');
 const unzip = require('./unzip').unzip;
-const extend = require('extend');
 const mime = require('mime');
 const https = require('https');
 const http = require('http');
@@ -25,6 +24,13 @@ const zlib = require('zlib');
 const util = require('util');
 const pkg = require('../../package.json');
 const RequestBase = require('../request-base');
+const CookieJar = require('cookiejar');
+let http2;
+try {
+  http2 = require('./http2wrapper');
+  console.warn('superagent: Enable experimental feature http2');
+} catch (_) {
+}
 
 function request(method, url) {
   // callback
@@ -80,6 +86,7 @@ mime.define({
 exports.protocols = {
   'http:': http,
   'https:': https,
+  'http2:': http2,
 };
 
 /**
@@ -107,6 +114,15 @@ exports.serialize = {
 
 exports.parse = require('./parsers');
 
+
+/**
+ * Default buffering map. Can be used to set certain
+ * response types to buffer/not buffer.
+ *
+ *     superagent.buffer['application/xml'] = true;
+ */
+exports.buffer = {};
+
 /**
  * Initialize internal header tracking properties on a request instance.
  *
@@ -134,6 +150,7 @@ function _initHeaders(req) {
 function Request(method, url) {
   Stream.call(this);
   if ('string' != typeof url) url = format(url);
+  this._enableHttp2 = !!process.env.HTTP2_TEST; // internal only
   this._agent = false;
   this._formData = null;
   this.method = method;
@@ -158,13 +175,50 @@ function Request(method, url) {
 util.inherits(Request, Stream);
 RequestBase(Request.prototype);
 
+/**
+ * Enable or Disable http2.
+ *
+ * Enable http2.
+ *
+ * ``` js
+ * request.get('http://localhost/')
+ *   .http2()
+ *   .end(callback);
+ *
+ * request.get('http://localhost/')
+ *   .http2(true)
+ *   .end(callback);
+ * ```
+ *
+ * Disable http2.
+ *
+ * ``` js
+ * request = request.http2();
+ * request.get('http://localhost/')
+ *   .http2(false)
+ *   .end(callback);
+ * ```
+ *
+ * @param {Boolean} enable
+ * @return {Request} for chaining
+ * @api public
+ */
+
+Request.prototype.http2 = function(bool){
+  if (exports.protocols['http2:'] === undefined){
+    throw new Error('superagent: does not support http2');
+  }
+  this._enableHttp2 = bool === undefined ? true : bool;
+  return this;
+}
+
 /**
  * Queue the given `file` as an attachment to the specified `field`,
  * with optional `options` (or filename).
  *
  * ``` js
  * request.post('http://localhost/upload')
- *   .attach(new Buffer('<b>Hello world</b>'), 'hello.html')
+ *   .attach('field', Buffer.from('<b>Hello world</b>'), 'hello.html')
  *   .end(callback);
  * ```
  *
@@ -234,7 +288,7 @@ Request.prototype.agent = function(agent){
 };
 
 /**
- * Set _Content-Type_ response header passed through `mime.lookup()`.
+ * Set _Content-Type_ response header passed through `mime.getType()`.
  *
  * Examples:
  *
@@ -261,12 +315,12 @@ Request.prototype.agent = function(agent){
 Request.prototype.type = function(type) {
   return this.set(
     'Content-Type',
-    ~type.indexOf('/') ? type : mime.lookup(type)
+    ~type.indexOf('/') ? type : mime.getType(type)
   );
 };
 
 /**
- * Set _Accept_ response header passed through `mime.lookup()`.
+ * Set _Accept_ response header passed through `mime.getType()`.
  *
  * Examples:
  *
@@ -288,7 +342,7 @@ Request.prototype.type = function(type) {
 Request.prototype.accept = function(type){
   return this.set('Accept', ~type.indexOf('/')
     ? type
-    : mime.lookup(type));
+    : mime.getType(type));
 };
 
 /**
@@ -309,7 +363,7 @@ Request.prototype.query = function(val){
   if ('string' == typeof val) {
     this._query.push(val);
   } else {
-    extend(this.qs, val);
+    Object.assign(this.qs, val);
   }
   return this;
 };
@@ -494,9 +548,7 @@ Request.prototype.auth = function(user, pass, options){
     options = { type: 'basic' };
   }
 
-  var encoder = function(string) {
-    return new Buffer(string).toString('base64');
-  };
+  const encoder = string => new Buffer.from(string).toString('base64');
 
   return this._auth(user, pass, options, encoder);
 };
@@ -587,10 +639,31 @@ Request.prototype.request = function(){
   let url = this.url;
   const retries = this._retries;
 
+  // Capture backticks as-is from the final query string built above.
+  // Note: this'll only find backticks entered in req.query(String)
+  // calls, because qs.stringify unconditionally encodes backticks.
+  let queryStringBackticks;
+  if(url.indexOf('`') > -1) {
+    const queryStartIndex = url.indexOf("?");
+
+    if(queryStartIndex !== -1) {
+      const queryString = url.substr(queryStartIndex + 1);
+      queryStringBackticks = queryString.match(/`|\%60/g);
+    }
+  }
+
   // default to http://
   if (0 != url.indexOf('http')) url = `http://${url}`;
   url = parse(url);
 
+  // See https://github.com/visionmedia/superagent/issues/1367
+  if(queryStringBackticks) {
+    let i = 0;
+    url.query = url.query.replace(/\%60/g, () => queryStringBackticks[i++]);
+    url.search = `?${url.query}`;
+    url.path = url.pathname + url.search;
+  }
+
   // support unix sockets
   if (/^https?\+unix:/.test(url.protocol) === true) {
     // get the protocol
@@ -614,8 +687,13 @@ Request.prototype.request = function(){
   options.passphrase = this._passphrase;
   options.agent = this._agent;
 
+  // Allows request.get('https://1.2.3.4/').set('Host', 'example.com')
+  if (this._header['host']) {
+    options.servername = this._header['host'].replace(/:[0-9]+$/,'');
+  }
+
   // initiate request
-  const mod = exports.protocols[url.protocol];
+  const mod = this._enableHttp2 ? exports.protocols['http2:'].setProtocol(url.protocol) : exports.protocols[url.protocol];
 
   // request
   const req = (this.req = mod.request(options));
@@ -632,7 +710,7 @@ Request.prototype.request = function(){
   // expose events
   req.once('drain', () => { this.emit('drain'); });
 
-  req.once('error', err => {
+  req.on('error', err => {
     // flag abortion here for out timeouts
     // because node will emit a faux-error "socket hang up"
     // when request is aborted before a connection is made
@@ -654,15 +732,24 @@ Request.prototype.request = function(){
   if (this.username && this.password) {
     this.auth(this.username, this.password);
   }
-
-  // add cookies
-  if (this.cookies) req.setHeader('Cookie', this.cookies);
-
   for (const key in this.header) {
     if (this.header.hasOwnProperty(key))
       req.setHeader(key, this.header[key]);
   }
 
+  // add cookies
+  if (this.cookies) {
+    if(this._header.hasOwnProperty('cookie')) {
+      // merge
+      const tmpJar = new CookieJar.CookieJar();
+      tmpJar.setCookies(this._header.cookie.split(';'));
+      tmpJar.setCookies(this.cookies.split(';'));
+      req.setHeader('Cookie',tmpJar.getCookies(CookieJar.CookieAccessInfo.All).toValueString());
+    } else {
+      req.setHeader('Cookie', this.cookies);
+    }
+  }
+
   return req;
 };
 
@@ -688,20 +775,23 @@ Request.prototype.callback = function(err, res){
 
   if (!err) {
     try {
-      if (this._isResponseOK(res)) {
-        return fn(err, res);
-      }
-
-      let msg = 'Unsuccessful HTTP response';
-      if (res) {
-        msg = http.STATUS_CODES[res.status] || msg;
+      if (!this._isResponseOK(res)) {
+        let msg = 'Unsuccessful HTTP response';
+        if (res) {
+          msg = http.STATUS_CODES[res.status] || msg;
+        }
+        err = new Error(msg);
+        err.status = res ? res.status : undefined;
       }
-      err = new Error(msg);
-      err.status = res ? res.status : undefined;
     } catch (new_err) {
       err = new_err;
     }
   }
+  // It's important that the callback is called outside try/catch
+  // to avoid double callback
+  if (!err) {
+    return fn(null, res);
+  }
 
   err.response = res;
   if (this._maxRetries) err.retries = this._retries - 1;
@@ -743,6 +833,11 @@ Request.prototype._emitResponse = function(body, files) {
     response.body = body;
   }
   response.files = files;
+  if (this._endCalled) {
+    response.pipe = function() {
+      throw Error("end() has already been called, so it's too late to start piping");
+    }
+  }
   this.emit('response', response);
   return response;
 };
@@ -752,8 +847,8 @@ Request.prototype.end = function(fn) {
   debug('%s %s', this.method, this.url);
 
   if (this._endCalled) {
-    console.warn(
-      'Warning: .end() was called twice. This is not supported in superagent'
+    throw Error(
+      '.end() was called twice. This is not supported in superagent'
     );
   }
   this._endCalled = true;
@@ -761,13 +856,14 @@ Request.prototype.end = function(fn) {
   // store callback
   this._callback = fn || noop;
 
-  return this._end();
+  this._end();
 };
 
 Request.prototype._end = function() {
+  if (this._aborted) return this.callback(Error("The request has been aborted even before .end() was called"));
+
   let data = this._data;
   const req = this.req;
-  let buffer = this._buffer;
   const method = this.method;
 
   this._setTimeouts();
@@ -779,7 +875,7 @@ Request.prototype._end = function() {
       let contentType = req.getHeader('Content-Type');
       // Parse out just the content type from the header (ignore the charset)
       if (contentType) contentType = contentType.split(';')[0];
-      let serialize = exports.serialize[contentType];
+      let serialize = this._serializer || exports.serialize[contentType];
       if (!serialize && isJSON(contentType)) {
         serialize = exports.serialize['application/json'];
       }
@@ -809,7 +905,6 @@ Request.prototype._end = function() {
     const type = mime.split('/')[0];
     const multipart = 'multipart' == type;
     const redirect = isRedirect(res.statusCode);
-    let parser = this._parser;
     const responseType = this._responseType;
 
     this.res = res;
@@ -830,6 +925,19 @@ Request.prototype._end = function() {
       unzip(req, res);
     }
 
+    let buffer = this._buffer;
+    if (buffer === undefined && mime in exports.buffer){
+      buffer = !!exports.buffer[mime];
+    }
+
+    let parser = this._parser;
+    if (undefined === buffer) {
+      if (parser) {
+        console.warn("A custom superagent parser has been set, but buffering strategy for the parser hasn't been configured. Call `req.buffer(true or false)` or set `superagent.buffer[mime] = true or false`");
+        buffer = true;
+      }
+    }
+
     if (!parser) {
       if (responseType) {
         parser = exports.parse.image; // It's actually a generic Buffer
@@ -853,6 +961,9 @@ Request.prototype._end = function() {
         buffer = (buffer !== false);
       } else if (buffer) {
         parser = exports.parse.text;
+      } else if (undefined === buffer) {
+        parser = exports.parse.image; // It's actually a generic Buffer
+        buffer = true;
       }
     }
 
@@ -861,6 +972,7 @@ Request.prototype._end = function() {
       buffer = true;
     }
 
+    this._resBuffered = buffer;
     let parserHandlesEnd = false;
     if (buffer) {
       // Protectiona against zip bombs and other nuisance
@@ -939,6 +1051,47 @@ Request.prototype._end = function() {
 
   this.emit('request', this);
 
+  const getProgressMonitor = () => {
+    const lengthComputable = true;
+    const total = req.getHeader('Content-Length');
+    let loaded = 0;
+
+    const progress = new Stream.Transform();
+    progress._transform = (chunk, encoding, cb) => {
+      loaded += chunk.length;
+      this.emit('progress', {
+        direction: 'upload',
+        lengthComputable,
+        loaded,
+        total,
+      });
+      cb(null, chunk);
+    };
+    return progress;
+  };
+
+  const bufferToChunks = (buffer) => {
+    const chunkSize = 16 * 1024; // default highWaterMark value
+    const chunking = new Stream.Readable();
+    const totalLength = buffer.length;
+    const remainder = totalLength % chunkSize;
+    const cutoff = totalLength - remainder;
+
+    for (let i = 0; i < cutoff; i += chunkSize) {
+      const chunk = buffer.slice(i, i + chunkSize);
+      chunking.push(chunk);
+    }
+
+    if (remainder > 0) {
+      const remainderBuffer = buffer.slice(-remainder);
+      chunking.push(remainderBuffer);
+    }
+
+    chunking.push(null); // no more data
+
+    return chunking;
+  }
+
   // if a FormData instance got created, then we send that as the request body
   const formData = this._formData;
   if (formData) {
@@ -959,31 +1112,13 @@ Request.prototype._end = function() {
         req.setHeader('Content-Length', length);
       }
 
-      const getProgressMonitor = () => {
-        const lengthComputable = true;
-        const total = req.getHeader('Content-Length');
-        let loaded = 0;
-
-        const progress = new Stream.Transform();
-        progress._transform = (chunk, encoding, cb) => {
-          loaded += chunk.length;
-          this.emit('progress', {
-            direction: 'upload',
-            lengthComputable,
-            loaded,
-            total,
-          });
-          cb(null, chunk);
-        };
-        return progress;
-      };
       formData.pipe(getProgressMonitor()).pipe(req);
     });
+  } else if (Buffer.isBuffer(data)) {
+    bufferToChunks(data).pipe(getProgressMonitor()).pipe(req);
   } else {
     req.end(data);
   }
-
-  return this;
 };
 
 /**
@@ -1064,7 +1199,9 @@ function isImageOrVideo(mime) {
  */
 
 function isJSON(mime) {
-  return /[\/+]json\b/.test(mime);
+  // should match /json or +json
+  // but not /json-seq
+  return /[\/+]json($|[^-\w])/.test(mime);
 }
 
 /**

package/lib/node/parsers/text.js

@@ -1,6 +1,6 @@
 'use strict';
 
-module.exports = function(res, fn){
+module.exports = (res, fn) => {
   res.text = '';
   res.setEncoding('utf8');
   res.on('data', chunk => {

package/lib/node/parsers/urlencoded.js

@@ -6,7 +6,7 @@
 
 const qs = require('qs');
 
-module.exports = function(res, fn){
+module.exports = (res, fn) => {
   res.text = '';
   res.setEncoding('ascii');
   res.on('data', chunk => {

package/lib/node/response.js

@@ -36,7 +36,7 @@ function Response(req) {
   this.text = res.text;
   this.body = res.body !== undefined ? res.body : {};
   this.files = res.files || {};
-  this.buffered = 'string' == typeof this.text;
+  this.buffered = req._resBuffered;
   this.header = this.headers = res.headers;
   this._setStatusProperties(res.statusCode);
   this._setHeaderProperties(this.header);

package/lib/node/unzip.js

@@ -59,9 +59,9 @@ exports.unzip = (req, res) => {
   const _on = res.on;
   res.on = function(type, fn) {
     if ('data' == type || 'end' == type) {
-      stream.on(type, fn);
+      stream.on(type, fn.bind(res));
     } else if ('error' == type) {
-      stream.on(type, fn);
+      stream.on(type, fn.bind(res));
       _on.call(res, type, fn);
     } else {
       _on.call(res, type, fn);