postman-runtime 7.19.0 → 7.21.0

package/.eslintrc

@@ -60,7 +60,7 @@
     "block-scoped-var": "error",
     "class-methods-use-this": "error",
     "complexity": "off",
-    "consistent-return": "warn",
+    "consistent-return": "off",
     "curly": "error",
     "default-case": "error",
     "dot-location": [

package/CHANGELOG.yaml

@@ -1,3 +1,48 @@
+7.21.0:
+  date: 2019-12-02
+  new features:
+    - >-
+      GH-937 Added ability to use OAuth2 tokens with unknown types as Bearer
+      token in OAuth2 authentication
+    - GH-934 Added ability to pull domain name from username for NTLM auth
+  fixed bugs:
+    - >-
+      GH-939 Fixed a bug where IPv6 localhost request through IPv4 localhost
+      proxy was failing.
+    - >-
+      GH-928 Fixed a bug where Basic Auth was failing if credentials had
+      non-ASCII characters
+    - >-
+      GH-929 Fixed a bug where error was thrown when `username` or `password`
+      fields were empty for NTLM auth
+    - >-
+      GH-933 Fixed a bug where NTLM could not complete authentication if
+      multiple `www-authenticate` headers were sent
+  chores:
+    - >-
+      GH-942 Convert response into a JSON serializable object before executing
+      the script
+    - Updated dependencies
+
+7.20.1:
+  date: 2019-11-13
+  chores:
+    - Updated dependencies
+
+7.20.0:
+  date: 2019-11-12
+  new features:
+    - GH-921 Added ability to preserve non-JSON data types in console logs
+    - >-
+      GH-921 Added an option `script.serializeLogs` to allow sending logs as a
+      serialized string which can be parsed using `teleport-javascript`
+  fixed bugs:
+    - >-
+      GH-918 Fixed a bug where requests having binary body with AWS, Hawk or
+      EdgeGrid authentication caused the Runtime to crash
+  chores:
+    - Updated dependencies
+
 7.19.0:
   date: 2019-10-16
   new features:

package/README.md

@@ -135,6 +135,14 @@ runner.run(collection, {
         }
     },
 
+    // Options specific to the script execution
+    script: {
+
+        // Option to set whether to send console logs in serialized format which can be parsed
+        // using the `teleport-javascript` serialization library.
+        serializeLogs: false
+    },
+
     // A ProxyConfigList, from the SDK
     proxies: new sdk.ProxyConfigList(),
 

package/lib/authorizer/aws4.js

@@ -63,7 +63,7 @@ var _ = require('lodash'),
                 return callback();
             }
 
-            originalReadStream.cloneReadStream(function (err, clonedStream) {
+            return originalReadStream.cloneReadStream(function (err, clonedStream) {
                 if (err) { return callback(); }
 
                 clonedStream.on('data', function (chunk) {

package/lib/authorizer/basic.js

@@ -1,5 +1,3 @@
-var btoa = require('btoa');
-
 /**
  * @implements {AuthHandlerInterface}
  */
@@ -70,7 +68,7 @@ module.exports = {
         request.removeHeader('Authorization', {ignoreCase: true});
         request.addHeader({
             key: 'Authorization',
-            value: 'Basic ' + btoa(username + ':' + password),
+            value: 'Basic ' + Buffer.from(`${username}:${password}`, 'utf8').toString('base64'),
             system: true
         });
 

package/lib/authorizer/edgegrid.js

@@ -142,7 +142,7 @@ var _ = require('lodash'),
                 return callback();
             }
 
-            originalReadStream.cloneReadStream(function (err, clonedStream) {
+            return originalReadStream.cloneReadStream(function (err, clonedStream) {
                 if (err) { return callback(); }
 
                 clonedStream.on('data', function (chunk) {

package/lib/authorizer/hawk.js

@@ -90,7 +90,7 @@ function computeBodyHash (body, algorithm, digestEncoding, contentType, callback
             return callback();
         }
 
-        originalReadStream.cloneReadStream(function (err, clonedStream) {
+        return originalReadStream.cloneReadStream(function (err, clonedStream) {
             if (err) { return callback(); }
 
             clonedStream.on('data', function (chunk) {

package/lib/authorizer/ntlm.js

@@ -31,6 +31,58 @@ var ntlmUtil = require('httpntlm').ntlm,
         T3_MSG_CREATED: 'T3_MSG_CREATED'
     };
 
+/**
+ * Parses the username to separate username and domain. It can handle two formats:
+ *   - Down-Level Logon name format `DOMAIN\USERNAME`
+ *   - User Principal Name format `USERNAME@DOMAIN`
+ *
+ * @param {String} username - Username string to parse from
+ * @return {Object} - An object with `username` and `domain` fields, which are `strings`.
+ */
+function parseParametersFromUsername (username) {
+    var dllParams,
+        upnParams;
+
+    if (!(username && typeof username === 'string')) {
+        return {
+            username: EMPTY,
+            domain: EMPTY
+        };
+    }
+
+    dllParams = username.split('\\');
+    upnParams = username.split('@');
+
+    // username should be either of the two formats, not both
+    if (dllParams.length > 1 && upnParams.length > 1) {
+        return {
+            username,
+            domain: EMPTY
+        };
+    }
+
+    // try to parse from "down level logon" format
+    if (dllParams.length === 2 && dllParams[0] && dllParams[1]) {
+        return {
+            username: dllParams[1],
+            domain: dllParams[0]
+        };
+    }
+
+    // try to parse from "user principal name" format
+    if (upnParams.length === 2 && upnParams[0] && upnParams[1]) {
+        return {
+            username: upnParams[0],
+            domain: upnParams[1]
+        };
+    }
+
+    return {
+        username,
+        domain: EMPTY
+    };
+}
+
 /**
  * NTLM auth while authenticating requires negotiateMessage (type 1) and authenticateMessage (type 3) to be stored.
  * Also it needs to know which stage is it in (INITIALIZED, T1_MSG_CREATED and T3_MSG_CREATED).
@@ -96,16 +148,26 @@ module.exports = {
         var state = auth.get(STATE),
             domain = auth.get(NTLM_PARAMETERS.DOMAIN) || EMPTY,
             workstation = auth.get(NTLM_PARAMETERS.WORKSTATION) || EMPTY,
-            username = auth.get(NTLM_PARAMETERS.USERNAME),
-            password = auth.get(NTLM_PARAMETERS.PASSWORD),
+            username = auth.get(NTLM_PARAMETERS.USERNAME) || EMPTY,
+            password = auth.get(NTLM_PARAMETERS.PASSWORD) || EMPTY,
             negotiateMessage, // type 1
             challengeMessage, // type 2
-            authenticateMessage; // type 3
+            authenticateMessage, // type 3
+            ntlmType2Header,
+            parsedParameters;
 
         if (response.code !== 401 && response.code !== 403) {
             return done(null, true);
         }
 
+        // we try to extract domain from username if not specified.
+        if (!domain) {
+            parsedParameters = parseParametersFromUsername(username) || {};
+
+            username = parsedParameters.username;
+            domain = parsedParameters.domain;
+        }
+
         if (state === STATES.INITIALIZED) {
             // Nothing to do if the server does not ask us for auth in the first place.
             if (!(response.headers.has(WWW_AUTHENTICATE, NTLM) ||
@@ -130,7 +192,20 @@ module.exports = {
         }
         else if (state === STATES.T1_MSG_CREATED) {
             // At this point, we can assume that the type 1 message was sent to the server
-            challengeMessage = ntlmUtil.parseType2Message(response.headers.get(WWW_AUTHENTICATE) || EMPTY, _.noop);
+
+            // there can be multiple headers present with key `www-authenticate`.
+            // iterate to get the one which has the NTLM hash. if multiple
+            // headers have the NTLM hash, use the first one.
+            ntlmType2Header = response.headers.find(function (header) {
+                return String(header.key).toLowerCase() === WWW_AUTHENTICATE &&
+                    header.valueOf().startsWith('NTLM ');
+            });
+
+            if (!ntlmType2Header) {
+                return done(new Error('ntlm: server did not send NTLM type 2 message'));
+            }
+
+            challengeMessage = ntlmUtil.parseType2Message(ntlmType2Header.valueOf(), _.noop);
 
             if (!challengeMessage) {
                 return done(new Error('ntlm: server did not correctly process authentication request'));

package/lib/authorizer/oauth2.js

@@ -3,6 +3,7 @@ var _ = require('lodash'),
     HEADER = 'header',
     QUERY_PARAMS = 'queryParams',
     BEARER = 'bearer',
+    MAC = 'mac',
     AUTHORIZATION = 'Authorization',
     ACCESS_TOKEN = 'access_token',
     AUTHORIZATION_PREFIX = 'Bearer ',
@@ -94,27 +95,31 @@ module.exports = {
         tokenType = _.toLower(params.tokenType);
 
         // @TODO Add support for HMAC
-        if (tokenType === BEARER) {
-            // clean conflicting headers and query params
-            // @todo: we should be able to get conflicting params from auth manifest
-            // and clear them before the sign step for any auth
-            request.removeHeader(AUTHORIZATION, {ignoreCase: true});
-            request.removeQueryParams([ACCESS_TOKEN]);
+        if (tokenType === MAC) {
+            return done();
+        }
 
-            if (params.addTokenTo === QUERY_PARAMS) {
-                request.addQueryParams({
-                    key: ACCESS_TOKEN,
-                    value: params.accessToken,
-                    system: true
-                });
-            }
-            else if (params.addTokenTo === HEADER) {
-                request.addHeader({
-                    key: AUTHORIZATION,
-                    value: AUTHORIZATION_PREFIX + params.accessToken,
-                    system: true
-                });
-            }
+        // treat every token types (other than MAC) as bearer token
+
+        // clean conflicting headers and query params
+        // @todo: we should be able to get conflicting params from auth manifest
+        // and clear them before the sign step for any auth
+        request.removeHeader(AUTHORIZATION, {ignoreCase: true});
+        request.removeQueryParams([ACCESS_TOKEN]);
+
+        if (params.addTokenTo === QUERY_PARAMS) {
+            request.addQueryParams({
+                key: ACCESS_TOKEN,
+                value: params.accessToken,
+                system: true
+            });
+        }
+        else if (params.addTokenTo === HEADER) {
+            request.addHeader({
+                key: AUTHORIZATION,
+                value: AUTHORIZATION_PREFIX + params.accessToken,
+                system: true
+            });
         }
 
         return done();

package/lib/requester/core.js

@@ -251,12 +251,14 @@ module.exports = {
             self = this,
             bodyParams,
             url = request.url && urlEncoder.toNodeUrl(request.url.toString(true)),
-            isSSL,
+            isSSL = _.startsWith(url.protocol, HTTPS),
+            isTunnelingProxy = request.proxy && (request.proxy.tunnel || isSSL),
             reqOption,
             portNumber,
             behaviorName,
             port = url && url.port,
-            hostname = url && url.hostname && url.hostname.toLowerCase();
+            hostname = url && url.hostname && url.hostname.toLowerCase(),
+            proxyHostname = request.proxy && request.proxy.host;
 
         !defaultOpts && (defaultOpts = {});
         !protocolProfileBehavior && (protocolProfileBehavior = {});
@@ -268,6 +270,10 @@ module.exports = {
             hostname = LOCALHOST;
         }
 
+        if (getTLD(proxyHostname) === LOCALHOST) {
+            proxyHostname = LOCALHOST;
+        }
+
         options.url = url;
         options.method = request.method;
         options.jar = defaultOpts.cookieJar || true;
@@ -334,9 +340,17 @@ module.exports = {
         self.ensureHeaderExists(options.headers, 'Host', url.host);
 
         // override DNS lookup
-        if (networkOptions.restrictedAddresses || hostname === LOCALHOST || networkOptions.hostLookup) {
-            isSSL = _.startsWith(request.url.protocol, HTTPS);
-            portNumber = Number(port) || (isSSL ? HTTPS_DEFAULT_PORT : HTTP_DEFAULT_PORT);
+        if (networkOptions.restrictedAddresses || hostname === LOCALHOST ||
+            (!isTunnelingProxy && proxyHostname === LOCALHOST) || networkOptions.hostLookup) {
+            // Use proxy port for localhost resolution in case of non-tunneling proxy
+            // because the request will be sent to proxy server by postman-request
+            if (request.proxy && !isTunnelingProxy) {
+                portNumber = Number(request.proxy.port);
+            }
+            // Otherwise, use request's port
+            else {
+                portNumber = Number(port) || (isSSL ? HTTPS_DEFAULT_PORT : HTTP_DEFAULT_PORT);
+            }
 
             _.isFinite(portNumber) && (options.lookup = lookup.bind(this, {
                 port: portNumber,

package/lib/runner/extensions/event.command.js

@@ -428,6 +428,7 @@ module.exports = {
                     timeout: payload.scriptTimeout, // @todo: Expose this as a property in Collection SDK's Script
                     cursor: scriptCursor,
                     context: _.pick(payload.context, SAFE_CONTEXT_VARIABLES),
+                    serializeLogs: _.get(this, 'options.script.serializeLogs'),
 
                     // legacy options
                     legacy: {
@@ -489,7 +490,13 @@ module.exports = {
                     result && result.collectionVariables &&
                         (result.collectionVariables = new sdk.VariableScope(result.collectionVariables));
                     result && result.request && (result.request = new sdk.Request(result.request));
-                    result && result.response && (result.response = new sdk.Response(result.response));
+
+                    // @note Since postman-sandbox@3.5.2, response object is not included in the execution result.
+                    // Refer: https://github.com/postmanlabs/postman-sandbox/pull/512
+                    // Adding back here to avoid breaking change in `script` callback.
+                    // @todo revisit script callback args in runtime v8.
+                    payload.context && payload.context.response &&
+                        (result.response = new sdk.Response(payload.context.response));
 
                     // persist the pm.variables for the next script
                     result && result._variables &&

package/lib/runner/extensions/item.command.js

@@ -1,9 +1,10 @@
 var _ = require('lodash'),
     uuid = require('uuid'),
+    Response = require('postman-collection').Response,
     visualizer = require('../../visualizer'),
 
     /**
-     * List of request properties which can be mutated via prerequest
+     * List of request properties which can be mutated via pre-request
      *
      * @private
      * @const
@@ -11,7 +12,8 @@ var _ = require('lodash'),
      */
     ALLOWED_REQUEST_MUTATIONS = ['url', 'method', 'headers'],
 
-    extractVisualizerData;
+    extractVisualizerData,
+    getResponseJSON;
 
 /**
  * Returns visualizer data from the latest execution result.
@@ -36,7 +38,7 @@ extractVisualizerData = function (prereqExecutions, testExecutions) {
     }
 
     if (_.isArray(prereqExecutions)) {
-        // extraxt visualizer data from pre-request script results if it is not found earlier
+        // extract visualizer data from pre-request script results if it is not found earlier
         for (i = prereqExecutions.length - 1; i >= 0; i--) {
             visualizerData = _.get(prereqExecutions[i], 'result.return.visualizer');
 
@@ -47,6 +49,31 @@ extractVisualizerData = function (prereqExecutions, testExecutions) {
     }
 };
 
+/**
+ * Convert response into a JSON serializable object.
+ * The stream property is converted to base64 string for performance reasons.
+ *
+ * @param {Object} response - SDK Response instance
+ * @returns {Object}
+ */
+getResponseJSON = function (response) {
+    if (!Response.isResponse(response)) {
+        return;
+    }
+
+    return {
+        id: response.id,
+        code: response.code,
+        status: response.status,
+        header: response.headers && response.headers.toJSON(),
+        stream: response.stream && {
+            type: 'Base64',
+            data: response.stream.toString('base64')
+        },
+        responseTime: response.responseTime
+    };
+};
+
 /**
  * Add options
  * stopOnError:Boolean
@@ -173,7 +200,15 @@ module.exports = {
 
                         // also the test object requires the updated request object (since auth helpers may modify it)
                         request && (ctxTemplate.request = request);
-                        response && (ctxTemplate.response = response);
+
+                        // @note convert response instance to plain object.
+                        // we want to avoid calling Response.toJSON() which triggers toJSON on Response.stream buffer.
+                        // Because that increases the size of stringified object by 3 times.
+                        // Also, that increases the total number of tokens (buffer.data) whereas Buffer.toString
+                        // generates a single string that is easier to stringify and sent over the UVM bridge.
+                        response && (ctxTemplate.response = getResponseJSON(response));
+
+                        // set cookies for this transaction
                         cookies && (ctxTemplate.cookies = cookies);
 
                         // the context template also has a test object to store assertions

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "postman-runtime",
-  "version": "7.19.0",
+  "version": "7.21.0",
   "description": "Underlying library of executing Postman Collections (used by Newman)",
   "main": "index.js",
   "directories": {
@@ -30,21 +30,20 @@
   "license": "Apache-2.0",
   "dependencies": {
     "async": "2.6.2",
-    "aws4": "1.8.0",
-    "btoa": "1.2.1",
+    "aws4": "1.9.0",
     "crypto-js": "3.1.9-1",
     "eventemitter3": "4.0.0",
-    "handlebars": "4.4.3",
+    "handlebars": "4.5.3",
     "http-reasons": "0.1.0",
     "httpntlm": "1.7.6",
     "inherits": "2.0.4",
     "lodash": "4.17.15",
     "node-oauth1": "1.2.2",
     "performance-now": "2.1.0",
-    "postman-collection": "3.5.4",
-    "postman-request": "2.88.1-postman.15",
-    "postman-sandbox": "3.4.0",
-    "postman-url-encoder": "1.0.2",
+    "postman-collection": "3.5.5",
+    "postman-request": "2.88.1-postman.16",
+    "postman-sandbox": "3.5.2",
+    "postman-url-encoder": "1.0.3",
     "resolve-from": "5.0.0",
     "serialised-error": "1.1.3",
     "tough-cookie": "3.0.1",
@@ -58,21 +57,22 @@
     "eslint": "5.16.0",
     "eslint-plugin-jsdoc": "8.7.0",
     "eslint-plugin-lodash": "5.1.0",
-    "eslint-plugin-mocha": "6.2.0",
+    "eslint-plugin-mocha": "6.2.2",
     "eslint-plugin-security": "1.4.0",
     "graphql": "14.5.8",
     "http-proxy": "1.18.0",
     "istanbul": "0.4.5",
     "js-yaml": "3.13.1",
     "jsdoc": "3.6.3",
-    "jsdoc-to-markdown": "5.0.2",
-    "mocha": "6.2.1",
+    "jsdoc-to-markdown": "5.0.3",
+    "mocha": "6.2.2",
     "parse-gitignore": "0.5.1",
     "postman-jsdoc-theme": "0.0.3",
     "recursive-readdir": "2.2.2",
     "server-destroy": "1.0.1",
     "shelljs": "0.8.3",
     "sinon": "7.5.0",
+    "teleport-javascript": "1.0.0",
     "tmp": "0.1.0",
     "yankee": "1.0.8"
   },

package/test/fixtures/server.js

@@ -1,11 +1,14 @@
 const fs = require('fs'),
     net = require('net'),
+    url = require('url'),
+    dns = require('dns'),
     _ = require('lodash'),
     path = require('path'),
     http = require('http'),
     https = require('https'),
     crypto = require('crypto'),
     GraphQL = require('graphql'),
+    ntlmUtils = require('httpntlm').ntlm,
     enableServerDestroy = require('server-destroy');
 
 /**
@@ -206,6 +209,7 @@ function createHTTPServer () {
  * @param {Object} [options] - Additional options to configure proxy server
  * @param {Object} [options.auth] - Proxy authentication, Basic auth
  * @param {String} [options.agent] - Agent used for http(s).request
+ * @param {Boolean} [options.useIPv6] - If true, force using IPv6 address while forwarding request.
  *
  * @example
  * var s = createProxyServer({
@@ -244,14 +248,22 @@ function createProxyServer (options) {
         req.headers = Object.assign(req.headers, options.headers || {});
 
         // forward request to the origin and pipe the response
-        var fwd = agent.request({
-            host: req.headers.host,
-            path: req.url,
-            method: req.method.toLowerCase(),
-            headers: req.headers
-        }, function (resp) {
-            resp.pipe(res);
-        });
+        var requestUrl = url.parse(req.url),
+            fwd = agent.request({
+                host: requestUrl.hostname,
+                path: requestUrl.path,
+                port: requestUrl.port,
+                method: req.method.toLowerCase(),
+                headers: req.headers,
+                lookup: options.useIPv6 && function (hostname, options, callback) {
+                    !options && (options = {});
+                    options.family = 6;
+
+                    return dns.lookup(hostname, options, callback);
+                }
+            }, function (resp) {
+                resp.pipe(res);
+            });
 
         req.pipe(fwd);
     });
@@ -497,6 +509,106 @@ function createEdgeGridAuthServer (options) {
     return server;
 }
 
+/**
+ * Creates an NTLM server.
+ *
+ * @param {Object} options - The options for the server
+ * @param {String} options.username - Username for authentication
+ * @param {String} options.password - Password for authentication
+ * @param {String} options.domain - Domain name for authentication
+ * @param {String} options.workstation - Workstation for authentication
+ * @param {Boolean} options.debug - Enable logging of requests
+ *
+ * @return {Object} - http server
+ */
+function createNTLMServer (options) {
+    options = options || {};
+
+    var type2Message = 'NTLM ' +
+            'TlRMTVNTUAACAAAAHgAeADgAAAAFgoqiBevywvJykjAAAAAAAAAAAJgAmABWAAAA' +
+            'CgC6RwAAAA9EAEUAUwBLAFQATwBQAC0ASgBTADQAVQBKAFQARAACAB4ARABFAFMA' +
+            'SwBUAE8AUAAtAEoAUwA0AFUASgBUAEQAAQAeAEQARQBTAEsAVABPAFAALQBKAFMA' +
+            'NABVAEoAVABEAAQAHgBEAEUAUwBLAFQATwBQAC0ASgBTADQAVQBKAFQARAADAB4A' +
+            'RABFAFMASwBUAE8AUAAtAEoAUwA0AFUASgBUAEQABwAIADmguzCHn9UBAAAAAA==',
+        parsedType2Message = ntlmUtils.parseType2Message(type2Message, _.noop),
+
+        username = options.username || 'username',
+        password = options.password || 'password',
+        domain = options.domain || '',
+        workstation = options.workstation || '',
+
+        type1Message = ntlmUtils.createType1Message({
+            domain,
+            workstation
+        }),
+        type3Message = ntlmUtils.createType3Message(parsedType2Message, {
+            domain,
+            workstation,
+            username,
+            password
+        }),
+
+        handler = function (req, res) {
+            var authHeaders = req.headers.authorization;
+
+            // send type2 message and ask for type3 message
+            if (authHeaders && authHeaders.startsWith(type1Message.slice(0, 20))) {
+                res.writeHead(401, {
+
+                    // @note we're sending a 'Negotiate' header here to make
+                    // sure that runtime can handle it.
+                    'www-authenticate': [type2Message, 'Negotiate']
+                });
+
+                options.debug && console.info('401: got type1 message');
+            }
+
+            // successful auth
+            // @note we don't check if the username and password are correct
+            // because I don't know how.
+            else if (authHeaders && authHeaders.startsWith(type3Message.slice(0, 100))) {
+                res.writeHead(200);
+
+                options.debug && console.info('200: got type3 message');
+            }
+
+            // no valid auth headers, ask for type1 message
+            else {
+                res.writeHead(401, {
+                    'www-authenticate': ['NTLM', 'Negotiate']
+                });
+
+                options.debug && console.info('401: got no authorization header');
+            }
+
+            res.end();
+        },
+        server = http.createServer(handler);
+
+    enableServerDestroy(server);
+
+    return server;
+}
+
+/**
+ * Custom junk bytes response server.
+ *
+ * `/${bytes}` returns binary response of given bytes size.
+ */
+function createBytesServer () {
+    var server = http.createServer(function (req, res) {
+        var bytes = Number(req.url.substr(1)) || 0; // remove leading /
+
+        res.writeHead(200);
+        res.write(Buffer.alloc(bytes));
+        res.end();
+    });
+
+    enableServerDestroy(server);
+
+    return server;
+}
+
 module.exports = {
     createSSLServer,
     createHTTPServer,
@@ -504,5 +616,7 @@ module.exports = {
     createRawEchoServer,
     createGraphQLServer,
     createRedirectServer,
-    createEdgeGridAuthServer
+    createEdgeGridAuthServer,
+    createNTLMServer,
+    createBytesServer
 };