zapier-platform-core 9.5.0 → 9.7.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zapier-platform-core",
-  "version": "9.5.0",
+  "version": "9.7.1",
   "description": "The core SDK for CLI apps in the Zapier Developer Platform.",
   "repository": "zapier/zapier-platform-core",
   "homepage": "https://zapier.com/",
@@ -41,16 +41,18 @@
     "bluebird": "3.5.5",
     "content-disposition": "0.5.3",
     "dotenv": "8.1.0",
-    "form-data": "2.5.0",
+    "form-data": "4.0.0",
     "lodash": "4.17.15",
-    "node-fetch": "2.6.0",
+    "mime-types": "2.1.34",
+    "node-fetch": "2.6.7",
     "oauth-sign": "0.9.0",
     "semver": "5.6.0",
-    "zapier-platform-schema": "9.5.0"
+    "zapier-platform-schema": "9.7.1"
   },
   "devDependencies": {
     "adm-zip": "0.4.13",
     "aws-sdk": "2.238.1",
+    "dicer": "0.3.0",
     "fs-extra": "8.1.0",
     "mock-fs": "4.10.1"
   },

package/src/tools/cleaner.js

@@ -7,7 +7,7 @@ const {
   flattenPaths,
   getObjectType,
   isPlainObj,
-  recurseReplace
+  recurseReplace,
 } = require('./data');
 
 const DEFAULT_BUNDLE = {
@@ -15,7 +15,7 @@ const DEFAULT_BUNDLE = {
   inputData: {},
   meta: {},
   subscribeData: {},
-  targetUrl: ''
+  targetUrl: '',
 };
 
 const isCurlies = /{{.*?}}/g;
@@ -34,7 +34,7 @@ const recurseCleanFuncs = (obj, path) => {
     });
   } else if (isPlainObj(obj)) {
     const newObj = {};
-    Object.keys(obj).forEach(key => {
+    Object.keys(obj).forEach((key) => {
       const value = obj[key];
       newObj[key] = recurseCleanFuncs(value, path.concat([key]));
     });
@@ -45,7 +45,7 @@ const recurseCleanFuncs = (obj, path) => {
 
 // Recurse a nested object replace all instances of keys->vals in the bank.
 const recurseReplaceBank = (obj, bank = {}) => {
-  const replacer = out => {
+  const replacer = (out) => {
     if (!['string', 'number'].includes(typeof out)) {
       return out;
     }
@@ -56,7 +56,7 @@ const recurseReplaceBank = (obj, bank = {}) => {
     const originalValueStr = String(out);
     let maybeChangedString = originalValueStr;
 
-    Object.keys(bank).forEach(key => {
+    Object.keys(bank).forEach((key) => {
       // Escape characters (ex. {{foo}} => \\{\\{foo\\}\\} )
       const escapedKey = key.replace(/[-[\]/{}()\\*+?.^$|]/g, '\\$&');
       const matchesKey = new RegExp(escapedKey, 'g');
@@ -106,12 +106,12 @@ const finalizeBundle = pipe(
 );
 
 // Takes a raw app and bundle and composes a bank of {{key}}->val
-const createBundleBank = (appRaw, event = {}, serializeFunc = x => x) => {
+const createBundleBank = (appRaw, event = {}, serializeFunc = (x) => x) => {
   const bank = {
     bundle: finalizeBundle(event.bundle),
     process: {
-      env: _.extend({}, process.env || {})
-    }
+      env: _.extend({}, process.env || {}),
+    },
   };
 
   const options = { preserve: { 'bundle.inputData': true } };
@@ -123,7 +123,7 @@ const createBundleBank = (appRaw, event = {}, serializeFunc = x => x) => {
   }, {});
 };
 
-const maskOutput = output => _.pick(output, 'results', 'status');
+const maskOutput = (output) => _.pick(output, 'results', 'status');
 
 // These normalize functions are called after the initial before middleware that
 // cleans the request. The reason is that we need to know why a value is empty
@@ -132,7 +132,7 @@ const maskOutput = output => _.pick(output, 'results', 'status');
 // an earlier Zap step? Or was it a null value? Each has different results depending
 // on how the partner has configued their integration.
 const normalizeEmptyRequestFields = (shouldCleanup, field, req) => {
-  const handleEmpty = key => {
+  const handleEmpty = (key) => {
     const value = req[field][key] || '';
     const cleaned = value.replace(isCurlies, '');
 
@@ -152,11 +152,11 @@ const normalizeEmptyRequestFields = (shouldCleanup, field, req) => {
   });
 };
 
-const isEmptyQueryParam = value =>
+const isEmptyQueryParam = (value) =>
   value === '' ||
   value === null ||
   value === undefined ||
-  isCurlies.test(value);
+  (typeof value === 'string' && value.search(isCurlies) >= 0);
 
 const normalizeEmptyParamFields = normalizeEmptyRequestFields.bind(
   null,
@@ -165,7 +165,7 @@ const normalizeEmptyParamFields = normalizeEmptyRequestFields.bind(
 );
 const normalizeEmptyBodyFields = normalizeEmptyRequestFields.bind(
   null,
-  v => isCurlies.test(v),
+  (v) => typeof v === 'string' && v.search(isCurlies) >= 0,
   'body'
 );
 
@@ -175,5 +175,5 @@ module.exports = {
   normalizeEmptyBodyFields,
   normalizeEmptyParamFields,
   recurseCleanFuncs,
-  recurseReplaceBank
+  recurseReplaceBank,
 };

package/src/tools/create-file-stasher.js

@@ -1,14 +1,18 @@
 'use strict';
 
-const _ = require('lodash');
+const fs = require('fs');
+const os = require('os');
 const path = require('path');
-const FormData = require('form-data');
+const { pipeline } = require('stream');
+const { promisify } = require('util');
+const { randomBytes } = require('crypto');
+
+const _ = require('lodash');
 const contentDisposition = require('content-disposition');
+const FormData = require('form-data');
+const mime = require('mime-types');
 
 const request = require('./request-client-internal');
-const ZapierPromise = require('./promise');
-
-const isPromise = obj => obj && typeof obj.then === 'function';
 
 const UPLOAD_MAX_SIZE = 1000 * 1000 * 150; // 150mb, in zapier backend too
 
@@ -19,33 +23,187 @@ const LENGTH_ERR_MESSAGE =
 const DEFAULT_FILE_NAME = 'unnamedfile';
 const DEFAULT_CONTENT_TYPE = 'application/octet-stream';
 
-const uploader = (
+const streamPipeline = promisify(pipeline);
+
+const filenameFromURL = (url) => {
+  try {
+    return decodeURIComponent(path.posix.basename(new URL(url).pathname));
+  } catch (error) {
+    return null;
+  }
+};
+
+const filenameFromHeader = (response) => {
+  const cd = response.headers.get('content-disposition');
+  let filename;
+  if (cd) {
+    try {
+      filename = contentDisposition.parse(cd).parameters.filename;
+    } catch (error) {
+      return null;
+    }
+  }
+  return filename || null;
+};
+
+const resolveRemoteStream = async (stream) => {
+  // Download to a temp file, get the file size, and create a readable stream
+  // from the temp file.
+  //
+  // The streamPipeline usage is taken from
+  // https://github.com/node-fetch/node-fetch#streams
+  const tmpFilePath = path.join(
+    os.tmpdir(),
+    'stash-' + randomBytes(16).toString('hex')
+  );
+
+  try {
+    await streamPipeline(stream, fs.createWriteStream(tmpFilePath));
+  } catch (error) {
+    try {
+      fs.unlinkSync(tmpFilePath);
+    } catch (e) {
+      // File doesn't exist? Probably okay
+    }
+    throw error;
+  }
+
+  const length = fs.statSync(tmpFilePath).size;
+  const readStream = fs.createReadStream(tmpFilePath);
+
+  readStream.on('end', () => {
+    // Burn after reading
+    try {
+      fs.unlinkSync(tmpFilePath);
+    } catch (e) {
+      // TODO: We probably want to log warning here
+    }
+  });
+
+  return {
+    streamOrData: readStream,
+    length,
+  };
+};
+
+const resolveResponseToStream = async (response) => {
+  // Get filename from content-disposition header or URL
+  let filename =
+    filenameFromHeader(response) ||
+    filenameFromURL(response.url || _.get(response, ['request', 'url'])) ||
+    DEFAULT_FILE_NAME;
+
+  const contentType = response.headers.get('content-type');
+  if (contentType && !path.extname(filename)) {
+    const ext = mime.extension(contentType);
+    if (ext && ext !== 'bin') {
+      filename += '.' + ext;
+    }
+  }
+
+  if (response.body && typeof response.body.pipe === 'function') {
+    // streamable response created by z.request({ raw: true })
+    return {
+      ...(await resolveRemoteStream(response.body)),
+      contentType: contentType || DEFAULT_CONTENT_TYPE,
+      filename,
+    };
+  }
+
+  // regular response created by z.request({ raw: false })
+  return {
+    streamOrData: response.content,
+    length: Buffer.byteLength(response.content),
+    contentType: contentType || DEFAULT_CONTENT_TYPE,
+    filename,
+  };
+};
+
+const resolveStreamWithMeta = async (stream) => {
+  const isLocalFile = stream.path && fs.existsSync(stream.path);
+  if (isLocalFile) {
+    const filename = path.basename(stream.path);
+    return {
+      streamOrData: stream,
+      length: fs.statSync(stream.path).size,
+      contentType: mime.lookup(filename) || DEFAULT_CONTENT_TYPE,
+      filename,
+    };
+  }
+
+  return {
+    ...(await resolveRemoteStream(stream)),
+    contentType: DEFAULT_CONTENT_TYPE,
+    filename: DEFAULT_FILE_NAME,
+  };
+};
+
+// Returns an object with fields:
+// * streamOrData: a readable stream, a string, or a Buffer
+// * length: content length in bytes
+// * contentType
+// * filename
+const resolveToBufferStringStream = async (responseOrData) => {
+  if (typeof responseOrData === 'string' || responseOrData instanceof String) {
+    // The .toString() call only makes a difference for the String object case.
+    // It converts a String object to a regular string.
+    const str = responseOrData.toString();
+    return {
+      streamOrData: str,
+      length: Buffer.byteLength(str),
+      contentType: 'text/plain',
+      filename: `${DEFAULT_FILE_NAME}.txt`,
+    };
+  } else if (Buffer.isBuffer(responseOrData)) {
+    return {
+      streamOrData: responseOrData,
+      length: responseOrData.length,
+      contentType: DEFAULT_CONTENT_TYPE,
+      filename: DEFAULT_FILE_NAME,
+    };
+  } else if (
+    (responseOrData.body && typeof responseOrData.body.pipe === 'function') ||
+    typeof responseOrData.content === 'string'
+  ) {
+    return resolveResponseToStream(responseOrData);
+  } else if (typeof responseOrData.pipe === 'function') {
+    return resolveStreamWithMeta(responseOrData);
+  }
+
+  throw new TypeError(
+    `z.stashFile() cannot stash type '${typeof responseOrData}'. ` +
+      'Pass it a request, readable stream, string, or Buffer.'
+  );
+};
+
+const uploader = async (
   signedPostData,
   bufferStringStream,
   knownLength,
   filename,
   contentType
 ) => {
-  const form = new FormData();
-
   if (knownLength && knownLength > UPLOAD_MAX_SIZE) {
-    return ZapierPromise.reject(
-      new Error(`${knownLength} is too big, ${UPLOAD_MAX_SIZE} is the max`)
-    );
+    throw new Error(`${knownLength} is too big, ${UPLOAD_MAX_SIZE} is the max`);
   }
+  filename = path.basename(filename).replace('"', '');
 
-  _.each(signedPostData.fields, (value, key) => {
-    form.append(key, value);
-  });
+  const fields = {
+    ...signedPostData.fields,
+    'Content-Disposition': contentDisposition(filename),
+    'Content-Type': contentType,
+  };
 
-  filename = path.basename(filename || DEFAULT_FILE_NAME).replace('"', '');
+  const form = new FormData();
 
-  form.append('Content-Disposition', contentDisposition(filename));
+  Object.entries(fields).forEach(([key, value]) => {
+    form.append(key, value);
+  });
 
   form.append('file', bufferStringStream, {
-    contentType,
     knownLength,
-    filename
+    contentType,
+    filename,
   });
 
   // Try to catch the missing length early, before upload to S3 fails.
@@ -56,120 +214,93 @@ const uploader = (
   }
 
   // Send to S3 with presigned request.
-  return request({
+  const response = await request({
     url: signedPostData.url,
     method: 'POST',
-    body: form
-  }).then(res => {
-    if (res.status === 204) {
-      return `${signedPostData.url}${signedPostData.fields.key}`;
-    }
-    if (
-      res.content.indexOf(
-        'You must provide the Content-Length HTTP header.'
-      ) !== -1
-    ) {
-      throw new Error(LENGTH_ERR_MESSAGE);
-    }
-    throw new Error(`Got ${res.status} - ${res.content}`);
+    body: form,
   });
+
+  if (response.status === 204) {
+    return new URL(signedPostData.fields.key, signedPostData.url).href;
+  }
+
+  if (
+    response.content &&
+    response.content.includes &&
+    response.content.includes(
+      'You must provide the Content-Length HTTP header.'
+    )
+  ) {
+    throw new Error(LENGTH_ERR_MESSAGE);
+  }
+
+  throw new Error(`Got ${response.status} - ${response.content}`);
 };
 
 // Designed to be some user provided function/api.
-const createFileStasher = input => {
+const createFileStasher = (input) => {
   const rpc = _.get(input, '_zapier.rpc');
 
-  return (bufferStringStream, knownLength, filename, contentType) => {
+  return async (requestOrData, knownLength, filename, contentType) => {
     // TODO: maybe this could be smart?
     // if it is already a public url, do we pass through? or upload?
     if (!rpc) {
-      return ZapierPromise.reject(new Error('rpc is not available'));
+      throw new Error('rpc is not available');
     }
 
-    const isRunningOnHydrator =
-      _.get(input, '_zapier.event.method', '').indexOf('hydrators.') === 0;
-    const isRunningOnCreate =
-      _.get(input, '_zapier.event.method', '').indexOf('creates.') === 0;
+    const isRunningOnHydrator = _.get(
+      input,
+      '_zapier.event.method',
+      ''
+    ).startsWith('hydrators.');
+    const isRunningOnCreate = _.get(
+      input,
+      '_zapier.event.method',
+      ''
+    ).startsWith('creates.');
 
     if (!isRunningOnHydrator && !isRunningOnCreate) {
-      return ZapierPromise.reject(
-        new Error(
-          'Files can only be stashed within a create or hydration function/method.'
-        )
+      throw new Error(
+        'Files can only be stashed within a create or hydration function/method.'
       );
     }
 
-    const fileContentType = contentType || DEFAULT_CONTENT_TYPE;
-
-    return rpc('get_presigned_upload_post_data', fileContentType).then(
-      result => {
-        if (isPromise(bufferStringStream)) {
-          return bufferStringStream.then(maybeResponse => {
-            const isStreamed = _.get(maybeResponse, 'request.raw', false);
-
-            const parseFinalResponse = response => {
-              let newBufferStringStream = response;
-              if (_.isString(response)) {
-                newBufferStringStream = response;
-              } else if (response) {
-                if (Buffer.isBuffer(response)) {
-                  newBufferStringStream = response;
-                } else if (Buffer.isBuffer(response.dataBuffer)) {
-                  newBufferStringStream = response.dataBuffer;
-                } else if (
-                  response.body &&
-                  typeof response.body.pipe === 'function'
-                ) {
-                  newBufferStringStream = response.body;
-                } else {
-                  newBufferStringStream = response.content;
-                }
-
-                if (response.headers) {
-                  knownLength =
-                    knownLength || response.getHeader('content-length');
-                  const cd = response.getHeader('content-disposition');
-                  if (cd) {
-                    filename =
-                      filename ||
-                      contentDisposition.parse(cd).parameters.filename;
-                  }
-                }
-              } else {
-                throw new Error(
-                  'Cannot stash a Promise wrapped file of unknown type.'
-                );
-              }
-
-              return uploader(
-                result,
-                newBufferStringStream,
-                knownLength,
-                filename,
-                fileContentType
-              );
-            };
-
-            if (isStreamed) {
-              maybeResponse.throwForStatus();
-              return maybeResponse.buffer().then(buffer => {
-                maybeResponse.dataBuffer = buffer;
-                return parseFinalResponse(maybeResponse);
-              });
-            } else {
-              return parseFinalResponse(maybeResponse);
-            }
-          });
-        } else {
-          return uploader(
-            result,
-            bufferStringStream,
-            knownLength,
-            filename,
-            fileContentType
-          );
-        }
-      }
+    // requestOrData can be one of these:
+    // * string
+    // * Buffer
+    // * z.request() - a Promise of a regular response
+    // * z.request({ raw: true }) - a Promise of a "streamable" response
+    // * await z.request() - a regular response
+    // * await z.request({ raw: true }) - a streamable response
+    //
+    // After the following, requestOrData is resolved to responseOrData, which
+    // is either:
+    // - string
+    // - Buffer
+    // - a regular response
+    // - a streamable response
+    const [signedPostData, responseOrData] = await Promise.all([
+      rpc('get_presigned_upload_post_data'),
+      requestOrData,
+    ]);
+
+    if (responseOrData.throwForStatus) {
+      responseOrData.throwForStatus();
+    }
+
+    const {
+      streamOrData,
+      length,
+      contentType: _contentType,
+      filename: _filename,
+    } = await resolveToBufferStringStream(responseOrData);
+
+    return uploader(
+      signedPostData,
+      streamOrData,
+      knownLength || length,
+      filename || _filename,
+      contentType || _contentType
     );
   };
 };

package/src/tools/create-http-patch.js

@@ -2,14 +2,14 @@ const zlib = require('zlib');
 const _ = require('lodash');
 const constants = require('../constants');
 
-const createHttpPatch = event => {
-  const createLogger = require('./create-logger');
-  const logBuffer = [];
-  const logger = createLogger(event, { logBuffer });
-
-  const httpPatch = object => {
+const createHttpPatch = (event) => {
+  const httpPatch = (object, logger) => {
     const originalRequest = object.request;
 
+    // Important not to reuse logger between calls, because we always destroy
+    // the logger at the end of a Lambda call.
+    object.zapierLogger = logger;
+
     // Avoids multiple patching and memory leaks (mostly when running tests locally)
     if (object.patchedByZapier) {
       return;
@@ -21,13 +21,15 @@ const createHttpPatch = event => {
     object.request = (options, callback) => {
       // `options` can be an object or a string. If options is a string, it is
       // automatically parsed with url.parse().
-      // See https://nodejs.org/docs/latest-v6.x/api/http.html#http_http_request_options_callback
+      // See https://nodejs.org/docs/latest-v12.x/api/http.html#http_http_request_options_callback
       let requestUrl;
       if (typeof options === 'string') {
         requestUrl = options;
       } else if (typeof options.url === 'string') {
-        // XXX: Somehow options.url is available for some requests although http.request doesn't really accept it.
-        // Without this else-if, many HTTP requests don't work. Should take a deeper look at this weirdness.
+        // XXX: Somehow options.url is available for some requests although
+        // http.request doesn't really accept it. Without this else-if, many
+        // HTTP requests don't work. Should take a deeper look at this
+        // weirdness.
         requestUrl = options.url;
       } else {
         requestUrl =
@@ -49,10 +51,10 @@ const createHttpPatch = event => {
       }
 
       // Proxy the callback to get the response
-      const newCallback = function(response) {
+      const newCallback = function (response) {
         const chunks = [];
 
-        const sendToLogger = responseBody => {
+        const sendToLogger = (responseBody) => {
           // Prepare data for GL
           const logData = {
             log_type: 'http',
@@ -64,10 +66,10 @@ const createHttpPatch = event => {
             request_via_client: false,
             response_status_code: response.statusCode,
             response_headers: response.headers,
-            response_content: responseBody
+            response_content: responseBody,
           };
 
-          logger(
+          object.zapierLogger(
             `${logData.response_status_code} ${logData.request_method} ${logData.request_url}`,
             logData
           );
@@ -85,14 +87,14 @@ const createHttpPatch = event => {
               sendToLogger(responseBody);
             });
           } else {
-            const responseBody = _.map(chunks, chunk => chunk.toString()).join(
-              '\n'
-            );
+            const responseBody = _.map(chunks, (chunk) =>
+              chunk.toString()
+            ).join('\n');
             sendToLogger(responseBody);
           }
         };
 
-        response.on('data', chunk => chunks.push(chunk));
+        response.on('data', (chunk) => chunks.push(chunk));
         response.on('end', logResponse);
         response.on('error', logResponse);
 

package/src/tools/create-lambda-handler.js

@@ -22,7 +22,7 @@ const ZapierPromise = require('./promise');
 const RequestSchema = require('zapier-platform-schema/lib/schemas/RequestSchema');
 const FunctionSchema = require('zapier-platform-schema/lib/schemas/FunctionSchema');
 
-const isRequestOrFunction = obj => {
+const isRequestOrFunction = (obj) => {
   return (
     RequestSchema.validate(obj).valid || FunctionSchema.validate(obj).valid
   );
@@ -34,7 +34,7 @@ const extendAppRaw = (base, extension) => {
     'perform',
     'performList',
     'performSubscribe',
-    'performUnsubscribe'
+    'performUnsubscribe',
   ];
   const concatArrayAndOverrideKeys = (objValue, srcValue, key) => {
     if (Array.isArray(objValue) && Array.isArray(srcValue)) {
@@ -99,7 +99,7 @@ const getAppRawOverride = (rpc, appRawOverride) => {
 
     // Otherwise just get it via RPC
     rpc('get_definition_override')
-      .then(fetchedOverride => {
+      .then((fetchedOverride) => {
         // "cache" it.
         fs.writeFileSync(hashPath, appRawOverride);
         fs.writeFileSync(overridePath, JSON.stringify(fetchedOverride));
@@ -108,7 +108,7 @@ const getAppRawOverride = (rpc, appRawOverride) => {
 
         resolve(fetchedOverride);
       })
-      .catch(err => reject(err));
+      .catch((err) => reject(err));
   });
 };
 
@@ -118,8 +118,8 @@ const loadApp = (event, rpc, appRawOrPath) => {
   return new ZapierPromise((resolve, reject) => {
     if (event && event.appRawOverride) {
       return getAppRawOverride(rpc, event.appRawOverride)
-        .then(appRawOverride => resolve(appRawOverride))
-        .catch(err => reject(err));
+        .then((appRawOverride) => resolve(appRawOverride))
+        .catch((err) => reject(err));
     }
 
     if (_.isString(appRawOrPath)) {
@@ -130,7 +130,7 @@ const loadApp = (event, rpc, appRawOrPath) => {
   });
 };
 
-const createLambdaHandler = appRawOrPath => {
+const createLambdaHandler = (appRawOrPath) => {
   const handler = (event, context, callback) => {
     // Wait for all async events to complete before callback returns.
     // This is not strictly necessary since this is the default now when
@@ -147,65 +147,61 @@ const createLambdaHandler = appRawOrPath => {
 
     environmentTools.cleanEnvironment();
 
+    // Copy bundle environment into process.env *before* creating the logger and
+    // loading app code, so that the logger gets the endpoint from process.env,
+    // and top level app code can get bundle environment vars via process.env.
+    environmentTools.applyEnvironment(event);
+
     // Create logger outside of domain, so we can use in both error and run callbacks.
     const logBuffer = [];
     const logger = createLogger(event, { logBuffer });
 
     let isCallbackCalled = false;
     const callbackOnce = (err, resp) => {
-      if (!isCallbackCalled) {
-        isCallbackCalled = true;
-        callback(err, resp);
-      }
+      logger.end().finally(() => {
+        if (!isCallbackCalled) {
+          isCallbackCalled = true;
+          callback(err, resp);
+        }
+      });
     };
 
     const logErrorAndCallbackOnce = (logMsg, logData, err) => {
-      // Wait for logger to complete before callback. This isn't
-      // strictly necessary because callbacksWaitsForEmptyLoop is
-      // the default behavior with callbacks anyway, but don't want
-      // to rely on that.
-      logger(logMsg, logData).then(() => {
-        // Check for `.message` in case someone did `throw "My Error"`
-        if (
-          !constants.IS_TESTING &&
-          err &&
-          !err.doNotContextify &&
-          err.message
-        ) {
-          err.message += `\n\nConsole logs:\n${logBuffer
-            .map(s => `  ${s.message}`)
-            .join('')}`;
-        }
-        callbackOnce(err);
-      });
+      logger(logMsg, logData);
+
+      // Check for `.message` in case someone did `throw "My Error"`
+      if (!constants.IS_TESTING && err && !err.doNotContextify && err.message) {
+        err.message += `\n\nConsole logs:\n${logBuffer
+          .map((s) => `  ${s.message}`)
+          .join('')}`;
+      }
+
+      callbackOnce(err);
     };
 
     const handlerDomain = domain.create();
 
-    handlerDomain.on('error', err => {
-      const logMsg = `Uncaught error: ${err}\n${(err && err.stack) ||
-        '<stack>'}`;
+    handlerDomain.on('error', (err) => {
+      const logMsg = `Uncaught error: ${err}\n${
+        (err && err.stack) || '<stack>'
+      }`;
       const logData = { err, log_type: 'error' };
       logErrorAndCallbackOnce(logMsg, logData, err);
     });
 
     handlerDomain.run(() => {
-      // Copy bundle environment into process.env *before* loading app code,
-      // so that top level app code can get bundle environment vars via process.env.
-      environmentTools.applyEnvironment(event);
-
       const rpc = createRpcClient(event);
 
       return loadApp(event, rpc, appRawOrPath)
-        .then(appRaw => {
+        .then((appRaw) => {
           const app = createApp(appRaw);
 
           const { skipHttpPatch } = appRaw.flags || {};
           // Adds logging for _all_ kinds of http(s) requests, no matter the library
-          if (!skipHttpPatch) {
+          if (!skipHttpPatch && !event.calledFromCli) {
             const httpPatch = createHttpPatch(event);
-            httpPatch(require('http'));
-            httpPatch(require('https')); // 'https' needs to be patched separately
+            httpPatch(require('http'), logger);
+            httpPatch(require('https'), logger); // 'https' needs to be patched separately
           }
 
           // TODO: Avoid calling prepareApp(appRaw) repeatedly here as createApp()
@@ -215,12 +211,13 @@ const createLambdaHandler = appRawOrPath => {
           const input = createInput(compiledApp, event, logger, logBuffer, rpc);
           return app(input);
         })
-        .then(output => {
+        .then((output) => {
           callbackOnce(null, cleaner.maskOutput(output));
         })
-        .catch(err => {
-          const logMsg = `Unhandled error: ${err}\n${(err && err.stack) ||
-            '<stack>'}`;
+        .catch((err) => {
+          const logMsg = `Unhandled error: ${err}\n${
+            (err && err.stack) || '<stack>'
+          }`;
           const logData = { err, log_type: 'error' };
           logErrorAndCallbackOnce(logMsg, logData, err);
         });

package/src/tools/create-legacy-scripting-runner.js

@@ -8,8 +8,11 @@ const semver = require('semver');
 const createLegacyScriptingRunner = (z, input) => {
   const app = _.get(input, '_zapier.app');
 
-  let source =
-    _.get(app, 'legacy.scriptingSource') || app.legacyScriptingSource;
+  // once we have node 14 everywhere, this can be:
+  // let source = _.get(app, 'legacy.scriptingSource') ?? app.legacyScriptingSource;
+  let source = _.get(app, 'legacy.scriptingSource');
+  source = source === undefined ? app.legacyScriptingSource : source;
+
   if (source === undefined) {
     // Don't initialize z.legacyScripting for a pure CLI app
     return null;
@@ -26,8 +29,8 @@ const createLegacyScriptingRunner = (z, input) => {
   let LegacyScriptingRunner, version;
   try {
     LegacyScriptingRunner = require('zapier-platform-legacy-scripting-runner');
-    version = require('zapier-platform-legacy-scripting-runner/package.json')
-      .version;
+    version =
+      require('zapier-platform-legacy-scripting-runner/package.json').version;
   } catch (e) {
     // Find it in cwd, in case we're developing legacy-scripting-runner itself
     const cwd = process.cwd();

package/src/tools/create-logger.js

@@ -1,16 +1,21 @@
 'use strict';
 
+const { Transform } = require('stream');
+
 const _ = require('lodash');
 
 const request = require('./request-client-internal');
 const cleaner = require('./cleaner');
 const dataTools = require('./data');
 const hashing = require('./hashing');
-const ZapierPromise = require('./promise');
 const constants = require('../constants');
 const { unheader } = require('./http');
 
-const truncate = str => dataTools.simpleTruncate(str, 3500, ' [...]');
+// The payload size per request to stream logs. This should be slighly lower
+// than the limit (16 MB) on the server side.
+const LOG_STREAM_BYTES_LIMIT = 15 * 1024 * 1024;
+
+const truncate = (str) => dataTools.simpleTruncate(str, 3500, ' [...]');
 
 const formatHeaders = (headers = {}) => {
   if (_.isEmpty(headers)) {
@@ -28,7 +33,7 @@ const formatHeaders = (headers = {}) => {
     .join('\n');
 };
 
-const maybeStringify = d => {
+const maybeStringify = (d) => {
   if (_.isPlainObject(d) || Array.isArray(d)) {
     return JSON.stringify(d);
   }
@@ -36,7 +41,7 @@ const maybeStringify = d => {
 };
 
 // format HTTP request details into string suitable for printing to stdout
-const httpDetailsLogMessage = data => {
+const httpDetailsLogMessage = (data) => {
   if (data.log_type !== 'http') {
     return '';
   }
@@ -58,9 +63,9 @@ const httpDetailsLogMessage = data => {
   }
 
   return `\
-${trimmedData.request_method || 'GET'} ${
-    trimmedData.request_url
-  }${trimmedData.request_params || ''}
+${trimmedData.request_method || 'GET'} ${trimmedData.request_url}${
+    trimmedData.request_params || ''
+  }
 ${formatHeaders(trimmedData.request_headers) || ''}
 
 ${maybeStringify(trimmedData.request_data) || ''}
@@ -93,12 +98,12 @@ const makeSensitiveBank = (event, data) => {
   const matcher = (key, value) => {
     if (_.isString(value)) {
       const lowerKey = key.toLowerCase();
-      return _.some(constants.SENSITIVE_KEYS, k => lowerKey.indexOf(k) >= 0);
+      return _.some(constants.SENSITIVE_KEYS, (k) => lowerKey.indexOf(k) >= 0);
     }
     return false;
   };
 
-  dataTools.recurseExtract(data, matcher).forEach(value => {
+  dataTools.recurseExtract(data, matcher).forEach((value) => {
     sensitiveValues.push(value);
   });
 
@@ -126,7 +131,75 @@ const makeSensitiveBank = (event, data) => {
   );
 };
 
-const sendLog = (options, event, message, data) => {
+class LogStream extends Transform {
+  constructor(options) {
+    super(options);
+    this.bytesWritten = 0;
+    this.request = this._newRequest(options.url, options.token);
+  }
+
+  _newRequest(url, token) {
+    const httpOptions = {
+      url,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-ndjson',
+        'X-Token': token,
+      },
+      body: this,
+    };
+    return request(httpOptions).catch((err) => {
+      // Swallow logging errors. This will show up in AWS logs at least.
+      console.error(
+        'Error making log request:',
+        err,
+        'http options:',
+        httpOptions
+      );
+    });
+  }
+
+  _transform(chunk, encoding, callback) {
+    this.push(chunk);
+    this.bytesWritten += Buffer.byteLength(chunk, encoding);
+    callback();
+  }
+}
+
+// Implements singleton for LogStream. The goal is for every sendLog() call we
+// reuse the same request until the request body grows too big and exceeds
+// LOG_STREAM_BYTES_LIMIT.
+class LogStreamFactory {
+  constructor() {
+    this._logStream = null;
+  }
+
+  getOrCreate(url, token) {
+    if (this._logStream) {
+      if (this._logStream.bytesWritten < LOG_STREAM_BYTES_LIMIT) {
+        // Reuse the same request for efficiency
+        return this._logStream;
+      }
+
+      // End this one before creating another
+      this._logStream.end();
+    }
+
+    this._logStream = new LogStream({ url, token });
+    return this._logStream;
+  }
+
+  async end() {
+    if (this._logStream) {
+      this._logStream.end();
+      const response = await this._logStream.request;
+      this._logStream = null;
+      return response;
+    }
+  }
+}
+
+const sendLog = async (logStreamFactory, options, event, message, data) => {
   data = _.extend({}, data || {}, event.logExtra || {});
   data.log_type = data.log_type || 'console';
 
@@ -144,7 +217,7 @@ const sendLog = (options, event, message, data) => {
   const unsafeData = dataTools.recurseReplace(data, truncate);
 
   // Keep safe log keys uncensored
-  Object.keys(safeData).forEach(key => {
+  Object.keys(safeData).forEach((key) => {
     if (constants.SAFE_LOG_KEYS.indexOf(key) !== -1) {
       safeData[key] = unsafeData[key];
     }
@@ -153,20 +226,6 @@ const sendLog = (options, event, message, data) => {
   safeData.request_headers = formatHeaders(safeData.request_headers);
   safeData.response_headers = formatHeaders(safeData.response_headers);
 
-  const body = {
-    message: safeMessage,
-    data: safeData,
-    token: options.token
-  };
-
-  const httpOptions = {
-    url: options.endpoint,
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(body),
-    timeout: 3000
-  };
-
   if (event.logToStdout) {
     toStdout(event, message, unsafeData);
   }
@@ -177,24 +236,36 @@ const sendLog = (options, event, message, data) => {
   }
 
   if (options.token) {
-    return request(httpOptions).catch(err => {
-      // Swallow logging errors.
-      // This will show up in AWS logs at least:
-      console.error(
-        'Error making log request:',
-        err,
-        'http options:',
-        httpOptions
-      );
-    });
-  } else {
-    return ZapierPromise.resolve();
+    const logStream = logStreamFactory.getOrCreate(
+      options.endpoint,
+      options.token
+    );
+    logStream.write(
+      // JSON Lines format: It's important the serialized JSON object itself has
+      // no line breaks, and after an object it ends with a line break.
+      JSON.stringify({ message: safeMessage, data: safeData }) + '\n'
+    );
   }
 };
 
 /*
   Creates low level logging function that POSTs to endpoint (GL by default).
   Use internally; do not expose to devs.
+
+  Usage:
+
+    const logger = createLogger(event, options);
+
+    // These will reuse the same request to the log server
+    logger('log message here', { log_type: 'console' });
+    logger('another log', { log_type: 'console' });
+    logger('200 GET https://example.com', { log_type: 'http' });
+
+    // After an invocation, the Lambda handler MUST call logger.end() to close
+    // the log stream. Otherwise, it will hang!
+    logger.end().finally(() => {
+      // anything else you want to do to finish an invocation
+    });
 */
 const createLogger = (event, options) => {
   options = options || {};
@@ -205,10 +276,16 @@ const createLogger = (event, options) => {
       process.env.LOGGING_ENDPOINT || constants.DEFAULT_LOGGING_HTTP_ENDPOINT,
     apiKey:
       process.env.LOGGING_API_KEY || constants.DEFAULT_LOGGING_HTTP_API_KEY,
-    token: process.env.LOGGING_TOKEN || event.token
+    token: process.env.LOGGING_TOKEN || event.token,
   });
 
-  return sendLog.bind(undefined, options, event);
+  const logStreamFactory = new LogStreamFactory();
+  const logger = sendLog.bind(undefined, logStreamFactory, options, event);
+
+  logger.end = async () => {
+    return logStreamFactory.end();
+  };
+  return logger;
 };
 
 module.exports = createLogger;

package/src/tools/fetch.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const { Writable } = require('stream');
+
 const fetch = require('node-fetch');
 
 // XXX: PatchedRequest is to get past node-fetch's check that forbids GET requests
@@ -7,10 +9,10 @@ const fetch = require('node-fetch');
 // https://github.com/node-fetch/node-fetch/blob/v2.6.0/src/request.js#L75-L78
 class PatchedRequest extends fetch.Request {
   constructor(url, opts) {
-    const origMethod = (opts.method || 'GET').toUpperCase();
+    const origMethod = ((opts && opts.method) || 'GET').toUpperCase();
 
     const isGetWithBody =
-      (origMethod === 'GET' || origMethod === 'HEAD') && opts.body;
+      (origMethod === 'GET' || origMethod === 'HEAD') && opts && opts.body;
     let newOpts = opts;
     if (isGetWithBody) {
       // Temporary remove body to fool fetch.Request constructor
@@ -50,9 +52,31 @@ class PatchedRequest extends fetch.Request {
 
 const newFetch = (url, opts) => {
   const request = new PatchedRequest(url, opts);
+
   // fetch actually accepts a Request object as an argument. It'll clone the
   // request internally, that's why the PatchedRequest.body hack works.
-  return fetch(request);
+  const responsePromise = fetch(request);
+
+  // node-fetch clones request.body and use the cloned body internally. We need
+  // to make sure to consume the original body stream so its internal buffer is
+  // not filled up, which causes it to pause.
+  // See https://github.com/node-fetch/node-fetch/issues/151
+  //
+  // Exclude form-data object to be consistent with
+  // https://github.com/node-fetch/node-fetch/blob/v2.6.6/src/body.js#L403-L412
+  if (
+    request.body &&
+    typeof request.body.pipe === 'function' &&
+    typeof request.body.getBoundary !== 'function'
+  ) {
+    const nullStream = new Writable();
+    nullStream._write = function (chunk, encoding, done) {
+      done();
+    };
+    request.body.pipe(nullStream);
+  }
+
+  return responsePromise;
 };
 
 newFetch.Promise = require('./promise');