zapier-platform-core 9.6.0 → 9.7.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zapier-platform-core",
-  "version": "9.6.0",
+  "version": "9.7.2",
   "description": "The core SDK for CLI apps in the Zapier Developer Platform.",
   "repository": "zapier/zapier-platform-core",
   "homepage": "https://zapier.com/",
@@ -44,10 +44,10 @@
     "form-data": "4.0.0",
     "lodash": "4.17.15",
     "mime-types": "2.1.34",
-    "node-fetch": "2.6.6",
+    "node-fetch": "2.6.7",
     "oauth-sign": "0.9.0",
     "semver": "5.6.0",
-    "zapier-platform-schema": "9.6.0"
+    "zapier-platform-schema": "9.7.2"
   },
   "devDependencies": {
     "adm-zip": "0.4.13",

package/src/http-middlewares/before/add-query-params.js

@@ -8,13 +8,30 @@ const hasQueryParams = ({ params = {} }) => Object.keys(params).length;
 // Take params off of req.params and append to url - "?a=1&b=2"".
 // This middleware should run *after* custom middlewares, because
 // custom middlewares might add params.
-const addQueryParams = req => {
+const addQueryParams = (req) => {
   if (hasQueryParams(req)) {
     const splitter = req.url.includes('?') ? '&' : '?';
 
     normalizeEmptyParamFields(req);
 
-    const stringifiedParams = querystring.stringify(req.params);
+    let stringifiedParams = querystring.stringify(req.params);
+
+    // it goes against spec, but for compatibility, some APIs want certain
+    // characters (mostly $) unencoded
+    if (req.skipEncodingChars) {
+      for (let i = 0; i < req.skipEncodingChars.length; i++) {
+        const char = req.skipEncodingChars.charAt(i);
+        const valToReplace = querystring.escape(char);
+        if (valToReplace === char) {
+          continue;
+        }
+        // no replaceAll in JS yet, coming in a node version soon!
+        stringifiedParams = stringifiedParams.replace(
+          new RegExp(valToReplace, 'g'),
+          char
+        );
+      }
+    }
 
     if (stringifiedParams) {
       req.url += `${splitter}${stringifiedParams}`;

package/src/tools/cleaner.js

@@ -7,7 +7,7 @@ const {
   flattenPaths,
   getObjectType,
   isPlainObj,
-  recurseReplace
+  recurseReplace,
 } = require('./data');
 
 const DEFAULT_BUNDLE = {
@@ -15,7 +15,7 @@ const DEFAULT_BUNDLE = {
   inputData: {},
   meta: {},
   subscribeData: {},
-  targetUrl: ''
+  targetUrl: '',
 };
 
 const isCurlies = /{{.*?}}/g;
@@ -34,7 +34,7 @@ const recurseCleanFuncs = (obj, path) => {
     });
   } else if (isPlainObj(obj)) {
     const newObj = {};
-    Object.keys(obj).forEach(key => {
+    Object.keys(obj).forEach((key) => {
       const value = obj[key];
       newObj[key] = recurseCleanFuncs(value, path.concat([key]));
     });
@@ -45,7 +45,7 @@ const recurseCleanFuncs = (obj, path) => {
 
 // Recurse a nested object replace all instances of keys->vals in the bank.
 const recurseReplaceBank = (obj, bank = {}) => {
-  const replacer = out => {
+  const replacer = (out) => {
     if (!['string', 'number'].includes(typeof out)) {
       return out;
     }
@@ -56,7 +56,7 @@ const recurseReplaceBank = (obj, bank = {}) => {
     const originalValueStr = String(out);
     let maybeChangedString = originalValueStr;
 
-    Object.keys(bank).forEach(key => {
+    Object.keys(bank).forEach((key) => {
       // Escape characters (ex. {{foo}} => \\{\\{foo\\}\\} )
       const escapedKey = key.replace(/[-[\]/{}()\\*+?.^$|]/g, '\\$&');
       const matchesKey = new RegExp(escapedKey, 'g');
@@ -106,12 +106,12 @@ const finalizeBundle = pipe(
 );
 
 // Takes a raw app and bundle and composes a bank of {{key}}->val
-const createBundleBank = (appRaw, event = {}, serializeFunc = x => x) => {
+const createBundleBank = (appRaw, event = {}, serializeFunc = (x) => x) => {
   const bank = {
     bundle: finalizeBundle(event.bundle),
     process: {
-      env: _.extend({}, process.env || {})
-    }
+      env: _.extend({}, process.env || {}),
+    },
   };
 
   const options = { preserve: { 'bundle.inputData': true } };
@@ -123,7 +123,7 @@ const createBundleBank = (appRaw, event = {}, serializeFunc = x => x) => {
   }, {});
 };
 
-const maskOutput = output => _.pick(output, 'results', 'status');
+const maskOutput = (output) => _.pick(output, 'results', 'status');
 
 // These normalize functions are called after the initial before middleware that
 // cleans the request. The reason is that we need to know why a value is empty
@@ -132,7 +132,7 @@ const maskOutput = output => _.pick(output, 'results', 'status');
 // an earlier Zap step? Or was it a null value? Each has different results depending
 // on how the partner has configued their integration.
 const normalizeEmptyRequestFields = (shouldCleanup, field, req) => {
-  const handleEmpty = key => {
+  const handleEmpty = (key) => {
     const value = req[field][key] || '';
     const cleaned = value.replace(isCurlies, '');
 
@@ -152,11 +152,11 @@ const normalizeEmptyRequestFields = (shouldCleanup, field, req) => {
   });
 };
 
-const isEmptyQueryParam = value =>
+const isEmptyQueryParam = (value) =>
   value === '' ||
   value === null ||
   value === undefined ||
-  isCurlies.test(value);
+  (typeof value === 'string' && value.search(isCurlies) >= 0);
 
 const normalizeEmptyParamFields = normalizeEmptyRequestFields.bind(
   null,
@@ -165,7 +165,7 @@ const normalizeEmptyParamFields = normalizeEmptyRequestFields.bind(
 );
 const normalizeEmptyBodyFields = normalizeEmptyRequestFields.bind(
   null,
-  v => isCurlies.test(v),
+  (v) => typeof v === 'string' && v.search(isCurlies) >= 0,
   'body'
 );
 
@@ -175,5 +175,5 @@ module.exports = {
   normalizeEmptyBodyFields,
   normalizeEmptyParamFields,
   recurseCleanFuncs,
-  recurseReplaceBank
+  recurseReplaceBank,
 };

package/src/tools/create-http-patch.js

@@ -2,14 +2,14 @@ const zlib = require('zlib');
 const _ = require('lodash');
 const constants = require('../constants');
 
-const createHttpPatch = event => {
-  const createLogger = require('./create-logger');
-  const logBuffer = [];
-  const logger = createLogger(event, { logBuffer });
-
-  const httpPatch = object => {
+const createHttpPatch = (event) => {
+  const httpPatch = (object, logger) => {
     const originalRequest = object.request;
 
+    // Important not to reuse logger between calls, because we always destroy
+    // the logger at the end of a Lambda call.
+    object.zapierLogger = logger;
+
     // Avoids multiple patching and memory leaks (mostly when running tests locally)
     if (object.patchedByZapier) {
       return;
@@ -21,13 +21,15 @@ const createHttpPatch = event => {
     object.request = (options, callback) => {
       // `options` can be an object or a string. If options is a string, it is
       // automatically parsed with url.parse().
-      // See https://nodejs.org/docs/latest-v6.x/api/http.html#http_http_request_options_callback
+      // See https://nodejs.org/docs/latest-v12.x/api/http.html#http_http_request_options_callback
       let requestUrl;
       if (typeof options === 'string') {
         requestUrl = options;
       } else if (typeof options.url === 'string') {
-        // XXX: Somehow options.url is available for some requests although http.request doesn't really accept it.
-        // Without this else-if, many HTTP requests don't work. Should take a deeper look at this weirdness.
+        // XXX: Somehow options.url is available for some requests although
+        // http.request doesn't really accept it. Without this else-if, many
+        // HTTP requests don't work. Should take a deeper look at this
+        // weirdness.
         requestUrl = options.url;
       } else {
         requestUrl =
@@ -49,10 +51,10 @@ const createHttpPatch = event => {
       }
 
       // Proxy the callback to get the response
-      const newCallback = function(response) {
+      const newCallback = function (response) {
         const chunks = [];
 
-        const sendToLogger = responseBody => {
+        const sendToLogger = (responseBody) => {
           // Prepare data for GL
           const logData = {
             log_type: 'http',
@@ -64,10 +66,10 @@ const createHttpPatch = event => {
             request_via_client: false,
             response_status_code: response.statusCode,
             response_headers: response.headers,
-            response_content: responseBody
+            response_content: responseBody,
           };
 
-          logger(
+          object.zapierLogger(
             `${logData.response_status_code} ${logData.request_method} ${logData.request_url}`,
             logData
           );
@@ -85,14 +87,14 @@ const createHttpPatch = event => {
               sendToLogger(responseBody);
             });
           } else {
-            const responseBody = _.map(chunks, chunk => chunk.toString()).join(
-              '\n'
-            );
+            const responseBody = _.map(chunks, (chunk) =>
+              chunk.toString()
+            ).join('\n');
             sendToLogger(responseBody);
           }
         };
 
-        response.on('data', chunk => chunks.push(chunk));
+        response.on('data', (chunk) => chunks.push(chunk));
         response.on('end', logResponse);
         response.on('error', logResponse);
 

package/src/tools/create-lambda-handler.js

@@ -22,7 +22,7 @@ const ZapierPromise = require('./promise');
 const RequestSchema = require('zapier-platform-schema/lib/schemas/RequestSchema');
 const FunctionSchema = require('zapier-platform-schema/lib/schemas/FunctionSchema');
 
-const isRequestOrFunction = obj => {
+const isRequestOrFunction = (obj) => {
   return (
     RequestSchema.validate(obj).valid || FunctionSchema.validate(obj).valid
   );
@@ -34,7 +34,7 @@ const extendAppRaw = (base, extension) => {
     'perform',
     'performList',
     'performSubscribe',
-    'performUnsubscribe'
+    'performUnsubscribe',
   ];
   const concatArrayAndOverrideKeys = (objValue, srcValue, key) => {
     if (Array.isArray(objValue) && Array.isArray(srcValue)) {
@@ -99,7 +99,7 @@ const getAppRawOverride = (rpc, appRawOverride) => {
 
     // Otherwise just get it via RPC
     rpc('get_definition_override')
-      .then(fetchedOverride => {
+      .then((fetchedOverride) => {
         // "cache" it.
         fs.writeFileSync(hashPath, appRawOverride);
         fs.writeFileSync(overridePath, JSON.stringify(fetchedOverride));
@@ -108,7 +108,7 @@ const getAppRawOverride = (rpc, appRawOverride) => {
 
         resolve(fetchedOverride);
       })
-      .catch(err => reject(err));
+      .catch((err) => reject(err));
   });
 };
 
@@ -118,8 +118,8 @@ const loadApp = (event, rpc, appRawOrPath) => {
   return new ZapierPromise((resolve, reject) => {
     if (event && event.appRawOverride) {
       return getAppRawOverride(rpc, event.appRawOverride)
-        .then(appRawOverride => resolve(appRawOverride))
-        .catch(err => reject(err));
+        .then((appRawOverride) => resolve(appRawOverride))
+        .catch((err) => reject(err));
     }
 
     if (_.isString(appRawOrPath)) {
@@ -130,7 +130,7 @@ const loadApp = (event, rpc, appRawOrPath) => {
   });
 };
 
-const createLambdaHandler = appRawOrPath => {
+const createLambdaHandler = (appRawOrPath) => {
   const handler = (event, context, callback) => {
     // Wait for all async events to complete before callback returns.
     // This is not strictly necessary since this is the default now when
@@ -147,65 +147,61 @@ const createLambdaHandler = appRawOrPath => {
 
     environmentTools.cleanEnvironment();
 
+    // Copy bundle environment into process.env *before* creating the logger and
+    // loading app code, so that the logger gets the endpoint from process.env,
+    // and top level app code can get bundle environment vars via process.env.
+    environmentTools.applyEnvironment(event);
+
     // Create logger outside of domain, so we can use in both error and run callbacks.
     const logBuffer = [];
     const logger = createLogger(event, { logBuffer });
 
     let isCallbackCalled = false;
     const callbackOnce = (err, resp) => {
-      if (!isCallbackCalled) {
-        isCallbackCalled = true;
-        callback(err, resp);
-      }
+      logger.end().finally(() => {
+        if (!isCallbackCalled) {
+          isCallbackCalled = true;
+          callback(err, resp);
+        }
+      });
     };
 
     const logErrorAndCallbackOnce = (logMsg, logData, err) => {
-      // Wait for logger to complete before callback. This isn't
-      // strictly necessary because callbacksWaitsForEmptyLoop is
-      // the default behavior with callbacks anyway, but don't want
-      // to rely on that.
-      logger(logMsg, logData).then(() => {
-        // Check for `.message` in case someone did `throw "My Error"`
-        if (
-          !constants.IS_TESTING &&
-          err &&
-          !err.doNotContextify &&
-          err.message
-        ) {
-          err.message += `\n\nConsole logs:\n${logBuffer
-            .map(s => `  ${s.message}`)
-            .join('')}`;
-        }
-        callbackOnce(err);
-      });
+      logger(logMsg, logData);
+
+      // Check for `.message` in case someone did `throw "My Error"`
+      if (!constants.IS_TESTING && err && !err.doNotContextify && err.message) {
+        err.message += `\n\nConsole logs:\n${logBuffer
+          .map((s) => `  ${s.message}`)
+          .join('')}`;
+      }
+
+      callbackOnce(err);
     };
 
     const handlerDomain = domain.create();
 
-    handlerDomain.on('error', err => {
-      const logMsg = `Uncaught error: ${err}\n${(err && err.stack) ||
-        '<stack>'}`;
+    handlerDomain.on('error', (err) => {
+      const logMsg = `Uncaught error: ${err}\n${
+        (err && err.stack) || '<stack>'
+      }`;
       const logData = { err, log_type: 'error' };
       logErrorAndCallbackOnce(logMsg, logData, err);
     });
 
     handlerDomain.run(() => {
-      // Copy bundle environment into process.env *before* loading app code,
-      // so that top level app code can get bundle environment vars via process.env.
-      environmentTools.applyEnvironment(event);
-
       const rpc = createRpcClient(event);
 
       return loadApp(event, rpc, appRawOrPath)
-        .then(appRaw => {
+        .then((appRaw) => {
           const app = createApp(appRaw);
 
           const { skipHttpPatch } = appRaw.flags || {};
           // Adds logging for _all_ kinds of http(s) requests, no matter the library
-          if (!skipHttpPatch) {
+          if (!skipHttpPatch && !event.calledFromCli) {
             const httpPatch = createHttpPatch(event);
-            httpPatch(require('http'));
-            httpPatch(require('https')); // 'https' needs to be patched separately
+            httpPatch(require('http'), logger);
+            httpPatch(require('https'), logger); // 'https' needs to be patched separately
           }
 
           // TODO: Avoid calling prepareApp(appRaw) repeatedly here as createApp()
@@ -215,12 +211,13 @@ const createLambdaHandler = appRawOrPath => {
           const input = createInput(compiledApp, event, logger, logBuffer, rpc);
           return app(input);
         })
-        .then(output => {
+        .then((output) => {
           callbackOnce(null, cleaner.maskOutput(output));
         })
-        .catch(err => {
-          const logMsg = `Unhandled error: ${err}\n${(err && err.stack) ||
-            '<stack>'}`;
+        .catch((err) => {
+          const logMsg = `Unhandled error: ${err}\n${
+            (err && err.stack) || '<stack>'
+          }`;
           const logData = { err, log_type: 'error' };
           logErrorAndCallbackOnce(logMsg, logData, err);
         });

package/src/tools/create-legacy-scripting-runner.js

@@ -8,8 +8,11 @@ const semver = require('semver');
 const createLegacyScriptingRunner = (z, input) => {
   const app = _.get(input, '_zapier.app');
 
-  let source =
-    _.get(app, 'legacy.scriptingSource') || app.legacyScriptingSource;
+  // once we have node 14 everywhere, this can be:
+  // let source = _.get(app, 'legacy.scriptingSource') ?? app.legacyScriptingSource;
+  let source = _.get(app, 'legacy.scriptingSource');
+  source = source === undefined ? app.legacyScriptingSource : source;
+
   if (source === undefined) {
     // Don't initialize z.legacyScripting for a pure CLI app
     return null;
@@ -26,8 +29,8 @@ const createLegacyScriptingRunner = (z, input) => {
   let LegacyScriptingRunner, version;
   try {
     LegacyScriptingRunner = require('zapier-platform-legacy-scripting-runner');
-    version = require('zapier-platform-legacy-scripting-runner/package.json')
-      .version;
+    version =
+      require('zapier-platform-legacy-scripting-runner/package.json').version;
   } catch (e) {
     // Find it in cwd, in case we're developing legacy-scripting-runner itself
     const cwd = process.cwd();

package/src/tools/create-logger.js

@@ -1,16 +1,21 @@
 'use strict';
 
+const { Transform } = require('stream');
+
 const _ = require('lodash');
 
 const request = require('./request-client-internal');
 const cleaner = require('./cleaner');
 const dataTools = require('./data');
 const hashing = require('./hashing');
-const ZapierPromise = require('./promise');
 const constants = require('../constants');
 const { unheader } = require('./http');
 
-const truncate = str => dataTools.simpleTruncate(str, 3500, ' [...]');
+// The payload size per request to stream logs. This should be slighly lower
+// than the limit (16 MB) on the server side.
+const LOG_STREAM_BYTES_LIMIT = 15 * 1024 * 1024;
+
+const truncate = (str) => dataTools.simpleTruncate(str, 3500, ' [...]');
 
 const formatHeaders = (headers = {}) => {
   if (_.isEmpty(headers)) {
@@ -28,7 +33,7 @@ const formatHeaders = (headers = {}) => {
     .join('\n');
 };
 
-const maybeStringify = d => {
+const maybeStringify = (d) => {
   if (_.isPlainObject(d) || Array.isArray(d)) {
     return JSON.stringify(d);
   }
@@ -36,7 +41,7 @@ const maybeStringify = d => {
 };
 
 // format HTTP request details into string suitable for printing to stdout
-const httpDetailsLogMessage = data => {
+const httpDetailsLogMessage = (data) => {
   if (data.log_type !== 'http') {
     return '';
   }
@@ -58,9 +63,9 @@ const httpDetailsLogMessage = data => {
   }
 
   return `\
-${trimmedData.request_method || 'GET'} ${
-    trimmedData.request_url
-  }${trimmedData.request_params || ''}
+${trimmedData.request_method || 'GET'} ${trimmedData.request_url}${
+    trimmedData.request_params || ''
+  }
 ${formatHeaders(trimmedData.request_headers) || ''}
 
 ${maybeStringify(trimmedData.request_data) || ''}
@@ -93,12 +98,12 @@ const makeSensitiveBank = (event, data) => {
   const matcher = (key, value) => {
     if (_.isString(value)) {
       const lowerKey = key.toLowerCase();
-      return _.some(constants.SENSITIVE_KEYS, k => lowerKey.indexOf(k) >= 0);
+      return _.some(constants.SENSITIVE_KEYS, (k) => lowerKey.indexOf(k) >= 0);
     }
     return false;
   };
 
-  dataTools.recurseExtract(data, matcher).forEach(value => {
+  dataTools.recurseExtract(data, matcher).forEach((value) => {
     sensitiveValues.push(value);
   });
 
@@ -126,7 +131,75 @@ const makeSensitiveBank = (event, data) => {
   );
 };
 
-const sendLog = (options, event, message, data) => {
+class LogStream extends Transform {
+  constructor(options) {
+    super(options);
+    this.bytesWritten = 0;
+    this.request = this._newRequest(options.url, options.token);
+  }
+
+  _newRequest(url, token) {
+    const httpOptions = {
+      url,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-ndjson',
+        'X-Token': token,
+      },
+      body: this,
+    };
+    return request(httpOptions).catch((err) => {
+      // Swallow logging errors. This will show up in AWS logs at least.
+      console.error(
+        'Error making log request:',
+        err,
+        'http options:',
+        httpOptions
+      );
+    });
+  }
+
+  _transform(chunk, encoding, callback) {
+    this.push(chunk);
+    this.bytesWritten += Buffer.byteLength(chunk, encoding);
+    callback();
+  }
+}
+
+// Implements singleton for LogStream. The goal is for every sendLog() call we
+// reuse the same request until the request body grows too big and exceeds
+// LOG_STREAM_BYTES_LIMIT.
+class LogStreamFactory {
+  constructor() {
+    this._logStream = null;
+  }
+
+  getOrCreate(url, token) {
+    if (this._logStream) {
+      if (this._logStream.bytesWritten < LOG_STREAM_BYTES_LIMIT) {
+        // Reuse the same request for efficiency
+        return this._logStream;
+      }
+
+      // End this one before creating another
+      this._logStream.end();
+    }
+
+    this._logStream = new LogStream({ url, token });
+    return this._logStream;
+  }
+
+  async end() {
+    if (this._logStream) {
+      this._logStream.end();
+      const response = await this._logStream.request;
+      this._logStream = null;
+      return response;
+    }
+  }
+}
+
+const sendLog = async (logStreamFactory, options, event, message, data) => {
   data = _.extend({}, data || {}, event.logExtra || {});
   data.log_type = data.log_type || 'console';
 
@@ -144,7 +217,7 @@ const sendLog = (options, event, message, data) => {
   const unsafeData = dataTools.recurseReplace(data, truncate);
 
   // Keep safe log keys uncensored
-  Object.keys(safeData).forEach(key => {
+  Object.keys(safeData).forEach((key) => {
     if (constants.SAFE_LOG_KEYS.indexOf(key) !== -1) {
       safeData[key] = unsafeData[key];
     }
@@ -153,20 +226,6 @@ const sendLog = (options, event, message, data) => {
   safeData.request_headers = formatHeaders(safeData.request_headers);
   safeData.response_headers = formatHeaders(safeData.response_headers);
 
-  const body = {
-    message: safeMessage,
-    data: safeData,
-    token: options.token
-  };
-
-  const httpOptions = {
-    url: options.endpoint,
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(body),
-    timeout: 3000
-  };
-
   if (event.logToStdout) {
     toStdout(event, message, unsafeData);
   }
@@ -177,24 +236,36 @@ const sendLog = (options, event, message, data) => {
   }
 
   if (options.token) {
-    return request(httpOptions).catch(err => {
-      // Swallow logging errors.
-      // This will show up in AWS logs at least:
-      console.error(
-        'Error making log request:',
-        err,
-        'http options:',
-        httpOptions
-      );
-    });
-  } else {
-    return ZapierPromise.resolve();
+    const logStream = logStreamFactory.getOrCreate(
+      options.endpoint,
+      options.token
+    );
+    logStream.write(
+      // JSON Lines format: It's important the serialized JSON object itself has
+      // no line breaks, and after an object it ends with a line break.
+      JSON.stringify({ message: safeMessage, data: safeData }) + '\n'
+    );
   }
 };
 
 /*
   Creates low level logging function that POSTs to endpoint (GL by default).
   Use internally; do not expose to devs.
+
+  Usage:
+
+    const logger = createLogger(event, options);
+
+    // These will reuse the same request to the log server
+    logger('log message here', { log_type: 'console' });
+    logger('another log', { log_type: 'console' });
+    logger('200 GET https://example.com', { log_type: 'http' });
+
+    // After an invocation, the Lambda handler MUST call logger.end() to close
+    // the log stream. Otherwise, it will hang!
+    logger.end().finally(() => {
+      // anything else you want to do to finish an invocation
+    });
 */
 const createLogger = (event, options) => {
   options = options || {};
@@ -205,10 +276,16 @@ const createLogger = (event, options) => {
       process.env.LOGGING_ENDPOINT || constants.DEFAULT_LOGGING_HTTP_ENDPOINT,
     apiKey:
       process.env.LOGGING_API_KEY || constants.DEFAULT_LOGGING_HTTP_API_KEY,
-    token: process.env.LOGGING_TOKEN || event.token
+    token: process.env.LOGGING_TOKEN || event.token,
   });
 
-  return sendLog.bind(undefined, options, event);
+  const logStreamFactory = new LogStreamFactory();
+  const logger = sendLog.bind(undefined, logStreamFactory, options, event);
+
+  logger.end = async () => {
+    return logStreamFactory.end();
+  };
+  return logger;
 };
 
 module.exports = createLogger;