zapier-platform-core 11.2.0 → 11.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zapier-platform-core",
-  "version": "11.2.0",
+  "version": "11.3.2",
   "description": "The core SDK for CLI apps in the Zapier Developer Platform.",
   "repository": "zapier/zapier-platform",
   "homepage": "https://platform.zapier.com/",
@@ -47,10 +47,10 @@
     "form-data": "4.0.0",
     "lodash": "4.17.21",
     "mime-types": "2.1.34",
-    "node-fetch": "2.6.6",
+    "node-fetch": "2.6.7",
     "oauth-sign": "0.9.0",
     "semver": "7.3.5",
-    "zapier-platform-schema": "11.2.0"
+    "zapier-platform-schema": "11.3.2"
   },
   "devDependencies": {
     "adm-zip": "0.5.5",

package/src/http-middlewares/before/add-query-params.js

@@ -14,7 +14,24 @@ const addQueryParams = (req) => {
 
     normalizeEmptyParamFields(req);
 
-    const stringifiedParams = querystring.stringify(req.params);
+    let stringifiedParams = querystring.stringify(req.params);
+
+    // it goes against spec, but for compatibility, some APIs want certain
+    // characters (mostly $) unencoded
+    if (req.skipEncodingChars) {
+      for (let i = 0; i < req.skipEncodingChars.length; i++) {
+        const char = req.skipEncodingChars.charAt(i);
+        const valToReplace = querystring.escape(char);
+        if (valToReplace === char) {
+          continue;
+        }
+        // no replaceAll in JS yet, coming in a node version soon!
+        stringifiedParams = stringifiedParams.replace(
+          new RegExp(valToReplace, 'g'),
+          char
+        );
+      }
+    }
 
     if (stringifiedParams) {
       req.url += `${splitter}${stringifiedParams}`;

package/src/tools/cleaner.js

@@ -158,7 +158,7 @@ const isEmptyQueryParam = (value) =>
   value === '' ||
   value === null ||
   value === undefined ||
-  isCurlies.test(value);
+  (typeof value === 'string' && value.search(isCurlies) >= 0);
 
 const normalizeEmptyParamFields = normalizeEmptyRequestFields.bind(
   null,
@@ -167,7 +167,7 @@ const normalizeEmptyParamFields = normalizeEmptyRequestFields.bind(
 );
 const normalizeEmptyBodyFields = normalizeEmptyRequestFields.bind(
   null,
-  (v) => isCurlies.test(v),
+  (v) => typeof v === 'string' && v.search(isCurlies) >= 0,
   'body'
 );
 

package/src/tools/create-http-patch.js

@@ -3,13 +3,13 @@ const _ = require('lodash');
 const constants = require('../constants');
 
 const createHttpPatch = (event) => {
-  const createLogger = require('./create-logger');
-  const logBuffer = [];
-  const logger = createLogger(event, { logBuffer });
-
-  const httpPatch = (object) => {
+  const httpPatch = (object, logger) => {
     const originalRequest = object.request;
 
+    // Important not to reuse logger between calls, because we always destroy
+    // the logger at the end of a Lambda call.
+    object.zapierLogger = logger;
+
     // Avoids multiple patching and memory leaks (mostly when running tests locally)
     if (object.patchedByZapier) {
       return;
@@ -21,13 +21,15 @@ const createHttpPatch = (event) => {
     object.request = (options, callback) => {
       // `options` can be an object or a string. If options is a string, it is
       // automatically parsed with url.parse().
-      // See https://nodejs.org/docs/latest-v6.x/api/http.html#http_http_request_options_callback
+      // See https://nodejs.org/docs/latest-v14.x/api/http.html#http_http_request_options_callback
       let requestUrl;
       if (typeof options === 'string') {
         requestUrl = options;
       } else if (typeof options.url === 'string') {
-        // XXX: Somehow options.url is available for some requests although http.request doesn't really accept it.
-        // Without this else-if, many HTTP requests don't work. Should take a deeper look at this weirdness.
+        // XXX: Somehow options.url is available for some requests although
+        // http.request doesn't really accept it. Without this else-if, many
+        // HTTP requests don't work. Should take a deeper look at this
+        // weirdness.
         requestUrl = options.url;
       } else {
         requestUrl =
@@ -67,7 +69,7 @@ const createHttpPatch = (event) => {
             response_content: responseBody,
           };
 
-          logger(
+          object.zapierLogger(
             `${logData.response_status_code} ${logData.request_method} ${logData.request_url}`,
             logData
           );

package/src/tools/create-lambda-handler.js

@@ -147,37 +147,36 @@ const createLambdaHandler = (appRawOrPath) => {
 
     environmentTools.cleanEnvironment();
 
+    // Copy bundle environment into process.env *before* creating the logger and
+    // loading app code, so that the logger gets the endpoint from process.env,
+    // and top level app code can get bundle environment vars via process.env.
+    environmentTools.applyEnvironment(event);
+
     // Create logger outside of domain, so we can use in both error and run callbacks.
     const logBuffer = [];
     const logger = createLogger(event, { logBuffer });
 
     let isCallbackCalled = false;
     const callbackOnce = (err, resp) => {
-      if (!isCallbackCalled) {
-        isCallbackCalled = true;
-        callback(err, resp);
-      }
+      logger.end().finally(() => {
+        if (!isCallbackCalled) {
+          isCallbackCalled = true;
+          callback(err, resp);
+        }
+      });
     };
 
     const logErrorAndCallbackOnce = (logMsg, logData, err) => {
-      // Wait for logger to complete before callback. This isn't
-      // strictly necessary because callbacksWaitsForEmptyLoop is
-      // the default behavior with callbacks anyway, but don't want
-      // to rely on that.
-      logger(logMsg, logData).then(() => {
-        // Check for `.message` in case someone did `throw "My Error"`
-        if (
-          !constants.IS_TESTING &&
-          err &&
-          !err.doNotContextify &&
-          err.message
-        ) {
-          err.message += `\n\nConsole logs:\n${logBuffer
-            .map((s) => `  ${s.message}`)
-            .join('')}`;
-        }
-        callbackOnce(err);
-      });
+      logger(logMsg, logData);
+
+      // Check for `.message` in case someone did `throw "My Error"`
+      if (!constants.IS_TESTING && err && !err.doNotContextify && err.message) {
+        err.message += `\n\nConsole logs:\n${logBuffer
+          .map((s) => `  ${s.message}`)
+          .join('')}`;
+      }
+
+      callbackOnce(err);
     };
 
     const handlerDomain = domain.create();
@@ -191,10 +190,6 @@ const createLambdaHandler = (appRawOrPath) => {
     });
 
     handlerDomain.run(() => {
-      // Copy bundle environment into process.env *before* loading app code,
-      // so that top level app code can get bundle environment vars via process.env.
-      environmentTools.applyEnvironment(event);
-
       const rpc = createRpcClient(event);
 
       return loadApp(event, rpc, appRawOrPath)
@@ -203,10 +198,10 @@ const createLambdaHandler = (appRawOrPath) => {
 
           const { skipHttpPatch } = appRaw.flags || {};
           // Adds logging for _all_ kinds of http(s) requests, no matter the library
-          if (!skipHttpPatch) {
+          if (!skipHttpPatch && !event.calledFromCli) {
             const httpPatch = createHttpPatch(event);
-            httpPatch(require('http'));
-            httpPatch(require('https')); // 'https' needs to be patched separately
+            httpPatch(require('http'), logger);
+            httpPatch(require('https'), logger); // 'https' needs to be patched separately
           }
 
           // TODO: Avoid calling prepareApp(appRaw) repeatedly here as createApp()

package/src/tools/create-legacy-scripting-runner.js

@@ -8,8 +8,11 @@ const semver = require('semver');
 const createLegacyScriptingRunner = (z, input) => {
   const app = _.get(input, '_zapier.app');
 
-  let source =
-    _.get(app, 'legacy.scriptingSource') || app.legacyScriptingSource;
+  // once we have node 14 everywhere, this can be:
+  // let source = _.get(app, 'legacy.scriptingSource') ?? app.legacyScriptingSource;
+  let source = _.get(app, 'legacy.scriptingSource');
+  source = source === undefined ? app.legacyScriptingSource : source;
+
   if (source === undefined) {
     // Don't initialize z.legacyScripting for a pure CLI app
     return null;
@@ -26,8 +29,8 @@ const createLegacyScriptingRunner = (z, input) => {
   let LegacyScriptingRunner, version;
   try {
     LegacyScriptingRunner = require('zapier-platform-legacy-scripting-runner');
-    version = require('zapier-platform-legacy-scripting-runner/package.json')
-      .version;
+    version =
+      require('zapier-platform-legacy-scripting-runner/package.json').version;
   } catch (e) {
     // Find it in cwd, in case we're developing legacy-scripting-runner itself
     const cwd = process.cwd();

package/src/tools/create-logger.js

@@ -1,10 +1,11 @@
 'use strict';
 
+const { Transform } = require('stream');
+
 const _ = require('lodash');
 
 const request = require('./request-client-internal');
 const { simpleTruncate, recurseReplace } = require('./data');
-const ZapierPromise = require('./promise');
 const {
   DEFAULT_LOGGING_HTTP_API_KEY,
   DEFAULT_LOGGING_HTTP_ENDPOINT,
@@ -19,6 +20,10 @@ const {
 // not really a public function, but it came from here originally
 const { isUrlWithSecrets } = require('@zapier/secret-scrubber/lib/convenience');
 
+// The payload size per request to stream logs. This should be slighly lower
+// than the limit (16 MB) on the server side.
+const LOG_STREAM_BYTES_LIMIT = 15 * 1024 * 1024;
+
 const isUrl = (url) => {
   try {
     // eslint-disable-next-line no-new
@@ -121,7 +126,75 @@ const buildSensitiveValues = (event, data) => {
   ];
 };
 
-const sendLog = (options, event, message, data) => {
+class LogStream extends Transform {
+  constructor(options) {
+    super(options);
+    this.bytesWritten = 0;
+    this.request = this._newRequest(options.url, options.token);
+  }
+
+  _newRequest(url, token) {
+    const httpOptions = {
+      url,
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/x-ndjson',
+        'X-Token': token,
+      },
+      body: this,
+    };
+    return request(httpOptions).catch((err) => {
+      // Swallow logging errors. This will show up in AWS logs at least.
+      console.error(
+        'Error making log request:',
+        err,
+        'http options:',
+        httpOptions
+      );
+    });
+  }
+
+  _transform(chunk, encoding, callback) {
+    this.push(chunk);
+    this.bytesWritten += Buffer.byteLength(chunk, encoding);
+    callback();
+  }
+}
+
+// Implements singleton for LogStream. The goal is for every sendLog() call we
+// reuse the same request until the request body grows too big and exceeds
+// LOG_STREAM_BYTES_LIMIT.
+class LogStreamFactory {
+  constructor() {
+    this._logStream = null;
+  }
+
+  getOrCreate(url, token) {
+    if (this._logStream) {
+      if (this._logStream.bytesWritten < LOG_STREAM_BYTES_LIMIT) {
+        // Reuse the same request for efficiency
+        return this._logStream;
+      }
+
+      // End this one before creating another
+      this._logStream.end();
+    }
+
+    this._logStream = new LogStream({ url, token });
+    return this._logStream;
+  }
+
+  async end() {
+    if (this._logStream) {
+      this._logStream.end();
+      const response = await this._logStream.request;
+      this._logStream = null;
+      return response;
+    }
+  }
+}
+
+const sendLog = async (logStreamFactory, options, event, message, data) => {
   data = _.extend({}, data || {}, event.logExtra || {});
   data.log_type = data.log_type || 'console';
 
@@ -148,20 +221,6 @@ const sendLog = (options, event, message, data) => {
   safeData.request_headers = formatHeaders(safeData.request_headers);
   safeData.response_headers = formatHeaders(safeData.response_headers);
 
-  const body = {
-    message: safeMessage,
-    data: safeData,
-    token: options.token,
-  };
-
-  const httpOptions = {
-    url: options.endpoint,
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify(body),
-    timeout: 3000,
-  };
-
   if (event.logToStdout) {
     toStdout(event, message, unsafeData);
   }
@@ -172,24 +231,36 @@ const sendLog = (options, event, message, data) => {
   }
 
   if (options.token) {
-    return request(httpOptions).catch((err) => {
-      // Swallow logging errors.
-      // This will show up in AWS logs at least:
-      console.error(
-        'Error making log request:',
-        err,
-        'http options:',
-        httpOptions
-      );
-    });
-  } else {
-    return ZapierPromise.resolve();
+    const logStream = logStreamFactory.getOrCreate(
+      options.endpoint,
+      options.token
+    );
+    logStream.write(
+      // JSON Lines format: It's important the serialized JSON object itself has
+      // no line breaks, and after an object it ends with a line break.
+      JSON.stringify({ message: safeMessage, data: safeData }) + '\n'
+    );
   }
 };
 
 /*
   Creates low level logging function that POSTs to endpoint (GL by default).
   Use internally; do not expose to devs.
+
+  Usage:
+
+    const logger = createLogger(event, options);
+
+    // These will reuse the same request to the log server
+    logger('log message here', { log_type: 'console' });
+    logger('another log', { log_type: 'console' });
+    logger('200 GET https://example.com', { log_type: 'http' });
+
+    // After an invocation, the Lambda handler MUST call logger.end() to close
+    // the log stream. Otherwise, it will hang!
+    logger.end().finally(() => {
+      // anything else you want to do to finish an invocation
+    });
 */
 const createLogger = (event, options) => {
   options = options || {};
@@ -201,7 +272,13 @@ const createLogger = (event, options) => {
     token: process.env.LOGGING_TOKEN || event.token,
   });
 
-  return sendLog.bind(undefined, options, event);
+  const logStreamFactory = new LogStreamFactory();
+  const logger = sendLog.bind(undefined, logStreamFactory, options, event);
+
+  logger.end = async () => {
+    return logStreamFactory.end();
+  };
+  return logger;
 };
 
 module.exports = createLogger;