zapier-platform-core 12.0.2 → 12.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zapier-platform-core",
-  "version": "12.0.2",
+  "version": "12.1.0",
   "description": "The core SDK for CLI apps in the Zapier Developer Platform.",
   "repository": "zapier/zapier-platform",
   "homepage": "https://platform.zapier.com/",
@@ -47,10 +47,11 @@
     "form-data": "4.0.0",
     "lodash": "4.17.21",
     "mime-types": "2.1.34",
+    "node-abort-controller": "3.0.1",
     "node-fetch": "2.6.7",
     "oauth-sign": "0.9.0",
     "semver": "7.3.5",
-    "zapier-platform-schema": "12.0.2"
+    "zapier-platform-schema": "12.1.0"
   },
   "devDependencies": {
     "adm-zip": "0.5.5",

package/src/http-middlewares/after/log-response.js

@@ -76,4 +76,4 @@ const logResponse = (resp) => {
     .catch(() => resp);
 };
 
-module.exports = logResponse;
+module.exports = { logResponse, prepareRequestLog };

package/src/tools/create-app-request-client.js

@@ -16,7 +16,7 @@ const oauth1SignRequest = require('../http-middlewares/before/oauth1-sign-reques
 const prepareRequest = require('../http-middlewares/before/prepare-request');
 
 // after middles
-const logResponse = require('../http-middlewares/after/log-response');
+const { logResponse } = require('../http-middlewares/after/log-response');
 const prepareResponse = require('../http-middlewares/after/prepare-response');
 const throwForStaleAuth = require('../http-middlewares/after/throw-for-stale-auth');
 const throwForStatusMiddleware = require('../http-middlewares/after/throw-for-status');

package/src/tools/create-logger.js

@@ -1,11 +1,14 @@
 'use strict';
 
+const { promisify } = require('util');
 const { Transform } = require('stream');
+const { parse: querystringParse } = require('querystring');
 
 const _ = require('lodash');
+const { AbortController } = require('node-abort-controller');
 
 const request = require('./request-client-internal');
-const { simpleTruncate, recurseReplace } = require('./data');
+const { simpleTruncate, recurseReplace, truncateData } = require('./data');
 const {
   DEFAULT_LOGGING_HTTP_API_KEY,
   DEFAULT_LOGGING_HTTP_ENDPOINT,
@@ -18,12 +21,19 @@ const {
   recurseExtract,
 } = require('@zapier/secret-scrubber');
 // not really a public function, but it came from here originally
-const { isUrlWithSecrets } = require('@zapier/secret-scrubber/lib/convenience');
+const {
+  isUrlWithSecrets,
+  isSensitiveKey,
+} = require('@zapier/secret-scrubber/lib/convenience');
 
 // The payload size per request to stream logs. This should be slighly lower
 // than the limit (16 MB) on the server side.
 const LOG_STREAM_BYTES_LIMIT = 15 * 1024 * 1024;
 
+const DEFAULT_LOGGER_TIMEOUT = 200;
+
+const sleep = promisify(setTimeout);
+
 const isUrl = (url) => {
   try {
     // eslint-disable-next-line no-new
@@ -34,7 +44,8 @@ const isUrl = (url) => {
   }
 };
 
-const truncate = (str) => simpleTruncate(str, 3500, ' [...]');
+const MAX_LENGTH = 3500;
+const truncateString = (str) => simpleTruncate(str, MAX_LENGTH, ' [...]');
 
 const formatHeaders = (headers = {}) => {
   if (_.isEmpty(headers)) {
@@ -70,7 +81,7 @@ const httpDetailsLogMessage = (data) => {
     (result, value, key) => {
       result[key] = value;
       if (typeof value === 'string') {
-        result[key] = truncate(value);
+        result[key] = truncateString(value);
       }
       return result;
     },
@@ -108,28 +119,61 @@ const toStdout = (event, msg, data) => {
   }
 };
 
+// try to parse json; if successful, find secrets in it
+const attemptFindSecretsInStr = (s) => {
+  let parsedRespContent;
+  try {
+    parsedRespContent = JSON.parse(s);
+  } catch {
+    return [];
+  }
+  return findSensitiveValues(parsedRespContent);
+};
+
 const buildSensitiveValues = (event, data) => {
   const bundle = event.bundle || {};
   const authData = bundle.authData || {};
   // for the most part, we should censor all the values from authData
   // the exception is safe urls, which should be filtered out - we want those to be logged
+  // but, we _should_ censor-no-matter-what sensitive keys, even if their value is a safe url
+  // this covers the case where someone's password is a valid url ¯\_(ツ)_/¯
   const sensitiveAuthData = recurseExtract(authData, (key, value) => {
+    if (isSensitiveKey(key)) {
+      return true;
+    }
+
     if (isUrl(value) && !isUrlWithSecrets(value)) {
       return false;
     }
     return true;
   });
-  return [
+
+  const result = [
     ...sensitiveAuthData,
     ...findSensitiveValues(process.env),
     ...findSensitiveValues(data),
   ];
+
+  // for our http logs (genrated by prepareRequestLog), make sure that we try to parse the content to find any new strings
+  // (such as what comes back in the response during an auth refresh)
+
+  for (const prop of ['response_content', 'request_data']) {
+    if (data[prop]) {
+      result.push(...attemptFindSecretsInStr(data[prop]));
+    }
+  }
+  if (data.request_params) {
+    result.push(...findSensitiveValues(querystringParse(data.request_params)));
+  }
+  // unique- no point in duplicates
+  return [...new Set(result)];
 };
 
 class LogStream extends Transform {
   constructor(options) {
     super(options);
     this.bytesWritten = 0;
+    this.controller = new AbortController();
     this.request = this._newRequest(options.url, options.token);
   }
 
@@ -142,15 +186,20 @@ class LogStream extends Transform {
         'X-Token': token,
       },
       body: this,
+      signal: this.controller.signal,
     };
     return request(httpOptions).catch((err) => {
+      if (err.name === 'AbortError') {
+        return {
+          status: 200,
+          content: 'aborted',
+        };
+      }
+
       // Swallow logging errors. This will show up in AWS logs at least.
-      console.error(
-        'Error making log request:',
-        err,
-        'http options:',
-        httpOptions
-      );
+      // Don't need to log for AbortError because that happens when we abort
+      // on purpose.
+      console.error('Error making log request:', err);
     });
   }
 
@@ -159,6 +208,10 @@ class LogStream extends Transform {
     this.bytesWritten += Buffer.byteLength(chunk, encoding);
     callback();
   }
+
+  abort() {
+    this.controller.abort();
+  }
 }
 
 // Implements singleton for LogStream. The goal is for every sendLog() call we
@@ -185,17 +238,35 @@ class LogStreamFactory {
     return this._logStream;
   }
 
-  async end() {
+  // Ends the logger and gets a response from the log server. Optionally takes
+  // timeoutToAbort to specify how many milliseconds we want to wait before
+  // force aborting the connection to the log server.
+  async end(timeoutToAbort = DEFAULT_LOGGER_TIMEOUT) {
     // Mark the factory as ended. This suggests that any logStream.write() that
     // follows should end() right away.
     this.ended = true;
+    let response;
 
     if (this._logStream) {
       this._logStream.end();
-      const response = await this._logStream.request;
+
+      const clock =
+        timeoutToAbort > 0 ? sleep(timeoutToAbort) : Promise.resolve(undefined);
+      const responsePromise = this._logStream.request;
+
+      const result = await Promise.race([clock, responsePromise]);
+      const isTimeout = !result;
+      if (isTimeout) {
+        this._logStream.abort();
+        // Expect to get a `{content: 'aborted'}` response
+        response = await responsePromise;
+      } else {
+        response = result;
+      }
+
       this._logStream = null;
-      return response;
     }
+    return response;
   }
 }
 
@@ -207,15 +278,21 @@ const sendLog = async (logStreamFactory, options, event, message, data) => {
   data.response_headers = unheader(data.response_headers);
 
   const sensitiveValues = buildSensitiveValues(event, data);
+
+  // data.input and data.output have the ability to grow unbounded; the following caps the size to a reasonable amount
+  if (data.log_type === 'bundle') {
+    data.input = truncateData(data.input, MAX_LENGTH);
+    data.output = truncateData(data.output, MAX_LENGTH);
+  }
   // scrub throws an error if there are no secrets
-  const safeMessage = truncate(
+  const safeMessage = truncateString(
     sensitiveValues.length ? scrub(message, sensitiveValues) : message
   );
   const safeData = recurseReplace(
     sensitiveValues.length ? scrub(data, sensitiveValues) : data,
-    truncate
+    truncateString
   );
-  const unsafeData = recurseReplace(data, truncate);
+  const unsafeData = recurseReplace(data, truncateString);
   // Keep safe log keys uncensored
   Object.keys(safeData).forEach((key) => {
     if (SAFE_LOG_KEYS.includes(key)) {
@@ -251,7 +328,7 @@ const sendLog = async (logStreamFactory, options, event, message, data) => {
       // (bad) callback that is still running after the Lambda handler returns?
       // We need to make sure the bad callback ends the logger as well.
       // Otherwise, it will hang!
-      logStreamFactory.end();
+      logStreamFactory.end(DEFAULT_LOGGER_TIMEOUT);
     }
   }
 };
@@ -288,8 +365,8 @@ const createLogger = (event, options) => {
   const logStreamFactory = new LogStreamFactory();
   const logger = sendLog.bind(undefined, logStreamFactory, options, event);
 
-  logger.end = async () => {
-    return logStreamFactory.end();
+  logger.end = async (timeoutToAbort = DEFAULT_LOGGER_TIMEOUT) => {
+    return logStreamFactory.end(timeoutToAbort);
   };
   return logger;
 };

package/src/tools/data.js

@@ -172,6 +172,162 @@ const simpleTruncate = (string, length, suffix) => {
   return string;
 };
 
+/**
+ * Adds an item to an object or array.
+ * If the parent is an object, the value will be set at the specified key.
+ * If the parent is an array, the value will be added to the end of the array (and the key will be ignored).
+ * Used by truncateData.
+ *
+ * @param {object | any[]} parent An object or array.
+ * @param {string} key The key to set the value at (objects only; ignored for arrays).
+ * @param {any} value The value to add.
+ */
+const _addItem = (parent, key, value) => {
+  if (Array.isArray(parent)) {
+    parent.push(value);
+  } else {
+    parent[key] = value;
+  }
+};
+
+/**
+ * Examines an object entry or array entry (`item`) and determines its "cost" (how many characters will it take to add it to `parent`).
+ * If the item is a string, `availableSpace` is used to determine if the string should be truncated.
+ * If the item is an object or array, its entries / values are added to `queue`.
+ * Used by truncateData.
+ *
+ * @param {any[]} queue
+ * @param {object | any[]} parent
+ * @param {string} key
+ * @param {any} item
+ * @param {number} availableSpace
+ * @returns
+ */
+const _processItem = (queue, parent, key, item, availableSpace) => {
+  let itemLength = 0;
+  let itemToAdd = item;
+  let wasTruncated = false;
+
+  itemLength += key.length; // array keys are empty strings, so this is a noop
+  itemLength += key.length ? 3 : 0; // objects get +2 for "" around the key and +1 for the :
+  itemLength += 1; // arrays and objects both have +1 for commas between entries
+  if (parent && typeof parent === 'object' && _.isEmpty(parent)) {
+    itemLength -= 1; // this is the first entry for an object or array; remove the count for a comma
+  }
+
+  if (typeof item === 'number' || typeof item === 'boolean' || item == null) {
+    itemLength += String(item).length;
+  } else if (typeof item === 'string') {
+    const overhead = itemLength + 2; // the minimum amount of space needed after truncation
+    if (item.length + overhead > availableSpace) {
+      // this string is going to push us over the edge; truncate it
+      itemToAdd = simpleTruncate(item, availableSpace - overhead, ' [...]');
+      wasTruncated = true;
+    }
+    itemLength += itemToAdd.length + 2; // 2 for quotes around the string value
+  } else if (typeof item === 'object') {
+    itemLength += 2; // '{}' or '[]'
+
+    let entries;
+    if (Array.isArray(item)) {
+      const newArr = [];
+      itemToAdd = newArr;
+      entries = item.map((subValue) => [newArr, '', subValue]);
+    } else {
+      const newObj = {};
+      itemToAdd = newObj;
+      entries = Object.entries(item).map(([subKey, subValue]) => [
+        newObj,
+        subKey,
+        subValue,
+      ]);
+    }
+    queue.unshift(...entries);
+  } else {
+    // JSON.stringify doesn't usually really do anything for any other typeofs
+    // we're just going to use `undefined` and hope for the best
+    itemLength += 'undefined'.length;
+    itemToAdd = undefined;
+  }
+  return [itemLength, itemToAdd, wasTruncated];
+};
+
+/**
+ * Takes a given `data` object or array and copies pieces of that data into `output` until its stringified length fits in `maxLength` characters.
+ *
+ * In general, output should track with `JSON.stringify(item).substring(0, maxLength)` (i.e. depth-first traversal of arrays and object entries), but in a JSON-aware way.
+ * If the item's initial stringified length is less than or equal to `maxLength`, the item is returned as-is.
+ * @param {object | any[]} data The JSON object or array to be truncated.
+ * @param {number} maxLength The maximum length of JSON.stringify(output). Note that this may not be the exact output length, but it serves as an upper bound. Minimum value is 40.
+ * @returns {object | any[]} The truncated object or array.
+ */
+const truncateData = (data, maxLength) => {
+  if (!data || typeof data !== 'object') {
+    // the following code is only meant to work on objects and arrays
+    return data;
+  }
+  if (JSON.stringify(data).length <= maxLength) {
+    // no need to truncate
+    return data;
+  }
+
+  const root = Array.isArray(data) ? [] : {};
+  let length = 2; // '{}' or '[]'
+  let dataWasTruncated = false; // used during iteration to track if a string was truncated
+  const truncateMessageSize = 39; // the overhead required to add a message about truncating data
+
+  if (maxLength < 40) {
+    // adding the truncate message takes 39 characters, but the minimum output (i.e. just the message wrapped in an object or array)
+    // is 40 characters due to the overhead of the {} or [] characters (+2) minus the comma (-1)
+    throw new Error(`maxLength must be at least 40`);
+  }
+
+  const queue = Array.isArray(data)
+    ? data.map((value) => [root, '', value])
+    : Object.entries(data).map(([key, value]) => [root, key, value]);
+
+  // iterate over the queue
+  while (queue.length > 0) {
+    const [parent, key, item] = queue.shift();
+    const [itemLength, processedItem, itemWasTruncated] = _processItem(
+      queue,
+      parent,
+      key,
+      item,
+      maxLength - length - truncateMessageSize
+    );
+
+    if (itemWasTruncated) {
+      // if a string was truncated, we mark the total data as truncated for messaging purposes
+      dataWasTruncated = true;
+    }
+
+    if (length + itemLength + truncateMessageSize < maxLength) {
+      // we're still under the max length, add this and keep going
+      _addItem(parent, key, processedItem);
+      length += itemLength;
+    } else {
+      if (length + itemLength + truncateMessageSize === maxLength) {
+        // we can fit this item + the truncate message, so let's add it before we stop
+        _addItem(parent, key, processedItem);
+      }
+      dataWasTruncated = true;
+      break;
+    }
+  }
+
+  // we can hit the following even if we got through all the items in the queue in the case that any strings were truncated
+  if (dataWasTruncated) {
+    if (Array.isArray(root)) {
+      root.push('NOTE : This data has been truncated.');
+    } else {
+      root.NOTE = 'This data has been truncated.';
+    }
+  }
+
+  return root;
+};
+
 const genId = () => parseInt(Math.random() * 100000000);
 
 module.exports = {
@@ -186,4 +342,5 @@ module.exports = {
   memoizedFindMapDeep,
   recurseReplace,
   simpleTruncate,
+  truncateData,
 };