zapier-platform-core 11.0.0 → 11.2.0

package/LICENSE

@@ -1,4 +1,3 @@
 Copyright (c) Zapier, Inc.
 
-This repository is part of Zapier Platform, of which license terms can be found at
-https://zapier.com/platform/tos.
+This repository is part of Zapier Platform.  By downloading, installing, accessing, or using any part of the Zapier Platform, including this repository, you agree to the Zapier Platform Agreement, which can be found at: https://zapier.com/platform/tos.  If you do not agree to the Zapier Platform Agreement, you may not download, install, access, or use any part of the Zapier Platform, including this repository.

package/index.d.ts

@@ -37,6 +37,7 @@ export interface Bundle<InputData = { [x: string]: any }> {
   inputData: InputData;
   inputDataRaw: { [x: string]: string };
   meta: {
+    isBulkRead: boolean;
     isFillingDynamicDropdown: boolean;
     isLoadingSample: boolean;
     isPopulatingDedupe: boolean;
@@ -70,6 +71,9 @@ declare class AppError extends Error {
 declare class HaltedError extends Error {}
 declare class ExpiredAuthError extends Error {}
 declare class RefreshAuthError extends Error {}
+declare class ThrottledError extends Error {
+  constructor(message: string, delay?: number);
+}
 
 // copied http stuff from external typings
 export interface HttpRequestOptions {
@@ -123,12 +127,13 @@ type DehydrateFunc = <T>(
 export interface ZObject {
   request: {
     // most specific overloads go first
-    (url: string, options: HttpRequestOptions & { raw: true }): Promise<
-      RawHttpResponse
-    >;
-    (options: HttpRequestOptions & { raw: true; url: string }): Promise<
-      RawHttpResponse
-    >;
+    (
+      url: string,
+      options: HttpRequestOptions & { raw: true }
+    ): Promise<RawHttpResponse>;
+    (
+      options: HttpRequestOptions & { raw: true; url: string }
+    ): Promise<RawHttpResponse>;
 
     (url: string, options?: HttpRequestOptions): Promise<HttpResponse>;
     (options: HttpRequestOptions & { url: string }): Promise<HttpResponse>;
@@ -186,5 +191,6 @@ export interface ZObject {
     HaltedError: typeof HaltedError;
     ExpiredAuthError: typeof ExpiredAuthError;
     RefreshAuthError: typeof RefreshAuthError;
+    ThrottledError: typeof ThrottledError;
   };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zapier-platform-core",
-  "version": "11.0.0",
+  "version": "11.2.0",
   "description": "The core SDK for CLI apps in the Zapier Developer Platform.",
   "repository": "zapier/zapier-platform",
   "homepage": "https://platform.zapier.com/",
@@ -18,10 +18,11 @@
     "preversion": "git pull && yarn test",
     "version": "node bin/bump-dependencies.js && yarn && git add package.json yarn.lock",
     "postversion": "git push && git push --tags",
-    "test": "mocha -t 20000 --recursive test",
+    "main-tests": "mocha -t 20000 --recursive test",
+    "solo-test": "test $(OPT_OUT_PATCH_TEST_ONLY=yes mocha --recursive test -g 'should be able to opt out of patch' -R json | jq '.stats.passes') -eq 1 && echo 'Ran 1 test and it passed!'",
+    "test": "yarn main-tests && yarn solo-test",
     "debug": "mocha -t 10000 --inspect-brk --recursive test",
     "test:w": "mocha -t 10000 --recursive test --watch",
-    "plain-test": "mocha -t 5000 --recursive test",
     "integration-test": "mocha -t 20000 integration-test",
     "local-integration-test": "mocha -t 10000 integration-test --local",
     "lambda-integration-test": "mocha -t 10000 integration-test --lambda",
@@ -34,24 +35,27 @@
     "validate": "yarn test && yarn smoke-test && yarn lint"
   },
   "engines": {
-    "node": ">=10",
+    "node": ">=12",
     "npm": ">=5.6.0"
   },
   "engineStrict": true,
   "dependencies": {
+    "@zapier/secret-scrubber": "^1.0.3",
     "bluebird": "3.7.2",
     "content-disposition": "0.5.3",
     "dotenv": "9.0.2",
     "form-data": "4.0.0",
     "lodash": "4.17.21",
-    "node-fetch": "2.6.1",
+    "mime-types": "2.1.34",
+    "node-fetch": "2.6.6",
     "oauth-sign": "0.9.0",
     "semver": "7.3.5",
-    "zapier-platform-schema": "11.0.0"
+    "zapier-platform-schema": "11.2.0"
   },
   "devDependencies": {
     "adm-zip": "0.5.5",
     "aws-sdk": "^2.905.0",
+    "dicer": "^0.3.0",
     "fs-extra": "^10.0.0",
     "mock-fs": "^4.14.0"
   },

package/src/constants.js

@@ -23,17 +23,6 @@ const REQUEST_OBJECT_SHORTHAND_OPTIONS = { isShorthand: true, replace: true };
 const DEFAULT_LOGGING_HTTP_ENDPOINT = 'https://httplogger.zapier.com/input';
 const DEFAULT_LOGGING_HTTP_API_KEY = 'R24hzu86v3jntwtX2DtYECeWAB'; // It's ok, this isn't PROD
 
-const SENSITIVE_KEYS = [
-  'api_key',
-  'apikey',
-  'auth',
-  'passwd',
-  'password',
-  'secret',
-  'signature',
-  'token',
-];
-
 const SAFE_LOG_KEYS = [
   'account_id',
   'api_title',
@@ -76,6 +65,5 @@ module.exports = {
   REQUEST_OBJECT_SHORTHAND_OPTIONS,
   RESPONSE_SIZE_LIMIT,
   SAFE_LOG_KEYS,
-  SENSITIVE_KEYS,
   STATUSES,
 };

package/src/create-command-handler.js

@@ -18,7 +18,7 @@ const commandHandlers = {
   commands like 'execute', 'validate', 'definition', 'request'.
 */
 const createCommandHandler = (compiledApp) => {
-  return (input) => {
+  return async (input) => {
     const command = input._zapier.event.command || 'execute'; // validate || definition || request
     const handler = commandHandlers[command];
     if (!handler) {
@@ -26,7 +26,7 @@ const createCommandHandler = (compiledApp) => {
     }
 
     try {
-      return handler(compiledApp, input);
+      return await handler(compiledApp, input);
     } catch (err) {
       return handleError(err);
     }

package/src/errors.js

@@ -32,6 +32,7 @@ class ResponseError extends Error {
         status: response.status,
         headers: {
           'content-type': response.headers.get('content-type'),
+          'retry-after': response.headers.get('retry-after'),
         },
         content,
         request: {
@@ -44,6 +45,19 @@ class ResponseError extends Error {
   }
 }
 
+class ThrottledError extends Error {
+  constructor(message, delay) {
+    super(
+      JSON.stringify({
+        message,
+        delay,
+      })
+    );
+    this.name = 'ThrottledError';
+    this.doNotContextify = true;
+  }
+}
+
 // Make some of the errors we'll use!
 const createError = (name) => {
   const NewError = function (message = '') {
@@ -77,6 +91,7 @@ const exceptions = _.reduce(
   {
     Error: AppError,
     ResponseError,
+    ThrottledError,
   }
 );
 

package/src/tools/cleaner.js

@@ -45,6 +45,12 @@ const recurseCleanFuncs = (obj, path) => {
 
 // Recurse a nested object replace all instances of keys->vals in the bank.
 const recurseReplaceBank = (obj, bank = {}) => {
+  const matchesCurlies = /({{.*?}})/;
+  const matchesKeyRegexMap = Object.keys(bank).reduce((acc, key) => {
+    // Escape characters (ex. {{foo}} => \\{\\{foo\\}\\} )
+    acc[key] = new RegExp(key.replace(/[-[\]/{}()\\*+?.^$|]/g, '\\$&'), 'g');
+    return acc;
+  }, {});
   const replacer = (out) => {
     if (!['string', 'number'].includes(typeof out)) {
       return out;
@@ -57,15 +63,11 @@ const recurseReplaceBank = (obj, bank = {}) => {
     let maybeChangedString = originalValueStr;
 
     Object.keys(bank).forEach((key) => {
-      // Escape characters (ex. {{foo}} => \\{\\{foo\\}\\} )
-      const escapedKey = key.replace(/[-[\]/{}()\\*+?.^$|]/g, '\\$&');
-      const matchesKey = new RegExp(escapedKey, 'g');
-
+      const matchesKey = matchesKeyRegexMap[key];
       if (!matchesKey.test(maybeChangedString)) {
         return;
       }
 
-      const matchesCurlies = /({{.*?}})/;
       const valueParts = maybeChangedString
         .split(matchesCurlies)
         .filter(Boolean);

package/src/tools/create-app-tester.js

@@ -3,32 +3,9 @@
 const createLambdaHandler = require('./create-lambda-handler');
 const resolveMethodPath = require('./resolve-method-path');
 const ZapierPromise = require('./promise');
-const { get } = require('lodash');
+const { isFunction } = require('lodash');
 const { genId } = require('./data');
-
-// this is (annoyingly) mirrored in cli/api_base, so that test functions only
-// have a storeKey when canPaginate is true. otherwise, a test would work but a
-// poll on site would fail. this is only used in test handlers
-
-// there are 2 places you can put a method that can interact with cursors:
-// triggers.contact.operation.perform, if it's a poll trigger
-// resources.contact.list.operation.perform if it's a resource
-// schema doesn't currently allow cursor use on hook trigger `performList`, so we don't need to account for it
-const shouldPaginate = (appRaw, method) => {
-  const methodParts = method.split('.');
-
-  if (
-    method.endsWith('perform') &&
-    ((methodParts[0] === 'resources' && methodParts[2] === 'list') ||
-      methodParts[0] === 'triggers' ||
-      methodParts[0] === 'bulkReads')
-  ) {
-    methodParts.pop();
-    return get(appRaw, [...methodParts, 'canPaginate']);
-  }
-
-  return false;
-};
+const { shouldPaginate } = require('./should-paginate');
 
 // Convert a app handler to promise for convenience.
 const promisifyHandler = (handler) => {
@@ -55,7 +32,21 @@ const createAppTester = (appRaw, { customStoreKey } = {}) => {
   return (methodOrFunc, bundle) => {
     bundle = bundle || {};
 
-    const method = resolveMethodPath(appRaw, methodOrFunc);
+    let method = resolveMethodPath(appRaw, methodOrFunc, false);
+    if (!method) {
+      if (isFunction(methodOrFunc)) {
+        // definitely have a function but didn't find it on the app; it's an adhoc
+        appRaw._testRequest = (z, bundle) => methodOrFunc(z, bundle);
+        method = resolveMethodPath(appRaw, appRaw._testRequest);
+      } else {
+        throw new Error(
+          `Unable to find the following on your App instance: ${JSON.stringify(
+            methodOrFunc
+          )}`
+        );
+      }
+    }
+
     const storeKey = shouldPaginate(appRaw, method)
       ? customStoreKey
         ? `testKey-${customStoreKey}`
@@ -77,7 +68,10 @@ const createAppTester = (appRaw, { customStoreKey } = {}) => {
       event.detailedLogToStdout = true;
     }
 
-    return createHandlerPromise(event).then((resp) => resp.results);
+    return createHandlerPromise(event).then((resp) => {
+      delete appRaw._testRequest; // clear adHocFunc so tests can't affect each other
+      return resp.results;
+    });
   };
 };
 

package/src/tools/create-file-stasher.js

@@ -1,14 +1,18 @@
 'use strict';
 
-const _ = require('lodash');
+const fs = require('fs');
+const os = require('os');
 const path = require('path');
-const FormData = require('form-data');
+const { pipeline } = require('stream');
+const { promisify } = require('util');
+const { randomBytes } = require('crypto');
+
+const _ = require('lodash');
 const contentDisposition = require('content-disposition');
+const FormData = require('form-data');
+const mime = require('mime-types');
 
 const request = require('./request-client-internal');
-const ZapierPromise = require('./promise');
-
-const isPromise = (obj) => obj && typeof obj.then === 'function';
 
 const UPLOAD_MAX_SIZE = 1000 * 1000 * 150; // 150mb, in zapier backend too
 
@@ -19,32 +23,186 @@ const LENGTH_ERR_MESSAGE =
 const DEFAULT_FILE_NAME = 'unnamedfile';
 const DEFAULT_CONTENT_TYPE = 'application/octet-stream';
 
-const uploader = (
+const streamPipeline = promisify(pipeline);
+
+const filenameFromURL = (url) => {
+  try {
+    return decodeURIComponent(path.posix.basename(new URL(url).pathname));
+  } catch (error) {
+    return null;
+  }
+};
+
+const filenameFromHeader = (response) => {
+  const cd = response.headers.get('content-disposition');
+  let filename;
+  if (cd) {
+    try {
+      filename = contentDisposition.parse(cd).parameters.filename;
+    } catch (error) {
+      return null;
+    }
+  }
+  return filename || null;
+};
+
+const resolveRemoteStream = async (stream) => {
+  // Download to a temp file, get the file size, and create a readable stream
+  // from the temp file.
+  //
+  // The streamPipeline usage is taken from
+  // https://github.com/node-fetch/node-fetch#streams
+  const tmpFilePath = path.join(
+    os.tmpdir(),
+    'stash-' + randomBytes(16).toString('hex')
+  );
+
+  try {
+    await streamPipeline(stream, fs.createWriteStream(tmpFilePath));
+  } catch (error) {
+    try {
+      fs.unlinkSync(tmpFilePath);
+    } catch (e) {
+      // File doesn't exist? Probably okay
+    }
+    throw error;
+  }
+
+  const length = fs.statSync(tmpFilePath).size;
+  const readStream = fs.createReadStream(tmpFilePath);
+
+  readStream.on('end', () => {
+    // Burn after reading
+    try {
+      fs.unlinkSync(tmpFilePath);
+    } catch (e) {
+      // TODO: We probably want to log warning here
+    }
+  });
+
+  return {
+    streamOrData: readStream,
+    length,
+  };
+};
+
+const resolveResponseToStream = async (response) => {
+  // Get filename from content-disposition header or URL
+  let filename =
+    filenameFromHeader(response) ||
+    filenameFromURL(response.url || _.get(response, ['request', 'url'])) ||
+    DEFAULT_FILE_NAME;
+
+  const contentType = response.headers.get('content-type');
+  if (contentType && !path.extname(filename)) {
+    const ext = mime.extension(contentType);
+    if (ext && ext !== 'bin') {
+      filename += '.' + ext;
+    }
+  }
+
+  if (response.body && typeof response.body.pipe === 'function') {
+    // streamable response created by z.request({ raw: true })
+    return {
+      ...(await resolveRemoteStream(response.body)),
+      contentType: contentType || DEFAULT_CONTENT_TYPE,
+      filename,
+    };
+  }
+
+  // regular response created by z.request({ raw: false })
+  return {
+    streamOrData: response.content,
+    length: Buffer.byteLength(response.content),
+    contentType: contentType || DEFAULT_CONTENT_TYPE,
+    filename,
+  };
+};
+
+const resolveStreamWithMeta = async (stream) => {
+  const isLocalFile = stream.path && fs.existsSync(stream.path);
+  if (isLocalFile) {
+    const filename = path.basename(stream.path);
+    return {
+      streamOrData: stream,
+      length: fs.statSync(stream.path).size,
+      contentType: mime.lookup(filename) || DEFAULT_CONTENT_TYPE,
+      filename,
+    };
+  }
+
+  return {
+    ...(await resolveRemoteStream(stream)),
+    contentType: DEFAULT_CONTENT_TYPE,
+    filename: DEFAULT_FILE_NAME,
+  };
+};
+
+// Returns an object with fields:
+// * streamOrData: a readable stream, a string, or a Buffer
+// * length: content length in bytes
+// * contentType
+// * filename
+const resolveToBufferStringStream = async (responseOrData) => {
+  if (typeof responseOrData === 'string' || responseOrData instanceof String) {
+    // The .toString() call only makes a difference for the String object case.
+    // It converts a String object to a regular string.
+    const str = responseOrData.toString();
+    return {
+      streamOrData: str,
+      length: Buffer.byteLength(str),
+      contentType: 'text/plain',
+      filename: `${DEFAULT_FILE_NAME}.txt`,
+    };
+  } else if (Buffer.isBuffer(responseOrData)) {
+    return {
+      streamOrData: responseOrData,
+      length: responseOrData.length,
+      contentType: DEFAULT_CONTENT_TYPE,
+      filename: DEFAULT_FILE_NAME,
+    };
+  } else if (
+    (responseOrData.body && typeof responseOrData.body.pipe === 'function') ||
+    typeof responseOrData.content === 'string'
+  ) {
+    return resolveResponseToStream(responseOrData);
+  } else if (typeof responseOrData.pipe === 'function') {
+    return resolveStreamWithMeta(responseOrData);
+  }
+
+  throw new TypeError(
+    `z.stashFile() cannot stash type '${typeof responseOrData}'. ` +
+      'Pass it a request, readable stream, string, or Buffer.'
+  );
+};
+
+const uploader = async (
   signedPostData,
   bufferStringStream,
   knownLength,
   filename,
   contentType
 ) => {
-  const form = new FormData();
-
   if (knownLength && knownLength > UPLOAD_MAX_SIZE) {
-    return ZapierPromise.reject(
-      new Error(`${knownLength} is too big, ${UPLOAD_MAX_SIZE} is the max`)
-    );
+    throw new Error(`${knownLength} is too big, ${UPLOAD_MAX_SIZE} is the max`);
   }
+  filename = path.basename(filename).replace('"', '');
 
-  _.each(signedPostData.fields, (value, key) => {
-    form.append(key, value);
-  });
+  const fields = {
+    ...signedPostData.fields,
+    'Content-Disposition': contentDisposition(filename),
+    'Content-Type': contentType,
+  };
 
-  filename = path.basename(filename || DEFAULT_FILE_NAME).replace('"', '');
+  const form = new FormData();
 
-  form.append('Content-Disposition', contentDisposition(filename));
+  Object.entries(fields).forEach(([key, value]) => {
+    form.append(key, value);
+  });
 
   form.append('file', bufferStringStream, {
-    contentType,
     knownLength,
+    contentType,
     filename,
   });
 
@@ -56,120 +214,93 @@ const uploader = (
   }
 
   // Send to S3 with presigned request.
-  return request({
+  const response = await request({
     url: signedPostData.url,
     method: 'POST',
     body: form,
-  }).then((res) => {
-    if (res.status === 204) {
-      return `${signedPostData.url}${signedPostData.fields.key}`;
-    }
-    if (
-      res.content.indexOf(
-        'You must provide the Content-Length HTTP header.'
-      ) !== -1
-    ) {
-      throw new Error(LENGTH_ERR_MESSAGE);
-    }
-    throw new Error(`Got ${res.status} - ${res.content}`);
   });
+
+  if (response.status === 204) {
+    return new URL(signedPostData.fields.key, signedPostData.url).href;
+  }
+
+  if (
+    response.content &&
+    response.content.includes &&
+    response.content.includes(
+      'You must provide the Content-Length HTTP header.'
+    )
+  ) {
+    throw new Error(LENGTH_ERR_MESSAGE);
+  }
+
+  throw new Error(`Got ${response.status} - ${response.content}`);
 };
 
 // Designed to be some user provided function/api.
 const createFileStasher = (input) => {
   const rpc = _.get(input, '_zapier.rpc');
 
-  return (bufferStringStream, knownLength, filename, contentType) => {
+  return async (requestOrData, knownLength, filename, contentType) => {
     // TODO: maybe this could be smart?
     // if it is already a public url, do we pass through? or upload?
     if (!rpc) {
-      return ZapierPromise.reject(new Error('rpc is not available'));
+      throw new Error('rpc is not available');
     }
 
-    const method = _.get(input, ['_zapier', 'event', 'method'], '');
-    const isValidSource =
-      method.startsWith('hydrators.') ||
-      method.startsWith('creates.') ||
-      // key regex from KeySchema
-      method.match(/^resources\.[a-zA-Z][a-zA-Z0-9_]*\.create\./);
-
-    if (!isValidSource) {
-      return ZapierPromise.reject(
-        new Error(
-          'Files can only be stashed within a create or hydration function/method.'
-        )
+    const isRunningOnHydrator = _.get(
+      input,
+      '_zapier.event.method',
+      ''
+    ).startsWith('hydrators.');
+    const isRunningOnCreate = _.get(
+      input,
+      '_zapier.event.method',
+      ''
+    ).startsWith('creates.');
+
+    if (!isRunningOnHydrator && !isRunningOnCreate) {
+      throw new Error(
+        'Files can only be stashed within a create or hydration function/method.'
       );
     }
 
-    const fileContentType = contentType || DEFAULT_CONTENT_TYPE;
-
-    return rpc('get_presigned_upload_post_data', fileContentType).then(
-      (result) => {
-        const parseFinalResponse = (stream) => {
-          let newBufferStringStream = stream;
-
-          // if stream is string, don't update headers, just return as is
-          if (_.isString(stream)) {
-            newBufferStringStream = stream;
-          } else if (stream) {
-            if (Buffer.isBuffer(stream)) {
-              newBufferStringStream = stream;
-            } else if (Buffer.isBuffer(stream.dataBuffer)) {
-              newBufferStringStream = stream.dataBuffer;
-            } else if (stream.body && typeof stream.body.pipe === 'function') {
-              newBufferStringStream = stream.body;
-            } else if (stream.content) {
-              newBufferStringStream = stream.content;
-            }
-
-            // if stream has headers update knownLength and filename to reflect the header values
-            if (stream.headers) {
-              knownLength = knownLength || stream.getHeader('content-length');
-              const cd = stream.getHeader('content-disposition');
-              if (cd) {
-                filename =
-                  filename || contentDisposition.parse(cd).parameters.filename;
-              }
-            }
-            // if stream is not defined, error
-          } else {
-            throw new Error('Cannot stash a file of unknown type.');
-          }
-
-          // send final response to uploader
-          return uploader(
-            result,
-            newBufferStringStream,
-            knownLength,
-            filename,
-            fileContentType
-          );
-        };
-
-        const formResponse = (response) => {
-          // determine if string is streamed based on if raw:true
-          const isStreamed = _.get(response, 'request.raw', false);
-
-          // if it's streamed, buffer first
-          if (isStreamed) {
-            return response.buffer().then((buffer) => {
-              response.dataBuffer = buffer;
-              return parseFinalResponse(response);
-            });
-          } else {
-            return parseFinalResponse(response);
-          }
-        };
-
-        // if stream is a promise, wait until resolved
-        if (isPromise(bufferStringStream)) {
-          return bufferStringStream.then((maybeResponse) => {
-            return formResponse(maybeResponse);
-          });
-        } else {
-          return formResponse(bufferStringStream);
-        }
-      }
+    // requestOrData can be one of these:
+    // * string
+    // * Buffer
+    // * z.request() - a Promise of a regular response
+    // * z.request({ raw: true }) - a Promise of a "streamable" response
+    // * await z.request() - a regular response
+    // * await z.request({ raw: true }) - a streamable response
+    //
+    // After the following, requestOrData is resolved to responseOrData, which
+    // is either:
+    // - string
+    // - Buffer
+    // - a regular response
+    // - a streamable response
+    const [signedPostData, responseOrData] = await Promise.all([
+      rpc('get_presigned_upload_post_data'),
+      requestOrData,
+    ]);
+
+    if (responseOrData.throwForStatus) {
+      responseOrData.throwForStatus();
+    }
+
+    const {
+      streamOrData,
+      length,
+      contentType: _contentType,
+      filename: _filename,
+    } = await resolveToBufferStringStream(responseOrData);
+
+    return uploader(
+      signedPostData,
+      streamOrData,
+      knownLength || length,
+      filename || _filename,
+      contentType || _contentType
     );
   };
 };

package/src/tools/create-lambda-handler.js

@@ -205,7 +205,8 @@ const createLambdaHandler = (appRawOrPath) => {
           // Adds logging for _all_ kinds of http(s) requests, no matter the library
           if (!skipHttpPatch) {
             const httpPatch = createHttpPatch(event);
-            httpPatch(require('http')); // 'https' uses 'http' under the hood
+            httpPatch(require('http'));
+            httpPatch(require('https')); // 'https' needs to be patched separately
           }
 
           // TODO: Avoid calling prepareApp(appRaw) repeatedly here as createApp()

package/src/tools/create-logger.js

@@ -3,19 +3,33 @@
 const _ = require('lodash');
 
 const request = require('./request-client-internal');
-const cleaner = require('./cleaner');
-const dataTools = require('./data');
-const hashing = require('./hashing');
+const { simpleTruncate, recurseReplace } = require('./data');
 const ZapierPromise = require('./promise');
 const {
   DEFAULT_LOGGING_HTTP_API_KEY,
   DEFAULT_LOGGING_HTTP_ENDPOINT,
   SAFE_LOG_KEYS,
-  SENSITIVE_KEYS,
 } = require('../constants');
 const { unheader } = require('./http');
+const {
+  scrub,
+  findSensitiveValues,
+  recurseExtract,
+} = require('@zapier/secret-scrubber');
+// not really a public function, but it came from here originally
+const { isUrlWithSecrets } = require('@zapier/secret-scrubber/lib/convenience');
+
+const isUrl = (url) => {
+  try {
+    // eslint-disable-next-line no-new
+    new URL(url);
+    return true;
+  } catch (e) {
+    return false;
+  }
+};
 
-const truncate = (str) => dataTools.simpleTruncate(str, 3500, ' [...]');
+const truncate = (str) => simpleTruncate(str, 3500, ' [...]');
 
 const formatHeaders = (headers = {}) => {
   if (_.isEmpty(headers)) {
@@ -89,66 +103,22 @@ const toStdout = (event, msg, data) => {
   }
 };
 
-const isSafeUrl = (value) => {
-  let url;
-  try {
-    url = new URL(value);
-  } catch (err) {
-    return false;
-  }
-  return !(url.username || url.password || url.search);
-};
-
-const makeSensitiveBank = (event, data) => {
+const buildSensitiveValues = (event, data) => {
   const bundle = event.bundle || {};
-
-  const bank = { ...bundle.authData, ...process.env };
-  const sensitiveValues = Object.entries(bank).reduce(
-    (values, [key, value]) => {
-      if (SENSITIVE_KEYS.includes(key) || !isSafeUrl(value)) {
-        return [...values, value];
-      }
-      // Allow uncensored if the value is a safe URL and the key is not in
-      // SENSITIVE_KEYS
-      return [...values];
-    },
-    []
-  );
-
-  const matcher = (key, value) => {
-    if (_.isString(value)) {
-      const lowerKey = key.toLowerCase();
-      return _.some(SENSITIVE_KEYS, (k) => lowerKey.indexOf(k) >= 0);
+  const authData = bundle.authData || {};
+  // for the most part, we should censor all the values from authData
+  // the exception is safe urls, which should be filtered out - we want those to be logged
+  const sensitiveAuthData = recurseExtract(authData, (key, value) => {
+    if (isUrl(value) && !isUrlWithSecrets(value)) {
+      return false;
     }
-    return false;
-  };
-
-  dataTools.recurseExtract(data, matcher).forEach((value) => {
-    sensitiveValues.push(value);
+    return true;
   });
-
-  return _.reduce(
-    sensitiveValues,
-    (bank, val) => {
-      // keeps short values from spamming censor strings in logs, < 6 chars is not a proper secret
-      // see https://github.com/zapier/zapier-platform-core/issues/4#issuecomment-277855071
-      if (val && String(val).length > 5) {
-        const censored = hashing.snipify(val);
-        bank[val] = censored;
-        bank[encodeURIComponent(val)] = censored;
-        try {
-          bank[Buffer.from(String(val)).toString('base64')] = censored;
-        } catch (e) {
-          if (e.name !== 'TypeError') {
-            throw e;
-          }
-          // ignore; Buffer is semi-selective about what types it takes
-        }
-      }
-      return bank;
-    },
-    {}
-  );
+  return [
+    ...sensitiveAuthData,
+    ...findSensitiveValues(process.env),
+    ...findSensitiveValues(data),
+  ];
 };
 
 const sendLog = (options, event, message, data) => {
@@ -158,16 +128,16 @@ const sendLog = (options, event, message, data) => {
   data.request_headers = unheader(data.request_headers);
   data.response_headers = unheader(data.response_headers);
 
-  const sensitiveBank = makeSensitiveBank(event, data);
+  const sensitiveValues = buildSensitiveValues(event, data);
+  // scrub throws an error if there are no secrets
   const safeMessage = truncate(
-    cleaner.recurseReplaceBank(message, sensitiveBank)
+    sensitiveValues.length ? scrub(message, sensitiveValues) : message
   );
-  const safeData = dataTools.recurseReplace(
-    cleaner.recurseReplaceBank(data, sensitiveBank),
+  const safeData = recurseReplace(
+    sensitiveValues.length ? scrub(data, sensitiveValues) : data,
     truncate
   );
-  const unsafeData = dataTools.recurseReplace(data, truncate);
-
+  const unsafeData = recurseReplace(data, truncate);
   // Keep safe log keys uncensored
   Object.keys(safeData).forEach((key) => {
     if (SAFE_LOG_KEYS.includes(key)) {

package/src/tools/data.js

@@ -123,22 +123,6 @@ const recurseReplace = (obj, replacer, options = {}) => {
   return obj;
 };
 
-// Recursively extract values from a nested object based on the matcher function.
-const recurseExtract = (obj, matcher) => {
-  const values = [];
-  Object.keys(obj).forEach((key) => {
-    const value = obj[key];
-    if (matcher(key, value)) {
-      values.push(value);
-    } else if (isPlainObj(value)) {
-      recurseExtract(value, matcher).forEach((v) => {
-        values.push(v);
-      });
-    }
-  });
-  return values;
-};
-
 const _IGNORE = {};
 
 // Flatten a nested object.
@@ -169,7 +153,7 @@ const flattenPaths = (data, { preserve = {} } = {}) => {
 
 // A simpler, and memory-friendlier version of _.truncate()
 const simpleTruncate = (string, length, suffix) => {
-  if (string === undefined) {
+  if (string == null) {
     return string;
   }
   if (!string || !string.toString) {
@@ -200,7 +184,6 @@ module.exports = {
   isPlainObj,
   jsonCopy,
   memoizedFindMapDeep,
-  recurseExtract,
   recurseReplace,
   simpleTruncate,
 };

package/src/tools/fetch.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const { Writable } = require('stream');
+
 const fetch = require('node-fetch');
 
 // XXX: PatchedRequest is to get past node-fetch's check that forbids GET requests
@@ -7,10 +9,10 @@ const fetch = require('node-fetch');
 // https://github.com/node-fetch/node-fetch/blob/v2.6.0/src/request.js#L75-L78
 class PatchedRequest extends fetch.Request {
   constructor(url, opts) {
-    const origMethod = (opts.method || 'GET').toUpperCase();
+    const origMethod = ((opts && opts.method) || 'GET').toUpperCase();
 
     const isGetWithBody =
-      (origMethod === 'GET' || origMethod === 'HEAD') && opts.body;
+      (origMethod === 'GET' || origMethod === 'HEAD') && opts && opts.body;
     let newOpts = opts;
     if (isGetWithBody) {
       // Temporary remove body to fool fetch.Request constructor
@@ -50,9 +52,31 @@ class PatchedRequest extends fetch.Request {
 
 const newFetch = (url, opts) => {
   const request = new PatchedRequest(url, opts);
+
   // fetch actually accepts a Request object as an argument. It'll clone the
   // request internally, that's why the PatchedRequest.body hack works.
-  return fetch(request);
+  const responsePromise = fetch(request);
+
+  // node-fetch clones request.body and use the cloned body internally. We need
+  // to make sure to consume the original body stream so its internal buffer is
+  // not filled up, which causes it to pause.
+  // See https://github.com/node-fetch/node-fetch/issues/151
+  //
+  // Exclude form-data object to be consistent with
+  // https://github.com/node-fetch/node-fetch/blob/v2.6.6/src/body.js#L403-L412
+  if (
+    request.body &&
+    typeof request.body.pipe === 'function' &&
+    typeof request.body.getBoundary !== 'function'
+  ) {
+    const nullStream = new Writable();
+    nullStream._write = function (chunk, encoding, done) {
+      done();
+    };
+    request.body.pipe(nullStream);
+  }
+
+  return responsePromise;
 };
 
 newFetch.Promise = require('./promise');

package/src/tools/resolve-method-path.js

@@ -1,45 +1,32 @@
 'use strict';
 
-const _ = require('lodash');
-const dataTools = require('./data');
+const { isEqual } = require('lodash');
+const { memoizedFindMapDeep } = require('./data');
 
 const isRequestMethod = (needle) =>
   typeof needle === 'object' && typeof needle.url === 'string';
 
-/*
-  Verifies a object in an app tree, returning the path to it if found (or throwing an error if not found):
-
-    // a findable function/array/method
-    resolveMethodPath(app, app.resources.contact.get.operation.perform)
-
+/**
+  Verifies a object exists in an app tree, returning the path to it if found (optionally throwing an error if not found)
 */
-const resolveMethodPath = (app, needle) => {
-  // temporary warning for all those with old code
-  if (typeof needle === 'string') {
-    console.log(
-      'In version 0.9.10 we removed string path resolution. Read more here https://github.com/zapier/zapier-platform-core/blob/master/CHANGELOG.md#0910'
-    );
-  }
-
+const resolveMethodPath = (app, needle, explodeIfMissing = true) => {
   if (
     !(
       typeof needle === 'function' ||
-      _.isArray(needle) ||
+      Array.isArray(needle) ||
       isRequestMethod(needle)
     )
   ) {
     throw new Error(
-      'You must pass in a function/array/object. We got ' +
-        typeof needle +
-        ' instead.'
+      `You must pass in a function/array/object. We got ${typeof needle} instead.`
     );
   }
 
   // incurs roughly ~10ms penalty for _.isEqual fallback on a === miss on an averagish app
   const path =
-    dataTools.memoizedFindMapDeep(app, needle) ||
-    dataTools.memoizedFindMapDeep(app, needle, _.isEqual);
-  if (!path) {
+    memoizedFindMapDeep(app, needle) ||
+    memoizedFindMapDeep(app, needle, isEqual);
+  if (!path && explodeIfMissing) {
     throw new Error(
       'We could not find your function/array/object anywhere on your App definition.'
     );

package/src/tools/schema-tools.js

@@ -2,9 +2,13 @@
 
 const dataTools = require('./data');
 
+// AsyncFunction is not a global object and can be obtained in this way
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AsyncFunction
+const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;
+
 const makeFunction = (source, args = []) => {
   try {
-    return Function(...args, source); // eslint-disable-line no-new-func
+    return new AsyncFunction(...args, source);
   } catch (err) {
     return () => {
       throw err;

package/src/tools/should-paginate.js

@@ -0,0 +1,47 @@
+const { get } = require('lodash');
+
+// this is (annoyingly) mirrored in cli/api_base, so that test functions only
+// have a storeKey when canPaginate is true. otherwise, a test would work but a
+// poll on site would fail. this is only used in test handlers
+
+// there are 4 places you can put a method that can interact with cursors:
+// triggers.contact.operation.perform, if it's a poll trigger
+// triggers.contact.operation.performList, if it's a hook trigger
+// resources.contact.list.operation.perform if it's a resource
+// resources.contact.hook.operation.performList if it's a resource
+
+const shouldPaginate = (appRaw, method) => {
+  const methodParts = method.split('.');
+  const methodName = methodParts.pop();
+  const operation = get(appRaw, methodParts);
+
+  if (methodParts[0] === 'triggers') {
+    // Polling operations may not specify type
+    if (
+      ['polling', undefined].includes(operation.type) &&
+      methodName === 'perform'
+    ) {
+      return !!operation.canPaginate;
+    }
+
+    if (operation.type === 'hook' && methodName === 'performList') {
+      return !!operation.canPaginate;
+    }
+  }
+
+  if (methodParts[0] === 'resources') {
+    if (methodParts[2] === 'list' && methodName === 'perform') {
+      return !!operation.canPaginate;
+    }
+
+    if (methodParts[2] === 'hook' && methodName === 'performList') {
+      return !!operation.canPaginate;
+    }
+  }
+
+  return false;
+};
+
+module.exports = {
+  shouldPaginate,
+};