zapier-platform-core 17.7.0 → 17.7.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zapier-platform-core",
-  "version": "17.7.0",
+  "version": "17.7.2",
   "description": "The core SDK for CLI apps in the Zapier Developer Platform.",
   "repository": "zapier/zapier-platform",
   "homepage": "https://platform.zapier.com/",
@@ -63,7 +63,7 @@
     "node-fetch": "2.7.0",
     "oauth-sign": "0.9.0",
     "semver": "7.7.1",
-    "zapier-platform-schema": "17.7.0"
+    "zapier-platform-schema": "17.7.2"
   },
   "devDependencies": {
     "@types/node-fetch": "^2.6.11",

package/src/http-middlewares/after/prepare-response.js

@@ -2,19 +2,44 @@
 
 const _ = require('lodash');
 const querystring = require('querystring');
+
+const { scrub, findSensitiveValues } = require('@zapier/secret-scrubber');
+
 const { replaceHeaders } = require('./middleware-utils');
 const { FORM_TYPE } = require('../../tools/http');
 const errors = require('../../errors');
+const {
+  findSensitiveValuesFromAuthData,
+} = require('../../tools/secret-scrubber');
 
-const _throwForStatus = (response) => {
+const buildSensitiveValues = (bundle) => {
+  const authData = bundle?.authData || {};
+  const result = [
+    ...findSensitiveValuesFromAuthData(authData),
+    ...findSensitiveValues(process.env),
+  ];
+  return [...new Set(result)];
+};
+
+const _throwForStatus = (response, bundle) => {
   // calling this always throws, regardless of the skipThrowForStatus value
   // eslint-disable-next-line yoda
   if (400 <= response.status && response.status < 600) {
+    // Create a cleaned version of the response to avoid sensitive data leaks
+    try {
+      // Find sensitive values from environment variables and bundle authData
+      const sensitiveValues = buildSensitiveValues(bundle);
+      if (sensitiveValues.length > 0 && response?.request?.url) {
+        response.request.url = scrub(response.request.url, sensitiveValues);
+      }
+    } catch (err) {
+      // don't fail the whole request if we can't scrub for some reason
+    }
     throw new errors.ResponseError(response);
   }
 };
 
-const prepareRawResponse = (resp, request) => {
+const prepareRawResponse = (resp, request, bundle) => {
   // TODO: if !2xx should we go ahead and get response.content for them?
   // retain the response signature for raw control
   const extendedResp = {
@@ -24,7 +49,7 @@ const prepareRawResponse = (resp, request) => {
   const outResp = _.extend(resp, extendedResp, replaceHeaders(resp));
 
   outResp.throwForStatus = () => {
-    _throwForStatus(outResp);
+    _throwForStatus(outResp, bundle);
   };
 
   Object.defineProperty(outResp, 'content', {
@@ -39,7 +64,7 @@ const prepareRawResponse = (resp, request) => {
   return outResp;
 };
 
-const prepareContentResponse = async (resp, request) => {
+const prepareContentResponse = async (resp, request, bundle) => {
   // TODO: does it make sense to not trim the signature? more equivalence to raw...
   const content = await resp.text();
 
@@ -67,14 +92,14 @@ const prepareContentResponse = async (resp, request) => {
   } catch (_e) {}
 
   outResp.throwForStatus = () => {
-    _throwForStatus(outResp);
+    _throwForStatus(outResp, bundle);
   };
 
   return outResp;
 };
 
 // Provide a standardized plain JS responseObj for common consumption, or raw response for streaming.
-const prepareResponse = (resp) => {
+const prepareResponse = (resp, z, bundle) => {
   const request = resp.input;
   delete resp.input;
 
@@ -82,7 +107,7 @@ const prepareResponse = (resp) => {
     ? prepareRawResponse
     : prepareContentResponse;
 
-  return responseFunc(resp, request);
+  return responseFunc(resp, request, bundle);
 };
 
 module.exports = prepareResponse;

package/src/tools/create-logger.js

@@ -15,16 +15,8 @@ const {
   SAFE_LOG_KEYS,
 } = require('../constants');
 const { unheader } = require('./http');
-const {
-  scrub,
-  findSensitiveValues,
-  recurseExtract,
-} = require('@zapier/secret-scrubber');
-// not really a public function, but it came from here originally
-const {
-  isUrlWithSecrets,
-  isSensitiveKey,
-} = require('@zapier/secret-scrubber/lib/convenience');
+const { scrub, findSensitiveValues } = require('@zapier/secret-scrubber');
+const { findSensitiveValuesFromAuthData } = require('./secret-scrubber');
 
 // The payload size per request to stream logs. This should be slighly lower
 // than the limit (16 MB) on the server side.
@@ -32,70 +24,8 @@ const LOG_STREAM_BYTES_LIMIT = 15 * 1024 * 1024;
 
 const DEFAULT_LOGGER_TIMEOUT = 200;
 
-// Will be initialized lazily
-const SAFE_AUTH_DATA_KEYS = new Set();
-
 const sleep = promisify(setTimeout);
 
-const initSafeAuthDataKeys = () => {
-  // An authData key in (safePrefixes x safeSuffixes) is considered safe to log
-  // uncensored
-  const safePrefixes = [
-    'account',
-    'bot_user',
-    'cloud',
-    'cloud_site',
-    'company',
-    'domain',
-    'email',
-    'environment',
-    'location',
-    'org',
-    'organization',
-    'project',
-    'region',
-    'scope',
-    'scopes',
-    'site',
-    'subdomain',
-    'team',
-    'token_type',
-    'user',
-    'workspace',
-  ];
-  const safeSuffixes = ['', '_id', '_name', 'id', 'name'];
-  for (const prefix of safePrefixes) {
-    for (const suffix of safeSuffixes) {
-      SAFE_AUTH_DATA_KEYS.add(prefix + suffix);
-    }
-  }
-};
-
-const isUrl = (value) => {
-  if (!value || typeof value !== 'string') {
-    return false;
-  }
-  const commonProtocols = [
-    'https://',
-    'http://',
-    'ftp://',
-    'ftps://',
-    'file://',
-  ];
-  for (const protocol of commonProtocols) {
-    if (value.startsWith(protocol)) {
-      try {
-        // eslint-disable-next-line no-new
-        new URL(value);
-        return true;
-      } catch (e) {
-        return false;
-      }
-    }
-  }
-  return false;
-};
-
 const MAX_LENGTH = 3500;
 const truncateString = (str) => simpleTruncate(str, MAX_LENGTH, ' [...]');
 
@@ -188,35 +118,12 @@ const attemptFindSecretsInStr = (s, isGettingNewSecret) => {
   return findSensitiveValues(parsedRespContent);
 };
 
-const isSafeAuthDataKey = (key) => {
-  if (SAFE_AUTH_DATA_KEYS.size === 0) {
-    initSafeAuthDataKeys();
-  }
-  return SAFE_AUTH_DATA_KEYS.has(key.toLowerCase());
-};
-
 const buildSensitiveValues = (event, data) => {
   const bundle = event.bundle || {};
   const authData = bundle.authData || {};
-  // for the most part, we should censor all the values from authData
-  // the exception is safe urls, which should be filtered out - we want those to be logged
-  // but, we _should_ censor-no-matter-what sensitive keys, even if their value is a safe url
-  // this covers the case where someone's password is a valid url ¯\_(ツ)_/¯
-  const sensitiveAuthData = recurseExtract(authData, (key, value) => {
-    if (isSensitiveKey(key)) {
-      return true;
-    }
-    if (isSafeAuthDataKey(key)) {
-      return false;
-    }
-    if (isUrl(value) && !isUrlWithSecrets(value)) {
-      return false;
-    }
-    return true;
-  });
 
   const result = [
-    ...sensitiveAuthData,
+    ...findSensitiveValuesFromAuthData(authData),
     ...findSensitiveValues(process.env),
     ...findSensitiveValues(data),
   ];

package/src/tools/secret-scrubber.js

@@ -0,0 +1,98 @@
+'use strict';
+
+const { recurseExtract } = require('@zapier/secret-scrubber');
+const {
+  isUrlWithSecrets,
+  isSensitiveKey,
+} = require('@zapier/secret-scrubber/lib/convenience');
+
+// Will be initialized lazily
+const SAFE_AUTH_DATA_KEYS = new Set();
+
+const initSafeAuthDataKeys = () => {
+  // An authData key in (safePrefixes x safeSuffixes) is considered safe to log
+  // uncensored
+  const safePrefixes = [
+    'account',
+    'bot_user',
+    'cloud',
+    'cloud_site',
+    'company',
+    'domain',
+    'email',
+    'environment',
+    'location',
+    'org',
+    'organization',
+    'project',
+    'region',
+    'scope',
+    'scopes',
+    'site',
+    'subdomain',
+    'team',
+    'token_type',
+    'user',
+    'workspace',
+  ];
+  const safeSuffixes = ['', '_id', '_name', 'id', 'name'];
+  for (const prefix of safePrefixes) {
+    for (const suffix of safeSuffixes) {
+      SAFE_AUTH_DATA_KEYS.add(prefix + suffix);
+    }
+  }
+};
+
+const isSafeAuthDataKey = (key) => {
+  if (SAFE_AUTH_DATA_KEYS.size === 0) {
+    initSafeAuthDataKeys();
+  }
+  return SAFE_AUTH_DATA_KEYS.has(key.toLowerCase());
+};
+
+const isUrl = (value) => {
+  if (!value || typeof value !== 'string') {
+    return false;
+  }
+  const commonProtocols = [
+    'https://',
+    'http://',
+    'ftp://',
+    'ftps://',
+    'file://',
+  ];
+  for (const protocol of commonProtocols) {
+    if (value.startsWith(protocol)) {
+      try {
+        // eslint-disable-next-line no-new
+        new URL(value);
+        return true;
+      } catch (e) {
+        return false;
+      }
+    }
+  }
+  return false;
+};
+
+const findSensitiveValuesFromAuthData = (authData) =>
+  recurseExtract(authData, (key, value) => {
+    // for the most part, we should censor all the values from authData
+    // the exception is safe urls, which should be filtered out - we want those to be logged
+    // but, we _should_ censor-no-matter-what sensitive keys, even if their value is a safe url
+    // this covers the case where someone's password is a valid url ¯\_(ツ)_/¯
+    if (isSensitiveKey(key)) {
+      return true;
+    }
+    if (isSafeAuthDataKey(key)) {
+      return false;
+    }
+    if (isUrl(value) && !isUrlWithSecrets(value)) {
+      return false;
+    }
+    return true;
+  });
+
+module.exports = {
+  findSensitiveValuesFromAuthData,
+};

package/types/custom.d.ts

@@ -112,6 +112,14 @@ export interface Bundle<
       };
     };
 
+    /**
+     * A token, such as a pagination cursor, that means this run should
+     * continue pulling results. Currently only used for search
+     * pagination. Set by previous invocations by returning `{results:
+     * […], paging_token: '…'}`
+     */
+    paging_token?: string;
+
     /**
      * Contains metadata about the input fields, optionally provided
      * by the inputField.meta property. Useful for storing extra data
@@ -142,7 +150,11 @@ export interface Bundle<
 
 // Error class types that match the runtime structure from src/errors.js
 type ErrorConstructor = new (message?: string) => Error;
-type AppErrorConstructor = new (message: string, code?: string, status?: number) => Error;
+type AppErrorConstructor = new (
+  message: string,
+  code?: string,
+  status?: number,
+) => Error;
 type ThrottledErrorConstructor = new (message: string, delay?: number) => Error;
 type ResponseErrorConstructor = new (response: HttpResponse) => Error;
 
@@ -163,7 +175,6 @@ interface ErrorsModule {
   handleError: (...args: any[]) => never;
 }
 
-
 // copied http stuff from external typings
 export interface HttpRequestOptions {
   agent?: Agent;

package/types/functions.d.ts

@@ -1,7 +1,20 @@
 import type { Bundle, ZObject } from './custom';
+import type { InferInputData, InputField } from './inputs';
 
 type DefaultInputData = Record<string, unknown>;
 
+/**
+ * Automatically infer the type of the bundle based on if a raw
+ * inputData object shape is given, or an array of input fields that
+ * should be automatically inferred into an inputData object.
+ */
+type AutoBundle<$InputDataOrFields extends DefaultInputData | InputField[]> =
+  $InputDataOrFields extends InputField[]
+    ? Bundle<InferInputData<$InputDataOrFields>>
+    : $InputDataOrFields extends DefaultInputData
+      ? Bundle<$InputDataOrFields>
+      : never;
+
 /**
  * Wraps a `perform` function that is used to poll for data from an API.
  * By default must return an array of objects with an `id` field, but
@@ -10,9 +23,12 @@ type DefaultInputData = Record<string, unknown>;
  * those keys.
  */
 export type PollingTriggerPerform<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   $Return extends {} = { id: string },
-> = (z: ZObject, bundle: Bundle<$InputData>) => $Return[] | Promise<$Return[]>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => $Return[] | Promise<$Return[]>;
 
 /**
  * Process the data of a webhook sent to Zapier.
@@ -21,62 +37,83 @@ export type PollingTriggerPerform<
  * in `bundle.cleanedRequest` (and `bundle.rawRequest`).
  */
 export type WebhookTriggerPerform<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   $Return extends {} = {},
-> = (z: ZObject, bundle: Bundle<$InputData>) => $Return[] | Promise<$Return[]>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => $Return[] | Promise<$Return[]>;
 
 /**
  * Pull sample data from the webhook trigger. Try to make these resemble
  * to data of the hooks as closely as possible.
  */
 export type WebhookTriggerPerformList<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   $Return extends {} = {},
-> = (z: ZObject, bundle: Bundle<$InputData>) => $Return[] | Promise<$Return[]>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => $Return[] | Promise<$Return[]>;
 
 /**
  * Return must be an object of subscription data. It will be passed as
  * `bundle.subscribeData` in the performUnsubscribe function.
  */
 export type WebhookTriggerPerformSubscribe<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   $Return extends {} = {},
-> = (z: ZObject, bundle: Bundle<$InputData>) => $Return | Promise<$Return>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => $Return | Promise<$Return>;
 
 /**
  * Unsubscribe from the webhook.
  * Data from the subscribe function is provided in `bundle.subscribeData`.
  */
 export type WebhookTriggerPerformUnsubscribe<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   $Return extends {} = {},
-> = (z: ZObject, bundle: Bundle<$InputData>) => $Return | Promise<$Return>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => $Return | Promise<$Return>;
 
 /**
  * Pull data from the API service, same as a polling trigger.
  */
 export type HookToPollTriggerPerformList<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   $Return extends {} = {},
-> = (z: ZObject, bundle: Bundle<$InputData>) => $Return[] | Promise<$Return[]>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => $Return[] | Promise<$Return[]>;
 
 /**
  * Return must be an object of subscription data. It will be passed as
  * `bundle.subscribeData` in the performUnsubscribe function.
  */
 export type HookToPollTriggerPerformSubscribe<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   $Return extends {} = {},
-> = (z: ZObject, bundle: Bundle<$InputData>) => $Return | Promise<$Return>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => $Return | Promise<$Return>;
 
 /**
  * Unsubscribe from the HookToPoll.
  * Data from the subscribe function is provided in `bundle.subscribeData`.
  */
 export type HookToPollTriggerPerformUnsubscribe<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   $Return extends {} = {},
-> = (z: ZObject, bundle: Bundle<$InputData>) => $Return | Promise<$Return>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => $Return | Promise<$Return>;
 
 /**
  * Create an item on a partner API.
@@ -85,9 +122,12 @@ export type HookToPollTriggerPerformUnsubscribe<
  * to populate more data.
  */
 export type CreatePerform<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   $Return extends {} = {},
-> = (z: ZObject, bundle: Bundle<$InputData>) => $Return | Promise<$Return>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => $Return | Promise<$Return>;
 
 /**
  * A `perform` function can setup a partner API to call back to this
@@ -104,18 +144,24 @@ export type CreatePerform<
  * - bundle.outputData: The output data from the original `perform`.
  */
 export type CreatePerformResume<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   // TODO: Type cleanedRequest & outputData on Bundle interface
   $Return extends {} = {},
-> = (z: ZObject, bundle: Bundle<$InputData>) => $Return | Promise<$Return>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => $Return | Promise<$Return>;
 
 /**
  * Look up an object to populate it after creation?
  */
 export type CreatePerformGet<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   $Return extends {} = {},
-> = (z: ZObject, bundle: Bundle<$InputData>) => $Return | Promise<$Return>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => $Return | Promise<$Return>;
 
 /**
  * Helper type for search results that can optionally include pagination.
@@ -125,7 +171,7 @@ export type CreatePerformGet<
  *
  * When `canPaginate` is true for the search, the object shape is required.
  */
-type SearchResult<T> = T[] | { results: T[], paging_token: string };
+type SearchResult<T> = T[] | { results: T[]; paging_token: string };
 
 /**
  * Search for objects on a partner API.
@@ -136,9 +182,12 @@ type SearchResult<T> = T[] | { results: T[], paging_token: string };
  * revisit this?
  */
 export type SearchPerform<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   $Return extends {} = {},
-> = (z: ZObject, bundle: Bundle<$InputData>) => SearchResult<$Return> | Promise<SearchResult<$Return>>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => SearchResult<$Return> | Promise<SearchResult<$Return>>;
 
 /**
  * Follow up a search's perform with additional data.
@@ -148,17 +197,23 @@ export type SearchPerform<
  * -> PROBABLY: Just the result of searchPerform, no inputFields.
  */
 export type SearchPerformGet<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   $Return extends {} = {},
-> = (z: ZObject, bundle: Bundle<$InputData>) => $Return | Promise<$Return>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => $Return | Promise<$Return>;
 
 /**
  *
  */
 export type SearchPerformResume<
-  $InputData extends DefaultInputData = DefaultInputData,
+  $InputDataOrFields extends DefaultInputData | InputField[] = DefaultInputData,
   $Return extends {} = {},
-> = (z: ZObject, bundle: Bundle<$InputData>) => $Return | Promise<$Return>;
+> = (
+  z: ZObject,
+  bundle: AutoBundle<$InputDataOrFields>,
+) => $Return | Promise<$Return>;
 
 /**
  * Produce the URL to send the user to authorise with the OAuth2 provider.

package/types/functions.test-d.ts

@@ -1,5 +1,8 @@
-import { expectAssignable } from 'tsd';
+import { expectAssignable, expectType } from 'tsd';
 import type { PollingTriggerPerform } from './functions';
+import { Bundle, ZObject } from './custom';
+import { InferInputData } from './inputs';
+import { defineInputFields } from './typeHelpers';
 
 const simplePerform = (async (z, bundle) => {
   return [{ id: '1', name: 'test' }];
@@ -16,3 +19,46 @@ const primaryKeyOverridePerform = (async (z, bundle) => {
 expectAssignable<PollingTriggerPerform<{}, { itemId: number; name: string }>>(
   primaryKeyOverridePerform,
 );
+
+//
+// Automatic inputData inference
+//
+// Pass shape directly
+const simplePerformWithInputFields = (async (z, bundle) => {
+  return [{ id: '1', name: 'test' }];
+}) satisfies PollingTriggerPerform<{ key: 'string' }>;
+expectType<
+  (
+    z: ZObject,
+    bundle: Bundle<{ key: 'string' }>,
+  ) => Promise<{ id: string; name: string }[]>
+>(simplePerformWithInputFields);
+
+// Use InputFields and manually infer inputData
+const inputFields = defineInputFields([
+  { key: 'key1', type: 'string', required: true },
+  { key: 'key2', type: 'number', required: false },
+]);
+const simplePerformWithInputFieldsAndManualInference = (async (z, bundle) => {
+  return [{ id: '1', name: 'test' }];
+}) satisfies PollingTriggerPerform<InferInputData<typeof inputFields>>;
+expectType<
+  (
+    z: ZObject,
+    bundle: Bundle<{ key1: string; key2?: number | undefined }>,
+  ) => Promise<{ id: string; name: string }[]>
+>(simplePerformWithInputFieldsAndManualInference);
+
+// Use inputFields and automatically infer inputData
+const simplePerformWithInputFieldsAndAutomaticInference = (async (
+  z,
+  bundle,
+) => {
+  return [{ id: '1', name: 'test' }];
+}) satisfies PollingTriggerPerform<typeof inputFields>;
+expectType<
+  (
+    z: ZObject,
+    bundle: Bundle<{ key1: string; key2?: number | undefined }>,
+  ) => Promise<{ id: string; name: string }[]>
+>(simplePerformWithInputFieldsAndAutomaticInference);