zapier-platform-core 15.8.0 → 15.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zapier-platform-core",
-  "version": "15.8.0",
+  "version": "15.9.0",
   "description": "The core SDK for CLI apps in the Zapier Developer Platform.",
   "repository": "zapier/zapier-platform",
   "homepage": "https://platform.zapier.com/",
@@ -52,7 +52,7 @@
     "node-fetch": "2.6.7",
     "oauth-sign": "0.9.0",
     "semver": "7.5.2",
-    "zapier-platform-schema": "15.8.0"
+    "zapier-platform-schema": "15.9.0"
   },
   "devDependencies": {
     "@types/node-fetch": "^2.6.11",

package/src/app-middlewares/after/large-response-cacher.js

@@ -2,19 +2,28 @@
 
 const constants = require('../../constants');
 const cleaner = require('../../tools/cleaner');
+const responseStasher = require('../../tools/create-response-stasher');
 
-/*
-  TODO: Some services _do not_ enjoy having 6mb+ responses returned, so
-  we may need to user/request pre-signed S3 URLs (or somewhere else?)
-  to stash the large response, and return a pointer.
-*/
-const largeResponseCachePointer = (output) => {
-  const size = JSON.stringify(cleaner.maskOutput(output)).length;
-  if (size > constants.RESPONSE_SIZE_LIMIT) {
-    console.log(
-      `Oh no! Payload is ${size}, which is larger than ${constants.RESPONSE_SIZE_LIMIT}.`
-    );
-    // TODO: use envelope feature and to build RPC to get signed S3 upload URL.
+const largeResponseCachePointer = async (output) => {
+  const response = cleaner.maskOutput(output);
+
+  const autostashLimit = output.input._zapier.event.autostashPayloadOutputLimit;
+
+  const payload = JSON.stringify(response.results);
+  const size = payload.length;
+
+  // If autostash limit is defined, and is within the range, stash the response
+  // If it is -1, stash the response regardless of size
+  // If the limit is defined and is out of range, let lambda deal with it
+  if (
+    (autostashLimit &&
+      size >= constants.RESPONSE_SIZE_LIMIT &&
+      size <= autostashLimit) ||
+    autostashLimit === -1
+  ) {
+    const url = await responseStasher(output.input, payload, size);
+    output.resultsUrl = url;
+    output.results = Array.isArray(output.results) ? [] : {};
   }
   return output;
 };

package/src/app-middlewares/before/z-object.js

@@ -29,7 +29,10 @@ const injectZObject = (input) => {
     generateCallbackUrl: createCallbackHigherOrderFunction(input),
     hash: hashing.hashify,
     JSON: createJSONtool(),
-    require: (moduleName) => require(moduleName),
+    require: (moduleName) =>
+      require(require.resolve(moduleName, {
+        paths: module.paths.concat([process.cwd()]),
+      })),
     stashFile: createFileStasher(input),
   };
 

package/src/tools/cleaner.js

@@ -130,7 +130,8 @@ const createBundleBank = (appRaw, event = {}, serializeFunc = (x) => x) => {
   }, {});
 };
 
-const maskOutput = (output) => _.pick(output, 'results', 'status');
+const maskOutput = (output) =>
+  _.pick(output, 'results', 'status', 'resultsUrl');
 
 // These normalize functions are called after the initial before middleware that
 // cleans the request. The reason is that we need to know why a value is empty

package/src/tools/create-file-stasher.js

@@ -9,15 +9,11 @@ const { randomBytes } = require('crypto');
 
 const _ = require('lodash');
 const contentDisposition = require('content-disposition');
-const FormData = require('form-data');
+
 const mime = require('mime-types');
 
-const request = require('./request-client-internal');
 const { UPLOAD_MAX_SIZE, NON_STREAM_UPLOAD_MAX_SIZE } = require('../constants');
-
-const LENGTH_ERR_MESSAGE =
-  'We could not calculate the length of your file - please ' +
-  'pass a knownLength like z.stashFile(f, knownLength)';
+const uploader = require('./uploader');
 
 const DEFAULT_FILE_NAME = 'unnamedfile';
 const DEFAULT_CONTENT_TYPE = 'application/octet-stream';
@@ -175,64 +171,6 @@ const resolveToBufferStringStream = async (responseOrData) => {
   );
 };
 
-const uploader = async (
-  signedPostData,
-  bufferStringStream,
-  knownLength,
-  filename,
-  contentType
-) => {
-  filename = path.basename(filename).replace('"', '');
-
-  const fields = {
-    ...signedPostData.fields,
-    'Content-Disposition': contentDisposition(filename),
-    'Content-Type': contentType,
-  };
-
-  const form = new FormData();
-
-  Object.entries(fields).forEach(([key, value]) => {
-    form.append(key, value);
-  });
-
-  form.append('file', bufferStringStream, {
-    knownLength,
-    contentType,
-    filename,
-  });
-
-  // Try to catch the missing length early, before upload to S3 fails.
-  try {
-    form.getLengthSync();
-  } catch (err) {
-    throw new Error(LENGTH_ERR_MESSAGE);
-  }
-
-  // Send to S3 with presigned request.
-  const response = await request({
-    url: signedPostData.url,
-    method: 'POST',
-    body: form,
-  });
-
-  if (response.status === 204) {
-    return new URL(signedPostData.fields.key, signedPostData.url).href;
-  }
-
-  if (
-    response.content &&
-    response.content.includes &&
-    response.content.includes(
-      'You must provide the Content-Length HTTP header.'
-    )
-  ) {
-    throw new Error(LENGTH_ERR_MESSAGE);
-  }
-
-  throw new Error(`Got ${response.status} - ${response.content}`);
-};
-
 const ensureUploadMaxSizeNotExceeded = (streamOrData, length) => {
   let uploadMaxSize = NON_STREAM_UPLOAD_MAX_SIZE;
   let uploadMethod = 'non-streaming';
@@ -242,7 +180,9 @@ const ensureUploadMaxSizeNotExceeded = (streamOrData, length) => {
   }
 
   if (length && length > uploadMaxSize) {
-    throw new Error(`${length} bytes is too big, ${uploadMaxSize} is the max for ${uploadMethod} data.`);
+    throw new Error(
+      `${length} bytes is too big, ${uploadMaxSize} is the max for ${uploadMethod} data.`
+    );
   }
 };
 

package/src/tools/create-response-stasher.js

@@ -0,0 +1,40 @@
+'use strict';
+
+const _ = require('lodash');
+const uploader = require('./uploader');
+const crypto = require('crypto');
+
+const withRetry = async (fn, retries = 3, delay = 100, attempt = 0) => {
+  try {
+    return await fn();
+  } catch (error) {
+    if (attempt >= retries) {
+      throw error;
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, delay));
+    return withRetry(fn, retries, delay, attempt + 1);
+  }
+};
+
+// responseStasher uploads the data and returns the URL that points to that data.
+const stashResponse = async (input, response, size) => {
+  const rpc = _.get(input, '_zapier.rpc');
+
+  if (!rpc) {
+    throw new Error('rpc is not available');
+  }
+  const signedPostData = await rpc('get_presigned_upload_post_data');
+  return withRetry(
+    _.partial(
+      uploader,
+      signedPostData,
+      response.toString(), // accept JSON string to send to uploader.
+      size,
+      crypto.randomUUID() + '.txt',
+      'text/plain'
+    )
+  );
+};
+
+module.exports = stashResponse;

package/src/tools/uploader.js

@@ -0,0 +1,69 @@
+const path = require('path');
+
+const FormData = require('form-data');
+const contentDisposition = require('content-disposition');
+
+const request = require('./request-client-internal');
+const LENGTH_ERR_MESSAGE =
+  'We could not calculate the length of your file - please ' +
+  'pass a knownLength like z.stashFile(f, knownLength)';
+
+const uploader = async (
+  signedPostData,
+  bufferStringStream,
+  knownLength,
+  filename,
+  contentType
+) => {
+  filename = path.basename(filename).replace('"', '');
+
+  const fields = {
+    ...signedPostData.fields,
+    'Content-Disposition': contentDisposition(filename),
+    'Content-Type': contentType,
+  };
+
+  const form = new FormData();
+
+  Object.entries(fields).forEach(([key, value]) => {
+    form.append(key, value);
+  });
+
+  form.append('file', bufferStringStream, {
+    knownLength,
+    contentType,
+    filename,
+  });
+
+  // Try to catch the missing length early, before upload to S3 fails.
+  try {
+    form.getLengthSync();
+  } catch (err) {
+    throw new Error(LENGTH_ERR_MESSAGE);
+  }
+
+  // Send to S3 with presigned request.
+  const response = await request({
+    url: signedPostData.url,
+    method: 'POST',
+    body: form,
+  });
+
+  if (response.status === 204) {
+    return new URL(signedPostData.fields.key, signedPostData.url).href;
+  }
+
+  if (
+    response.content &&
+    response.content.includes &&
+    response.content.includes(
+      'You must provide the Content-Length HTTP header.'
+    )
+  ) {
+    throw new Error(LENGTH_ERR_MESSAGE);
+  }
+
+  throw new Error(`Got ${response.status} - ${response.content}`);
+};
+
+module.exports = uploader;