zapier-platform-core 11.1.1 → 11.2.0

package/LICENSE

@@ -1,4 +1,3 @@
 Copyright (c) Zapier, Inc.
 
-This repository is part of Zapier Platform, of which license terms can be found at
-https://zapier.com/platform/tos.
+This repository is part of Zapier Platform.  By downloading, installing, accessing, or using any part of the Zapier Platform, including this repository, you agree to the Zapier Platform Agreement, which can be found at: https://zapier.com/platform/tos.  If you do not agree to the Zapier Platform Agreement, you may not download, install, access, or use any part of the Zapier Platform, including this repository.

package/index.d.ts

@@ -71,6 +71,9 @@ declare class AppError extends Error {
 declare class HaltedError extends Error {}
 declare class ExpiredAuthError extends Error {}
 declare class RefreshAuthError extends Error {}
+declare class ThrottledError extends Error {
+  constructor(message: string, delay?: number);
+}
 
 // copied http stuff from external typings
 export interface HttpRequestOptions {
@@ -188,5 +191,6 @@ export interface ZObject {
     HaltedError: typeof HaltedError;
     ExpiredAuthError: typeof ExpiredAuthError;
     RefreshAuthError: typeof RefreshAuthError;
+    ThrottledError: typeof ThrottledError;
   };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zapier-platform-core",
-  "version": "11.1.1",
+  "version": "11.2.0",
   "description": "The core SDK for CLI apps in the Zapier Developer Platform.",
   "repository": "zapier/zapier-platform",
   "homepage": "https://platform.zapier.com/",
@@ -35,7 +35,7 @@
     "validate": "yarn test && yarn smoke-test && yarn lint"
   },
   "engines": {
-    "node": ">=10",
+    "node": ">=12",
     "npm": ">=5.6.0"
   },
   "engineStrict": true,
@@ -46,14 +46,16 @@
     "dotenv": "9.0.2",
     "form-data": "4.0.0",
     "lodash": "4.17.21",
-    "node-fetch": "2.6.1",
+    "mime-types": "2.1.34",
+    "node-fetch": "2.6.6",
     "oauth-sign": "0.9.0",
     "semver": "7.3.5",
-    "zapier-platform-schema": "11.1.1"
+    "zapier-platform-schema": "11.2.0"
   },
   "devDependencies": {
     "adm-zip": "0.5.5",
     "aws-sdk": "^2.905.0",
+    "dicer": "^0.3.0",
     "fs-extra": "^10.0.0",
     "mock-fs": "^4.14.0"
   },

package/src/errors.js

@@ -32,6 +32,7 @@ class ResponseError extends Error {
         status: response.status,
         headers: {
           'content-type': response.headers.get('content-type'),
+          'retry-after': response.headers.get('retry-after'),
         },
         content,
         request: {
@@ -44,6 +45,19 @@ class ResponseError extends Error {
   }
 }
 
+class ThrottledError extends Error {
+  constructor(message, delay) {
+    super(
+      JSON.stringify({
+        message,
+        delay,
+      })
+    );
+    this.name = 'ThrottledError';
+    this.doNotContextify = true;
+  }
+}
+
 // Make some of the errors we'll use!
 const createError = (name) => {
   const NewError = function (message = '') {
@@ -77,6 +91,7 @@ const exceptions = _.reduce(
   {
     Error: AppError,
     ResponseError,
+    ThrottledError,
   }
 );
 

package/src/tools/create-file-stasher.js

@@ -1,14 +1,18 @@
 'use strict';
 
-const _ = require('lodash');
+const fs = require('fs');
+const os = require('os');
 const path = require('path');
-const FormData = require('form-data');
+const { pipeline } = require('stream');
+const { promisify } = require('util');
+const { randomBytes } = require('crypto');
+
+const _ = require('lodash');
 const contentDisposition = require('content-disposition');
+const FormData = require('form-data');
+const mime = require('mime-types');
 
 const request = require('./request-client-internal');
-const ZapierPromise = require('./promise');
-
-const isPromise = (obj) => obj && typeof obj.then === 'function';
 
 const UPLOAD_MAX_SIZE = 1000 * 1000 * 150; // 150mb, in zapier backend too
 
@@ -19,32 +23,186 @@ const LENGTH_ERR_MESSAGE =
 const DEFAULT_FILE_NAME = 'unnamedfile';
 const DEFAULT_CONTENT_TYPE = 'application/octet-stream';
 
-const uploader = (
+const streamPipeline = promisify(pipeline);
+
+const filenameFromURL = (url) => {
+  try {
+    return decodeURIComponent(path.posix.basename(new URL(url).pathname));
+  } catch (error) {
+    return null;
+  }
+};
+
+const filenameFromHeader = (response) => {
+  const cd = response.headers.get('content-disposition');
+  let filename;
+  if (cd) {
+    try {
+      filename = contentDisposition.parse(cd).parameters.filename;
+    } catch (error) {
+      return null;
+    }
+  }
+  return filename || null;
+};
+
+const resolveRemoteStream = async (stream) => {
+  // Download to a temp file, get the file size, and create a readable stream
+  // from the temp file.
+  //
+  // The streamPipeline usage is taken from
+  // https://github.com/node-fetch/node-fetch#streams
+  const tmpFilePath = path.join(
+    os.tmpdir(),
+    'stash-' + randomBytes(16).toString('hex')
+  );
+
+  try {
+    await streamPipeline(stream, fs.createWriteStream(tmpFilePath));
+  } catch (error) {
+    try {
+      fs.unlinkSync(tmpFilePath);
+    } catch (e) {
+      // File doesn't exist? Probably okay
+    }
+    throw error;
+  }
+
+  const length = fs.statSync(tmpFilePath).size;
+  const readStream = fs.createReadStream(tmpFilePath);
+
+  readStream.on('end', () => {
+    // Burn after reading
+    try {
+      fs.unlinkSync(tmpFilePath);
+    } catch (e) {
+      // TODO: We probably want to log warning here
+    }
+  });
+
+  return {
+    streamOrData: readStream,
+    length,
+  };
+};
+
+const resolveResponseToStream = async (response) => {
+  // Get filename from content-disposition header or URL
+  let filename =
+    filenameFromHeader(response) ||
+    filenameFromURL(response.url || _.get(response, ['request', 'url'])) ||
+    DEFAULT_FILE_NAME;
+
+  const contentType = response.headers.get('content-type');
+  if (contentType && !path.extname(filename)) {
+    const ext = mime.extension(contentType);
+    if (ext && ext !== 'bin') {
+      filename += '.' + ext;
+    }
+  }
+
+  if (response.body && typeof response.body.pipe === 'function') {
+    // streamable response created by z.request({ raw: true })
+    return {
+      ...(await resolveRemoteStream(response.body)),
+      contentType: contentType || DEFAULT_CONTENT_TYPE,
+      filename,
+    };
+  }
+
+  // regular response created by z.request({ raw: false })
+  return {
+    streamOrData: response.content,
+    length: Buffer.byteLength(response.content),
+    contentType: contentType || DEFAULT_CONTENT_TYPE,
+    filename,
+  };
+};
+
+const resolveStreamWithMeta = async (stream) => {
+  const isLocalFile = stream.path && fs.existsSync(stream.path);
+  if (isLocalFile) {
+    const filename = path.basename(stream.path);
+    return {
+      streamOrData: stream,
+      length: fs.statSync(stream.path).size,
+      contentType: mime.lookup(filename) || DEFAULT_CONTENT_TYPE,
+      filename,
+    };
+  }
+
+  return {
+    ...(await resolveRemoteStream(stream)),
+    contentType: DEFAULT_CONTENT_TYPE,
+    filename: DEFAULT_FILE_NAME,
+  };
+};
+
+// Returns an object with fields:
+// * streamOrData: a readable stream, a string, or a Buffer
+// * length: content length in bytes
+// * contentType
+// * filename
+const resolveToBufferStringStream = async (responseOrData) => {
+  if (typeof responseOrData === 'string' || responseOrData instanceof String) {
+    // The .toString() call only makes a difference for the String object case.
+    // It converts a String object to a regular string.
+    const str = responseOrData.toString();
+    return {
+      streamOrData: str,
+      length: Buffer.byteLength(str),
+      contentType: 'text/plain',
+      filename: `${DEFAULT_FILE_NAME}.txt`,
+    };
+  } else if (Buffer.isBuffer(responseOrData)) {
+    return {
+      streamOrData: responseOrData,
+      length: responseOrData.length,
+      contentType: DEFAULT_CONTENT_TYPE,
+      filename: DEFAULT_FILE_NAME,
+    };
+  } else if (
+    (responseOrData.body && typeof responseOrData.body.pipe === 'function') ||
+    typeof responseOrData.content === 'string'
+  ) {
+    return resolveResponseToStream(responseOrData);
+  } else if (typeof responseOrData.pipe === 'function') {
+    return resolveStreamWithMeta(responseOrData);
+  }
+
+  throw new TypeError(
+    `z.stashFile() cannot stash type '${typeof responseOrData}'. ` +
+      'Pass it a request, readable stream, string, or Buffer.'
+  );
+};
+
+const uploader = async (
   signedPostData,
   bufferStringStream,
   knownLength,
   filename,
   contentType
 ) => {
-  const form = new FormData();
-
   if (knownLength && knownLength > UPLOAD_MAX_SIZE) {
-    return ZapierPromise.reject(
-      new Error(`${knownLength} is too big, ${UPLOAD_MAX_SIZE} is the max`)
-    );
+    throw new Error(`${knownLength} is too big, ${UPLOAD_MAX_SIZE} is the max`);
   }
+  filename = path.basename(filename).replace('"', '');
 
-  _.each(signedPostData.fields, (value, key) => {
-    form.append(key, value);
-  });
+  const fields = {
+    ...signedPostData.fields,
+    'Content-Disposition': contentDisposition(filename),
+    'Content-Type': contentType,
+  };
 
-  filename = path.basename(filename || DEFAULT_FILE_NAME).replace('"', '');
+  const form = new FormData();
 
-  form.append('Content-Disposition', contentDisposition(filename));
+  Object.entries(fields).forEach(([key, value]) => {
+    form.append(key, value);
+  });
 
   form.append('file', bufferStringStream, {
-    contentType,
     knownLength,
+    contentType,
     filename,
   });
 
@@ -56,120 +214,93 @@ const uploader = (
   }
 
   // Send to S3 with presigned request.
-  return request({
+  const response = await request({
     url: signedPostData.url,
     method: 'POST',
     body: form,
-  }).then((res) => {
-    if (res.status === 204) {
-      return `${signedPostData.url}${signedPostData.fields.key}`;
-    }
-    if (
-      res.content.indexOf(
-        'You must provide the Content-Length HTTP header.'
-      ) !== -1
-    ) {
-      throw new Error(LENGTH_ERR_MESSAGE);
-    }
-    throw new Error(`Got ${res.status} - ${res.content}`);
   });
+
+  if (response.status === 204) {
+    return new URL(signedPostData.fields.key, signedPostData.url).href;
+  }
+
+  if (
+    response.content &&
+    response.content.includes &&
+    response.content.includes(
+      'You must provide the Content-Length HTTP header.'
+    )
+  ) {
+    throw new Error(LENGTH_ERR_MESSAGE);
+  }
+
+  throw new Error(`Got ${response.status} - ${response.content}`);
 };
 
 // Designed to be some user provided function/api.
 const createFileStasher = (input) => {
   const rpc = _.get(input, '_zapier.rpc');
 
-  return (bufferStringStream, knownLength, filename, contentType) => {
+  return async (requestOrData, knownLength, filename, contentType) => {
     // TODO: maybe this could be smart?
     // if it is already a public url, do we pass through? or upload?
     if (!rpc) {
-      return ZapierPromise.reject(new Error('rpc is not available'));
+      throw new Error('rpc is not available');
     }
 
-    const method = _.get(input, ['_zapier', 'event', 'method'], '');
-    const isValidSource =
-      method.startsWith('hydrators.') ||
-      method.startsWith('creates.') ||
-      // key regex from KeySchema
-      method.match(/^resources\.[a-zA-Z][a-zA-Z0-9_]*\.create\./);
-
-    if (!isValidSource) {
-      return ZapierPromise.reject(
-        new Error(
-          'Files can only be stashed within a create or hydration function/method.'
-        )
+    const isRunningOnHydrator = _.get(
+      input,
+      '_zapier.event.method',
+      ''
+    ).startsWith('hydrators.');
+    const isRunningOnCreate = _.get(
+      input,
+      '_zapier.event.method',
+      ''
+    ).startsWith('creates.');
+
+    if (!isRunningOnHydrator && !isRunningOnCreate) {
+      throw new Error(
+        'Files can only be stashed within a create or hydration function/method.'
       );
     }
 
-    const fileContentType = contentType || DEFAULT_CONTENT_TYPE;
-
-    return rpc('get_presigned_upload_post_data', fileContentType).then(
-      (result) => {
-        const parseFinalResponse = (stream) => {
-          let newBufferStringStream = stream;
-
-          // if stream is string, don't update headers, just return as is
-          if (_.isString(stream)) {
-            newBufferStringStream = stream;
-          } else if (stream) {
-            if (Buffer.isBuffer(stream)) {
-              newBufferStringStream = stream;
-            } else if (Buffer.isBuffer(stream.dataBuffer)) {
-              newBufferStringStream = stream.dataBuffer;
-            } else if (stream.body && typeof stream.body.pipe === 'function') {
-              newBufferStringStream = stream.body;
-            } else if (stream.content) {
-              newBufferStringStream = stream.content;
-            }
-
-            // if stream has headers update knownLength and filename to reflect the header values
-            if (stream.headers) {
-              knownLength = knownLength || stream.getHeader('content-length');
-              const cd = stream.getHeader('content-disposition');
-              if (cd) {
-                filename =
-                  filename || contentDisposition.parse(cd).parameters.filename;
-              }
-            }
-            // if stream is not defined, error
-          } else {
-            throw new Error('Cannot stash a file of unknown type.');
-          }
-
-          // send final response to uploader
-          return uploader(
-            result,
-            newBufferStringStream,
-            knownLength,
-            filename,
-            fileContentType
-          );
-        };
-
-        const formResponse = (response) => {
-          // determine if string is streamed based on if raw:true
-          const isStreamed = _.get(response, 'request.raw', false);
-
-          // if it's streamed, buffer first
-          if (isStreamed) {
-            return response.buffer().then((buffer) => {
-              response.dataBuffer = buffer;
-              return parseFinalResponse(response);
-            });
-          } else {
-            return parseFinalResponse(response);
-          }
-        };
-
-        // if stream is a promise, wait until resolved
-        if (isPromise(bufferStringStream)) {
-          return bufferStringStream.then((maybeResponse) => {
-            return formResponse(maybeResponse);
-          });
-        } else {
-          return formResponse(bufferStringStream);
-        }
-      }
+    // requestOrData can be one of these:
+    // * string
+    // * Buffer
+    // * z.request() - a Promise of a regular response
+    // * z.request({ raw: true }) - a Promise of a "streamable" response
+    // * await z.request() - a regular response
+    // * await z.request({ raw: true }) - a streamable response
+    //
+    // After the following, requestOrData is resolved to responseOrData, which
+    // is either:
+    // - string
+    // - Buffer
+    // - a regular response
+    // - a streamable response
+    const [signedPostData, responseOrData] = await Promise.all([
+      rpc('get_presigned_upload_post_data'),
+      requestOrData,
+    ]);
+
+    if (responseOrData.throwForStatus) {
+      responseOrData.throwForStatus();
+    }
+
+    const {
+      streamOrData,
+      length,
+      contentType: _contentType,
+      filename: _filename,
+    } = await resolveToBufferStringStream(responseOrData);
+
+    return uploader(
+      signedPostData,
+      streamOrData,
+      knownLength || length,
+      filename || _filename,
+      contentType || _contentType
     );
   };
 };

package/src/tools/fetch.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const { Writable } = require('stream');
+
 const fetch = require('node-fetch');
 
 // XXX: PatchedRequest is to get past node-fetch's check that forbids GET requests
@@ -7,10 +9,10 @@ const fetch = require('node-fetch');
 // https://github.com/node-fetch/node-fetch/blob/v2.6.0/src/request.js#L75-L78
 class PatchedRequest extends fetch.Request {
   constructor(url, opts) {
-    const origMethod = (opts.method || 'GET').toUpperCase();
+    const origMethod = ((opts && opts.method) || 'GET').toUpperCase();
 
     const isGetWithBody =
-      (origMethod === 'GET' || origMethod === 'HEAD') && opts.body;
+      (origMethod === 'GET' || origMethod === 'HEAD') && opts && opts.body;
     let newOpts = opts;
     if (isGetWithBody) {
       // Temporary remove body to fool fetch.Request constructor
@@ -50,9 +52,31 @@ class PatchedRequest extends fetch.Request {
 
 const newFetch = (url, opts) => {
   const request = new PatchedRequest(url, opts);
+
   // fetch actually accepts a Request object as an argument. It'll clone the
   // request internally, that's why the PatchedRequest.body hack works.
-  return fetch(request);
+  const responsePromise = fetch(request);
+
+  // node-fetch clones request.body and use the cloned body internally. We need
+  // to make sure to consume the original body stream so its internal buffer is
+  // not filled up, which causes it to pause.
+  // See https://github.com/node-fetch/node-fetch/issues/151
+  //
+  // Exclude form-data object to be consistent with
+  // https://github.com/node-fetch/node-fetch/blob/v2.6.6/src/body.js#L403-L412
+  if (
+    request.body &&
+    typeof request.body.pipe === 'function' &&
+    typeof request.body.getBoundary !== 'function'
+  ) {
+    const nullStream = new Writable();
+    nullStream._write = function (chunk, encoding, done) {
+      done();
+    };
+    request.body.pipe(nullStream);
+  }
+
+  return responsePromise;
 };
 
 newFetch.Promise = require('./promise');