zapier-platform-core 9.4.0 → 9.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zapier-platform-core",
-  "version": "9.4.0",
+  "version": "9.6.0",
   "description": "The core SDK for CLI apps in the Zapier Developer Platform.",
   "repository": "zapier/zapier-platform-core",
   "homepage": "https://zapier.com/",
@@ -41,16 +41,18 @@
     "bluebird": "3.5.5",
     "content-disposition": "0.5.3",
     "dotenv": "8.1.0",
-    "form-data": "2.5.0",
+    "form-data": "4.0.0",
     "lodash": "4.17.15",
-    "node-fetch": "2.6.0",
+    "mime-types": "2.1.34",
+    "node-fetch": "2.6.6",
     "oauth-sign": "0.9.0",
     "semver": "5.6.0",
-    "zapier-platform-schema": "9.4.0"
+    "zapier-platform-schema": "9.6.0"
   },
   "devDependencies": {
     "adm-zip": "0.4.13",
     "aws-sdk": "2.238.1",
+    "dicer": "0.3.0",
     "fs-extra": "8.1.0",
     "mock-fs": "4.10.1"
   },

package/src/create-command-handler.js

@@ -18,7 +18,7 @@ const commandHandlers = {
   commands like 'execute', 'validate', 'definition', 'request'.
 */
 const createCommandHandler = compiledApp => {
-  return input => {
+  return async input => {
     const command = input._zapier.event.command || 'execute'; // validate || definition || request
     const handler = commandHandlers[command];
     if (!handler) {
@@ -26,7 +26,7 @@ const createCommandHandler = compiledApp => {
     }
 
     try {
-      return handler(compiledApp, input);
+      return await handler(compiledApp, input);
     } catch (err) {
       return handleError(err);
     }

package/src/errors.js

@@ -9,7 +9,7 @@ class AppError extends Error {
       JSON.stringify({
         message,
         code,
-        status
+        status,
       })
     );
     this.name = 'AppError';
@@ -19,16 +19,23 @@ class AppError extends Error {
 
 class ResponseError extends Error {
   constructor(response) {
+    let content;
+    try {
+      content = response.content;
+    } catch (err) {
+      // Stream request (z.request({raw: true})) doesn't have response.content
+      content = null;
+    }
     super(
       JSON.stringify({
         status: response.status,
         headers: {
-          'content-type': response.headers.get('content-type')
+          'content-type': response.headers.get('content-type'),
         },
-        content: response.content,
+        content,
         request: {
-          url: response.request.url
-        }
+          url: response.request.url,
+        },
       })
     );
     this.name = 'ResponseError';
@@ -37,8 +44,8 @@ class ResponseError extends Error {
 }
 
 // Make some of the errors we'll use!
-const createError = name => {
-  const NewError = function(message = '') {
+const createError = (name) => {
+  const NewError = function (message = '') {
     this.name = name;
     this.message = message;
     Error.call(this);
@@ -57,7 +64,7 @@ const names = [
   'NotImplementedError',
   'RefreshAuthError',
   'RequireModuleError',
-  'StopRequestError'
+  'StopRequestError',
 ];
 
 const exceptions = _.reduce(
@@ -68,7 +75,7 @@ const exceptions = _.reduce(
   },
   {
     Error: AppError,
-    ResponseError
+    ResponseError,
   }
 );
 
@@ -89,5 +96,5 @@ const handleError = (...args) => {
 
 module.exports = {
   ...exceptions,
-  handleError
+  handleError,
 };

package/src/http-middlewares/before/disable-ssl-cert-check.js

@@ -1,12 +1,20 @@
 'use strict';
 
+const http = require('http');
 const https = require('https');
 
-const disableSSLCertCheck = req => {
+const httpAgent = new http.Agent({ rejectUnauthorized: false });
+const httpsAgent = new https.Agent({ rejectUnauthorized: false });
+
+const disableSSLCertCheck = (req) => {
   if (req.agent) {
     req.agent.options.rejectUnauthorized = false;
   } else if (req.url.startsWith('https://')) {
-    req.agent = new https.Agent({ rejectUnauthorized: false });
+    // Need to dynamically choose a different agent because redirection can be
+    // across HTTPS and HTTP.
+    // See https://github.com/node-fetch/node-fetch/tree/6ee9d318#custom-agent
+    req.agent = (parsedURL) =>
+      parsedURL.protocol === 'http:' ? httpAgent : httpsAgent;
   }
   return req;
 };

package/src/tools/create-file-stasher.js

@@ -1,14 +1,18 @@
 'use strict';
 
-const _ = require('lodash');
+const fs = require('fs');
+const os = require('os');
 const path = require('path');
-const FormData = require('form-data');
+const { pipeline } = require('stream');
+const { promisify } = require('util');
+const { randomBytes } = require('crypto');
+
+const _ = require('lodash');
 const contentDisposition = require('content-disposition');
+const FormData = require('form-data');
+const mime = require('mime-types');
 
 const request = require('./request-client-internal');
-const ZapierPromise = require('./promise');
-
-const isPromise = obj => obj && typeof obj.then === 'function';
 
 const UPLOAD_MAX_SIZE = 1000 * 1000 * 150; // 150mb, in zapier backend too
 
@@ -19,33 +23,187 @@ const LENGTH_ERR_MESSAGE =
 const DEFAULT_FILE_NAME = 'unnamedfile';
 const DEFAULT_CONTENT_TYPE = 'application/octet-stream';
 
-const uploader = (
+const streamPipeline = promisify(pipeline);
+
+const filenameFromURL = (url) => {
+  try {
+    return decodeURIComponent(path.posix.basename(new URL(url).pathname));
+  } catch (error) {
+    return null;
+  }
+};
+
+const filenameFromHeader = (response) => {
+  const cd = response.headers.get('content-disposition');
+  let filename;
+  if (cd) {
+    try {
+      filename = contentDisposition.parse(cd).parameters.filename;
+    } catch (error) {
+      return null;
+    }
+  }
+  return filename || null;
+};
+
+const resolveRemoteStream = async (stream) => {
+  // Download to a temp file, get the file size, and create a readable stream
+  // from the temp file.
+  //
+  // The streamPipeline usage is taken from
+  // https://github.com/node-fetch/node-fetch#streams
+  const tmpFilePath = path.join(
+    os.tmpdir(),
+    'stash-' + randomBytes(16).toString('hex')
+  );
+
+  try {
+    await streamPipeline(stream, fs.createWriteStream(tmpFilePath));
+  } catch (error) {
+    try {
+      fs.unlinkSync(tmpFilePath);
+    } catch (e) {
+      // File doesn't exist? Probably okay
+    }
+    throw error;
+  }
+
+  const length = fs.statSync(tmpFilePath).size;
+  const readStream = fs.createReadStream(tmpFilePath);
+
+  readStream.on('end', () => {
+    // Burn after reading
+    try {
+      fs.unlinkSync(tmpFilePath);
+    } catch (e) {
+      // TODO: We probably want to log warning here
+    }
+  });
+
+  return {
+    streamOrData: readStream,
+    length,
+  };
+};
+
+const resolveResponseToStream = async (response) => {
+  // Get filename from content-disposition header or URL
+  let filename =
+    filenameFromHeader(response) ||
+    filenameFromURL(response.url || _.get(response, ['request', 'url'])) ||
+    DEFAULT_FILE_NAME;
+
+  const contentType = response.headers.get('content-type');
+  if (contentType && !path.extname(filename)) {
+    const ext = mime.extension(contentType);
+    if (ext && ext !== 'bin') {
+      filename += '.' + ext;
+    }
+  }
+
+  if (response.body && typeof response.body.pipe === 'function') {
+    // streamable response created by z.request({ raw: true })
+    return {
+      ...(await resolveRemoteStream(response.body)),
+      contentType: contentType || DEFAULT_CONTENT_TYPE,
+      filename,
+    };
+  }
+
+  // regular response created by z.request({ raw: false })
+  return {
+    streamOrData: response.content,
+    length: Buffer.byteLength(response.content),
+    contentType: contentType || DEFAULT_CONTENT_TYPE,
+    filename,
+  };
+};
+
+const resolveStreamWithMeta = async (stream) => {
+  const isLocalFile = stream.path && fs.existsSync(stream.path);
+  if (isLocalFile) {
+    const filename = path.basename(stream.path);
+    return {
+      streamOrData: stream,
+      length: fs.statSync(stream.path).size,
+      contentType: mime.lookup(filename) || DEFAULT_CONTENT_TYPE,
+      filename,
+    };
+  }
+
+  return {
+    ...(await resolveRemoteStream(stream)),
+    contentType: DEFAULT_CONTENT_TYPE,
+    filename: DEFAULT_FILE_NAME,
+  };
+};
+
+// Returns an object with fields:
+// * streamOrData: a readable stream, a string, or a Buffer
+// * length: content length in bytes
+// * contentType
+// * filename
+const resolveToBufferStringStream = async (responseOrData) => {
+  if (typeof responseOrData === 'string' || responseOrData instanceof String) {
+    // The .toString() call only makes a difference for the String object case.
+    // It converts a String object to a regular string.
+    const str = responseOrData.toString();
+    return {
+      streamOrData: str,
+      length: Buffer.byteLength(str),
+      contentType: 'text/plain',
+      filename: `${DEFAULT_FILE_NAME}.txt`,
+    };
+  } else if (Buffer.isBuffer(responseOrData)) {
+    return {
+      streamOrData: responseOrData,
+      length: responseOrData.length,
+      contentType: DEFAULT_CONTENT_TYPE,
+      filename: DEFAULT_FILE_NAME,
+    };
+  } else if (
+    (responseOrData.body && typeof responseOrData.body.pipe === 'function') ||
+    typeof responseOrData.content === 'string'
+  ) {
+    return resolveResponseToStream(responseOrData);
+  } else if (typeof responseOrData.pipe === 'function') {
+    return resolveStreamWithMeta(responseOrData);
+  }
+
+  throw new TypeError(
+    `z.stashFile() cannot stash type '${typeof responseOrData}'. ` +
+      'Pass it a request, readable stream, string, or Buffer.'
+  );
+};
+
+const uploader = async (
   signedPostData,
   bufferStringStream,
   knownLength,
   filename,
   contentType
 ) => {
-  const form = new FormData();
-
   if (knownLength && knownLength > UPLOAD_MAX_SIZE) {
-    return ZapierPromise.reject(
-      new Error(`${knownLength} is too big, ${UPLOAD_MAX_SIZE} is the max`)
-    );
+    throw new Error(`${knownLength} is too big, ${UPLOAD_MAX_SIZE} is the max`);
   }
+  filename = path.basename(filename).replace('"', '');
 
-  _.each(signedPostData.fields, (value, key) => {
-    form.append(key, value);
-  });
+  const fields = {
+    ...signedPostData.fields,
+    'Content-Disposition': contentDisposition(filename),
+    'Content-Type': contentType,
+  };
 
-  filename = path.basename(filename || DEFAULT_FILE_NAME).replace('"', '');
+  const form = new FormData();
 
-  form.append('Content-Disposition', contentDisposition(filename));
+  Object.entries(fields).forEach(([key, value]) => {
+    form.append(key, value);
+  });
 
   form.append('file', bufferStringStream, {
-    contentType,
     knownLength,
-    filename
+    contentType,
+    filename,
   });
 
   // Try to catch the missing length early, before upload to S3 fails.
@@ -56,120 +214,93 @@ const uploader = (
   }
 
   // Send to S3 with presigned request.
-  return request({
+  const response = await request({
     url: signedPostData.url,
     method: 'POST',
-    body: form
-  }).then(res => {
-    if (res.status === 204) {
-      return `${signedPostData.url}${signedPostData.fields.key}`;
-    }
-    if (
-      res.content.indexOf(
-        'You must provide the Content-Length HTTP header.'
-      ) !== -1
-    ) {
-      throw new Error(LENGTH_ERR_MESSAGE);
-    }
-    throw new Error(`Got ${res.status} - ${res.content}`);
+    body: form,
   });
+
+  if (response.status === 204) {
+    return new URL(signedPostData.fields.key, signedPostData.url).href;
+  }
+
+  if (
+    response.content &&
+    response.content.includes &&
+    response.content.includes(
+      'You must provide the Content-Length HTTP header.'
+    )
+  ) {
+    throw new Error(LENGTH_ERR_MESSAGE);
+  }
+
+  throw new Error(`Got ${response.status} - ${response.content}`);
 };
 
 // Designed to be some user provided function/api.
-const createFileStasher = input => {
+const createFileStasher = (input) => {
   const rpc = _.get(input, '_zapier.rpc');
 
-  return (bufferStringStream, knownLength, filename, contentType) => {
+  return async (requestOrData, knownLength, filename, contentType) => {
     // TODO: maybe this could be smart?
     // if it is already a public url, do we pass through? or upload?
     if (!rpc) {
-      return ZapierPromise.reject(new Error('rpc is not available'));
+      throw new Error('rpc is not available');
     }
 
-    const isRunningOnHydrator =
-      _.get(input, '_zapier.event.method', '').indexOf('hydrators.') === 0;
-    const isRunningOnCreate =
-      _.get(input, '_zapier.event.method', '').indexOf('creates.') === 0;
+    const isRunningOnHydrator = _.get(
+      input,
+      '_zapier.event.method',
+      ''
+    ).startsWith('hydrators.');
+    const isRunningOnCreate = _.get(
+      input,
+      '_zapier.event.method',
+      ''
+    ).startsWith('creates.');
 
     if (!isRunningOnHydrator && !isRunningOnCreate) {
-      return ZapierPromise.reject(
-        new Error(
-          'Files can only be stashed within a create or hydration function/method.'
-        )
+      throw new Error(
+        'Files can only be stashed within a create or hydration function/method.'
       );
     }
 
-    const fileContentType = contentType || DEFAULT_CONTENT_TYPE;
-
-    return rpc('get_presigned_upload_post_data', fileContentType).then(
-      result => {
-        if (isPromise(bufferStringStream)) {
-          return bufferStringStream.then(maybeResponse => {
-            const isStreamed = _.get(maybeResponse, 'request.raw', false);
-
-            const parseFinalResponse = response => {
-              let newBufferStringStream = response;
-              if (_.isString(response)) {
-                newBufferStringStream = response;
-              } else if (response) {
-                if (Buffer.isBuffer(response)) {
-                  newBufferStringStream = response;
-                } else if (Buffer.isBuffer(response.dataBuffer)) {
-                  newBufferStringStream = response.dataBuffer;
-                } else if (
-                  response.body &&
-                  typeof response.body.pipe === 'function'
-                ) {
-                  newBufferStringStream = response.body;
-                } else {
-                  newBufferStringStream = response.content;
-                }
-
-                if (response.headers) {
-                  knownLength =
-                    knownLength || response.getHeader('content-length');
-                  const cd = response.getHeader('content-disposition');
-                  if (cd) {
-                    filename =
-                      filename ||
-                      contentDisposition.parse(cd).parameters.filename;
-                  }
-                }
-              } else {
-                throw new Error(
-                  'Cannot stash a Promise wrapped file of unknown type.'
-                );
-              }
-
-              return uploader(
-                result,
-                newBufferStringStream,
-                knownLength,
-                filename,
-                fileContentType
-              );
-            };
-
-            if (isStreamed) {
-              maybeResponse.throwForStatus();
-              return maybeResponse.buffer().then(buffer => {
-                maybeResponse.dataBuffer = buffer;
-                return parseFinalResponse(maybeResponse);
-              });
-            } else {
-              return parseFinalResponse(maybeResponse);
-            }
-          });
-        } else {
-          return uploader(
-            result,
-            bufferStringStream,
-            knownLength,
-            filename,
-            fileContentType
-          );
-        }
-      }
+    // requestOrData can be one of these:
+    // * string
+    // * Buffer
+    // * z.request() - a Promise of a regular response
+    // * z.request({ raw: true }) - a Promise of a "streamable" response
+    // * await z.request() - a regular response
+    // * await z.request({ raw: true }) - a streamable response
+    //
+    // After the following, requestOrData is resolved to responseOrData, which
+    // is either:
+    // - string
+    // - Buffer
+    // - a regular response
+    // - a streamable response
+    const [signedPostData, responseOrData] = await Promise.all([
+      rpc('get_presigned_upload_post_data'),
+      requestOrData,
+    ]);
+
+    if (responseOrData.throwForStatus) {
+      responseOrData.throwForStatus();
+    }
+
+    const {
+      streamOrData,
+      length,
+      contentType: _contentType,
+      filename: _filename,
+    } = await resolveToBufferStringStream(responseOrData);
+
+    return uploader(
+      signedPostData,
+      streamOrData,
+      knownLength || length,
+      filename || _filename,
+      contentType || _contentType
     );
   };
 };

package/src/tools/create-lambda-handler.js

@@ -204,7 +204,8 @@ const createLambdaHandler = appRawOrPath => {
           // Adds logging for _all_ kinds of http(s) requests, no matter the library
           if (!skipHttpPatch) {
             const httpPatch = createHttpPatch(event);
-            httpPatch(require('http')); // 'https' uses 'http' under the hood
+            httpPatch(require('http'));
+            httpPatch(require('https')); // 'https' needs to be patched separately
           }
 
           // TODO: Avoid calling prepareApp(appRaw) repeatedly here as createApp()

package/src/tools/fetch.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const { Writable } = require('stream');
+
 const fetch = require('node-fetch');
 
 // XXX: PatchedRequest is to get past node-fetch's check that forbids GET requests
@@ -7,10 +9,10 @@ const fetch = require('node-fetch');
 // https://github.com/node-fetch/node-fetch/blob/v2.6.0/src/request.js#L75-L78
 class PatchedRequest extends fetch.Request {
   constructor(url, opts) {
-    const origMethod = (opts.method || 'GET').toUpperCase();
+    const origMethod = ((opts && opts.method) || 'GET').toUpperCase();
 
     const isGetWithBody =
-      (origMethod === 'GET' || origMethod === 'HEAD') && opts.body;
+      (origMethod === 'GET' || origMethod === 'HEAD') && opts && opts.body;
     let newOpts = opts;
     if (isGetWithBody) {
       // Temporary remove body to fool fetch.Request constructor
@@ -50,9 +52,31 @@ class PatchedRequest extends fetch.Request {
 
 const newFetch = (url, opts) => {
   const request = new PatchedRequest(url, opts);
+
   // fetch actually accepts a Request object as an argument. It'll clone the
   // request internally, that's why the PatchedRequest.body hack works.
-  return fetch(request);
+  const responsePromise = fetch(request);
+
+  // node-fetch clones request.body and use the cloned body internally. We need
+  // to make sure to consume the original body stream so its internal buffer is
+  // not filled up, which causes it to pause.
+  // See https://github.com/node-fetch/node-fetch/issues/151
+  //
+  // Exclude form-data object to be consistent with
+  // https://github.com/node-fetch/node-fetch/blob/v2.6.6/src/body.js#L403-L412
+  if (
+    request.body &&
+    typeof request.body.pipe === 'function' &&
+    typeof request.body.getBoundary !== 'function'
+  ) {
+    const nullStream = new Writable();
+    nullStream._write = function (chunk, encoding, done) {
+      done();
+    };
+    request.body.pipe(nullStream);
+  }
+
+  return responsePromise;
 };
 
 newFetch.Promise = require('./promise');

package/src/tools/schema-tools.js

@@ -2,9 +2,13 @@
 
 const dataTools = require('./data');
 
+// AsyncFunction is not a global object and can be obtained in this way
+// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AsyncFunction
+const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor;
+
 const makeFunction = (source, args = []) => {
   try {
-    return Function(...args, source); // eslint-disable-line no-new-func
+    return new AsyncFunction(...args, source);
   } catch (err) {
     return () => {
       throw err;