zapier-platform-core 9.5.0 → 9.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zapier-platform-core",
-  "version": "9.5.0",
+  "version": "9.6.0",
   "description": "The core SDK for CLI apps in the Zapier Developer Platform.",
   "repository": "zapier/zapier-platform-core",
   "homepage": "https://zapier.com/",
@@ -41,16 +41,18 @@
     "bluebird": "3.5.5",
     "content-disposition": "0.5.3",
     "dotenv": "8.1.0",
-    "form-data": "2.5.0",
+    "form-data": "4.0.0",
     "lodash": "4.17.15",
-    "node-fetch": "2.6.0",
+    "mime-types": "2.1.34",
+    "node-fetch": "2.6.6",
     "oauth-sign": "0.9.0",
     "semver": "5.6.0",
-    "zapier-platform-schema": "9.5.0"
+    "zapier-platform-schema": "9.6.0"
   },
   "devDependencies": {
     "adm-zip": "0.4.13",
     "aws-sdk": "2.238.1",
+    "dicer": "0.3.0",
     "fs-extra": "8.1.0",
     "mock-fs": "4.10.1"
   },

package/src/tools/create-file-stasher.js

@@ -1,14 +1,18 @@
 'use strict';
 
-const _ = require('lodash');
+const fs = require('fs');
+const os = require('os');
 const path = require('path');
-const FormData = require('form-data');
+const { pipeline } = require('stream');
+const { promisify } = require('util');
+const { randomBytes } = require('crypto');
+
+const _ = require('lodash');
 const contentDisposition = require('content-disposition');
+const FormData = require('form-data');
+const mime = require('mime-types');
 
 const request = require('./request-client-internal');
-const ZapierPromise = require('./promise');
-
-const isPromise = obj => obj && typeof obj.then === 'function';
 
 const UPLOAD_MAX_SIZE = 1000 * 1000 * 150; // 150mb, in zapier backend too
 
@@ -19,33 +23,187 @@ const LENGTH_ERR_MESSAGE =
 const DEFAULT_FILE_NAME = 'unnamedfile';
 const DEFAULT_CONTENT_TYPE = 'application/octet-stream';
 
-const uploader = (
+const streamPipeline = promisify(pipeline);
+
+const filenameFromURL = (url) => {
+  try {
+    return decodeURIComponent(path.posix.basename(new URL(url).pathname));
+  } catch (error) {
+    return null;
+  }
+};
+
+const filenameFromHeader = (response) => {
+  const cd = response.headers.get('content-disposition');
+  let filename;
+  if (cd) {
+    try {
+      filename = contentDisposition.parse(cd).parameters.filename;
+    } catch (error) {
+      return null;
+    }
+  }
+  return filename || null;
+};
+
+const resolveRemoteStream = async (stream) => {
+  // Download to a temp file, get the file size, and create a readable stream
+  // from the temp file.
+  //
+  // The streamPipeline usage is taken from
+  // https://github.com/node-fetch/node-fetch#streams
+  const tmpFilePath = path.join(
+    os.tmpdir(),
+    'stash-' + randomBytes(16).toString('hex')
+  );
+
+  try {
+    await streamPipeline(stream, fs.createWriteStream(tmpFilePath));
+  } catch (error) {
+    try {
+      fs.unlinkSync(tmpFilePath);
+    } catch (e) {
+      // File doesn't exist? Probably okay
+    }
+    throw error;
+  }
+
+  const length = fs.statSync(tmpFilePath).size;
+  const readStream = fs.createReadStream(tmpFilePath);
+
+  readStream.on('end', () => {
+    // Burn after reading
+    try {
+      fs.unlinkSync(tmpFilePath);
+    } catch (e) {
+      // TODO: We probably want to log warning here
+    }
+  });
+
+  return {
+    streamOrData: readStream,
+    length,
+  };
+};
+
+const resolveResponseToStream = async (response) => {
+  // Get filename from content-disposition header or URL
+  let filename =
+    filenameFromHeader(response) ||
+    filenameFromURL(response.url || _.get(response, ['request', 'url'])) ||
+    DEFAULT_FILE_NAME;
+
+  const contentType = response.headers.get('content-type');
+  if (contentType && !path.extname(filename)) {
+    const ext = mime.extension(contentType);
+    if (ext && ext !== 'bin') {
+      filename += '.' + ext;
+    }
+  }
+
+  if (response.body && typeof response.body.pipe === 'function') {
+    // streamable response created by z.request({ raw: true })
+    return {
+      ...(await resolveRemoteStream(response.body)),
+      contentType: contentType || DEFAULT_CONTENT_TYPE,
+      filename,
+    };
+  }
+
+  // regular response created by z.request({ raw: false })
+  return {
+    streamOrData: response.content,
+    length: Buffer.byteLength(response.content),
+    contentType: contentType || DEFAULT_CONTENT_TYPE,
+    filename,
+  };
+};
+
+const resolveStreamWithMeta = async (stream) => {
+  const isLocalFile = stream.path && fs.existsSync(stream.path);
+  if (isLocalFile) {
+    const filename = path.basename(stream.path);
+    return {
+      streamOrData: stream,
+      length: fs.statSync(stream.path).size,
+      contentType: mime.lookup(filename) || DEFAULT_CONTENT_TYPE,
+      filename,
+    };
+  }
+
+  return {
+    ...(await resolveRemoteStream(stream)),
+    contentType: DEFAULT_CONTENT_TYPE,
+    filename: DEFAULT_FILE_NAME,
+  };
+};
+
+// Returns an object with fields:
+// * streamOrData: a readable stream, a string, or a Buffer
+// * length: content length in bytes
+// * contentType
+// * filename
+const resolveToBufferStringStream = async (responseOrData) => {
+  if (typeof responseOrData === 'string' || responseOrData instanceof String) {
+    // The .toString() call only makes a difference for the String object case.
+    // It converts a String object to a regular string.
+    const str = responseOrData.toString();
+    return {
+      streamOrData: str,
+      length: Buffer.byteLength(str),
+      contentType: 'text/plain',
+      filename: `${DEFAULT_FILE_NAME}.txt`,
+    };
+  } else if (Buffer.isBuffer(responseOrData)) {
+    return {
+      streamOrData: responseOrData,
+      length: responseOrData.length,
+      contentType: DEFAULT_CONTENT_TYPE,
+      filename: DEFAULT_FILE_NAME,
+    };
+  } else if (
+    (responseOrData.body && typeof responseOrData.body.pipe === 'function') ||
+    typeof responseOrData.content === 'string'
+  ) {
+    return resolveResponseToStream(responseOrData);
+  } else if (typeof responseOrData.pipe === 'function') {
+    return resolveStreamWithMeta(responseOrData);
+  }
+
+  throw new TypeError(
+    `z.stashFile() cannot stash type '${typeof responseOrData}'. ` +
+      'Pass it a request, readable stream, string, or Buffer.'
+  );
+};
+
+const uploader = async (
   signedPostData,
   bufferStringStream,
   knownLength,
   filename,
   contentType
 ) => {
-  const form = new FormData();
-
   if (knownLength && knownLength > UPLOAD_MAX_SIZE) {
-    return ZapierPromise.reject(
-      new Error(`${knownLength} is too big, ${UPLOAD_MAX_SIZE} is the max`)
-    );
+    throw new Error(`${knownLength} is too big, ${UPLOAD_MAX_SIZE} is the max`);
   }
+  filename = path.basename(filename).replace('"', '');
 
-  _.each(signedPostData.fields, (value, key) => {
-    form.append(key, value);
-  });
+  const fields = {
+    ...signedPostData.fields,
+    'Content-Disposition': contentDisposition(filename),
+    'Content-Type': contentType,
+  };
 
-  filename = path.basename(filename || DEFAULT_FILE_NAME).replace('"', '');
+  const form = new FormData();
 
-  form.append('Content-Disposition', contentDisposition(filename));
+  Object.entries(fields).forEach(([key, value]) => {
+    form.append(key, value);
+  });
 
   form.append('file', bufferStringStream, {
-    contentType,
     knownLength,
-    filename
+    contentType,
+    filename,
   });
 
   // Try to catch the missing length early, before upload to S3 fails.
@@ -56,120 +214,93 @@ const uploader = (
   }
 
   // Send to S3 with presigned request.
-  return request({
+  const response = await request({
     url: signedPostData.url,
     method: 'POST',
-    body: form
-  }).then(res => {
-    if (res.status === 204) {
-      return `${signedPostData.url}${signedPostData.fields.key}`;
-    }
-    if (
-      res.content.indexOf(
-        'You must provide the Content-Length HTTP header.'
-      ) !== -1
-    ) {
-      throw new Error(LENGTH_ERR_MESSAGE);
-    }
-    throw new Error(`Got ${res.status} - ${res.content}`);
+    body: form,
   });
+
+  if (response.status === 204) {
+    return new URL(signedPostData.fields.key, signedPostData.url).href;
+  }
+
+  if (
+    response.content &&
+    response.content.includes &&
+    response.content.includes(
+      'You must provide the Content-Length HTTP header.'
+    )
+  ) {
+    throw new Error(LENGTH_ERR_MESSAGE);
+  }
+
+  throw new Error(`Got ${response.status} - ${response.content}`);
 };
 
 // Designed to be some user provided function/api.
-const createFileStasher = input => {
+const createFileStasher = (input) => {
   const rpc = _.get(input, '_zapier.rpc');
 
-  return (bufferStringStream, knownLength, filename, contentType) => {
+  return async (requestOrData, knownLength, filename, contentType) => {
     // TODO: maybe this could be smart?
     // if it is already a public url, do we pass through? or upload?
     if (!rpc) {
-      return ZapierPromise.reject(new Error('rpc is not available'));
+      throw new Error('rpc is not available');
     }
 
-    const isRunningOnHydrator =
-      _.get(input, '_zapier.event.method', '').indexOf('hydrators.') === 0;
-    const isRunningOnCreate =
-      _.get(input, '_zapier.event.method', '').indexOf('creates.') === 0;
+    const isRunningOnHydrator = _.get(
+      input,
+      '_zapier.event.method',
+      ''
+    ).startsWith('hydrators.');
+    const isRunningOnCreate = _.get(
+      input,
+      '_zapier.event.method',
+      ''
+    ).startsWith('creates.');
 
     if (!isRunningOnHydrator && !isRunningOnCreate) {
-      return ZapierPromise.reject(
-        new Error(
-          'Files can only be stashed within a create or hydration function/method.'
-        )
+      throw new Error(
+        'Files can only be stashed within a create or hydration function/method.'
       );
     }
 
-    const fileContentType = contentType || DEFAULT_CONTENT_TYPE;
-
-    return rpc('get_presigned_upload_post_data', fileContentType).then(
-      result => {
-        if (isPromise(bufferStringStream)) {
-          return bufferStringStream.then(maybeResponse => {
-            const isStreamed = _.get(maybeResponse, 'request.raw', false);
-
-            const parseFinalResponse = response => {
-              let newBufferStringStream = response;
-              if (_.isString(response)) {
-                newBufferStringStream = response;
-              } else if (response) {
-                if (Buffer.isBuffer(response)) {
-                  newBufferStringStream = response;
-                } else if (Buffer.isBuffer(response.dataBuffer)) {
-                  newBufferStringStream = response.dataBuffer;
-                } else if (
-                  response.body &&
-                  typeof response.body.pipe === 'function'
-                ) {
-                  newBufferStringStream = response.body;
-                } else {
-                  newBufferStringStream = response.content;
-                }
-
-                if (response.headers) {
-                  knownLength =
-                    knownLength || response.getHeader('content-length');
-                  const cd = response.getHeader('content-disposition');
-                  if (cd) {
-                    filename =
-                      filename ||
-                      contentDisposition.parse(cd).parameters.filename;
-                  }
-                }
-              } else {
-                throw new Error(
-                  'Cannot stash a Promise wrapped file of unknown type.'
-                );
-              }
-
-              return uploader(
-                result,
-                newBufferStringStream,
-                knownLength,
-                filename,
-                fileContentType
-              );
-            };
-
-            if (isStreamed) {
-              maybeResponse.throwForStatus();
-              return maybeResponse.buffer().then(buffer => {
-                maybeResponse.dataBuffer = buffer;
-                return parseFinalResponse(maybeResponse);
-              });
-            } else {
-              return parseFinalResponse(maybeResponse);
-            }
-          });
-        } else {
-          return uploader(
-            result,
-            bufferStringStream,
-            knownLength,
-            filename,
-            fileContentType
-          );
-        }
-      }
+    // requestOrData can be one of these:
+    // * string
+    // * Buffer
+    // * z.request() - a Promise of a regular response
+    // * z.request({ raw: true }) - a Promise of a "streamable" response
+    // * await z.request() - a regular response
+    // * await z.request({ raw: true }) - a streamable response
+    //
+    // After the following, requestOrData is resolved to responseOrData, which
+    // is either:
+    // - string
+    // - Buffer
+    // - a regular response
+    // - a streamable response
+    const [signedPostData, responseOrData] = await Promise.all([
+      rpc('get_presigned_upload_post_data'),
+      requestOrData,
+    ]);
+
+    if (responseOrData.throwForStatus) {
+      responseOrData.throwForStatus();
+    }
+
+    const {
+      streamOrData,
+      length,
+      contentType: _contentType,
+      filename: _filename,
+    } = await resolveToBufferStringStream(responseOrData);
+
+    return uploader(
+      signedPostData,
+      streamOrData,
+      knownLength || length,
+      filename || _filename,
+      contentType || _contentType
     );
   };
 };

package/src/tools/fetch.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const { Writable } = require('stream');
+
 const fetch = require('node-fetch');
 
 // XXX: PatchedRequest is to get past node-fetch's check that forbids GET requests
@@ -7,10 +9,10 @@ const fetch = require('node-fetch');
 // https://github.com/node-fetch/node-fetch/blob/v2.6.0/src/request.js#L75-L78
 class PatchedRequest extends fetch.Request {
   constructor(url, opts) {
-    const origMethod = (opts.method || 'GET').toUpperCase();
+    const origMethod = ((opts && opts.method) || 'GET').toUpperCase();
 
     const isGetWithBody =
-      (origMethod === 'GET' || origMethod === 'HEAD') && opts.body;
+      (origMethod === 'GET' || origMethod === 'HEAD') && opts && opts.body;
     let newOpts = opts;
     if (isGetWithBody) {
       // Temporary remove body to fool fetch.Request constructor
@@ -50,9 +52,31 @@ class PatchedRequest extends fetch.Request {
 
 const newFetch = (url, opts) => {
   const request = new PatchedRequest(url, opts);
+
   // fetch actually accepts a Request object as an argument. It'll clone the
   // request internally, that's why the PatchedRequest.body hack works.
-  return fetch(request);
+  const responsePromise = fetch(request);
+
+  // node-fetch clones request.body and use the cloned body internally. We need
+  // to make sure to consume the original body stream so its internal buffer is
+  // not filled up, which causes it to pause.
+  // See https://github.com/node-fetch/node-fetch/issues/151
+  //
+  // Exclude form-data object to be consistent with
+  // https://github.com/node-fetch/node-fetch/blob/v2.6.6/src/body.js#L403-L412
+  if (
+    request.body &&
+    typeof request.body.pipe === 'function' &&
+    typeof request.body.getBoundary !== 'function'
+  ) {
+    const nullStream = new Writable();
+    nullStream._write = function (chunk, encoding, done) {
+      done();
+    };
+    request.body.pipe(nullStream);
+  }
+
+  return responsePromise;
 };
 
 newFetch.Promise = require('./promise');