scratch-storage 2.1.0 → 2.2.0

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "scratch-storage",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "Load and store project and asset files for Scratch 3.0",
   "license": "BSD-3-Clause",
   "homepage": "https://github.com/LLK/scratch-storage#readme",
   "repository": {
     "type": "git",
     "url": "https://github.com/LLK/scratch-storage.git",
-    "sha": "2f6a562d46c4393cc70ab8c1b4350beaf6c07675"
+    "sha": "5c6588073bb3c9ba4500403054d6409db2379286"
   },
   "main": "./dist/node/scratch-storage.js",
   "browser": "./src/index.js",
@@ -24,9 +24,14 @@
     "watch": "webpack --progress --colors --watch",
     "semantic-release": "semantic-release"
   },
+  "tap": {
+    "check-coverage": false
+  },
   "dependencies": {
+    "@babel/runtime": "7.21.0",
     "arraybuffer-loader": "^1.0.3",
     "base64-js": "1.3.0",
+    "cross-fetch": "3.1.5",
     "fastestsmallesttextencoderdecoder": "^1.0.7",
     "js-md5": "0.7.3",
     "minilog": "3.1.0",
@@ -34,6 +39,7 @@
   },
   "devDependencies": {
     "@babel/core": "7.14.8",
+    "@babel/plugin-transform-runtime": "7.21.0",
     "@babel/polyfill": "7.12.1",
     "@babel/preset-env": "7.14.8",
     "@commitlint/cli": "8.2.0",
@@ -48,9 +54,8 @@
     "file-loader": "4.1.0",
     "husky": "1.3.1",
     "json": "^9.0.4",
-    "node-fetch": "2.6.1",
     "semantic-release": "^15.10.5",
-    "tap": "12.1.1",
+    "tap": "16.3.4",
     "uglifyjs-webpack-plugin": "2.2.0",
     "webpack": "4.46.0",
     "webpack-cli": "3.1.2"

package/src/FetchTool.js

@@ -1,48 +1,52 @@
-/* eslint-env browser */
+const {scratchFetch} = require('./scratchFetch');
+
+/**
+ * @typedef {Request & {withCredentials: boolean}} ScratchSendRequest
+ */
 
 /**
  * Get and send assets with the fetch standard web api.
  */
 class FetchTool {
     /**
-     * Is get supported? false if the environment does not support fetch.
+     * Is get supported?
+     * Always true for `FetchTool` because `scratchFetch` ponyfills `fetch` if necessary.
      * @returns {boolean} Is get supported?
      */
     get isGetSupported () {
-        return typeof fetch !== 'undefined';
+        return true;
     }
 
     /**
      * Request data from a server with fetch.
-     * @param {{url:string}} reqConfig - Request configuration for data to get.
-     * @param {{method:string}} options - Additional options to configure fetch.
-     * @returns {Promise.<Uint8Array>} Resolve to Buffer of data from server.
+     * @param {Request} reqConfig - Request configuration for data to get.
+     * @returns {Promise.<Uint8Array?>} Resolve to Buffer of data from server.
      */
     get ({url, ...options}) {
-        return fetch(url, Object.assign({method: 'GET'}, options))
+        return scratchFetch(url, Object.assign({method: 'GET'}, options))
             .then(result => {
                 if (result.ok) return result.arrayBuffer().then(b => new Uint8Array(b));
                 if (result.status === 404) return null;
-                return Promise.reject(result.status);
+                return Promise.reject(result.status); // TODO: we should throw a proper error
             });
     }
 
     /**
-     * Is sending supported? false if the environment does not support sending
-     * with fetch.
+     * Is sending supported?
+     * Always true for `FetchTool` because `scratchFetch` ponyfills `fetch` if necessary.
      * @returns {boolean} Is sending supported?
      */
     get isSendSupported () {
-        return typeof fetch !== 'undefined';
+        return true;
     }
 
     /**
      * Send data to a server with fetch.
-     * @param {Request} reqConfig - Request configuration for data to send.
+     * @param {ScratchSendRequest} reqConfig - Request configuration for data to send.
      * @returns {Promise.<string>} Server returned metadata.
      */
     send ({url, withCredentials = false, ...options}) {
-        return fetch(url, Object.assign({
+        return scratchFetch(url, Object.assign({
             credentials: withCredentials ? 'include' : 'omit'
         }, options))
             .then(response => {

package/src/FetchWorkerTool.js

@@ -1,3 +1,5 @@
+const {applyMetadata} = require('./scratchFetch');
+
 /**
  * Get and send assets with a worker that uses fetch.
  */
@@ -13,13 +15,13 @@ class PrivateFetchWorkerTool {
 
         /**
          * A possible error occurred standing up the worker.
-         * @type {!Error}
+         * @type {Error?}
          */
         this._supportError = null;
 
         /**
          * The worker that runs fetch and returns data for us.
-         * @type {!Worker}
+         * @type {Worker?}
          */
         this.worker = null;
 
@@ -34,9 +36,9 @@ class PrivateFetchWorkerTool {
                 // eslint-disable-next-line global-require
                 const FetchWorker = require('worker-loader?{"inline":true,"fallback":true}!./FetchWorkerTool.worker');
 
-                this.worker = new FetchWorker();
+                const worker = new FetchWorker();
 
-                this.worker.addEventListener('message', ({data}) => {
+                worker.addEventListener('message', ({data}) => {
                     if (data.support) {
                         this._workerSupport = data.support;
                         return;
@@ -52,6 +54,8 @@ class PrivateFetchWorkerTool {
                         }
                     }
                 });
+
+                this.worker = worker;
             }
         } catch (error) {
             this._supportError = error;
@@ -78,17 +82,20 @@ class PrivateFetchWorkerTool {
      * Request data from a server with a worker using fetch.
      * @param {{url:string}} reqConfig - Request configuration for data to get.
      * @param {{method:string}} options - Additional options to configure fetch.
-     * @returns {Promise.<Buffer>} Resolve to Buffer of data from server.
+     * @returns {Promise.<Buffer|Uint8Array|null>} Resolve to Buffer of data from server.
      */
     get ({url, ...options}) {
         return new Promise((resolve, reject) => {
             // TODO: Use a Scratch standard ID generator ...
             const id = Math.random().toString(16)
                 .substring(2);
+            const augmentedOptions = applyMetadata(
+                Object.assign({method: 'GET'}, options)
+            );
             this.worker.postMessage({
                 id,
                 url,
-                options: Object.assign({method: 'GET'}, options)
+                options: augmentedOptions
             });
             this.jobs[id] = {
                 id,
@@ -153,7 +160,7 @@ class PublicFetchWorkerTool {
     /**
      * Request data from a server with a worker that uses fetch.
      * @param {{url:string}} reqConfig - Request configuration for data to get.
-     * @returns {Promise.<Buffer>} Resolve to Buffer of data from server.
+     * @returns {Promise.<Buffer|Uint8Array|null>} Resolve to Buffer of data from server.
      */
     get (reqConfig) {
         return this.inner.get(reqConfig);

package/src/FetchWorkerTool.worker.js

@@ -1,5 +1,7 @@
 /* eslint-env worker */
 
+const crossFetch = require('cross-fetch').default;
+
 let jobsActive = 0;
 const complete = [];
 
@@ -48,7 +50,7 @@ const onMessage = ({data: job}) => {
 
     jobsActive++;
 
-    fetch(job.url, job.options)
+    crossFetch(job.url, job.options)
         .then(result => {
             if (result.ok) return result.arrayBuffer();
             if (result.status === 404) return null;
@@ -59,12 +61,6 @@ const onMessage = ({data: job}) => {
         .then(() => jobsActive--);
 };
 
-if (self.fetch) {
-    postMessage({support: {fetch: true}});
-    self.addEventListener('message', onMessage);
-} else {
-    postMessage({support: {fetch: false}});
-    self.addEventListener('message', ({data: job}) => {
-        postMessage([{id: job.id, error: 'fetch is unavailable'}]);
-    });
-}
+// crossFetch means "fetch" is now always supported
+postMessage({support: {fetch: true}});
+self.addEventListener('message', onMessage);

package/src/scratchFetch.js

@@ -0,0 +1,86 @@
+const {fetch, Headers} = require('cross-fetch');
+
+/**
+ * Metadata header names
+ * @enum {string} The enum value is the name of the associated header.
+ * @readonly
+ */
+const RequestMetadata = {
+    /** The ID of the project associated with this request */
+    ProjectId: 'X-ProjectId',
+    /** The ID of the project run associated with this request */
+    RunId: 'X-RunId'
+};
+
+/**
+ * Metadata for requests
+ * @type {Map<string, string>}
+ */
+const metadata = new Map();
+
+/**
+ * Non-destructively merge any metadata state (if any) with the provided options object (if any).
+ * If there is metadata state but no options object is provided, make a new object.
+ * If there is no metadata state, return the provided options parameter without modification.
+ * If there is metadata and an options object is provided, modify a copy and return it.
+ * Headers in the provided options object may override headers generated from metadata state.
+ * @param {RequestInit} [options] The initial request options. May be null or undefined.
+ * @returns {RequestInit|undefined} the provided options parameter without modification, or a new options object.
+ */
+const applyMetadata = options => {
+    if (metadata.size > 0) {
+        const augmentedOptions = Object.assign({}, options);
+        augmentedOptions.headers = new Headers(Array.from(metadata));
+        if (options && options.headers) {
+            const overrideHeaders =
+                options.headers instanceof Headers ? options.headers : new Headers(options.headers);
+            for (const [name, value] of overrideHeaders.entries()) {
+                augmentedOptions.headers.set(name, value);
+            }
+        }
+        return augmentedOptions;
+    }
+    return options;
+};
+
+/**
+ * Make a network request.
+ * This is a wrapper for the global fetch method, adding some Scratch-specific functionality.
+ * @param {RequestInfo|URL} resource The resource to fetch.
+ * @param {RequestInit} options Optional object containing custom settings for this request.
+ * @see {@link https://developer.mozilla.org/docs/Web/API/fetch} for more about the fetch API.
+ * @returns {Promise<Response>} A promise for the response to the request.
+ */
+const scratchFetch = (resource, options) => {
+    const augmentedOptions = applyMetadata(options);
+    return fetch(resource, augmentedOptions);
+};
+
+/**
+ * Set the value of a named request metadata item.
+ * Setting the value to `null` or `undefined` will NOT remove the item.
+ * Use `unsetMetadata` for that.
+ * @param {RequestMetadata} name The name of the metadata item to set.
+ * @param {any} value The value to set (will be converted to a string).
+ */
+const setMetadata = (name, value) => {
+    metadata.set(name, value);
+};
+
+/**
+ * Remove a named request metadata item.
+ * @param {RequestMetadata} name The name of the metadata item to remove.
+ */
+const unsetMetadata = name => {
+    metadata.delete(name);
+};
+
+module.exports = {
+    default: scratchFetch,
+
+    RequestMetadata,
+    applyMetadata,
+    scratchFetch,
+    setMetadata,
+    unsetMetadata
+};

package/test/mocks/mockFetch.js

@@ -0,0 +1,64 @@
+const TextEncoder = require('util').TextEncoder;
+const {Headers} = require('cross-fetch');
+
+const successText = 'successful response';
+
+/**
+ * @typedef MockFetchResponse The Response-like object returned by mockFetch.
+ * @property {boolean} ok True if the simulated request was successful, false otherwise.
+ * @property {number} status The HTTP status code of the simulated request.
+ * @property {() => Promise<string>} [text] A success string if the simulated request succeeded, undefined otherwise.
+ * @property {() => Promise<Uint8Array>} [arrayBuffer] Same as `text`, but encoded with UTF-8 if present.
+ */
+
+/**
+ * @typedef {RequestInit & {mockFetchTestData: MockFetchTestData}} MockFetchRequestInit
+ */
+
+/**
+ * @typedef MockFetchTestData
+ * @property {Headers} [headers] A Headers object initialized with the header info received by mockFetch.
+ * @property {Number} [headersCount] The number of headers in the 'headers' property.
+ */
+
+/**
+ * Mock the 'fetch' method from browsers.
+ * @param {RequestInfo|URL} resource The (mock) resource to fetch, which will determine the response.
+ * @param {MockFetchRequestInit} [options] Optional object containing custom settings for this request.
+ * @returns {Promise<MockFetchResponse>} A promise for a Response-like object. Does not fully implement Response.
+ */
+const mockFetch = (resource, options) => {
+    /** @type MockFetchResponse */
+    const results = {
+        ok: false,
+        status: 0
+    };
+    if (options?.mockFetchTestData) {
+        options.mockFetchTestData.headers = new Headers(options.headers);
+        options.mockFetchTestData.headersCount = Array.from(options.mockFetchTestData.headers).length;
+    }
+    switch (resource) {
+    case '200':
+        results.ok = true;
+        results.status = 200;
+        results.text = () => Promise.resolve(successText);
+        results.arrayBuffer = () => Promise.resolve(new TextEncoder().encode(successText));
+        break;
+    case '404':
+        results.ok = false;
+        results.status = 404;
+        break;
+    case '500':
+        results.ok = false;
+        results.status = 500;
+        break;
+    default:
+        throw new Error('unimplemented');
+    }
+    return Promise.resolve(results);
+};
+
+module.exports = {
+    mockFetch,
+    successText
+};

package/test/unit/fetch-tool.js

@@ -1,77 +1,60 @@
-const test = require('tap').test;
-const TextEncoder = require('util').TextEncoder;
+const tap = require('tap');
 const TextDecoder = require('util').TextDecoder;
 
-const FetchTool = require('../../src/FetchTool');
-
-test('send success returns response.text()', t => {
-    global.fetch = () => Promise.resolve({
-        ok: true,
-        text: () => Promise.resolve('successful response')
-    });
+const {mockFetch, successText} = require('../mocks/mockFetch.js');
+
+/**
+ * This is the real FetchTool, but the 'cross-fetch' module has been replaced with the mockFetch function.
+ * @type {typeof import('../../src/FetchTool')}
+ */
+const FetchTool = tap.mock('../../src/FetchTool', {
+    'cross-fetch': {
+        default: mockFetch,
+        fetch: mockFetch
+    }
+});
 
+tap.test('send success returns response.text()', t => {
     const tool = new FetchTool();
-    
+
     return t.resolves(
-        tool.send('url').then(result => {
-            t.equal(result, 'successful response');
+        tool.send({url: '200'}).then(result => {
+            t.equal(result, successText);
         })
     );
 });
 
-test('send failure returns response.status', t => {
-    global.fetch = () => Promise.resolve({
-        ok: false,
-        status: 500
-    });
-
+tap.test('send failure returns response.status', t => {
     const tool = new FetchTool();
 
-    return t.rejects(tool.send('url'), 500);
+    return t.rejects(tool.send({url: '500'}), 500);
 });
 
-test('get success returns Uint8Array.body(response.arrayBuffer())', t => {
-    const text = 'successful response';
+tap.test('get success returns Uint8Array.body(response.arrayBuffer())', t => {
     const encoding = 'utf-8';
-    const encoded = new TextEncoder().encode(text);
     const decoder = new TextDecoder(encoding);
 
-    global.fetch = () => Promise.resolve({
-        ok: true,
-        arrayBuffer: () => Promise.resolve(encoded.buffer)
-    });
-
     const tool = new FetchTool();
-    
+
     return t.resolves(
-        tool.get({url: 'url'}).then(result => {
-            t.equal(decoder.decode(result), text);
+        tool.get({url: '200'}).then(result => {
+            t.equal(decoder.decode(result), successText);
         })
     );
 });
 
-test('get with 404 response returns null data', t => {
-    global.fetch = () => Promise.resolve({
-        ok: false,
-        status: 404
-    });
-
+tap.test('get with 404 response returns null data', t => {
     const tool = new FetchTool();
 
     return t.resolves(
-        tool.get('url').then(result => {
+        tool.get({url: '404'}).then(result => {
             t.equal(result, null);
         })
     );
 });
 
-test('get failure returns response.status', t => {
-    global.fetch = () => Promise.resolve({
-        ok: false,
-        status: 500
-    });
-
+tap.test('get failure returns response.status', t => {
     const tool = new FetchTool();
 
-    return t.rejects(tool.get({url: 'url'}), 500);
+    return t.rejects(tool.get({url: '500'}), 500);
 });

package/test/unit/metadata.js

@@ -0,0 +1,136 @@
+const tap = require('tap');
+
+const crossFetch = require('cross-fetch');
+
+const {mockFetch} = require('../mocks/mockFetch.js');
+
+const mockFetchModule = {
+    ...crossFetch, // Headers, Request, Response, etc.
+    default: mockFetch,
+    fetch: mockFetch
+};
+
+// Call this separately from each test to ensure that metadata gets reset.
+// This is especially important when parallelizing tests!
+const setupModules = () => {
+    /**
+     * This instance of scratchFetch will be shared between this file and FetchTool.
+     * By sharing the same instance, the test can affect the metadata that FetchTool will use.
+     */
+    const scratchFetchModule = tap.mock('../../src/scratchFetch', {
+        'cross-fetch': mockFetchModule
+    });
+
+    /**
+     * This is the real FetchTool, but the 'cross-fetch' module has been replaced with the mockFetch function.
+     * @type {typeof import('../../src/FetchTool')}
+     */
+    const FetchTool = tap.mock('../../src/FetchTool', {
+        'cross-fetch': mockFetchModule,
+        // Make sure FetchTool uses the same scratchFetch instance
+        '../../src/scratchFetch': scratchFetchModule
+    });
+
+    return {scratchFetchModule, FetchTool};
+};
+
+tap.test('get without metadata', async t => {
+    const {FetchTool} = setupModules();
+
+    const tool = new FetchTool();
+
+    /** @type import('../mocks/mockFetch.js').MockFetchTestData */
+    const mockFetchTestData = {};
+    const result = await tool.get({url: '200', mockFetchTestData});
+
+    t.type(result, Uint8Array);
+    t.ok(mockFetchTestData.headers, 'mockFetch did not report headers');
+    t.equal(mockFetchTestData.headersCount, 0);
+});
+
+tap.test('get with metadata', async t => {
+    const {scratchFetchModule, FetchTool} = setupModules();
+    const {RequestMetadata, setMetadata} = scratchFetchModule;
+
+    const tool = new FetchTool();
+
+    setMetadata(RequestMetadata.ProjectId, 1234);
+    setMetadata(RequestMetadata.RunId, 5678);
+
+    /** @type import('../mocks/mockFetch.js').MockFetchTestData */
+    const mockFetchTestData = {};
+    const result = await tool.get({url: '200', mockFetchTestData});
+
+    t.type(result, Uint8Array);
+    t.ok(mockFetchTestData.headers, 'mockFetch did not report headers');
+    t.equal(mockFetchTestData.headersCount, 2);
+    t.equal(mockFetchTestData.headers?.get(RequestMetadata.ProjectId), '1234');
+    t.equal(mockFetchTestData.headers?.get(RequestMetadata.RunId), '5678');
+});
+
+tap.test('send without metadata', async t => {
+    const {FetchTool} = setupModules();
+
+    const tool = new FetchTool();
+
+    /** @type import('../mocks/mockFetch.js').MockFetchTestData */
+    const mockFetchTestData = {};
+    const result = await tool.send({url: '200', mockFetchTestData});
+
+    t.type(result, 'string');
+    t.ok(mockFetchTestData.headers, 'mockFetch did not report headers');
+    t.equal(mockFetchTestData.headersCount, 0);
+});
+
+tap.test('send with metadata', async t => {
+    const {scratchFetchModule, FetchTool} = setupModules();
+    const {RequestMetadata, setMetadata} = scratchFetchModule;
+
+    const tool = new FetchTool();
+
+    setMetadata(RequestMetadata.ProjectId, 4321);
+    setMetadata(RequestMetadata.RunId, 8765);
+
+    /** @type import('../mocks/mockFetch.js').MockFetchTestData */
+    const mockFetchTestData = {};
+    const result = await tool.send({url: '200', mockFetchTestData});
+
+    t.type(result, 'string');
+    t.ok(mockFetchTestData.headers, 'mockFetch did not report headers');
+    t.equal(mockFetchTestData.headersCount, 2);
+    t.equal(mockFetchTestData.headers?.get(RequestMetadata.ProjectId), '4321');
+    t.equal(mockFetchTestData.headers?.get(RequestMetadata.RunId), '8765');
+});
+
+tap.test('selectively delete metadata', async t => {
+    const {scratchFetchModule, FetchTool} = setupModules();
+    const {RequestMetadata, setMetadata, unsetMetadata} = scratchFetchModule;
+
+    // verify that these special values are preserved and not interpreted as "delete"
+    setMetadata(RequestMetadata.ProjectId, null);
+    setMetadata(RequestMetadata.RunId, void 0); // void 0 = undefined
+
+    const tool = new FetchTool();
+
+    /** @type import('../mocks/mockFetch.js').MockFetchTestData */
+    const mockFetchTestData = {};
+
+    const result1 = await tool.send({url: '200', mockFetchTestData});
+    t.type(result1, 'string');
+    t.ok(mockFetchTestData.headers, 'mockFetch did not report headers');
+
+    t.equal(mockFetchTestData.headersCount, 2);
+    t.equal(mockFetchTestData.headers?.get(RequestMetadata.ProjectId), 'null'); // string "null" means it's present
+    t.equal(mockFetchTestData.headers?.get(RequestMetadata.RunId), 'undefined');
+
+    // remove the Project ID from metadata
+    unsetMetadata(RequestMetadata.ProjectId);
+
+    const result2 = await tool.send({url: '200', mockFetchTestData});
+    t.type(result2, 'string');
+    t.ok(mockFetchTestData.headers, 'mockFetch did not report headers');
+
+    t.equal(mockFetchTestData.headersCount, 1);
+    t.equal(mockFetchTestData.headers?.get(RequestMetadata.ProjectId), null); // value `null` means it's missing
+    t.equal(mockFetchTestData.headers?.get(RequestMetadata.RunId), 'undefined');
+});

package/webpack.config.js

@@ -1,5 +1,4 @@
 const path = require('path');
-const {ProvidePlugin} = require('webpack');
 const UglifyJsPlugin = require('uglifyjs-webpack-plugin');
 
 const base = {
@@ -14,9 +13,16 @@ const base = {
                 test: /\.js$/,
                 loader: 'babel-loader',
                 options: {
+                    plugins: [
+                        '@babel/plugin-transform-runtime'
+                    ],
                     presets: [
                         ['@babel/preset-env', {targets: {browsers: ['last 3 versions', 'Safari >= 8', 'iOS >= 8']}}]
-                    ]
+                    ],
+                    // Consider a file a "module" if import/export statements are present, or else consider it a
+                    // "script". Fixes "Cannot assign to read only property 'exports'" when using
+                    // @babel/plugin-transform-runtime with CommonJS files.
+                    sourceType: 'unambiguous'
                 }
             }
         ]
@@ -65,11 +71,6 @@ module.exports = [
             'js-md5': true,
             'localforage': true,
             'text-encoding': true
-        },
-        plugins: [
-            new ProvidePlugin({
-                fetch: ['node-fetch', 'default']
-            })
-        ]
+        }
     })
 ];