scratch-storage 2.0.2 → 2.2.0

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "scratch-storage",
-  "version": "2.0.2",
+  "version": "2.2.0",
   "description": "Load and store project and asset files for Scratch 3.0",
   "license": "BSD-3-Clause",
   "homepage": "https://github.com/LLK/scratch-storage#readme",
   "repository": {
     "type": "git",
     "url": "https://github.com/LLK/scratch-storage.git",
-    "sha": "1592e27d02d61c89172517cfb0832cdbb4f95dea"
+    "sha": "5c6588073bb3c9ba4500403054d6409db2379286"
   },
   "main": "./dist/node/scratch-storage.js",
   "browser": "./src/index.js",
@@ -24,9 +24,14 @@
     "watch": "webpack --progress --colors --watch",
     "semantic-release": "semantic-release"
   },
+  "tap": {
+    "check-coverage": false
+  },
   "dependencies": {
+    "@babel/runtime": "7.21.0",
     "arraybuffer-loader": "^1.0.3",
     "base64-js": "1.3.0",
+    "cross-fetch": "3.1.5",
     "fastestsmallesttextencoderdecoder": "^1.0.7",
     "js-md5": "0.7.3",
     "minilog": "3.1.0",
@@ -34,6 +39,7 @@
   },
   "devDependencies": {
     "@babel/core": "7.14.8",
+    "@babel/plugin-transform-runtime": "7.21.0",
     "@babel/polyfill": "7.12.1",
     "@babel/preset-env": "7.14.8",
     "@commitlint/cli": "8.2.0",
@@ -48,9 +54,8 @@
     "file-loader": "4.1.0",
     "husky": "1.3.1",
     "json": "^9.0.4",
-    "node-fetch": "2.6.1",
     "semantic-release": "^15.10.5",
-    "tap": "12.1.1",
+    "tap": "16.3.4",
     "uglifyjs-webpack-plugin": "2.2.0",
     "webpack": "4.46.0",
     "webpack-cli": "3.1.2"

package/src/FetchTool.js

@@ -1,48 +1,52 @@
-/* eslint-env browser */
+const {scratchFetch} = require('./scratchFetch');
+
+/**
+ * @typedef {Request & {withCredentials: boolean}} ScratchSendRequest
+ */
 
 /**
  * Get and send assets with the fetch standard web api.
  */
 class FetchTool {
     /**
-     * Is get supported? false if the environment does not support fetch.
+     * Is get supported?
+     * Always true for `FetchTool` because `scratchFetch` ponyfills `fetch` if necessary.
      * @returns {boolean} Is get supported?
      */
     get isGetSupported () {
-        return typeof fetch !== 'undefined';
+        return true;
     }
 
     /**
      * Request data from a server with fetch.
-     * @param {{url:string}} reqConfig - Request configuration for data to get.
-     * @param {{method:string}} options - Additional options to configure fetch.
-     * @returns {Promise.<Uint8Array>} Resolve to Buffer of data from server.
+     * @param {Request} reqConfig - Request configuration for data to get.
+     * @returns {Promise.<Uint8Array?>} Resolve to Buffer of data from server.
      */
     get ({url, ...options}) {
-        return fetch(url, Object.assign({method: 'GET'}, options))
+        return scratchFetch(url, Object.assign({method: 'GET'}, options))
             .then(result => {
                 if (result.ok) return result.arrayBuffer().then(b => new Uint8Array(b));
                 if (result.status === 404) return null;
-                return Promise.reject(result.status);
+                return Promise.reject(result.status); // TODO: we should throw a proper error
             });
     }
 
     /**
-     * Is sending supported? false if the environment does not support sending
-     * with fetch.
+     * Is sending supported?
+     * Always true for `FetchTool` because `scratchFetch` ponyfills `fetch` if necessary.
      * @returns {boolean} Is sending supported?
      */
     get isSendSupported () {
-        return typeof fetch !== 'undefined';
+        return true;
     }
 
     /**
      * Send data to a server with fetch.
-     * @param {Request} reqConfig - Request configuration for data to send.
+     * @param {ScratchSendRequest} reqConfig - Request configuration for data to send.
      * @returns {Promise.<string>} Server returned metadata.
      */
     send ({url, withCredentials = false, ...options}) {
-        return fetch(url, Object.assign({
+        return scratchFetch(url, Object.assign({
             credentials: withCredentials ? 'include' : 'omit'
         }, options))
             .then(response => {

package/src/FetchWorkerTool.js

@@ -1,3 +1,5 @@
+const {applyMetadata} = require('./scratchFetch');
+
 /**
  * Get and send assets with a worker that uses fetch.
  */
@@ -13,13 +15,13 @@ class PrivateFetchWorkerTool {
 
         /**
          * A possible error occurred standing up the worker.
-         * @type {!Error}
+         * @type {Error?}
          */
         this._supportError = null;
 
         /**
          * The worker that runs fetch and returns data for us.
-         * @type {!Worker}
+         * @type {Worker?}
          */
         this.worker = null;
 
@@ -34,9 +36,9 @@ class PrivateFetchWorkerTool {
                 // eslint-disable-next-line global-require
                 const FetchWorker = require('worker-loader?{"inline":true,"fallback":true}!./FetchWorkerTool.worker');
 
-                this.worker = new FetchWorker();
+                const worker = new FetchWorker();
 
-                this.worker.addEventListener('message', ({data}) => {
+                worker.addEventListener('message', ({data}) => {
                     if (data.support) {
                         this._workerSupport = data.support;
                         return;
@@ -52,6 +54,8 @@ class PrivateFetchWorkerTool {
                         }
                     }
                 });
+
+                this.worker = worker;
             }
         } catch (error) {
             this._supportError = error;
@@ -78,17 +82,20 @@ class PrivateFetchWorkerTool {
      * Request data from a server with a worker using fetch.
      * @param {{url:string}} reqConfig - Request configuration for data to get.
      * @param {{method:string}} options - Additional options to configure fetch.
-     * @returns {Promise.<Buffer>} Resolve to Buffer of data from server.
+     * @returns {Promise.<Buffer|Uint8Array|null>} Resolve to Buffer of data from server.
      */
     get ({url, ...options}) {
         return new Promise((resolve, reject) => {
             // TODO: Use a Scratch standard ID generator ...
             const id = Math.random().toString(16)
                 .substring(2);
+            const augmentedOptions = applyMetadata(
+                Object.assign({method: 'GET'}, options)
+            );
             this.worker.postMessage({
                 id,
                 url,
-                options: Object.assign({method: 'GET'}, options)
+                options: augmentedOptions
             });
             this.jobs[id] = {
                 id,
@@ -153,7 +160,7 @@ class PublicFetchWorkerTool {
     /**
      * Request data from a server with a worker that uses fetch.
      * @param {{url:string}} reqConfig - Request configuration for data to get.
-     * @returns {Promise.<Buffer>} Resolve to Buffer of data from server.
+     * @returns {Promise.<Buffer|Uint8Array|null>} Resolve to Buffer of data from server.
      */
     get (reqConfig) {
         return this.inner.get(reqConfig);

package/src/FetchWorkerTool.worker.js

@@ -1,5 +1,7 @@
 /* eslint-env worker */
 
+const crossFetch = require('cross-fetch').default;
+
 let jobsActive = 0;
 const complete = [];
 
@@ -48,7 +50,7 @@ const onMessage = ({data: job}) => {
 
     jobsActive++;
 
-    fetch(job.url, job.options)
+    crossFetch(job.url, job.options)
         .then(result => {
             if (result.ok) return result.arrayBuffer();
             if (result.status === 404) return null;
@@ -59,12 +61,6 @@ const onMessage = ({data: job}) => {
         .then(() => jobsActive--);
 };
 
-if (self.fetch) {
-    postMessage({support: {fetch: true}});
-    self.addEventListener('message', onMessage);
-} else {
-    postMessage({support: {fetch: false}});
-    self.addEventListener('message', ({data: job}) => {
-        postMessage([{id: job.id, error: 'fetch is unavailable'}]);
-    });
-}
+// crossFetch means "fetch" is now always supported
+postMessage({support: {fetch: true}});
+self.addEventListener('message', onMessage);

package/src/scratchFetch.js

@@ -0,0 +1,86 @@
+const {fetch, Headers} = require('cross-fetch');
+
+/**
+ * Metadata header names
+ * @enum {string} The enum value is the name of the associated header.
+ * @readonly
+ */
+const RequestMetadata = {
+    /** The ID of the project associated with this request */
+    ProjectId: 'X-ProjectId',
+    /** The ID of the project run associated with this request */
+    RunId: 'X-RunId'
+};
+
+/**
+ * Metadata for requests
+ * @type {Map<string, string>}
+ */
+const metadata = new Map();
+
+/**
+ * Non-destructively merge any metadata state (if any) with the provided options object (if any).
+ * If there is metadata state but no options object is provided, make a new object.
+ * If there is no metadata state, return the provided options parameter without modification.
+ * If there is metadata and an options object is provided, modify a copy and return it.
+ * Headers in the provided options object may override headers generated from metadata state.
+ * @param {RequestInit} [options] The initial request options. May be null or undefined.
+ * @returns {RequestInit|undefined} the provided options parameter without modification, or a new options object.
+ */
+const applyMetadata = options => {
+    if (metadata.size > 0) {
+        const augmentedOptions = Object.assign({}, options);
+        augmentedOptions.headers = new Headers(Array.from(metadata));
+        if (options && options.headers) {
+            const overrideHeaders =
+                options.headers instanceof Headers ? options.headers : new Headers(options.headers);
+            for (const [name, value] of overrideHeaders.entries()) {
+                augmentedOptions.headers.set(name, value);
+            }
+        }
+        return augmentedOptions;
+    }
+    return options;
+};
+
+/**
+ * Make a network request.
+ * This is a wrapper for the global fetch method, adding some Scratch-specific functionality.
+ * @param {RequestInfo|URL} resource The resource to fetch.
+ * @param {RequestInit} options Optional object containing custom settings for this request.
+ * @see {@link https://developer.mozilla.org/docs/Web/API/fetch} for more about the fetch API.
+ * @returns {Promise<Response>} A promise for the response to the request.
+ */
+const scratchFetch = (resource, options) => {
+    const augmentedOptions = applyMetadata(options);
+    return fetch(resource, augmentedOptions);
+};
+
+/**
+ * Set the value of a named request metadata item.
+ * Setting the value to `null` or `undefined` will NOT remove the item.
+ * Use `unsetMetadata` for that.
+ * @param {RequestMetadata} name The name of the metadata item to set.
+ * @param {any} value The value to set (will be converted to a string).
+ */
+const setMetadata = (name, value) => {
+    metadata.set(name, value);
+};
+
+/**
+ * Remove a named request metadata item.
+ * @param {RequestMetadata} name The name of the metadata item to remove.
+ */
+const unsetMetadata = name => {
+    metadata.delete(name);
+};
+
+module.exports = {
+    default: scratchFetch,
+
+    RequestMetadata,
+    applyMetadata,
+    scratchFetch,
+    setMetadata,
+    unsetMetadata
+};

package/test/integration/download-known-assets.js

@@ -20,16 +20,17 @@ test('constructor', t => {
  * @property {DataFormat} [ext] - Optional: the asset's data format / file extension.
  */
 const testAssets = [
-    {
-        type: storage.AssetType.Project,
-        id: '117504922',
-        md5: null // don't check MD5 for project without revision ID
-    },
-    {
-        type: storage.AssetType.Project,
-        id: '117504922.d6ae1ffb76f2bc83421cd3f40fc4fd57',
-        md5: '1225460702e149727de28bff4cfd9e23'
-    },
+    // TODO: mock project download, since we can no longer download projects directly
+    // {
+    //     type: storage.AssetType.Project,
+    //     id: '117504922',
+    //     md5: null // don't check MD5 for project without revision ID
+    // },
+    // {
+    //     type: storage.AssetType.Project,
+    //     id: '117504922.d6ae1ffb76f2bc83421cd3f40fc4fd57',
+    //     md5: '1225460702e149727de28bff4cfd9e23'
+    // },
     {
         type: storage.AssetType.ImageVector,
         id: 'f88bf1935daea28f8ca098462a31dbb0', // cat1-a
@@ -65,9 +66,9 @@ const testAssets = [
     }
 ];
 
-test('addWebSource', t => {
+test('addWebStore', t => {
     t.doesNotThrow(() => {
-        storage.addWebSource(
+        storage.addWebStore(
             [storage.AssetType.Project],
             asset => {
                 const idParts = asset.assetId.split('.');
@@ -77,7 +78,7 @@ test('addWebSource', t => {
             });
     });
     t.doesNotThrow(() => {
-        storage.addWebSource(
+        storage.addWebStore(
             [storage.AssetType.ImageVector, storage.AssetType.ImageBitmap, storage.AssetType.Sound],
             asset => `https://cdn.assets.scratch.mit.edu/internalapi/asset/${asset.assetId}.${asset.dataFormat}/get/`
         );
@@ -86,29 +87,25 @@ test('addWebSource', t => {
 });
 
 test('load', t => {
-    const promises = [];
-    const checkAsset = (assetInfo, asset) => {
+    const assetChecks = testAssets.map(async assetInfo => {
+        const asset = await storage.load(assetInfo.type, assetInfo.id, assetInfo.ext)
+            .catch(e => {
+                // tap's output isn't great if we just let it catch the unhandled promise rejection
+                // wrapping it like this makes a failure much easier to read in the test output
+                throw new Error(`failed to load ${assetInfo.type.name} asset with id=${assetInfo.id} (e=${e})`);
+            });
         t.type(asset, storage.Asset);
-        t.strictEqual(asset.assetId, assetInfo.id);
-        t.strictEqual(asset.assetType, assetInfo.type);
+        t.equal(asset.assetId, assetInfo.id);
+        t.equal(asset.assetType, assetInfo.type);
         t.ok(asset.data.length);
 
         // Web assets should come back as clean
-        t.true(asset.clean);
+        t.ok(asset.clean);
 
         if (assetInfo.md5) {
-            t.strictEqual(md5(asset.data), assetInfo.md5);
+            t.equal(md5(asset.data), assetInfo.md5);
         }
-    };
-    for (let i = 0; i < testAssets.length; ++i) {
-        const assetInfo = testAssets[i];
-
-        let promise = storage.load(assetInfo.type, assetInfo.id, assetInfo.ext);
-        t.type(promise, 'Promise');
-
-        promise = promise.then(asset => checkAsset(assetInfo, asset));
-        promises.push(promise);
-    }
+    });
 
-    return Promise.all(promises);
+    return Promise.all(assetChecks);
 });

package/test/mocks/mockFetch.js

@@ -0,0 +1,64 @@
+const TextEncoder = require('util').TextEncoder;
+const {Headers} = require('cross-fetch');
+
+const successText = 'successful response';
+
+/**
+ * @typedef MockFetchResponse The Response-like object returned by mockFetch.
+ * @property {boolean} ok True if the simulated request was successful, false otherwise.
+ * @property {number} status The HTTP status code of the simulated request.
+ * @property {() => Promise<string>} [text] A success string if the simulated request succeeded, undefined otherwise.
+ * @property {() => Promise<Uint8Array>} [arrayBuffer] Same as `text`, but encoded with UTF-8 if present.
+ */
+
+/**
+ * @typedef {RequestInit & {mockFetchTestData: MockFetchTestData}} MockFetchRequestInit
+ */
+
+/**
+ * @typedef MockFetchTestData
+ * @property {Headers} [headers] A Headers object initialized with the header info received by mockFetch.
+ * @property {Number} [headersCount] The number of headers in the 'headers' property.
+ */
+
+/**
+ * Mock the 'fetch' method from browsers.
+ * @param {RequestInfo|URL} resource The (mock) resource to fetch, which will determine the response.
+ * @param {MockFetchRequestInit} [options] Optional object containing custom settings for this request.
+ * @returns {Promise<MockFetchResponse>} A promise for a Response-like object. Does not fully implement Response.
+ */
+const mockFetch = (resource, options) => {
+    /** @type MockFetchResponse */
+    const results = {
+        ok: false,
+        status: 0
+    };
+    if (options?.mockFetchTestData) {
+        options.mockFetchTestData.headers = new Headers(options.headers);
+        options.mockFetchTestData.headersCount = Array.from(options.mockFetchTestData.headers).length;
+    }
+    switch (resource) {
+    case '200':
+        results.ok = true;
+        results.status = 200;
+        results.text = () => Promise.resolve(successText);
+        results.arrayBuffer = () => Promise.resolve(new TextEncoder().encode(successText));
+        break;
+    case '404':
+        results.ok = false;
+        results.status = 404;
+        break;
+    case '500':
+        results.ok = false;
+        results.status = 500;
+        break;
+    default:
+        throw new Error('unimplemented');
+    }
+    return Promise.resolve(results);
+};
+
+module.exports = {
+    mockFetch,
+    successText
+};

package/test/unit/fetch-tool.js

@@ -1,77 +1,60 @@
-const test = require('tap').test;
-const TextEncoder = require('util').TextEncoder;
+const tap = require('tap');
 const TextDecoder = require('util').TextDecoder;
 
-const FetchTool = require('../../src/FetchTool');
-
-test('send success returns response.text()', t => {
-    global.fetch = () => Promise.resolve({
-        ok: true,
-        text: () => Promise.resolve('successful response')
-    });
+const {mockFetch, successText} = require('../mocks/mockFetch.js');
+
+/**
+ * This is the real FetchTool, but the 'cross-fetch' module has been replaced with the mockFetch function.
+ * @type {typeof import('../../src/FetchTool')}
+ */
+const FetchTool = tap.mock('../../src/FetchTool', {
+    'cross-fetch': {
+        default: mockFetch,
+        fetch: mockFetch
+    }
+});
 
+tap.test('send success returns response.text()', t => {
     const tool = new FetchTool();
-    
+
     return t.resolves(
-        tool.send('url').then(result => {
-            t.equal(result, 'successful response');
+        tool.send({url: '200'}).then(result => {
+            t.equal(result, successText);
         })
     );
 });
 
-test('send failure returns response.status', t => {
-    global.fetch = () => Promise.resolve({
-        ok: false,
-        status: 500
-    });
-
+tap.test('send failure returns response.status', t => {
     const tool = new FetchTool();
 
-    return t.rejects(tool.send('url'), 500);
+    return t.rejects(tool.send({url: '500'}), 500);
 });
 
-test('get success returns Uint8Array.body(response.arrayBuffer())', t => {
-    const text = 'successful response';
+tap.test('get success returns Uint8Array.body(response.arrayBuffer())', t => {
     const encoding = 'utf-8';
-    const encoded = new TextEncoder().encode(text);
     const decoder = new TextDecoder(encoding);
 
-    global.fetch = () => Promise.resolve({
-        ok: true,
-        arrayBuffer: () => Promise.resolve(encoded.buffer)
-    });
-
     const tool = new FetchTool();
-    
+
     return t.resolves(
-        tool.get({url: 'url'}).then(result => {
-            t.equal(decoder.decode(result), text);
+        tool.get({url: '200'}).then(result => {
+            t.equal(decoder.decode(result), successText);
         })
     );
 });
 
-test('get with 404 response returns null data', t => {
-    global.fetch = () => Promise.resolve({
-        ok: false,
-        status: 404
-    });
-
+tap.test('get with 404 response returns null data', t => {
     const tool = new FetchTool();
 
     return t.resolves(
-        tool.get('url').then(result => {
+        tool.get({url: '404'}).then(result => {
             t.equal(result, null);
         })
     );
 });
 
-test('get failure returns response.status', t => {
-    global.fetch = () => Promise.resolve({
-        ok: false,
-        status: 500
-    });
-
+tap.test('get failure returns response.status', t => {
     const tool = new FetchTool();
 
-    return t.rejects(tool.get({url: 'url'}), 500);
+    return t.rejects(tool.get({url: '500'}), 500);
 });