scratch-storage 5.0.11 → 6.0.0

package/src/{scratchFetch.js → scratchFetch.ts}

@@ -1,29 +1,44 @@
-const crossFetch = require('cross-fetch');
+import {type QueueOptions} from '@scratch/task-herder';
+import {hostQueueManager} from './HostQueues';
+
+export const Headers = globalThis.Headers;
 
 /**
- * Metadata header names
- * @enum {string} The enum value is the name of the associated header.
- * @readonly
+ * Metadata header names.
+ * The enum value is the name of the associated header.
  */
-const RequestMetadata = {
+export enum RequestMetadata {
     /** The ID of the project associated with this request */
-    ProjectId: 'X-Project-ID',
+    ProjectId = 'X-Project-ID',
     /** The ID of the project run associated with this request */
-    RunId: 'X-Run-ID'
+    RunId = 'X-Run-ID'
+}
+
+export type ScratchFetchOptions = {
+    /**
+     * The name of the queue to use for this request.
+     * If absent, the hostname of the requested URL will be used as the queue name.
+     * This is a Scratch-specific extension to the standard RequestInit type.
+     */
+    queueName?: string;
+
+    /**
+     * The options to use when creating the queue for this request.
+     * Ignored if a queue with the specified name already exists.
+     */
+    queueOptions?: QueueOptions;
 };
 
 /**
- * Metadata headers for requests
- * @type {Headers}
+ * Metadata headers for requests.
  */
-const metadata = new crossFetch.Headers();
+const metadata = new Headers();
 
 /**
  * Check if there is any metadata to apply.
  * @returns {boolean} true if `metadata` has contents, or false if it is empty.
  */
-const hasMetadata = () => {
-    /* global self */
+export const hasMetadata = (): boolean => {
     const searchParams = (
         typeof self !== 'undefined' &&
         self &&
@@ -51,16 +66,16 @@ const hasMetadata = () => {
  * @param {RequestInit} [options] The initial request options. May be null or undefined.
  * @returns {RequestInit|undefined} the provided options parameter without modification, or a new options object.
  */
-const applyMetadata = options => {
+export const applyMetadata = (options?: globalThis.RequestInit): globalThis.RequestInit | undefined => {
     if (hasMetadata()) {
         const augmentedOptions = Object.assign({}, options);
-        augmentedOptions.headers = new crossFetch.Headers(metadata);
+        augmentedOptions.headers = new Headers(metadata);
         if (options && options.headers) {
             // the Fetch spec says options.headers could be:
             // "A Headers object, an object literal, or an array of two-item arrays to set request's headers."
             // turn it into a Headers object to be sure of how to interact with it
-            const overrideHeaders = options.headers instanceof crossFetch.Headers ?
-                options.headers : new crossFetch.Headers(options.headers);
+            const overrideHeaders = options.headers instanceof Headers ?
+                options.headers : new Headers(options.headers);
             for (const [name, value] of overrideHeaders.entries()) {
                 augmentedOptions.headers.set(name, value);
             }
@@ -74,13 +89,27 @@ const applyMetadata = options => {
  * Make a network request.
  * This is a wrapper for the global fetch method, adding some Scratch-specific functionality.
  * @param {RequestInfo|URL} resource The resource to fetch.
- * @param {RequestInit} options Optional object containing custom settings for this request.
+ * @param {RequestInit} [requestOptions] Optional object containing custom settings for this request.
+ * @param {ScratchFetchOptions} [scratchOptions] Optional Scratch-specific settings for this request.
  * @see {@link https://developer.mozilla.org/docs/Web/API/fetch} for more about the fetch API.
  * @returns {Promise<Response>} A promise for the response to the request.
  */
-const scratchFetch = (resource, options) => {
-    const augmentedOptions = applyMetadata(options);
-    return crossFetch(resource, augmentedOptions);
+export const scratchFetch = (
+    resource: RequestInfo | URL,
+    requestOptions?: globalThis.RequestInit,
+    scratchOptions?: ScratchFetchOptions
+): Promise<Response> => {
+    requestOptions = applyMetadata(requestOptions);
+
+    let queueName = scratchOptions?.queueName;
+    if (!queueName) {
+        // Normalize resource to a Request object. The `fetch` call will do this anyway, so it's not much extra work,
+        // but it guarantees availability of the URL for queue naming.
+        resource = new Request(resource, requestOptions);
+        queueName = new URL(resource.url).hostname;
+    }
+    const queue = hostQueueManager.getOrCreate(queueName, scratchOptions?.queueOptions);
+    return queue.do(() => fetch(resource, requestOptions));
 };
 
 /**
@@ -90,7 +119,7 @@ const scratchFetch = (resource, options) => {
  * @param {RequestMetadata} name The name of the metadata item to set.
  * @param {any} value The value to set (will be converted to a string).
  */
-const setMetadata = (name, value) => {
+export const setMetadata = (name: RequestMetadata, value: any): void => {
     metadata.set(name, value);
 };
 
@@ -98,7 +127,7 @@ const setMetadata = (name, value) => {
  * Remove a named request metadata item.
  * @param {RequestMetadata} name The name of the metadata item to remove.
  */
-const unsetMetadata = name => {
+export const unsetMetadata = (name: RequestMetadata): void => {
     metadata.delete(name);
 };
 
@@ -106,16 +135,6 @@ const unsetMetadata = name => {
  * Retrieve a named request metadata item.
  * Only for use in tests. At the time of writing, used in scratch-vm tests.
  * @param {RequestMetadata} name The name of the metadata item to retrieve.
- * @returns {any} value The value of the metadata item, or `undefined` if it was not found.
+ * @returns {string|null} The value of the metadata item, or `null` if it was not found.
  */
-const getMetadata = name => metadata.get(name);
-
-module.exports = {
-    Headers: crossFetch.Headers,
-    RequestMetadata,
-    applyMetadata,
-    scratchFetch,
-    setMetadata,
-    unsetMetadata,
-    getMetadata
-};
+export const getMetadata = (name: RequestMetadata): string | null => metadata.get(name);

package/src/types.d.ts

@@ -1,3 +1,4 @@
-declare module '*.png?arrayBuffer';
-declare module '*.wav?arrayBuffer';
-declare module '*.svg?arrayBuffer';
+declare module '*?arrayBuffer' {
+    const value: ArrayBuffer;
+    export default value;
+}

package/test/build/api.test.js

@@ -0,0 +1,60 @@
+/**
+ * @file Tests the build output to verify the general parts of the public API.
+ */
+
+const ScratchStorageModule = require('../../dist/node/scratch-storage.js');
+
+/** @type {ScratchStorageModule.ScratchStorage} */
+let storage;
+
+beforeEach(() => {
+    const {ScratchStorage} = ScratchStorageModule;
+    storage = new ScratchStorage();
+});
+
+test('constructor', () => {
+    const {ScratchStorage} = ScratchStorageModule;
+    expect(storage).toBeInstanceOf(ScratchStorage);
+});
+
+test('DataFormat', () => {
+    const {DataFormat} = ScratchStorageModule;
+    expect(DataFormat).toBeDefined();
+    expect(DataFormat.JPG).toBe('jpg');
+    expect(DataFormat.JSON).toBe('json');
+    expect(DataFormat.MP3).toBe('mp3');
+    expect(DataFormat.PNG).toBe('png');
+    expect(DataFormat.SB2).toBe('sb2');
+    expect(DataFormat.SB3).toBe('sb3');
+    expect(DataFormat.SVG).toBe('svg');
+    expect(DataFormat.WAV).toBe('wav');
+});
+
+test('AssetType', () => {
+    const {AssetType, DataFormat} = ScratchStorageModule;
+    expect(AssetType).toBeDefined();
+    expect(AssetType.ImageBitmap.contentType).toBe('image/png');
+    expect(AssetType.ImageVector.contentType).toBe('image/svg+xml');
+    expect(AssetType.Project.runtimeFormat).toBe(DataFormat.JSON);
+    expect(AssetType.Sound.immutable).toBe(true);
+    expect(AssetType.Sprite.name).toBe('Sprite');
+});
+
+test('Asset', () => {
+    const {Asset, AssetType} = ScratchStorageModule;
+    expect(Asset).toBeDefined();
+    const asset = new Asset(
+        AssetType.ImageVector,
+        'some-hash'
+    );
+    expect(asset).toBeInstanceOf(Asset);
+    expect(asset.dataFormat).toBe('svg');
+    expect(asset.assetId).toBe('some-hash');
+});
+
+test('Helper', () => {
+    const {Helper} = ScratchStorageModule;
+    expect(Helper).toBeDefined();
+    const helper = new Helper();
+    expect(helper).toBeInstanceOf(Helper);
+});

package/test/build/scratchFetch.test.js

@@ -0,0 +1,40 @@
+/**
+ * @file Tests the build output to verify the scratchFetch portion of the public API.
+ */
+
+const ScratchStorageModule = require('../../dist/node/scratch-storage.js');
+
+/** @type {ScratchStorageModule.ScratchStorage} */
+let storage;
+
+beforeEach(() => {
+    const {ScratchStorage} = ScratchStorageModule;
+    storage = new ScratchStorage();
+});
+
+test('scratchFetch accessor', () => {
+    expect(storage.scratchFetch).toBeDefined();
+});
+
+test('Headers', () => {
+    expect(storage.scratchFetch).toBeDefined();
+    const headers = new storage.scratchFetch.Headers();
+    expect(headers).toBeInstanceOf(storage.scratchFetch.Headers);
+});
+
+test('RequestMetadata enum', () => {
+    expect(storage.scratchFetch.RequestMetadata).toBeDefined();
+    expect(typeof storage.scratchFetch.RequestMetadata.ProjectId).toBe('string');
+    expect(typeof storage.scratchFetch.RequestMetadata.RunId).toBe('string');
+});
+
+test('scratchFetch function', () => {
+    expect(typeof storage.scratchFetch.scratchFetch).toBe('function');
+});
+
+test('metadata functions', () => {
+    expect(typeof storage.scratchFetch.applyMetadata).toBe('function');
+    expect(typeof storage.scratchFetch.setMetadata).toBe('function');
+    expect(typeof storage.scratchFetch.unsetMetadata).toBe('function');
+    expect(typeof storage.scratchFetch.getMetadata).toBe('function');
+});

package/test/fixtures/known-assets.js

@@ -3,6 +3,16 @@ const path = require('path');
 
 const md5 = require('js-md5');
 
+/**
+ * @typedef {object} KnownAsset
+ * @property {Buffer} content - The content of the asset.
+ * @property {string} hash - The MD5 hash of the asset content.
+ */
+
+/**
+ * @typedef {{[id: string]: KnownAsset}} KnownAssetCollection
+ */
+
 const projects = [
     '117504922'
 ];
@@ -15,6 +25,11 @@ const assets = [
     'fe5e3566965f9de793beeffce377d054.jpg'
 ];
 
+/**
+ * Load a file from disk, then return its content and hash.
+ * @param {string} filename - The file to load
+ * @returns {KnownAsset} The loaded asset
+ */
 const loadSomething = filename => {
     const fullPath = path.join(__dirname, 'assets', filename);
     const content = fs.readFileSync(fullPath);
@@ -25,6 +40,11 @@ const loadSomething = filename => {
     };
 };
 
+/**
+ * Load a project from disk, ensure it's valid JSON, then return its content and hash.
+ * @param {string} id - The project ID
+ * @returns {KnownAsset} The loaded project asset
+ */
 const loadProject = id => {
     const filename = `${id}.json`;
     const result = loadSomething(filename);
@@ -35,6 +55,11 @@ const loadProject = id => {
     return result;
 };
 
+/**
+ * Load an asset from disk, ensuring its hash matches its filename.
+ * @param {string} filename - The file to load
+ * @returns {KnownAsset} The loaded asset
+ */
 const loadAsset = filename => {
     const result = loadSomething(filename);
 
@@ -46,15 +71,18 @@ const loadAsset = filename => {
     return result;
 };
 
+/**
+ * @type {KnownAssetCollection}
+ */
 const knownAssets = Object.assign({},
     projects.reduce((bag, id) => {
         bag[id] = loadProject(id);
         return bag;
-    }, {}),
+    }, /** @type {KnownAssetCollection} */ ({})),
     assets.reduce((bag, filename) => {
         bag[filename] = loadAsset(filename);
         return bag;
-    }, {})
+    }, /** @type {KnownAssetCollection} */ ({}))
 );
 
 module.exports = knownAssets;

package/test/{__mocks__/cross-fetch.js → fixtures/mockFetch.js}

@@ -1,8 +1,15 @@
+/**
+ * Mock implementation of the fetch function for testing.
+ * Since `fetch` is a global, Jest will not automatically mock it from `__mocks__`.
+ *
+ * // In your test setup file or at the top of your test files:
+ * global.fetch = require('../mocks/fetch').default;
+ */
+
 const TextEncoder = require('util').TextEncoder;
-const crossFetch = jest.requireActual('cross-fetch');
 const knownAssets = require('../fixtures/known-assets.js');
 
-const Headers = crossFetch.Headers;
+const Headers = global.Headers;
 const successText = 'successful response';
 
 /**
@@ -30,7 +37,7 @@ const successText = 'successful response';
  * @returns {Promise<MockFetchResponse>} A promise for a Response-like object. Does not fully implement Response.
  */
 const mockFetch = (resource, options) => {
-    /** @type MockFetchResponse */
+    /** @type {MockFetchResponse} */
     const results = {
         ok: false,
         status: 0
@@ -39,14 +46,15 @@ const mockFetch = (resource, options) => {
         options.mockFetchTestData.headers = new Headers(options.headers);
         options.mockFetchTestData.headersCount = Array.from(options.mockFetchTestData.headers).length;
     }
-
-    const assetInfo = knownAssets[resource];
+    const request = new Request(resource, options);
+    const path = new URL(request.url).pathname.slice(1); // remove leading '/'
+    const assetInfo = knownAssets[path];
     if (assetInfo) {
         results.ok = true;
         results.status = 200;
         results.arrayBuffer = () => Promise.resolve(assetInfo.content);
     } else {
-        switch (resource) {
+        switch (path) {
         case '200':
             results.ok = true;
             results.status = 200;
@@ -68,14 +76,7 @@ const mockFetch = (resource, options) => {
     return Promise.resolve(results);
 };
 
-// Mimic the cross-fetch module, but replace its `fetch` with `mockFetch` and add a few extras
-
-module.exports = exports = mockFetch;
-exports.fetch = mockFetch;
-exports.Headers = crossFetch.Headers;
-exports.Request = crossFetch.Request;
-exports.Response = crossFetch.Response;
-exports.successText = successText;
-
-// Needed for TypeScript consumers without esModuleInterop.
-exports.default = mockFetch;
+module.exports = {
+    fetch: mockFetch,
+    successText
+};

package/test/integration/download-known-assets.test.js

@@ -1,7 +1,10 @@
 const md5 = require('js-md5');
 
+const mockFetch = require('../fixtures/mockFetch.js');
 const ScratchStorage = require('../../src/index').ScratchStorage;
 
+jest.spyOn(global, 'fetch').mockImplementation(mockFetch.fetch);
+
 test('constructor', () => {
     const storage = new ScratchStorage();
     expect(storage).toBeInstanceOf(ScratchStorage);
@@ -68,15 +71,15 @@ const getTestAssets = storage => [
 ];
 
 const addWebStores = storage => {
-    // these `asset => ...` callbacks generate values specifically for the cross-fetch mock
+    // these `asset => ...` callbacks generate values specifically for the fetch mock
     // in the real world they would generate proper URIs
     storage.addWebStore(
         [storage.AssetType.Project],
-        asset => asset.assetId,
+        asset => `http://example.com/${asset.assetId}`,
         null, null);
     storage.addWebStore(
         [storage.AssetType.ImageVector, storage.AssetType.ImageBitmap, storage.AssetType.Sound],
-        asset => `${asset.assetId}.${asset.dataFormat}`,
+        asset => `http://example.com/${asset.assetId}.${asset.dataFormat}`,
         null, null
     );
 };

package/test/unit/fetch-tool.test.js

@@ -1,13 +1,14 @@
 const TextDecoder = require('util').TextDecoder;
 
-jest.mock('cross-fetch');
-const mockFetch = require('cross-fetch');
+const mockFetch = require('../fixtures/mockFetch.js');
+jest.spyOn(global, 'fetch').mockImplementation(mockFetch.fetch);
+
 const {FetchTool} = require('../../src/FetchTool');
 
 test('send success returns response.text()', async () => {
     const tool = new FetchTool();
 
-    const result = await tool.send({url: '200'});
+    const result = await tool.send({url: 'http://example.com/200'});
     expect(result).toBe(mockFetch.successText);
 });
 
@@ -17,7 +18,7 @@ test('send failure returns response.status', async () => {
     const catcher = jest.fn();
 
     try {
-        await tool.send({url: '500'});
+        await tool.send({url: 'http://example.com/500'});
     } catch (e) {
         catcher(e);
     }
@@ -31,14 +32,14 @@ test('get success returns Uint8Array.body(response.arrayBuffer())', async () =>
 
     const tool = new FetchTool();
 
-    const result = await tool.get({url: '200'});
+    const result = await tool.get({url: 'http://example.com/200'});
     expect(decoder.decode(result)).toBe(mockFetch.successText);
 });
 
 test('get with 404 response returns null data', async () => {
     const tool = new FetchTool();
 
-    const result = await tool.get({url: '404'});
+    const result = await tool.get({url: 'http://example.com/404'});
     expect(result).toBeNull();
 });
 
@@ -47,7 +48,7 @@ test('get failure returns response.status', async () => {
     const catcher = jest.fn();
 
     try {
-        await tool.get({url: '500'});
+        await tool.get({url: 'http://example.com/500'});
     } catch (e) {
         catcher(e);
     }

package/test/unit/metadata.test.js

@@ -1,4 +1,5 @@
-jest.mock('cross-fetch');
+const mockFetch = require('../fixtures/mockFetch.js');
+jest.spyOn(global, 'fetch').mockImplementation(mockFetch.fetch);
 
 beforeEach(() => {
     // reset the metadata container to ensure the tests don't interfere with each other
@@ -17,7 +18,7 @@ test('get without metadata', async () => {
     const tool = new FetchTool();
 
     const mockFetchTestData = {};
-    const result = await tool.get({url: '200', mockFetchTestData});
+    const result = await tool.get({url: 'http://example.com/200', mockFetchTestData});
 
     expect(result).toBeInstanceOf(Uint8Array);
     expect(mockFetchTestData.headers).toBeTruthy();
@@ -35,7 +36,7 @@ test('get with metadata', async () => {
     setMetadata(RequestMetadata.RunId, 5678);
 
     const mockFetchTestData = {};
-    const result = await tool.get({url: '200', mockFetchTestData});
+    const result = await tool.get({url: 'http://example.com/200', mockFetchTestData});
 
     expect(result).toBeInstanceOf(Uint8Array);
     expect(mockFetchTestData.headers).toBeTruthy();
@@ -49,7 +50,7 @@ test('send without metadata', async () => {
     const tool = new FetchTool();
 
     const mockFetchTestData = {};
-    const result = await tool.send({url: '200', mockFetchTestData});
+    const result = await tool.send({url: 'http://example.com/200', mockFetchTestData});
 
     expect(typeof result).toBe('string');
     expect(mockFetchTestData.headers).toBeTruthy();
@@ -67,7 +68,7 @@ test('send with metadata', async () => {
     setMetadata(RequestMetadata.RunId, 8765);
 
     const mockFetchTestData = {};
-    const result = await tool.send({url: '200', mockFetchTestData});
+    const result = await tool.send({url: 'http://example.com/200', mockFetchTestData});
 
     expect(typeof result).toBe('string');
     expect(mockFetchTestData.headers).toBeTruthy();
@@ -89,7 +90,7 @@ test('selectively delete metadata', async () => {
 
     const mockFetchTestData = {};
 
-    const result1 = await tool.send({url: '200', mockFetchTestData});
+    const result1 = await tool.send({url: 'http://example.com/200', mockFetchTestData});
     expect(typeof result1).toBe('string');
     expect(mockFetchTestData.headers).toBeTruthy();
 
@@ -100,7 +101,7 @@ test('selectively delete metadata', async () => {
     // remove the Project ID from metadata
     unsetMetadata(RequestMetadata.ProjectId);
 
-    const result2 = await tool.send({url: '200', mockFetchTestData});
+    const result2 = await tool.send({url: 'http://example.com/200', mockFetchTestData});
     expect(typeof result2).toBe('string');
     expect(mockFetchTestData.headers).toBeTruthy();
 
@@ -120,7 +121,7 @@ test('metadata has case-insensitive keys', async () => {
     const tool = new FetchTool();
 
     const mockFetchTestData = {};
-    await tool.get({url: '200', mockFetchTestData});
+    await tool.get({url: 'http://example.com/200', mockFetchTestData});
 
     expect(mockFetchTestData.headers).toBeTruthy();
     expect(mockFetchTestData.headersCount).toBe(1);

package/tsconfig.json

@@ -9,7 +9,6 @@
 
     /* Modules */
     "module": "Preserve",
-    "types": ["./src/types.d.ts"],
 
     /* Emit */
     "declaration": true,

package/tsconfig.test.json

@@ -5,6 +5,7 @@
     /* Visit https://aka.ms/tsconfig to read more about this file */
 
     /* Modules */
+    "isolatedModules": true, // for ts-jest
     "module": "CommonJS",
     "types": ["jest", "./src/types.d.ts"],
   }