scratch-storage 6.1.6 → 6.1.8

package/.github/workflows/ci-cd.yml

@@ -18,9 +18,9 @@ jobs:
     ci-cd:
         runs-on: ubuntu-latest
         steps:
-            - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+            - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
 
-            - uses: wagoid/commitlint-github-action@5ce82f5d814d4010519d15f0552aec4f17a1e1fe # v5
+            - uses: wagoid/commitlint-github-action@b948419dd99f3fd78a6548d48f94e3df7f6bf3ed # v6
               if: github.event_name == 'pull_request'
 
             - uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6

package/.github/workflows/commitlint.yml

@@ -8,5 +8,5 @@ jobs:
   commitlint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
-      - uses: wagoid/commitlint-github-action@5ce82f5d814d4010519d15f0552aec4f17a1e1fe # v5
+      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6
+      - uses: wagoid/commitlint-github-action@b948419dd99f3fd78a6548d48f94e3df7f6bf3ed # v6

package/dist/node/chunks/{fetch-worker.35b372c60d5dbced60fa.js → fetch-worker.1b76af6e6dc6197e4c43.js}

@@ -181,12 +181,13 @@ const AssetQueueOptions = {
  * Central registry of per-host queues.
  * Uses strict limits by default. Override these strict limits as needed for specific hosts.
  */
-const HostQueues_hostQueueManager = new QueueManager({
+const hostQueueManager = new QueueManager({
   burstLimit: 5,
   sustainRate: 1,
   concurrency: 1
 });
 ;// ./src/scratchFetch.ts
+/* unused harmony import specifier */ var scratchFetch_hostQueueManager;
 
 const Headers = globalThis.Headers;
 /**
@@ -264,7 +265,7 @@ const scratchFetch = (resource, requestOptions, scratchOptions) => {
     resource = new Request(resource, requestOptions);
     queueName = new URL(resource.url).hostname;
   }
-  const queue = HostQueues_hostQueueManager.getOrCreate(queueName, scratchOptions === null || scratchOptions === void 0 ? void 0 : scratchOptions.queueOptions);
+  const queue = hostQueueManager.getOrCreate(queueName, scratchOptions === null || scratchOptions === void 0 ? void 0 : scratchOptions.queueOptions);
   return queue.do(() => fetch(resource, requestOptions));
 };
 /**
@@ -278,7 +279,7 @@ const scratchFetch = (resource, requestOptions, scratchOptions) => {
  * @param overrides Optional overrides for the default QueueOptions for this specific queue.
  */
 const createQueue = (queueName, overrides) => {
-  hostQueueManager.create(queueName, overrides);
+  scratchFetch_hostQueueManager.create(queueName, overrides);
 };
 /**
  * Set the value of a named request metadata item.
@@ -393,4 +394,4 @@ self.addEventListener('message', onMessage);
 module.exports = __webpack_exports__;
 /******/ })()
 ;
-//# sourceMappingURL=fetch-worker.35b372c60d5dbced60fa.js.map
+//# sourceMappingURL=fetch-worker.1b76af6e6dc6197e4c43.js.map

package/dist/node/chunks/fetch-worker.1b76af6e6dc6197e4c43.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunks/fetch-worker.1b76af6e6dc6197e4c43.js","mappings":";;;;;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;;ACrJA;AAEA;;;;;;;;;;;;;AAaA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEA;;;;AAIA;AACA;AACA;AACA;AACA;;;AC/BA;AAEA;AAEA;;;;AAIA;AAAA;AACA;AACA;AACA;AACA;AACA;AAiBA;;;AAGA;AAEA;;;;AAIA;AACA;AAOA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEA;;;;;;;;;AASA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEA;;;;;;;;;AASA;AAKA;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEA;;;;;;;;;;AAUA;AACA;AACA;AAEA;;;;;;;AAOA;AACA;AACA;AAEA;;;;AAIA;AACA;AACA;AAEA;;;;;;AAMA;;ACzJA;AACA;AACA;AAEA;AACA;AACA;AACA;AACA;AACA;AAcA;AACA;AAEA;AAEA;;;;;;;AAOA;AACA;AACA;AACA;AACA;AACA;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAEA;;;;;AAKA;AAAA;AAAA;AAAA;AACA;AACA;AACA;AAEA;AAEA;AACA;AAAA;AAAA;AAEA;AAAA;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AAAA;AAAA;AAAA;AACA;AACA;AACA;AACA;AAEA;AACA;AAAA;AAAA;AAAA;AAAA;AACA","sources":["webpack://scratch-storage/./node_modules/@scratch/task-herder/dist/task-herder.js","webpack://scratch-storage/./src/HostQueues.ts","webpack://scratch-storage/./src/scratchFetch.ts","webpack://scratch-storage/./src/FetchWorkerTool.worker.ts"],"sourcesContent":["const CancelReason = {\n\tQueueCostLimitExceeded: \"Queue cost limit exceeded\",\n\tAborted: \"Task aborted\",\n\tCancel: \"Task cancelled\",\n\tTaskTooExpensive: \"Task cost exceeds maximum bucket size\"\n};\nfunction PromiseWithResolvers() {\n\tlet e, t;\n\treturn {\n\t\tpromise: new Promise((n, r) => {\n\t\t\te = n, t = r;\n\t\t}),\n\t\tresolve: e,\n\t\treject: t\n\t};\n}\nvar TaskRecord = class {\n\tcost;\n\tpromise;\n\trun;\n\tcancel;\n\tconstructor(e, n = {}) {\n\t\tthis.cost = n.cost ?? 1;\n\t\tlet { promise: r, resolve: i, reject: a } = PromiseWithResolvers();\n\t\tthis.promise = r, this.cancel = (e) => {\n\t\t\ta(e);\n\t\t}, this.run = async () => {\n\t\t\ttry {\n\t\t\t\ti(await e());\n\t\t\t} catch (e) {\n\t\t\t\ta(e);\n\t\t\t}\n\t\t};\n\t}\n}, TaskQueue = class {\n\tburstLimit;\n\tsustainRate;\n\tqueueCostLimit;\n\tconcurrencyLimit;\n\ttokenCount;\n\trunningTasks = 0;\n\tpendingTaskRecords = [];\n\tlastRefillTime = Date.now();\n\tonTaskAdded = PromiseWithResolvers().resolve;\n\tonTaskFinished = PromiseWithResolvers().resolve;\n\tconstructor(e) {\n\t\tthis.burstLimit = e.burstLimit, this.sustainRate = e.sustainRate, this.tokenCount = e.startingTokens ?? e.burstLimit, this.queueCostLimit = e.queueCostLimit ?? Infinity, this.concurrencyLimit = e.concurrency ?? 1, this.runTasks();\n\t}\n\tget length() {\n\t\treturn this.pendingTaskRecords.length;\n\t}\n\tget options() {\n\t\treturn {\n\t\t\tburstLimit: this.burstLimit,\n\t\t\tsustainRate: this.sustainRate,\n\t\t\tstartingTokens: this.tokenCount,\n\t\t\tqueueCostLimit: this.queueCostLimit,\n\t\t\tconcurrency: this.concurrencyLimit\n\t\t};\n\t}\n\tdo(t, r = {}) {\n\t\tlet i = new TaskRecord(t, r);\n\t\treturn i.cost > this.burstLimit ? Promise.reject(Error(CancelReason.TaskTooExpensive)) : this.queueCostLimit < Infinity && this.pendingTaskRecords.reduce((e, t) => e + t.cost, i.cost) > this.queueCostLimit ? Promise.reject(Error(CancelReason.QueueCostLimitExceeded)) : (this.pendingTaskRecords.push(i), r.signal?.addEventListener(\"abort\", () => {\n\t\t\tthis.cancel(i.promise, Error(CancelReason.Aborted));\n\t\t}), this.onTaskAdded(), i.promise);\n\t}\n\tcancel(t, n) {\n\t\tlet r = this.pendingTaskRecords.findIndex((e) => e.promise === t);\n\t\tif (r !== -1) {\n\t\t\tlet [t] = this.pendingTaskRecords.splice(r, 1);\n\t\t\treturn t.cancel(n ?? Error(CancelReason.Cancel)), !0;\n\t\t}\n\t\treturn !1;\n\t}\n\tcancelAll(t) {\n\t\tlet n = this.pendingTaskRecords;\n\t\treturn this.pendingTaskRecords = [], t ??= Error(CancelReason.Cancel), n.forEach((e) => {\n\t\t\te.cancel(t);\n\t\t}), n.length;\n\t}\n\trefillAndSpend(e) {\n\t\treturn this.refill(), this.spend(e);\n\t}\n\trefill() {\n\t\tlet e = Date.now(), t = e - this.lastRefillTime;\n\t\tif (t <= 0) return;\n\t\tthis.lastRefillTime = e;\n\t\tlet n = t / 1e3 * this.sustainRate;\n\t\tthis.tokenCount = Math.min(this.burstLimit, this.tokenCount + n);\n\t}\n\tspend(e) {\n\t\treturn this.tokenCount >= e ? (this.tokenCount -= e, !0) : !1;\n\t}\n\tasync runTasks() {\n\t\tfor (;;) {\n\t\t\tlet n = this.pendingTaskRecords.shift();\n\t\t\tif (!n) {\n\t\t\t\tlet { promise: e, resolve: n } = PromiseWithResolvers();\n\t\t\t\tthis.onTaskAdded = n, await e;\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (n.cost > this.burstLimit) {\n\t\t\t\tn.cancel(Error(CancelReason.TaskTooExpensive));\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (this.refillAndSpend(n.cost)) {\n\t\t\t\tif (this.runningTasks >= this.concurrencyLimit) {\n\t\t\t\t\tlet { promise: e, resolve: n } = PromiseWithResolvers();\n\t\t\t\t\tthis.onTaskFinished = n, await e;\n\t\t\t\t}\n\t\t\t\tthis.runTask(n);\n\t\t\t} else {\n\t\t\t\tthis.pendingTaskRecords.unshift(n);\n\t\t\t\tlet e = Math.max(n.cost - this.tokenCount, 0), t = Math.ceil(1e3 * e / this.sustainRate);\n\t\t\t\tawait new Promise((e) => setTimeout(e, t));\n\t\t\t}\n\t\t}\n\t}\n\tasync runTask(e) {\n\t\tthis.runningTasks++;\n\t\ttry {\n\t\t\tawait e.run();\n\t\t} finally {\n\t\t\tthis.runningTasks--, this.onTaskFinished();\n\t\t}\n\t}\n}, QueueManager = class {\n\tqueues;\n\tdefaultOptions;\n\tconstructor(e, t) {\n\t\tthis.queues = new Map(t), this.defaultOptions = e;\n\t}\n\tcreate(e, t = {}) {\n\t\tlet n = new TaskQueue({\n\t\t\t...this.defaultOptions,\n\t\t\t...t\n\t\t});\n\t\treturn this.queues.set(e, n), n;\n\t}\n\tget(e) {\n\t\treturn this.queues.get(e);\n\t}\n\tgetOrCreate(e, t = {}) {\n\t\treturn this.get(e) ?? this.create(e, t);\n\t}\n\toptions() {\n\t\treturn { ...this.defaultOptions };\n\t}\n};\nexport { CancelReason, QueueManager, TaskQueue };\n","import {QueueManager, type QueueOptions} from '@scratch/task-herder';\n\n/**\n * @summary A set of generous limits, for things like downloading assets from CDN.\n * @description\n * In practice, these limits seem to lead to slightly better performance than no limits at all, mostly due to the\n * concurrency limit. For example, on my development computer & my relatively fast residential connection, a\n * concurrency limit of 4 loads a particular test project in 21 seconds, as opposed to 25 seconds when I bypass the\n * queue and call `fetch` directly. In that test, my setup downloads about 50 assets per second, so this set of options\n * only affects concurrency and doesn't actually throttle the downloads. Limiting concurrency also fixes the issue\n * where very large projects (thousands of assets) can lead to browser failures like `net::ERR_INSUFFICIENT_RESOURCES`.\n * The exact concurrency limit doesn't seem to matter much since the browser limits parallel connections itself. It\n * just needs to be high enough to avoid bubbles in the download pipeline and low enough to avoid resource exhaustion.\n * @see {@link https://github.com/scratchfoundation/scratch-gui/issues/7111}\n */\nexport const AssetQueueOptions: QueueOptions = {\n    burstLimit: 64,\n    sustainRate: 64,\n    // WARNING: asset download concurrency >=5 can lead to corrupted buffers on Chrome (December 2025, Chrome 142.0)\n    // when using Scratch's bitmap load pipeline. Marking the canvas context as `{willReadFrequently: true}` seems to\n    // eliminate that issue, so maybe the problem is related to hardware acceleration.\n    concurrency: 64\n};\n\n/**\n * Central registry of per-host queues.\n * Uses strict limits by default. Override these strict limits as needed for specific hosts.\n */\nexport const hostQueueManager = new QueueManager({\n    burstLimit: 5,\n    sustainRate: 1,\n    concurrency: 1\n});\n","import {type QueueOptions} from '@scratch/task-herder';\nimport {hostQueueManager} from './HostQueues';\n\nexport const Headers = globalThis.Headers;\n\n/**\n * Metadata header names.\n * The enum value is the name of the associated header.\n */\nexport enum RequestMetadata {\n    /** The ID of the project associated with this request */\n    ProjectId = 'X-Project-ID',\n    /** The ID of the project run associated with this request */\n    RunId = 'X-Run-ID'\n}\n\nexport type ScratchFetchOptions = {\n    /**\n     * The name of the queue to use for this request.\n     * If absent, the hostname of the requested URL will be used as the queue name.\n     * This is a Scratch-specific extension to the standard RequestInit type.\n     */\n    queueName?: string;\n\n    /**\n     * The options to use when creating the queue for this request.\n     * Ignored if a queue with the specified name already exists.\n     */\n    queueOptions?: QueueOptions;\n};\n\n/**\n * Metadata headers for requests.\n */\nconst metadata = new Headers();\n\n/**\n * Check if there is any metadata to apply.\n * @returns {boolean} true if `metadata` has contents, or false if it is empty.\n */\nexport const hasMetadata = (): boolean => {\n    const searchParams = (\n        typeof self !== 'undefined' &&\n        self &&\n        self.location &&\n        self.location.search &&\n        self.location.search.split(/[?&]/)\n    ) || [];\n    if (!searchParams.includes('scratchMetadata=1')) {\n        // for now, disable this feature unless scratchMetadata=1\n        // TODO: remove this check once we're sure the feature works correctly in production\n        return false;\n    }\n    for (const _ of metadata) {\n        return true;\n    }\n    return false;\n};\n\n/**\n * Non-destructively merge any metadata state (if any) with the provided options object (if any).\n * If there is metadata state but no options object is provided, make a new object.\n * If there is no metadata state, return the provided options parameter without modification.\n * If there is metadata and an options object is provided, modify a copy and return it.\n * Headers in the provided options object may override headers generated from metadata state.\n * @param {RequestInit} [options] The initial request options. May be null or undefined.\n * @returns {RequestInit|undefined} the provided options parameter without modification, or a new options object.\n */\nexport const applyMetadata = (options?: globalThis.RequestInit): globalThis.RequestInit | undefined => {\n    if (hasMetadata()) {\n        const augmentedOptions = Object.assign({}, options);\n        augmentedOptions.headers = new Headers(metadata);\n        if (options && options.headers) {\n            // the Fetch spec says options.headers could be:\n            // \"A Headers object, an object literal, or an array of two-item arrays to set request's headers.\"\n            // turn it into a Headers object to be sure of how to interact with it\n            const overrideHeaders = options.headers instanceof Headers ?\n                options.headers : new Headers(options.headers);\n            for (const [name, value] of overrideHeaders.entries()) {\n                augmentedOptions.headers.set(name, value);\n            }\n        }\n        return augmentedOptions;\n    }\n    return options;\n};\n\n/**\n * Make a network request.\n * This is a wrapper for the global fetch method, adding some Scratch-specific functionality.\n * @param {RequestInfo|URL} resource The resource to fetch.\n * @param {RequestInit} [requestOptions] Optional object containing custom settings for this request.\n * @param {ScratchFetchOptions} [scratchOptions] Optional Scratch-specific settings for this request.\n * @see {@link https://developer.mozilla.org/docs/Web/API/fetch} for more about the fetch API.\n * @returns {Promise<Response>} A promise for the response to the request.\n */\nexport const scratchFetch = (\n    resource: RequestInfo | URL,\n    requestOptions?: globalThis.RequestInit,\n    scratchOptions?: ScratchFetchOptions\n): Promise<Response> => {\n    requestOptions = applyMetadata(requestOptions);\n\n    let queueName = scratchOptions?.queueName;\n    if (!queueName) {\n        // Normalize resource to a Request object. The `fetch` call will do this anyway, so it's not much extra work,\n        // but it guarantees availability of the URL for queue naming.\n        resource = new Request(resource, requestOptions);\n        queueName = new URL(resource.url).hostname;\n    }\n    const queue = hostQueueManager.getOrCreate(queueName, scratchOptions?.queueOptions);\n    return queue.do(() => fetch(resource, requestOptions));\n};\n\n/**\n * Create a new fetch queue with the given identifier and option overrides.\n * If a queue with that identifier already exists, it will be replaced.\n * Queues are automatically created as needed with default options, so\n * there's no need to call this unless you need to override the default queue options.\n * WARNING: If the old queue has is not empty, it may continue to run its tasks in the background.\n * If you need to cancel fetch tasks in that queue before replacing it, do so manually first.\n * @param queueName The name of the queue to create.\n * @param overrides Optional overrides for the default QueueOptions for this specific queue.\n */\nexport const createQueue = (queueName: string, overrides: Partial<QueueOptions>): void => {\n    hostQueueManager.create(queueName, overrides);\n};\n\n/**\n * Set the value of a named request metadata item.\n * Setting the value to `null` or `undefined` will NOT remove the item.\n * Use `unsetMetadata` for that.\n * @param {RequestMetadata} name The name of the metadata item to set.\n * @param {any} value The value to set (will be converted to a string).\n */\nexport const setMetadata = (name: RequestMetadata, value: any): void => {\n    metadata.set(name, value);\n};\n\n/**\n * Remove a named request metadata item.\n * @param {RequestMetadata} name The name of the metadata item to remove.\n */\nexport const unsetMetadata = (name: RequestMetadata): void => {\n    metadata.delete(name);\n};\n\n/**\n * Retrieve a named request metadata item.\n * Only for use in tests. At the time of writing, used in scratch-vm tests.\n * @param {RequestMetadata} name The name of the metadata item to retrieve.\n * @returns {string|null} The value of the metadata item, or `null` if it was not found.\n */\nexport const getMetadata = (name: RequestMetadata): string | null => metadata.get(name);\n","/* eslint-env worker */\n/* eslint-disable-next-line spaced-comment */\n/// <reference lib=\"webworker\" />\n\n// This worker won't share the same queue as the main thread, but throttling should be okay\n// as long as we don't use FetchTool and FetchWorkerTool at the same time.\n// TODO: Communicate metadata from the main thread to workers or move the worker boundary \"into\" `scratchFetch`.\n// Make sure to benchmark any changes to avoid performance regressions, especially for large project loads.\nimport {AssetQueueOptions} from './HostQueues';\nimport {scratchFetch} from './scratchFetch';\n\ninterface JobMessage {\n    id: string;\n    url: string;\n    options: RequestInit | undefined;\n}\n\ninterface CompletionMessage {\n    id: string;\n    buffer?: ArrayBuffer | null;\n    error?: string;\n}\n\nlet jobsActive = 0;\nconst complete: CompletionMessage[] = [];\n\nlet intervalId: ReturnType<typeof setInterval> | undefined = void 0;\n\n/**\n * Register a step function.\n *\n * Step checks if there are completed jobs and if there are sends them to the\n * parent. Then it checks the jobs count. If there are no further jobs, clear\n * the step.\n */\nconst registerStep = function () {\n    intervalId = setInterval(() => {\n        if (complete.length) {\n            // Send our chunk of completed requests and instruct postMessage to\n            // transfer the buffers instead of copying them.\n            postMessage(\n                complete.slice(),\n                // Instruct postMessage that these buffers in the sent message\n                // should use their Transferable trait. After the postMessage\n                // call the \"buffers\" will still be in complete if you looked,\n                // but they will all be length 0 as the data they reference has\n                // been sent to the window. This lets us send a lot of data\n                // without the normal postMessage behaviour of making a copy of\n                // all of the data for the window.\n                complete.map(response => response.buffer).filter(Boolean) as Transferable[]\n            );\n            complete.length = 0;\n        }\n        if (jobsActive === 0) {\n            clearInterval(intervalId);\n            intervalId = void 0;\n        }\n    }, 1);\n};\n\n/**\n * Receive a job from the parent and fetch the requested data.\n * @param message The message from the parent.\n * @param message.data A job id, url, and options descriptor to perform.\n */\nconst onMessage = async ({data: job}: MessageEvent<JobMessage>) => {\n    if (jobsActive === 0 && !intervalId) {\n        registerStep();\n    }\n\n    jobsActive++;\n\n    try {\n        const response = await scratchFetch(job.url, job.options, {queueOptions: AssetQueueOptions});\n\n        const result: CompletionMessage = {id: job.id};\n        if (response.ok) {\n            result.buffer = await response.arrayBuffer();\n        } else if (response.status === 404) {\n            result.buffer = null;\n        } else {\n            throw response.status;\n        }\n        complete.push(result);\n    } catch (error) {\n        complete.push({id: job.id, error: ((error as Error)?.message) || `Failed request: ${job.url}`});\n    } finally {\n        jobsActive--;\n    }\n};\n\n// \"fetch\" is supported in Node.js as of 16.15 and our target browsers as of ~2017\npostMessage({support: {fetch: true}});\nself.addEventListener('message', onMessage);\n"],"names":[],"sourceRoot":""}