sourcemonkey 0.0.0-stage → 0.5.0

package/dist/extension/value-store.js

@@ -0,0 +1,204 @@
+// The worker's one store for `GM_setValue` values: every read and write
+// of a script's values in the worker goes through here, so what it
+// hands out always includes every write it has accepted, whether or
+// not that write has reached disk yet.
+//
+// - State: each script's whole value map, read from storage the first
+//   time it's needed and kept until the worker shuts down.
+// - Writes change the map at once and get a sequence number, then go
+//   to disk behind it: one `chrome.storage.local` write in flight per
+//   script, with writes arriving meanwhile combined into the next one.
+// - Sequence numbers are how documents order what they hear about a
+//   key (their own write's reply, other tabs' broadcasts, snapshots).
+//   They come from one counter, started from the clock, so a later
+//   worker's numbers are above an earlier one's unless the clock has
+//   stepped back by more than the earlier worker's lifetime (TODO.md).
+//
+// `gm-values.ts` is the row layer underneath; nothing else in the
+// worker may use it for values (`tests/unit/value-store.test.mjs`).
+// See docs/GM-APIs-storage.md.
+import { applyBatchValues, countStoredValues, DELETED_SENTINEL, loadAllValues, wipeScriptValues, } from './gm-values.js';
+// Microseconds since the epoch at worker start: a worker would need a
+// million writes a second, sustained, to run into its successor's
+// numbers. Well inside a double's exact integers.
+let lastSeq = Date.now() * 1000;
+// Numbers at or below this were handed out by an earlier worker, whose
+// writes this one knows nothing about.
+const workerStartSeq = lastSeq;
+function nextSeq() {
+    return ++lastSeq;
+}
+const DELETED = Symbol('sm:value-deleted');
+let scripts = new Map();
+function stateFor(scriptId) {
+    let st = scripts.get(scriptId);
+    if (!st) {
+        st = { values: null, loading: null, unsaved: new Map(), saving: null, lastChange: 0, saveFailed: false };
+        scripts.set(scriptId, st);
+    }
+    return st;
+}
+// The script's map, loading it on first use. Callers that arrive while
+// it loads share the load, and resume in the order they asked.
+async function loaded(scriptId, st) {
+    if (st.values)
+        return st.values;
+    if (!st.loading) {
+        const load = loadAllValues(scriptId).then((obj) => {
+            // A reset while this read was out leaves `st` orphaned; the
+            // caller still gets the map, but nothing new lands in it.
+            st.values = new Map(Object.entries(obj));
+            st.loading = null;
+            return st.values;
+        }, (err) => {
+            st.loading = null;
+            throw err;
+        });
+        st.loading = load;
+    }
+    return st.loading;
+}
+// The script's values and the sequence number they're current to.
+export async function readValues(scriptId) {
+    const map = await loaded(scriptId, stateFor(scriptId));
+    return { values: Object.fromEntries(map), seq: lastSeq };
+}
+// Whether a script's values may have changed after sequence number
+// `seq`. True when `seq` predates this worker, which can't say.
+export function valuesChangedSince(scriptId, seq) {
+    if (seq <= workerStartSeq)
+        return true;
+    return (scripts.get(scriptId)?.lastChange ?? 0) > seq;
+}
+// A script's values, without keeping a map loaded for it: the loaded
+// map when there is one, which holds writes not yet saved, else a read
+// of storage. For reading many scripts at once (the backup export).
+export async function peekValues(scriptId) {
+    const st = scripts.get(scriptId);
+    if (st?.values)
+        return Object.fromEntries(st.values);
+    if (st?.loading)
+        return Object.fromEntries(await st.loading);
+    return loadAllValues(scriptId);
+}
+// How many values a script has, without keeping a map loaded for it.
+export async function countValues(scriptId) {
+    const st = scripts.get(scriptId);
+    if (st?.values)
+        return st.values.size;
+    if (st?.loading)
+        return (await st.loading).size;
+    return countStoredValues(scriptId);
+}
+let onSaveFailed = () => undefined;
+export function onValueSaveFailed(handler) {
+    onSaveFailed = handler;
+}
+// Apply one document's writes, in order, and return their sequence
+// number. The map changes before this returns; disk follows.
+export async function writeValues(scriptId, writes) {
+    const st = stateFor(scriptId);
+    let map;
+    try {
+        map = await loaded(scriptId, st);
+    }
+    catch (err) {
+        // No map to apply them to, so the writes are refused: the page
+        // hears so in the reply, and the script's Log the same way as a
+        // failed save.
+        console.info(`[SourceMonkey] Reading stored values for script ${scriptId} failed, so its writes were not saved:`, err);
+        reportSaveFailure(st, scriptId, err, 'load');
+        throw err;
+    }
+    const seq = nextSeq();
+    st.lastChange = seq;
+    for (const w of writes) {
+        if (w.deleted) {
+            map.delete(w.key);
+            st.unsaved.set(w.key, DELETED);
+        }
+        else {
+            map.set(w.key, w.value);
+            st.unsaved.set(w.key, w.value);
+        }
+    }
+    if (!st.saving)
+        st.saving = saveLoop(scriptId, st);
+    return seq;
+}
+function reportSaveFailure(st, scriptId, err, kind) {
+    if (st.saveFailed)
+        return;
+    st.saveFailed = true;
+    onSaveFailed(scriptId, err, kind);
+}
+// Hands `unsaved` to storage until it stays empty. Started only with
+// something to save, so the first `await` comes before it can finish
+// and clear `saving`.
+async function saveLoop(scriptId, st) {
+    try {
+        while (st.unsaved.size > 0) {
+            const batch = st.unsaved;
+            st.unsaved = new Map();
+            const writes = [];
+            for (const [key, value] of batch) {
+                writes.push({ key, value: value === DELETED ? DELETED_SENTINEL : value });
+            }
+            try {
+                await applyBatchValues(scriptId, writes);
+                st.saveFailed = false;
+            }
+            catch (err) {
+                // Most likely the storage quota. The documents running the
+                // script, and this map, keep the values until the worker shuts
+                // down; only disk is behind. Not `console.warn`: see CLAUDE.md.
+                console.info(`[SourceMonkey] Saving stored values for script ${scriptId} failed:`, err);
+                reportSaveFailure(st, scriptId, err, 'save');
+            }
+        }
+    }
+    finally {
+        st.saving = null;
+    }
+}
+// Remove all of a script's values, saved and unsaved.
+export async function wipeValues(scriptId) {
+    const st = stateFor(scriptId);
+    // Let a load or save in flight finish first, so neither can put
+    // values back.
+    if (st.loading)
+        await st.loading.catch(() => undefined);
+    while (st.saving)
+        await st.saving;
+    st.unsaved.clear();
+    st.values = new Map();
+    st.lastChange = nextSeq();
+    return wipeScriptValues(scriptId);
+}
+// Saves still running for states `forgetLoadedValues` dropped.
+const forgottenSaves = new Set();
+// Resolves once every accepted write is on disk.
+export async function valueWritesSaved() {
+    for (;;) {
+        const pending = [
+            ...[...scripts.values()].map((st) => st.saving).filter((p) => p != null),
+            ...forgottenSaves,
+        ];
+        if (pending.length === 0)
+            return;
+        await Promise.all(pending);
+    }
+}
+// Tests only: forget every loaded map, after something changed
+// `gmval:` rows behind the store's back (clearing or seeding storage).
+export function forgetLoadedValues() {
+    // Their saves still land, so `valueWritesSaved` keeps waiting for them.
+    for (const st of scripts.values()) {
+        const saving = st.saving;
+        if (!saving)
+            continue;
+        forgottenSaves.add(saving);
+        void saving.finally(() => forgottenSaves.delete(saving));
+    }
+    scripts = new Map();
+}

package/dist/extension/version-probe-fetch.js

@@ -0,0 +1,68 @@
+// The fetching half of update probes. Split from `version-probe.ts` so
+// the pure half — target selection, the outdated verdict, the staleness
+// gate — can be imported by display code (`script-display.ts`,
+// `collections.ts`, `script-details.ts`) without dragging the whole fetch
+// stack into every extension page. Only `script-updater.ts` needs this
+// module. See [docs/updates.md].
+import { mapWithConcurrency, PROBE_URLS_PER_SCRIPT_CONCURRENCY } from './concurrency.js';
+import { fetchResourceText } from '../file-loader.js';
+import { getValue, parseHeader } from '../userscript-header.js';
+// A `.meta.js` is a couple of KB; a `@downloadURL` is a whole
+// userscript. Past 1 MB we're not reading a userscript header, we're
+// downloading something else.
+const PROBE_MAX_BYTES = 1024 * 1024;
+const PROBE_TIMEOUT_MS = 5000;
+// Fetch one target and read its `@version`. Never throws and never
+// reports anything but a `VersionProbe` — a probe failure is a display
+// annotation, not an error the refresh pipeline should react to.
+//
+// `prior` is this target's previous row, when there is one. A failed
+// attempt carries its `version` forward: retracting a verdict we
+// already established, because the host happened to 500 this
+// afternoon, would flap the OUTDATED badge and re-log the transition
+// every time the host recovered.
+export async function runProbe(target, now, prior) {
+    const failed = (error) => ({
+        kinds: target.kinds,
+        url: target.url,
+        checkedAt: now,
+        error,
+        ...(prior?.version ? { version: prior.version } : {}),
+    });
+    // The bytes fetcher does report which failure it was, but
+    // `fetchResourceText` drops that on purpose: `probe.error` renders as
+    // a value in a definition-list cell (`script-details.ts`) with no room
+    // for a sentence, so this row's job is "we couldn't check", not
+    // diagnosis. Tracked in TODO.md with the rest of the version-probe
+    // wording exception.
+    const text = await fetchResourceText(target.url, {
+        maxBytes: PROBE_MAX_BYTES,
+        timeoutMs: PROBE_TIMEOUT_MS,
+    });
+    if (text === null)
+        return failed('could not fetch');
+    // Both `.meta.js` (header only) and a full `.user.js` parse the same
+    // way — parseHeader reads the metadata block wherever it sits and
+    // ignores everything after it. Managers accept either shape at an
+    // `@updateURL`, so we do too.
+    const header = parseHeader(text);
+    if (!header)
+        return failed('no metadata block');
+    const version = getValue(header, 'version');
+    if (!version?.trim())
+        return failed('no @version');
+    return { kinds: target.kinds, url: target.url, checkedAt: now, version: version.trim() };
+}
+// Probe every target of ONE script, at most
+// PROBE_URLS_PER_SCRIPT_CONCURRENCY at a time — a guard that cannot bind
+// at today's two-targets-per-script maximum. What actually caps a
+// Refresh All burst is PROBE_SCRIPTS_CONCURRENCY, one level up in
+// `runVersionProbes`.
+//
+// Takes the targets rather than re-deriving them from the header,
+// because the caller already needed them for the staleness check and the
+// two must agree. Result order matches `targets`.
+export async function probeVersions(targets, now, prior) {
+    const priorByUrl = new Map((prior ?? []).map((p) => [p.url, p]));
+    return await mapWithConcurrency(targets, PROBE_URLS_PER_SCRIPT_CONCURRENCY, (target) => runProbe(target, now, priorByUrl.get(target.url)));
+}

package/dist/extension/worker-client.js

@@ -0,0 +1,109 @@
+// Page side of the worker's operations: every write an extension page
+// makes goes through here, to run in the service worker
+// (docs/worker-writes.md). Each export has the name and arguments of the
+// worker function it runs, so a call site only changes its import.
+//
+// One port per call:
+//   - The page sends the operation's name and arguments, then a
+//     heartbeat while it waits. A port's messages keep the worker alive,
+//     which a long refresh needs (Chrome stops an idle worker after
+//     30 s).
+//   - The worker answers once, with the result or the error's message,
+//     which is thrown here as a new Error with that message.
+//   - Closing the page doesn't stop the work: the worker carries on and
+//     drops the answer.
+//
+// Arguments and results cross as JSON: no Maps, Sets or functions, and
+// an `undefined` argument or array entry arrives as `null`.
+// worker-ops.ts adapts any function whose own signature has them.
+import { isPageContext } from '../run-context.js';
+export const WORKER_OP_PORT = 'sm:op';
+// Under Chrome's 30 s idle limit, with room to spare.
+export const WORKER_OP_HEARTBEAT_MS = 20_000;
+// The worker went away mid-call: it crashed, or Chrome stopped it.
+// It may have saved the change first.
+export const WORKER_STOPPED = 'Internal error: SourceMonkey\'s background worker stopped before answering. The change may not have been saved.';
+export function callWorker(op, ...args) {
+    // A port the worker opens reaches no listener of its own: code running
+    // in the worker calls the store directly.
+    if (!isPageContext())
+        return Promise.reject(new Error(`Internal error: The background worker called its own ${op} through a port`));
+    return new Promise((resolve, reject) => {
+        const port = chrome.runtime.connect({ name: WORKER_OP_PORT });
+        let settled = false;
+        const beat = setInterval(() => {
+            try {
+                port.postMessage({ beat: true });
+            }
+            catch { /* port gone */ }
+        }, WORKER_OP_HEARTBEAT_MS);
+        const finish = () => {
+            settled = true;
+            clearInterval(beat);
+            try {
+                port.disconnect();
+            }
+            catch { /* already gone */ }
+        };
+        port.onMessage.addListener((message) => {
+            if (settled)
+                return;
+            finish();
+            if (message.ok)
+                resolve(message.result);
+            else
+                reject(new Error(message.error));
+        });
+        port.onDisconnect.addListener(() => {
+            if (settled)
+                return;
+            finish();
+            reject(new Error(WORKER_STOPPED));
+        });
+        port.postMessage({ op, args });
+    });
+}
+// Trailing `undefined` arguments are left off, so the worker's default
+// parameters apply: JSON would send them as `null`.
+function op(name) {
+    return (...args) => {
+        let n = args.length;
+        while (n > 0 && args[n - 1] === undefined)
+            n--;
+        return callWorker(name, ...args.slice(0, n));
+    };
+}
+// ---------- Enable toggles ----------
+export const setScriptEnabledExclusive = op('setScriptEnabledExclusive');
+export const setCollectionEnabledExclusive = op('setCollectionEnabledExclusive');
+export const setAllScriptsEnabled = op('setAllScriptsEnabled');
+export const enableWithin = op('enableWithin');
+export const setScriptsEnabled = op('setScriptsEnabled');
+export const turnOffOtherCopies = op('turnOffOtherCopies');
+// ---------- Small writes ----------
+export const removeScript = op('removeScript');
+export const removeCollection = op('removeCollection');
+export const moveCollection = op('moveCollection');
+export const moveScript = op('moveScript');
+export const createCollection = op('createCollection');
+export const setCollectionOptions = op('setCollectionOptions');
+export const revokeConnectHost = op('revokeConnectHost');
+export const forgetDeniedHost = op('forgetDeniedHost');
+export const clearScriptLog = op('clearScriptLog');
+export const markFirstRunComplete = op('markFirstRunComplete');
+export const writeSetting = op('writeSetting');
+export const backfillCollectionDisplayNames = op('backfillCollectionDisplayNames');
+export const reconcileScriptMembership = op('reconcileScriptMembership');
+export const syncAfterUserScriptsAllowed = op('syncAfterUserScriptsAllowed');
+// ---------- Refresh, add, install ----------
+export const updateScript = op('updateScript');
+export const updateCollection = op('updateCollection');
+export const updateAllCollections = op('updateAllCollections');
+export const addCollection = op('addCollection');
+export const installScript = op('installScript');
+// ---------- Reads ----------
+export const readScriptValues = op('readScriptValues');
+// ---------- Site icons (the debug icons page) ----------
+export const refreshCachedIcon = op('refreshCachedIcon');
+export const deleteCachedIcon = op('deleteCachedIcon');
+export const deleteAllCachedIcons = op('deleteAllCachedIcons');

package/dist/extension/worker-ops.js

@@ -0,0 +1,171 @@
+// Worker side of worker-client.ts: the operations extension pages ask
+// the service worker to run, since only it writes storage
+// (docs/worker-writes.md).
+//
+// Each entry is the function a page would have called itself, or an
+// adapter where that function's own arguments or result can't cross as
+// JSON (a timer, a Map) or carry more than the page uses (script
+// bodies).
+import { addCollection, backfillCollectionDisplayNames, createCollection, describeError, installScript, markFirstRunComplete, migrateCollectionKeysForPage, withoutBodies, moveCollection, moveScript, reconcileScriptMembership, removeCollection, removeScript, setCollectionOptions, setScriptsEnabled, } from './script-store.js';
+import { enableWithin, setAllScriptsEnabled, setCollectionEnabledExclusive, setScriptEnabledExclusive, } from './script-enablement.js';
+import { turnOffOtherCopies } from './script-identity.js';
+import { syncRegistrations } from './injector.js';
+import { probeInstalledScript, updateAllCollections, updateCollection, updateScript, } from './script-updater.js';
+import { forgetDeniedHost, revokeConnectHost } from './connect-grants.js';
+import { clearScriptLog } from './script-log.js';
+import { peekValues } from './value-store.js';
+import { DEBUG_LOGGING_KEY } from '../debug-log.js';
+import { GITHUB_AUTH_TOKEN_KEY, setGithubAuthToken } from '../github-auth.js';
+import { LOCAL_ROOTS_KEY, readRootsSetting, setConfiguredLocalRoots } from '../local-file-overrides.js';
+import { INSTALL_CAPTURE_KEY } from './install-capture.js';
+import { DOWNLOADS_COPIES_KEY, registerLocalCopy } from './local-copy.js';
+import { deleteAllCachedIcons, deleteCachedIcon, ensureCachedIcon, refreshCachedIcon } from './site-icon.js';
+import { startPhaseTimer } from '../phase-timing.js';
+import { WORKER_OP_PORT } from './worker-client.js';
+// The settings a page may change: Options, and the local copy's
+// Downloads folder. Each a whole value, not a read-modify-write, so
+// writing it is all the worker does.
+const SETTING_KEYS = [
+    INSTALL_CAPTURE_KEY, DEBUG_LOGGING_KEY, LOCAL_ROOTS_KEY, GITHUB_AUTH_TOKEN_KEY, DOWNLOADS_COPIES_KEY,
+];
+const WORKER_OPS = {
+    // ---------- Enable toggles ----------
+    setScriptEnabledExclusive,
+    setCollectionEnabledExclusive,
+    setAllScriptsEnabled,
+    // The viewer's versions table: `scope` is the copies it shows.
+    async enableWithin(id, scope) {
+        const timer = startPhaseTimer('enable script');
+        await enableWithin(id, scope, timer);
+        timer.done();
+    },
+    async setScriptsEnabled(updates) {
+        await setScriptsEnabled(updates);
+    },
+    async turnOffOtherCopies(scriptId) {
+        await turnOffOtherCopies(scriptId);
+    },
+    // ---------- Small writes ----------
+    removeScript,
+    removeCollection,
+    moveCollection,
+    // The moved row, without the bodies a refresh of it would carry.
+    async moveScript(scriptId, target) {
+        const moved = await moveScript(scriptId, target);
+        return moved && 'bodyText' in moved ? withoutBodies(moved) : moved;
+    },
+    createCollection,
+    setCollectionOptions,
+    revokeConnectHost,
+    forgetDeniedHost,
+    clearScriptLog,
+    markFirstRunComplete,
+    // `null` removes the setting.
+    async writeSetting(key, value) {
+        if (!SETTING_KEYS.includes(key))
+            throw new Error(`Internal error: ${key} is not a setting a page may change`);
+        if (value === null)
+            await chrome.storage.local.remove(key);
+        else
+            await chrome.storage.local.set({ [key]: value });
+        // Settings the worker keeps in memory take the new value now, not
+        // when the change event arrives: a page saving the roots refreshes
+        // straight after.
+        if (key === LOCAL_ROOTS_KEY)
+            setConfiguredLocalRoots(readRootsSetting(value));
+        if (key === GITHUB_AUTH_TOKEN_KEY)
+            setGithubAuthToken(typeof value === 'string' ? value : '');
+    },
+    // ---------- Repairs the collections page asks for ----------
+    backfillCollectionDisplayNames,
+    // From the worker's own read of the rows, not the page's.
+    async reconcileScriptMembership() {
+        return await reconcileScriptMembership();
+    },
+    // A page that finds rows in the old spelling waits for this before
+    // reading.
+    migrateCollectionKeys: migrateCollectionKeysForPage,
+    // The collections page saw "Allow User Scripts" turned on. Chrome
+    // tells the worker nothing when that happens, and the syncs that ran
+    // while it was off registered nothing, so this places the whole set.
+    async syncAfterUserScriptsAllowed() {
+        await syncRegistrations();
+    },
+    // ---------- Refresh, add, install ----------
+    // The page reads storage for what changed, so these answer with no
+    // rows: a refresh's carry every body it loaded.
+    //
+    // Whether the script is still there afterwards: a refresh drops a
+    // row whose file is gone.
+    async updateScript(scriptId, mode, probe) {
+        return (await updateScript(scriptId, mode, probe)) !== null;
+    },
+    async updateCollection(collectionId, mode, probe) {
+        await updateCollection(collectionId, mode, probe);
+    },
+    async updateAllCollections(mode, probe) {
+        await updateAllCollections(mode, probe);
+    },
+    // A single script installed (or a `.user.js` added) is checked for a
+    // newer version at once, without the page waiting for it.
+    async addCollection(rawPath) {
+        const result = await addCollection(rawPath);
+        probeInstalledScript(result);
+        return result;
+    },
+    async installScript(rawLocation, target) {
+        const result = await installScript(rawLocation, target);
+        probeInstalledScript(result);
+        return result;
+    },
+    // ---------- Reads ----------
+    // GM values come from the worker's value store, which can hold writes
+    // not yet saved to storage, so a page reads them here rather than
+    // from storage (the backup export).
+    async readScriptValues(scriptIds) {
+        const read = await Promise.all(scriptIds.map((id) => peekValues(id)));
+        return Object.fromEntries(scriptIds.map((id, i) => [id, read[i]]));
+    },
+    // ---------- Local copy and site icons ----------
+    // After the page has written the files: it alone can download.
+    registerLocalCopy,
+    // One icon row's data URL: cached if still usable, else fetched and
+    // cached. `refreshCachedIcon` always fetches (the debug icons page).
+    ensureCachedIcon,
+    refreshCachedIcon,
+    deleteCachedIcon,
+    deleteAllCachedIcons,
+};
+function isWorkerOp(name) {
+    return Object.hasOwn(WORKER_OPS, name);
+}
+async function run(request) {
+    if (!isWorkerOp(request.op) || !Array.isArray(request.args)) {
+        throw new Error(`Internal error: There is no worker operation named ${request.op}`);
+    }
+    const fn = WORKER_OPS[request.op];
+    return await fn(...request.args);
+}
+// Answer pages' calls. `fromPage` is background.ts's sender check: only
+// our own pages may ask, as for every page relay.
+export function installWorkerOps(fromPage) {
+    chrome.runtime.onConnect.addListener((port) => {
+        if (port.name !== WORKER_OP_PORT || !port.sender || !fromPage(port.sender))
+            return;
+        let started = false;
+        port.onMessage.addListener((message) => {
+            // Later messages are the page's heartbeat: receiving them is what
+            // keeps this worker alive.
+            if (started || 'beat' in message)
+                return;
+            started = true;
+            const reply = (answer) => {
+                try {
+                    port.postMessage(answer);
+                }
+                catch { /* the page closed */ }
+            };
+            run(message).then((result) => reply({ ok: true, result: result ?? null }), (err) => reply({ ok: false, error: describeError(err) }));
+        });
+    });
+}

package/dist/extension/zip-writer.js

@@ -0,0 +1,24 @@
+// Writes the backup export's zip (docs/export-format.md), through the
+// vendored fflate library (src/vendor/fflate.d.ts).
+//
+// fflate deflates each entry and marks non-ASCII names as UTF-8, which
+// Tampermonkey's and Violentmonkey's readers both honor.
+import { zipSync } from '../vendor/fflate.js';
+// The archive's bytes. A repeated path is the caller's bug, refused
+// rather than letting the later one replace the earlier.
+//
+// fflate takes the entries as an object, so an integer-like path (`7`)
+// is written before the others; the order otherwise holds.
+export function buildZip(entries, now = Date.now()) {
+    // fflate writes no zip64, so past this the entry count would wrap.
+    if (entries.length > 0xffff)
+        throw new Error('The export has more files than a zip file can hold (65,535)');
+    // No prototype, so a path like `__proto__` is an ordinary key.
+    const files = Object.create(null);
+    for (const entry of entries) {
+        if (Object.hasOwn(files, entry.path))
+            throw new Error(`Internal error: The zip has two entries named ${entry.path}`);
+        files[entry.path] = [entry.data, { mtime: entry.modified ?? now }];
+    }
+    return zipSync(files);
+}