sourcemonkey 0.0.0-stage → 0.5.0

package/dist/extension/script-blobs.js

@@ -0,0 +1,413 @@
+// Storage format 2: script rows without their bodies. Each body (a
+// script's `bodyText`, each `@require` source, each `@resource`'s bytes)
+// is a `blob:<sha256>` row, shared by every row whose body has the same
+// bytes, and the script row points at it with a `BlobRef`. See
+// docs/storage.md#content-blobs.
+//
+// This module converts between the two shapes:
+//   - `toStorage`: a full `StoredScript` (bodies inline, as the rest of
+//     the code uses it) → the small row plus the blobs it needs.
+//   - `fromStorage`: small rows → full `StoredScript`s, reading their
+//     blobs in one call.
+// and runs the one-time migration of format 1 rows (bodies inline in
+// storage too), and the cleanup of blobs no row points at any more.
+//
+// Both directions take each body as it finds it: one already inline (a
+// format 1 row, or a full script handed to a write) stays as it is.
+import { assertWorkerContext } from '../run-context.js';
+import { base64ToBytes, bytesToBase64, sha256Hex } from '../bytes.js';
+import { SCRIPT_PREFIX } from './structural-keys.js';
+import { scanAllStorage } from './storage-scan.js';
+import { dlog } from '../debug-log.js';
+import { formatMs } from '../phase-timing.js';
+export const BLOB_PREFIX = 'blob:';
+export const STORE_FORMAT_KEY = 'storeFormat';
+export const STORE_FORMAT = 2;
+export const blobKey = (hash) => `${BLOB_PREFIX}${hash}`;
+// Shown on a script whose row points at a blob that isn't there (or
+// that this build can't decode): a row edited by hand, or a cleanup bug
+// (see `sweepUnusedBlobs`). Refresh fetches the file again, as for one
+// that failed to load.
+export const MISSING_BLOB_MESSAGE = 'Part of the stored copy of this script is missing. Refresh the script to load it again.';
+const encoder = new TextEncoder();
+// `ignoreBOM` keeps a leading U+FEFF: the blob's bytes are exactly what
+// was stored, and a require reading a blob a resource wrote first must
+// get back the same text the require itself would have stored.
+const decoder = new TextDecoder('utf-8', { ignoreBOM: true });
+// The ref and stored data behind each object `fromStorage` built (a
+// script, a require or resource entry), so writing it back unchanged
+// doesn't hash its body again. Entries survive the `{...script}` copies
+// the store's writers make; a script object often doesn't, and its body
+// is hashed again then.
+const knownRefs = new WeakMap();
+// Lets `to` reuse the body ref `from` was read with, when its body is
+// the same text: a refresh rebuilds a new script object whose body came
+// from the copy it started from, and would otherwise hash it again.
+export function carryBodyRef(from, to) {
+    const known = from ? knownRefs.get(from) : undefined;
+    if (known && known.data === to.bodyText)
+        knownRefs.set(to, known);
+}
+function reuse(owner, data, encoding, blobs) {
+    const known = knownRefs.get(owner);
+    if (known?.data !== data)
+        return null;
+    // Offered again: the write only stores it if it's gone.
+    blobs.set(blobKey(known.ref.hash), { encoding, data });
+    return known.ref;
+}
+async function textBlob(owner, text, blobs) {
+    const known = reuse(owner, text, 'text', blobs);
+    if (known)
+        return known;
+    const bytes = encoder.encode(text);
+    const hash = await sha256Hex(bytes);
+    blobs.set(blobKey(hash), { encoding: 'text', data: text });
+    return { hash, size: bytes.length };
+}
+async function base64Blob(owner, base64, blobs) {
+    const known = reuse(owner, base64, 'base64', blobs);
+    if (known)
+        return known;
+    const bytes = base64ToBytes(base64);
+    const hash = await sha256Hex(bytes);
+    blobs.set(blobKey(hash), { encoding: 'base64', data: base64 });
+    return { hash, size: bytes.byteLength };
+}
+// Does a stored row carry any body inline (format 1, or a full script
+// handed to a write)?
+export function hasInlineBodies(value) {
+    if (value === null || typeof value !== 'object')
+        return false;
+    const row = value;
+    return typeof row.bodyText === 'string'
+        || (row.requires ?? []).some((r) => typeof r.source === 'string')
+        || (row.resources ?? []).some((r) => typeof r.bytesBase64 === 'string');
+}
+// A full script (or a format 1 row) → its small row, adding the blobs it
+// needs to `blobs`. A body already in blob form passes through.
+export async function toStorage(script, blobs) {
+    const { bodyText, requires, resources, ...rest } = script;
+    const row = { ...rest };
+    // A row that never loaded has no body to point at (`size` is null
+    // exactly then); every other row does, an empty body included.
+    if (typeof bodyText === 'string' && (bodyText !== '' || script.size !== null)) {
+        row.bodyBlobRef = await textBlob(script, bodyText, blobs);
+    }
+    if (requires) {
+        row.requires = await Promise.all(requires.map(async (r) => {
+            const { source, ...meta } = r;
+            return typeof source === 'string'
+                ? { ...meta, blobRef: await textBlob(r, source, blobs) }
+                : r;
+        }));
+    }
+    if (resources) {
+        row.resources = await Promise.all(resources.map(async (r) => {
+            const { bytesBase64, ...meta } = r;
+            return typeof bytesBase64 === 'string'
+                ? { ...meta, blobRef: await base64Blob(r, bytesBase64, blobs) }
+                : r;
+        }));
+    }
+    return row;
+}
+// A blob's content in the form a reader wants, or null for an encoding
+// this build doesn't know (written by a newer one), which reads like a
+// missing blob rather than as garbage.
+function blobText(blob) {
+    if (blob.encoding === 'text')
+        return blob.data;
+    if (blob.encoding === 'base64')
+        return decoder.decode(base64ToBytes(blob.data));
+    return null;
+}
+function blobBase64(blob) {
+    if (blob.encoding === 'base64')
+        return blob.data;
+    if (blob.encoding === 'text')
+        return bytesToBase64(encoder.encode(blob.data));
+    return null;
+}
+// The blob keys a stored row points at. A format 1 row has none.
+export function blobKeysOf(value) {
+    if (value === null || typeof value !== 'object')
+        return [];
+    const row = value;
+    const keys = [];
+    if (row.bodyBlobRef)
+        keys.push(blobKey(row.bodyBlobRef.hash));
+    for (const r of row.requires ?? [])
+        if (r.blobRef)
+            keys.push(blobKey(r.blobRef.hash));
+    for (const r of row.resources ?? [])
+        if (r.blobRef)
+            keys.push(blobKey(r.blobRef.hash));
+    return keys;
+}
+// A row whose blob is gone keeps its metadata but can't run. As an
+// `invalid` row, the next refresh fetches it whatever its validators
+// say; dropping them (or the file entry, which a refresh then fetches
+// like a new one) is a backstop for that.
+//
+// Only readers that load bodies see this. The registration sync is one,
+// and it saves the row this way (`markMissingBlobs`), so the readers of
+// small rows (pages, the match index) see it from then on.
+//
+// A write of a full script built here stores it as it is: an invalid
+// row with an empty body. That's consistent, and the refresh repairs it
+// the same way.
+function withMissingBlob(script, bodyMissing) {
+    const out = { ...script, status: 'invalid', errorMessage: MISSING_BLOB_MESSAGE };
+    delete out.errorLocation;
+    if (bodyMissing) {
+        delete out.contentHash;
+        delete out.fsMtime;
+        delete out.httpEtag;
+        delete out.httpLastModified;
+    }
+    return out;
+}
+function hydrate(row, blobs) {
+    const get = (ref) => blobs[blobKey(ref.hash)];
+    // `bodyText` is there only on a row with its body inline.
+    const { bodyBlobRef, requires, resources, ...rest } = row;
+    let missing = false;
+    let bodyMissing = false;
+    const script = { ...rest, bodyText: rest.bodyText ?? '' };
+    // An inline body wins over a ref: it's what a format 1 row, or a test
+    // writing one, put there.
+    if (typeof rest.bodyText !== 'string' && bodyBlobRef) {
+        const blob = get(bodyBlobRef);
+        const text = blob ? blobText(blob) : null;
+        if (text === null)
+            missing = bodyMissing = true;
+        else {
+            script.bodyText = text;
+            knownRefs.set(script, { data: text, ref: bodyBlobRef });
+        }
+    }
+    if (requires) {
+        script.requires = requires.flatMap((r) => {
+            const inline = r;
+            if (typeof inline.source === 'string' || !r.blobRef)
+                return [inline];
+            const blob = get(r.blobRef);
+            const source = blob ? blobText(blob) : null;
+            if (source === null) {
+                missing = true;
+                return [];
+            }
+            const { blobRef, ...meta } = r;
+            const entry = { ...meta, source };
+            knownRefs.set(entry, { data: source, ref: blobRef });
+            return [entry];
+        });
+    }
+    if (resources) {
+        script.resources = resources.flatMap((r) => {
+            const inline = r;
+            if (typeof inline.bytesBase64 === 'string' || !r.blobRef)
+                return [inline];
+            const blob = get(r.blobRef);
+            const bytesBase64 = blob ? blobBase64(blob) : null;
+            if (bytesBase64 === null) {
+                missing = true;
+                return [];
+            }
+            const { blobRef, ...meta } = r;
+            const entry = { ...meta, bytesBase64 };
+            knownRefs.set(entry, { data: bytesBase64, ref: blobRef });
+            return [entry];
+        });
+    }
+    return missing ? withMissingBlob(script, bodyMissing) : script;
+}
+// A stored row as reading it shows when a blob it points at is missing
+// or unreadable (see `withMissingBlob`), for saving it that way; null
+// when all of its blobs are there. `blobs` holds what's stored under
+// the row's blob keys. Bodies inline on the row (format 1) count as
+// there.
+export function markMissingBlobs(row, blobs) {
+    const there = (ref) => {
+        if (!ref)
+            return true;
+        const blob = blobs[blobKey(ref.hash)];
+        return blob?.encoding === 'text' || blob?.encoding === 'base64';
+    };
+    const bodyMissing = typeof row.bodyText !== 'string' && !there(row.bodyBlobRef);
+    const requires = row.requires?.filter((r) => there(r.blobRef));
+    const resources = row.resources?.filter((r) => there(r.blobRef));
+    if (!bodyMissing && requires?.length === row.requires?.length && resources?.length === row.resources?.length) {
+        return null;
+    }
+    const out = withMissingBlob({ ...row }, bodyMissing);
+    if (requires)
+        out.requires = requires;
+    if (resources)
+        out.resources = resources;
+    return out;
+}
+// Small rows → full scripts. `have` is blobs the caller already read
+// (a whole-store scan has them all); the rest are read in one call.
+export async function fromStorage(rows, have = {}) {
+    const need = [...new Set(rows.flatMap(blobKeysOf))].filter((k) => !(k in have));
+    const blobs = need.length > 0 ? { ...have, ...await chrome.storage.local.get(need) } : have;
+    return rows.map((row) => hydrate(row, blobs));
+}
+// In place over a `get` result: every `script:` value becomes a full
+// script. For readers that take rows by key from a mixed read.
+export async function hydrateScriptRows(got) {
+    const keys = Object.keys(got).filter((k) => k.startsWith(SCRIPT_PREFIX) && got[k] != null);
+    if (keys.length === 0)
+        return got;
+    const full = await fromStorage(keys.map((k) => got[k]), got);
+    keys.forEach((k, i) => { got[k] = full[i]; });
+    return got;
+}
+// A write's items with every full `script:` value split into its small
+// row plus blobs. A blob already stored isn't written again: same key,
+// same bytes. Rows and their blobs go out in the caller's one `set`, so
+// no reader sees a row before its blobs.
+export async function withBlobs(items) {
+    const keys = Object.keys(items).filter((k) => k.startsWith(SCRIPT_PREFIX) && hasInlineBodies(items[k]));
+    if (keys.length === 0)
+        return items;
+    const blobs = new Map();
+    const out = { ...items };
+    for (const k of keys)
+        out[k] = await toStorage(items[k], blobs);
+    return { ...out, ...newBlobItems(blobs, new Set(await chrome.storage.local.getKeys())) };
+}
+// A full script's stored form: its small row, and the blobs it points
+// at. For a writer that compares the row with what's stored before
+// deciding to write: it then writes the row and the blobs not yet
+// stored, rather than the full script, which `withBlobs` would hash
+// again.
+export async function storedForm(script) {
+    const blobs = new Map();
+    return { row: await toStorage(script, blobs), blobs };
+}
+// The blobs among `blobs` whose keys aren't in `stored` (the keys in
+// storage), as items for a `set`. A stored one has the same bytes under
+// the same key, so it isn't written again.
+export function newBlobItems(blobs, stored) {
+    const out = {};
+    for (const [k, blob] of blobs)
+        if (!stored.has(k))
+            out[k] = blob;
+    return out;
+}
+// What the given scripts' rows take in storage, without their bodies,
+// as Chrome counts it: what a metadata read of them loads. For debug
+// timing lines.
+export async function scriptRowBytes(scriptIds) {
+    if (scriptIds.length === 0)
+        return 0;
+    return await chrome.storage.local.getBytesInUse(scriptIds.map((id) => SCRIPT_PREFIX + id));
+}
+// ---------- Migration from format 1 ----------
+// Rows per `set`. Each call replaces its rows' inline bodies with blobs,
+// so storage use barely moves within one call, and an interrupted run
+// keeps what earlier calls converted.
+export const MIGRATE_BATCH = 25;
+// Set in session storage when a run fails, so a store too full to
+// convert isn't rescanned on every worker wake. The next browser
+// session tries again.
+const MIGRATION_FAILED_KEY = 'storeFormatMigrationFailed';
+// Converts every format 1 row, then records format 2. Safe to re-run:
+// converted rows are skipped. Only the service worker runs it, before
+// anything else there writes (see `storeFormatReady` in script-store).
+//
+// Logged in full whenever it moves anything, since that's once per
+// profile; finding nothing to move, at each worker start or on a new
+// profile, only with debug logging on.
+export async function migrateToBlobs() {
+    const format = (await chrome.storage.local.get(STORE_FORMAT_KEY))[STORE_FORMAT_KEY];
+    if (typeof format === 'number' && format >= STORE_FORMAT) {
+        dlog('[SourceMonkey] storage format 2: already in place');
+        return;
+    }
+    if ((await chrome.storage.session.get(MIGRATION_FAILED_KEY))[MIGRATION_FAILED_KEY]) {
+        console.info('[SourceMonkey] storage format 2: not moving script bodies again until the browser restarts, since it failed earlier');
+        return;
+    }
+    const started = performance.now();
+    let moved = 0;
+    let written = 0;
+    try {
+        const all = await scanAllStorage('blob migration');
+        const stored = new Set(Object.keys(all));
+        const old = Object.entries(all).filter(([k, v]) => k.startsWith(SCRIPT_PREFIX) && hasInlineBodies(v));
+        if (old.length > 0) {
+            console.info(`[SourceMonkey] storage format 2: moving the bodies of ${old.length} script(s) to shared storage`);
+        }
+        for (let i = 0; i < old.length; i += MIGRATE_BATCH) {
+            const blobs = new Map();
+            const items = {};
+            const batch = old.slice(i, i + MIGRATE_BATCH);
+            for (const [k, v] of batch)
+                items[k] = await toStorage(v, blobs);
+            let fresh = 0;
+            for (const [k, blob] of blobs) {
+                if (stored.has(k))
+                    continue;
+                items[k] = blob;
+                stored.add(k);
+                fresh++;
+            }
+            assertWorkerContext('write storage');
+            await chrome.storage.local.set(items);
+            moved += batch.length;
+            written += fresh;
+        }
+        await chrome.storage.local.set({ [STORE_FORMAT_KEY]: STORE_FORMAT });
+        if (old.length === 0) {
+            // A new profile, or one whose rows were all converted already.
+            dlog('[SourceMonkey] storage format 2: no script bodies to move');
+        }
+        else {
+            console.info(`[SourceMonkey] storage format 2: moved ${moved} script(s), ${written} stored file(s), `
+                + `in ${formatMs(performance.now() - started)}`);
+        }
+    }
+    catch (err) {
+        console.info(`[SourceMonkey] storage format 2: stopped after moving ${moved} script(s), ${written} stored file(s); `
+            + 'the rest stay as they were until the browser restarts');
+        await chrome.storage.session.set({ [MIGRATION_FAILED_KEY]: true });
+        throw err;
+    }
+}
+// Only for the test surface: forget a failed run, as a new browser
+// session would.
+export async function clearMigrationFailureForTests() {
+    await chrome.storage.session.remove(MIGRATION_FAILED_KEY);
+}
+// `rows`: every script row, when the caller has them (the worker's row
+// cache); read from storage otherwise.
+export async function sweepUnusedBlobs(rows) {
+    const keys = await chrome.storage.local.getKeys();
+    const blobKeys = keys.filter((k) => k.startsWith(BLOB_PREFIX));
+    if (blobKeys.length === 0)
+        return { checked: 0, removed: 0 };
+    // The small rows only: refs are all a check needs.
+    const scriptRows = rows
+        ?? await chrome.storage.local.get(keys.filter((k) => k.startsWith(SCRIPT_PREFIX)));
+    const used = new Set(Object.values(scriptRows).flatMap(blobKeysOf));
+    const remove = blobKeys.filter((k) => !used.has(k));
+    if (remove.length > 0) {
+        assertWorkerContext('write storage');
+        await chrome.storage.local.remove(remove);
+    }
+    return { checked: blobKeys.length, removed: remove.length };
+}
+// Did a change to a script row leave a blob it pointed at unused by it:
+// a removed row, or one now pointing elsewhere? The worker's cue to
+// sweep.
+export function droppedBlobRefs(oldValue, newValue) {
+    const before = blobKeysOf(oldValue);
+    if (before.length === 0)
+        return false;
+    const after = new Set(blobKeysOf(newValue));
+    return before.some((k) => !after.has(k));
+}