sealkeep 0.9.0 → 0.11.0

package/dist/src/index-publish.js

@@ -0,0 +1,540 @@
+/**
+ * Publishing segments to the account index, and pulling a peer's.
+ *
+ * A segment is already the right unit to share: it is sealed once and never
+ * rewritten, so "publish" is just "upload the ones a peer has not seen" and
+ * "pull" is just "download the ones this machine has not seen" — no merge,
+ * no whole-index transfer. Scope is the account only (the same person's
+ * machines, identified by `machineIdentity`); a team shard keeps publishing
+ * the single legacy blob untouched.
+ *
+ * Remote layout, next to the blob's own keys (`segmentKeys` in
+ * `index-sync.ts`):
+ *   …/segments/<machineId>/<segmentId>.skseg[.json]/.skdir  the sealed files,
+ *     bytes unchanged — they are sealed to the phrase already.
+ *   …/segments/<machineId>/head.json   plaintext: this machine's own
+ *     (`origin: "local"`) segments and its current tombstones.
+ *   …/segments/heads.json              plaintext phonebook of every
+ *     machine's `{ generation, publishedAt }` — the only way a peer is
+ *     discovered, since the store has no listing.
+ *
+ * `heads.json` is a plain read-modify-write with no lock: two publishers at
+ * once can drop each other's entry. That is accepted, not a bug — a machine
+ * re-adds itself the next time IT publishes something, so the map heals.
+ */
+import { randomUUID } from "node:crypto";
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { machineIdentity } from "./cloud.js";
+import { readManifest, updateManifest } from "./index-manifest.js";
+import { openSegment, removeSegment, segmentPaths, writeSegment } from "./index-segments.js";
+import { normalizeContentIndex, loadContentIndexFromFile, pullRemoteIndexIfNewerVerified } from "./search.js";
+const segmentsDir = (dataDir) => join(dataDir, "index", "segments");
+const publishedStatePath = (dataDir) => join(dataDir, "index", "segments", "published.json");
+const pulledStatePath = (dataDir) => join(dataDir, "index", "segments", "pulled.json");
+const publishedBlobStatePath = (dataDir) => join(dataDir, "index", "segments", "published-blob.json");
+const pulledBlobStatePath = (dataDir) => join(dataDir, "index", "segments", "pulled-blob.json");
+/** Atomic publication that never leaves this call's temp behind on error.
+ * Mirrors the private `writeIndexFile` in src/search.ts, reimplemented here
+ * since that helper is not exported. */
+async function writeJsonFile(target, value) {
+    await mkdir(dirname(target), { recursive: true, mode: 0o700 });
+    const temp = `${target}.${randomUUID()}.tmp`;
+    try {
+        await writeFile(temp, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
+        await rename(temp, target);
+    }
+    finally {
+        await rm(temp, { force: true }).catch(() => undefined);
+    }
+}
+async function readPublishedBlobState(dataDir) {
+    try {
+        const parsed = JSON.parse(await readFile(publishedBlobStatePath(dataDir), "utf8"));
+        if (typeof parsed.generation !== "string" || typeof parsed.builtAt !== "string")
+            return null;
+        return { version: 1, generation: parsed.generation, builtAt: parsed.builtAt };
+    }
+    catch {
+        return null;
+    }
+}
+/** Called by `publishFromSegments` (src/index-store.ts) after a successful
+ * push, so `publishPending` and the blob-fallback `pullSegments` both know
+ * which generation and builtAt reached the store last. */
+export async function writePublishedBlobState(dataDir, state) {
+    await writeJsonFile(publishedBlobStatePath(dataDir), state);
+}
+async function readPulledBlobState(dataDir) {
+    try {
+        const parsed = JSON.parse(await readFile(pulledBlobStatePath(dataDir), "utf8"));
+        if (typeof parsed.builtAt !== "string")
+            return null;
+        return { version: 1, builtAt: parsed.builtAt };
+    }
+    catch {
+        return null;
+    }
+}
+async function writePulledBlobState(dataDir, state) {
+    await writeJsonFile(pulledBlobStatePath(dataDir), state);
+}
+async function readPublishedState(dataDir) {
+    try {
+        const parsed = JSON.parse(await readFile(publishedStatePath(dataDir), "utf8"));
+        if (typeof parsed.machineId !== "string" || typeof parsed.generation !== "string" || !Array.isArray(parsed.segments))
+            return null;
+        return {
+            version: 1,
+            machineId: parsed.machineId,
+            generation: parsed.generation,
+            segments: parsed.segments.filter((id) => typeof id === "string"),
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Uploads every local segment this machine has not yet published, deletes the
+ * remote copies of ids compaction or retirement has since replaced locally,
+ * republishes this machine's `head.json`, and re-registers it in the shared
+ * `heads.json`. Idempotent by construction: `head.json` is written only after
+ * every object it lists has actually landed, so a crash between any two steps
+ * is simply repaired by the next call, and nothing is re-uploaded because
+ * `published.json` is the last thing written.
+ */
+export async function publishSegments(dataDir, phrase, store, options = {}) {
+    const segments = store?.segments;
+    // No generic keyed storage (managed/team today) or no account machine
+    // identity (offline, or no Sealkeep Cloud account at all): either way there
+    // is no `<machineId>/…` this call could address. The next successful call
+    // catches up in full, since `published.json` still names the old machine.
+    const machine = segments ? await machineIdentity(dataDir, undefined, { signal: options.signal }).catch(() => null) : null;
+    if (!segments || !machine)
+        return { published: 0, removed: 0, skipped: true };
+    const machineId = String(machine.device_id);
+    const manifest = await readManifest(dataDir);
+    const state = await readPublishedState(dataDir);
+    if (state && state.machineId === machineId && state.generation === manifest.generation) {
+        return { published: 0, removed: 0 };
+    }
+    // Only this machine's own rows are ever this machine's to publish or
+    // retract — a segment learned from a peer (`origin: "remote"`) is that
+    // peer's to publish, never republished here.
+    const local = manifest.segments.filter((segment) => segment.origin !== "remote");
+    const localIds = new Set(local.map((segment) => segment.id));
+    const previouslyPublished = state && state.machineId === machineId ? state.segments : [];
+    const toUpload = local.filter((segment) => !previouslyPublished.includes(segment.id));
+    const toRemove = previouslyPublished.filter((id) => !localIds.has(id));
+    let published = 0;
+    for (const segment of toUpload) {
+        options.signal?.throwIfAborted();
+        const paths = await segmentPaths(dataDir, segment.id);
+        // The three local files, as paths: `put` streams the body instead of
+        // buffering a potentially large segment whole.
+        await segments.put(`${machineId}/${segment.id}.skseg`, paths.body, { signal: options.signal });
+        await segments.put(`${machineId}/${segment.id}.skseg.json`, paths.envelope, { signal: options.signal });
+        await segments.put(`${machineId}/${segment.id}.skdir`, paths.directory, { signal: options.signal });
+        published += 1;
+        await options.onYield?.();
+    }
+    let removed = 0;
+    for (const id of toRemove) {
+        options.signal?.throwIfAborted();
+        await segments.del(`${machineId}/${id}.skseg`, { signal: options.signal });
+        await segments.del(`${machineId}/${id}.skseg.json`, { signal: options.signal });
+        await segments.del(`${machineId}/${id}.skdir`, { signal: options.signal });
+        removed += 1;
+        await options.onYield?.();
+    }
+    const head = {
+        version: 1,
+        machineId,
+        generation: manifest.generation,
+        publishedAt: new Date().toISOString(),
+        segments: local.map((segment) => ({
+            id: segment.id,
+            builtAt: segment.builtAt,
+            archives: segment.archives,
+            tokens: segment.tokens,
+            bytes: segment.bytes,
+            archiveIds: segment.archiveIds ?? [],
+        })),
+        tombstones: manifest.tombstones,
+    };
+    await segments.put(`${machineId}/head.json`, Buffer.from(`${JSON.stringify(head, null, 2)}\n`, "utf8"), { signal: options.signal });
+    await healHeads(segments, machineId, { generation: manifest.generation, publishedAt: head.publishedAt }, options.signal);
+    await writeJsonFile(publishedStatePath(dataDir), {
+        version: 1, machineId, generation: manifest.generation, segments: local.map((segment) => segment.id),
+    });
+    return { published, removed };
+}
+/**
+ * True when a segments-mode vault has local segment content the account
+ * store has not seen yet — the daemon's cue to run a catch-up pass even when
+ * nothing is left to index. `published.json` (a segment store) and
+ * `published-blob.json` (the blob fallback) each name the manifest
+ * generation their own kind of publish last reached; either matching the
+ * current manifest counts as "nothing pending", since a vault only ever uses
+ * one or the other depending on what the account store supports. A blob-mode
+ * vault (not segments) always answers false: `indexArchiveTokens` already
+ * publishes inline at seal time there, so the daemon needs no extra nudge.
+ */
+export async function publishPending(dataDir) {
+    const { segmentsEnabled } = await import("./index-store.js");
+    if (!await segmentsEnabled(dataDir))
+        return false;
+    const manifest = await readManifest(dataDir);
+    const [published, publishedBlob] = await Promise.all([readPublishedState(dataDir), readPublishedBlobState(dataDir)]);
+    const upToDate = published?.generation === manifest.generation || publishedBlob?.generation === manifest.generation;
+    return !upToDate;
+}
+function parseHeadsFile(raw) {
+    if (!raw)
+        return { version: 1, machines: {} };
+    try {
+        const parsed = JSON.parse(raw.toString("utf8"));
+        const machines = {};
+        for (const [id, value] of Object.entries(parsed.machines ?? {})) {
+            const entry = value;
+            if (typeof entry?.generation === "string" && typeof entry?.publishedAt === "string") {
+                machines[id] = { generation: entry.generation, publishedAt: entry.publishedAt };
+            }
+        }
+        return { version: 1, machines };
+    }
+    catch {
+        return { version: 1, machines: {} };
+    }
+}
+async function healHeads(segments, machineId, entry, signal) {
+    const current = parseHeadsFile(await segments.get("heads.json", { signal }));
+    current.machines[machineId] = entry;
+    await segments.put("heads.json", Buffer.from(`${JSON.stringify(current, null, 2)}\n`, "utf8"), { signal });
+}
+function parseMachineHead(raw) {
+    try {
+        const parsed = JSON.parse(raw.toString("utf8"));
+        if (typeof parsed.machineId !== "string" || !Array.isArray(parsed.segments) || !Array.isArray(parsed.tombstones))
+            return null;
+        const segments = [];
+        for (const candidate of parsed.segments) {
+            const entry = candidate;
+            if (typeof entry?.id === "string" && typeof entry.builtAt === "string"
+                && typeof entry.archives === "number" && typeof entry.tokens === "number" && typeof entry.bytes === "number") {
+                segments.push({
+                    id: entry.id, builtAt: entry.builtAt, archives: entry.archives, tokens: entry.tokens, bytes: entry.bytes,
+                    archiveIds: Array.isArray(entry.archiveIds) ? entry.archiveIds.filter((id) => typeof id === "string") : [],
+                });
+            }
+        }
+        return {
+            version: 1,
+            machineId: parsed.machineId,
+            generation: typeof parsed.generation === "string" ? parsed.generation : "",
+            publishedAt: typeof parsed.publishedAt === "string" ? parsed.publishedAt : new Date(0).toISOString(),
+            segments,
+            tombstones: parsed.tombstones.filter((id) => typeof id === "string"),
+        };
+    }
+    catch {
+        return null;
+    }
+}
+async function readPulledState(dataDir) {
+    try {
+        const parsed = JSON.parse(await readFile(pulledStatePath(dataDir), "utf8"));
+        const machines = {};
+        for (const [id, generation] of Object.entries(parsed.machines ?? {}))
+            if (typeof generation === "string")
+                machines[id] = generation;
+        return { version: 1, machines };
+    }
+    catch {
+        return { version: 1, machines: {} };
+    }
+}
+/** Downloads a not-yet-local segment and verifies it opens under this
+ * phrase before trusting it. A segment that will not open (truncated
+ * transfer, a peer's write this machine caught mid-publish) is removed and
+ * skipped rather than left half-written: the peer's head still lists it, so
+ * the next pull retries it once the peer's own upload has finished. */
+async function downloadAndVerifySegment(dataDir, phrase, segments, machineId, segmentId, signal) {
+    const paths = await segmentPaths(dataDir, segmentId);
+    const [body, envelope, directory] = await Promise.all([
+        segments.get(`${machineId}/${segmentId}.skseg`, { signal }),
+        segments.get(`${machineId}/${segmentId}.skseg.json`, { signal }),
+        segments.get(`${machineId}/${segmentId}.skdir`, { signal }),
+    ]);
+    if (!body || !envelope || !directory) {
+        await removeSegment(dataDir, segmentId);
+        return false;
+    }
+    await mkdir(segmentsDir(dataDir), { recursive: true, mode: 0o700 });
+    await writeFile(paths.body, body, { mode: 0o600 });
+    await writeFile(paths.envelope, envelope, { mode: 0o600 });
+    await writeFile(paths.directory, directory, { mode: 0o600 });
+    try {
+        const reader = await openSegment(dataDir, phrase, segmentId);
+        await reader.close();
+        return true;
+    }
+    catch {
+        await removeSegment(dataDir, segmentId);
+        return false;
+    }
+}
+/** A decrypted index with any id this machine must never adopt from a peer's
+ * blob dropped: its own local records (already covered by its own segments)
+ * and anything it has tombstoned (retired, and not to be resurrected by a
+ * copy of the vault from before the retirement). */
+function withoutIds(index, exclude) {
+    const archives = {};
+    for (const [id, line] of Object.entries(index.archives))
+        if (!exclude.has(id))
+            archives[id] = line;
+    const tokens = {};
+    for (const [token, postings] of Object.entries(index.tokens)) {
+        const kept = postings.filter((posting) => !exclude.has(posting.split("@")[0]));
+        if (kept.length > 0)
+            tokens[token] = kept;
+    }
+    const meta = {};
+    for (const [id, value] of Object.entries(index.meta ?? {}))
+        if (!exclude.has(id))
+            meta[id] = value;
+    return { version: 2, extractionPolicy: index.extractionPolicy, builtAt: index.builtAt, archives, tokens, meta };
+}
+// A blob this size is a big vault's own foreground publication (segments
+// mode never produces one this large itself — see the background child's
+// `publishMaxTokens`), not something worth decrypting and re-deriving from
+// inside whatever process happened to call pullSegments (a hook, a daemon
+// tick): it would not fit that process's memory budget.
+const MAX_BLOB_FALLBACK_BYTES = 64 * 1024 * 1024;
+const blobSentinelMachineId = "blob";
+/** The published account index's size, without downloading it: the managed
+ * plane's CAS head carries the blob receipt, and a plain object store's
+ * envelope carries the chunk table. `null` when neither can say. */
+async function remoteBlobBytes(store, signal) {
+    if (store.cas) {
+        const head = await store.cas.head({ signal }).catch(() => null);
+        if (head && head.status === "ok" && head.head)
+            return head.head.receipt.blob.bytes;
+    }
+    const envelope = await store.meta({ signal }).catch(() => null);
+    if (!envelope)
+        return null;
+    return envelope.chunks.reduce((sum, chunk) => sum + chunk.storedBytes, 0);
+}
+/**
+ * Step 2 addendum: a store with no generic keyed storage (managed today)
+ * cannot hold segments at all — see docs/design/index-publish.md. Its single
+ * legacy blob slot is still readable, though, so this converts THAT into one
+ * ordinary `origin: "remote"` segment instead of a segments-mode vault on
+ * such a store seeing nothing a peer published. There is exactly one blob,
+ * so exactly one such entry ever exists — the sentinel `machineId: "blob"` —
+ * replaced whole (never merged with its own previous self) on every
+ * conversion, the same "remove old, add new, then clean up files" order
+ * `pullSegments`'s per-peer loop uses.
+ */
+async function pullFromBlobFallback(dataDir, phrase, store, options) {
+    const nothing = { machines: 0, added: 0, removed: 0 };
+    let tempPath = null;
+    try {
+        const [published, pulledBefore] = await Promise.all([readPublishedBlobState(dataDir), readPulledBlobState(dataDir)]);
+        // Size first, from metadata alone: a blob too large to convert must not be
+        // downloaded at all. The managed plane answers from its CAS head receipt,
+        // a plain object store from the envelope's chunk table.
+        const remoteBytes = await remoteBlobBytes(store, options.signal);
+        if (remoteBytes !== null && remoteBytes > MAX_BLOB_FALLBACK_BYTES)
+            return { ...nothing, skipped: true };
+        await mkdir(segmentsDir(dataDir), { recursive: true, mode: 0o700 });
+        tempPath = join(segmentsDir(dataDir), `.blob-pull-${randomUUID()}.tmp`);
+        // The account index is fetched exactly the way the legacy blob build
+        // fetched it: a managed plane publishes through its CAS generation lane
+        // and leaves the plain `meta`/`blob` objects weeks stale, so asking those
+        // directly reads an ancient envelope and concludes there is nothing new.
+        // The sink lands the bytes in this temp file; the blob's own path is
+        // never written in segments mode.
+        let envelope = null;
+        let tooLarge = false;
+        // "Newer than what we already converted", not "newer than our own blob":
+        // in segments mode this machine's blob is only ever a publication of its
+        // own segments.
+        const knownBuiltAt = pulledBefore?.builtAt ?? published?.builtAt ?? null;
+        const outcome = await pullRemoteIndexIfNewerVerified(dataDir, phrase, store, knownBuiltAt, async (remoteEnvelope, ciphertext) => {
+            // A second guard for a store whose metadata did not say: the bytes are
+            // already here, so this only avoids converting them.
+            const storedBytes = remoteEnvelope.chunks.reduce((sum, chunk) => sum + chunk.storedBytes, 0);
+            if (storedBytes > MAX_BLOB_FALLBACK_BYTES) {
+                tooLarge = true;
+                return;
+            }
+            envelope = remoteEnvelope;
+            await writeFile(tempPath, ciphertext, { mode: 0o600 });
+        }, { signal: options.signal, onYield: options.onYield }, true);
+        if (tooLarge)
+            return { ...nothing, skipped: true };
+        if (!outcome.pulled || !envelope)
+            return nothing;
+        const remoteBuiltAt = outcome.remoteBuiltAt ?? envelope.manifest?.createdAt ?? null;
+        if (!remoteBuiltAt)
+            return nothing;
+        // The identical safe reader loadContentIndex uses for its own local
+        // blob — streaming with a bounded legacy fallback — pointed at the
+        // downloaded temp instead: this is never written to the blob's own path.
+        const decoded = normalizeContentIndex(await loadContentIndexFromFile(envelope, tempPath, phrase, { signal: options.signal, onYield: options.onYield }));
+        const { listArchives } = await import("./vault.js");
+        const manifest = await readManifest(dataDir);
+        const exclude = new Set([...(await listArchives(dataDir)).map((record) => record.id), ...manifest.tombstones]);
+        const filtered = withoutIds(decoded, exclude);
+        const remoteArchiveIds = Object.keys(filtered.archives);
+        if (remoteArchiveIds.length === 0) {
+            // Every archive this blob carries is already ours or tombstoned —
+            // including a blob that is really just OUR OWN publication echoed
+            // back by a peer that also has no `segments` support: that peer's own
+            // publish step merges everything it holds (its local rows plus
+            // whatever it converted from us) into the one shared slot, unasked,
+            // which changes the slot's builtAt without changing what it means for
+            // ITS bytes to be "ours". Filtering by content rather than trusting
+            // builtAt alone is what makes "a build on A never adopts its own
+            // blob" hold even through that echo. Nothing to convert, but this
+            // builtAt has been seen, so the next unchanged call is a fast no-op.
+            await writePulledBlobState(dataDir, { version: 1, builtAt: remoteBuiltAt });
+            return { machines: 1, added: 0, removed: 0 };
+        }
+        const { segmentRowsFromIndex, writeCoverageFromManifest } = await import("./index-store.js");
+        const entry = await writeSegment(dataDir, phrase, {
+            segmentId: randomUUID(),
+            builtAt: new Date().toISOString(),
+            rows: segmentRowsFromIndex(filtered),
+        });
+        const withEntry = {
+            ...entry, origin: "remote", machineId: blobSentinelMachineId, archiveIds: [], remoteArchiveIds,
+        };
+        let previousId = null;
+        await updateManifest(dataDir, (current) => {
+            const existing = current.segments.find((segment) => segment.origin === "remote" && segment.machineId === blobSentinelMachineId);
+            previousId = existing?.id ?? null;
+            return {
+                ...current,
+                segments: [
+                    ...current.segments.filter((segment) => !(segment.origin === "remote" && segment.machineId === blobSentinelMachineId)),
+                    withEntry,
+                ],
+            };
+        });
+        if (previousId)
+            await removeSegment(dataDir, previousId);
+        await writePulledBlobState(dataDir, { version: 1, builtAt: remoteBuiltAt });
+        await writeCoverageFromManifest(dataDir);
+        await options.onYield?.();
+        return { machines: 1, added: 1, removed: previousId ? 1 : 0 };
+    }
+    catch (error) {
+        if (options.signal?.aborted)
+            throw error;
+        // A bad transfer, a blob that will not decrypt, or a network hiccup is
+        // not a hard failure: the next call's meta() check simply tries again.
+        return nothing;
+    }
+    finally {
+        if (tempPath)
+            await rm(tempPath, { force: true }).catch(() => undefined);
+    }
+}
+/**
+ * Reads the shared `heads.json`, then for every other machine whose
+ * generation has moved since this machine last reconciled it: fetches that
+ * machine's `head.json`, downloads whichever listed segments are not already
+ * local, verifies each, and replaces that machine's `origin: "remote"`
+ * entries in the local manifest with the current set in one `updateManifest`
+ * — dropping local files for ids it no longer lists only after the manifest
+ * no longer references them. A machine whose `head.json` has vanished (404)
+ * loses its remote entries entirely.
+ *
+ * A store with no generic keyed storage (managed today) has no `<machineId>`
+ * objects to read at all; `pullFromBlobFallback` converts its single legacy
+ * blob slot instead (step 2 addendum — see docs/design/index-publish.md).
+ */
+export async function pullSegments(dataDir, phrase, store, options = {}) {
+    if (!store)
+        return { machines: 0, added: 0, removed: 0, skipped: true };
+    if (!store.segments)
+        return pullFromBlobFallback(dataDir, phrase, store, options);
+    const segments = store.segments;
+    const machine = await machineIdentity(dataDir, undefined, { signal: options.signal }).catch(() => null);
+    if (!machine)
+        return { machines: 0, added: 0, removed: 0, skipped: true };
+    const thisMachineId = String(machine.device_id);
+    const heads = parseHeadsFile(await segments.get("heads.json", { signal: options.signal }));
+    const pulled = await readPulledState(dataDir);
+    let machinesProcessed = 0;
+    let added = 0;
+    let removed = 0;
+    for (const [peerId, summary] of Object.entries(heads.machines)) {
+        options.signal?.throwIfAborted();
+        if (peerId === thisMachineId)
+            continue;
+        if (pulled.machines[peerId] === summary.generation)
+            continue;
+        // Re-read every iteration: an earlier peer in this same loop may have
+        // just rewritten the manifest, and `priorForPeer`/`knownElsewhere` below
+        // must reflect it.
+        const manifest = await readManifest(dataDir);
+        const priorForPeer = manifest.segments.filter((entry) => entry.origin === "remote" && entry.machineId === peerId);
+        const priorById = new Map(priorForPeer.map((entry) => [entry.id, entry]));
+        // A ready-made SegmentEntry never overwritten below: this peer's own
+        // segment ids the manifest already carries some other way (never
+        // expected — ids are random — but never trusted blindly either).
+        const knownElsewhere = new Set(manifest.segments.filter((entry) => !(entry.origin === "remote" && entry.machineId === peerId)).map((entry) => entry.id));
+        const headRaw = await segments.get(`${peerId}/head.json`, { signal: options.signal });
+        const head = headRaw ? parseMachineHead(headRaw) : null;
+        // The peer's FULL current set, not just what changed this call: a
+        // tombstone-only republish (nothing added or removed) must still keep
+        // every segment this machine already trusts from this peer, or it would
+        // read as "everything just went stale".
+        const current = [];
+        if (head) {
+            for (const listed of head.segments) {
+                options.signal?.throwIfAborted();
+                const existing = priorById.get(listed.id);
+                if (existing) {
+                    current.push(existing);
+                    continue;
+                }
+                if (knownElsewhere.has(listed.id))
+                    continue;
+                if (!await downloadAndVerifySegment(dataDir, phrase, segments, peerId, listed.id, options.signal))
+                    continue;
+                current.push({
+                    id: listed.id, builtAt: listed.builtAt, archives: listed.archives, tokens: listed.tokens, bytes: listed.bytes,
+                    origin: "remote", machineId: peerId, archiveIds: [], remoteArchiveIds: listed.archiveIds,
+                });
+                added += 1;
+                await options.onYield?.();
+            }
+        }
+        const keptIds = new Set(current.map((entry) => entry.id));
+        const staleIds = priorForPeer.map((entry) => entry.id).filter((id) => !keptIds.has(id));
+        await updateManifest(dataDir, (manifestNow) => ({
+            ...manifestNow,
+            segments: [
+                ...manifestNow.segments.filter((entry) => !(entry.origin === "remote" && entry.machineId === peerId)),
+                ...current,
+            ],
+            tombstones: head ? [...new Set([...manifestNow.tombstones, ...head.tombstones])] : manifestNow.tombstones,
+        }));
+        // Only once the manifest no longer points at them: a crash before this
+        // point leaves an orphan file for the next compaction's orphan sweep.
+        for (const id of staleIds) {
+            await removeSegment(dataDir, id);
+            removed += 1;
+        }
+        pulled.machines[peerId] = summary.generation;
+        machinesProcessed += 1;
+    }
+    await writeJsonFile(pulledStatePath(dataDir), pulled);
+    return { machines: machinesProcessed, added, removed };
+}

package/dist/src/index-scratch-cleanup.d.ts

@@ -0,0 +1,19 @@
+/**
+ * One daemon-owned historical child gets one private scratch namespace.
+ * Foreground/manual builders deliberately keep using the ordinary index
+ * directory, so recovery of a dead child can never unlink their live files.
+ */
+export declare function backgroundIndexScratchDirectory(dataDir: string, generation: string): string;
+/** Create or reopen only the exact namespace assigned to this worker. */
+export declare function ensureBackgroundIndexScratch(dataDir: string, generation: string): Promise<string>;
+/**
+ * Remove scratch for one process generation after its durable child barrier
+ * has proved that exact worker dead. The caller owns that proof; this helper
+ * intentionally cannot scan legacy/global names because another foreground
+ * builder may be using them.
+ *
+ * Preflight every entry before deleting any. An unknown name, symlink or
+ * directory leaves the entire namespace intact for diagnosis. Concurrent
+ * successor contenders may safely repeat the same cleanup.
+ */
+export declare function cleanupInterruptedIndexScratch(dataDir: string, generation: string): Promise<void>;