sealkeep 0.8.1 → 0.11.0

package/dist/src/rehydrate.js

@@ -1,49 +1,345 @@
-import { access, constants } from "node:fs/promises";
-import { basename, resolve } from "node:path";
-import { listArchives } from "./vault.js";
-import { restoreArchive } from "./restore.js";
+import { randomUUID } from "node:crypto";
+import { createReadStream, createWriteStream } from "node:fs";
+import { access, constants, open, rm, stat, utimes } from "node:fs/promises";
+import { basename, dirname, join, resolve } from "node:path";
+import { pipeline } from "node:stream/promises";
+import { createGunzip } from "node:zlib";
+import { listArchives, assertPhraseOpens, findArchive, readConfig, restoreRecordToFile } from "./vault.js";
+import { homedir } from "node:os";
+import { nativeRestoreTarget, validateNativeRestoreOutput } from "./restore.js";
 import { recordAudit } from "./audit.js";
+import { readUntouchedCodexResumeStub } from "./resume-stub.js";
+import { canonicalPhrase } from "./mnemonic.js";
+import { MAX_FRAME_METADATA_BYTES } from "./byte-stream.js";
+import { hashFileRange, openArchiveToFile } from "../packages/sealkeep-crypto/src/index.js";
+import { parkExistingPathNoClobber, publishTempCreateIfAbsent, restoreParkedCreateIfAbsent, } from "./reclaim-transaction.js";
+const expectedStubFile = (stub) => ({ bytes: stub.bytes, sha256: stub.sha256 });
+/**
+ * Commit restored bytes without overwriting anything written during the slow
+ * decrypt/download. Existing pointer bytes are first parked under a unique
+ * hidden name, then re-read and digest-checked by the no-clobber primitive.
+ * The parked inode is retained: an agent with an already-open file descriptor
+ * may append after it moves, and those bytes must remain recoverable even when
+ * the restore succeeds.
+ */
+async function commitRestoredTranscript(temp, output, expectedStub, originalTimes) {
+    if (!expectedStub) {
+        const restored = await hashFileRange(temp);
+        await publishTempCreateIfAbsent(temp, output, restored);
+        return {};
+    }
+    const parked = join(dirname(output), `.${basename(output)}.${randomUUID()}.sealkeep-pointer-backup`);
+    try {
+        await parkExistingPathNoClobber(output, parked, expectedStubFile(expectedStub));
+    }
+    catch (error) {
+        throw new Error(`The resume pointer changed while the transcript was restoring; its bytes were preserved and the restore was refused (${error instanceof Error ? error.message : "conflict"})`);
+    }
+    try {
+        const restored = await hashFileRange(temp);
+        await publishTempCreateIfAbsent(temp, output, restored);
+    }
+    catch (error) {
+        await restoreParkedCreateIfAbsent(parked, output).catch(() => undefined);
+        throw new Error(`The native transcript path was recreated while the transcript was restoring; both existing copies were preserved (${error instanceof Error ? error.message : "conflict"})`);
+    }
+    if (originalTimes)
+        await utimes(output, originalTimes.atime, originalTimes.mtime).catch(() => undefined);
+    return { pointerBackup: parked };
+}
+function verifiedLocalPointer(record, wanted, stub) {
+    return record.id === stub.archiveId
+        && record.source.path === wanted
+        && record.reclaimed?.resumeStub?.version === 1
+        && record.reclaimed.resumeStub.sessionId === stub.sessionId
+        && record.reclaimed.resumeStub.bytes === stub.bytes
+        && record.reclaimed.resumeStub.sha256 === stub.sha256;
+}
 export async function rehydrateSession(dataDir, target, phrase, options = {}) {
     const wanted = target.path ? resolve(target.path) : null;
     if (!wanted && !target.sessionId)
         return { rehydrated: false, reason: "no-target" };
+    let resumeStub = null;
     if (wanted) {
         const present = await access(wanted, constants.R_OK).then(() => true, () => false);
-        if (present)
-            return { rehydrated: false, reason: "already-present" };
+        if (present) {
+            resumeStub = await readUntouchedCodexResumeStub(wanted);
+            if (!resumeStub)
+                return { rehydrated: false, reason: "already-present" };
+        }
     }
     const records = (await listArchives(dataDir)).filter((record) => record.version === 2);
-    const byPath = wanted ? records.filter((record) => record.source.path === wanted) : [];
-    const byName = byPath.length > 0 ? byPath
+    // A pointer names the exact verified snapshot it replaced. Never substitute
+    // a different archive just because it shares a path: a later archive may be
+    // a continuation, and an earlier one may omit the end of the session.
+    const byStub = resumeStub
+        ? records.filter((record) => verifiedLocalPointer(record, wanted, resumeStub))
+        : [];
+    const localKnowsPointerArchive = Boolean(resumeStub && records.some((record) => record.id === resumeStub.archiveId));
+    if (resumeStub && localKnowsPointerArchive && byStub.length === 0) {
+        return { rehydrated: false, reason: "restore-failed", note: "The resume pointer does not match its immutable reclaim record; nothing was overwritten." };
+    }
+    const byPath = resumeStub ? byStub : wanted ? records.filter((record) => record.source.path === wanted) : [];
+    const byName = resumeStub ? byPath : byPath.length > 0 ? byPath
         : wanted ? records.filter((record) => basename(record.source.path) === basename(wanted)) : [];
-    const bySession = byName.length > 0 ? byName
+    const bySession = resumeStub ? byName : byName.length > 0 ? byName
         : target.sessionId ? records.filter((record) => basename(record.source.path).includes(target.sessionId)) : [];
-    if (bySession.length === 0)
-        return { rehydrated: false, reason: "not-archived" };
+    if (bySession.length === 0) {
+        try {
+            const remote = await rehydrateManaged(dataDir, target, phrase, options, resumeStub);
+            if (remote)
+                return remote;
+            return { rehydrated: false, reason: phrase ? "not-archived" : "no-phrase" };
+        }
+        catch (error) {
+            return { rehydrated: false, reason: "restore-failed", note: error instanceof Error ? error.message.split("\n")[0] : "managed restore failed" };
+        }
+    }
     const newest = bySession.sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt))[0];
-    // If the match came by name and the file exists at ITS recorded path, the
-    // catalog and the disk already agree — nothing to do.
-    if (!wanted || newest.source.path !== wanted) {
+    if (!resumeStub && (!wanted || newest.source.path !== wanted)) {
         const there = await access(newest.source.path, constants.R_OK).then(() => true, () => false);
         if (there)
             return { rehydrated: false, reason: "already-present" };
     }
     if (!phrase)
         return { rehydrated: false, reason: "no-phrase", note: "No recovery phrase is available to this machine (keystore or SEALKEEP_RECOVERY_PHRASE), so the sealed copy stays sealed." };
+    let temp = null;
     try {
-        const outcome = await restoreArchive(dataDir, newest.id, phrase, { native: true, overwrite: "refuse", home: options.home });
-        await recordAudit(dataDir, "archive.restore", "allowed", { archiveId: newest.id, path: outcome.output, rehydrated: true });
-        return { rehydrated: true, archiveId: newest.id, output: outcome.output, bytes: outcome.bytes };
+        const record = await findArchive(dataDir, newest.id);
+        const output = nativeRestoreTarget(record, options.home);
+        if (wanted && output !== wanted)
+            throw new Error("The archived native path does not match the requested transcript path");
+        assertPhraseOpens(await readConfig(dataDir), record, phrase);
+        await validateNativeRestoreOutput(output, options.home ?? homedir(), resumeStub ? "file" : "missing-or-file");
+        temp = join(dirname(output), `.${basename(output)}.${randomUUID()}.rehydrate-partial`);
+        const restored = await restoreRecordToFile(dataDir, record, phrase, temp);
+        const originalTimes = resumeStub && wanted ? await stat(wanted).catch(() => null) : null;
+        await options.operations?.beforeCommit?.({ archiveId: newest.id, output, managed: false });
+        const committed = await commitRestoredTranscript(temp, output, resumeStub, originalTimes);
+        temp = null;
+        await recordAudit(dataDir, "archive.restore", "allowed", { archiveId: newest.id, path: output, rehydrated: true });
+        return { rehydrated: true, archiveId: newest.id, output, bytes: restored.bytes, ...committed };
     }
     catch (error) {
         return { rehydrated: false, reason: "restore-failed", note: error instanceof Error ? error.message.split("\n")[0] : "restore failed" };
     }
+    finally {
+        if (temp)
+            await rm(temp, { force: true }).catch(() => undefined);
+    }
 }
-/**
- * Lenient hook-payload reading, distinct from `hookEventFromStdin` on purpose:
- * that parser refuses when the transcript is unreadable, and an ABSENT
- * transcript is this feature's entire subject.
- */
+function validCatalogFacts(value) {
+    return value?.v === 1
+        && typeof value.createdAt === "string"
+        && Number.isSafeInteger(value.sourceBytes) && value.sourceBytes >= 0
+        && /^[0-9a-f]{64}$/i.test(value.sourceSha256)
+        && Number.isSafeInteger(value.storedBytes) && value.storedBytes >= 0
+        && typeof value.algorithm === "string"
+        && Number.isSafeInteger(value.chunks) && value.chunks >= 0
+        && (!value.delta || (typeof value.delta.baseArchiveId === "string"
+            && Number.isSafeInteger(value.delta.baseBytes) && value.delta.baseBytes > 0));
+}
+/** Resolve oldest-to-newest and refuse every incomplete, cyclic or inconsistent delta chain. */
+function managedDeltaChain(selected, entries, durable) {
+    const newestToOldest = [];
+    const seen = new Set();
+    let cursor = selected;
+    while (cursor) {
+        if (seen.has(cursor.id) || !durable.has(cursor.id) || !validCatalogFacts(cursor.facts))
+            return null;
+        seen.add(cursor.id);
+        newestToOldest.push(cursor);
+        const delta = cursor.facts.delta;
+        if (!delta)
+            break;
+        const base = entries.get(delta.baseArchiveId);
+        if (!base || base.facts.sourceBytes !== delta.baseBytes || cursor.facts.sourceBytes <= delta.baseBytes)
+            return null;
+        cursor = base;
+    }
+    const chain = newestToOldest.reverse();
+    if (chain.length === 0 || chain[0].facts.delta)
+        return null;
+    return chain;
+}
+const FRAME_MAGIC = Buffer.from("VLA1", "ascii");
+async function downloadManagedRef(dataDir, ref, destination) {
+    const { pullCiphertextStream } = await import("./cloud.js");
+    await pipeline(await pullCiphertextStream(dataDir, ref), createWriteStream(destination, { flags: "wx", mode: 0o600 }));
+}
+async function framedEnvelope(path) {
+    const handle = await open(path, "r");
+    try {
+        const info = await handle.stat();
+        const prefix = Buffer.alloc(Math.min(8, info.size));
+        if (prefix.length > 0)
+            await handle.read(prefix, 0, prefix.length, 0);
+        if (prefix.length < 8 || !prefix.subarray(0, 4).equals(FRAME_MAGIC)) {
+            return { envelope: null, bodyOffset: 0, gzip: prefix.length >= 2 && prefix[0] === 0x1f && prefix[1] === 0x8b };
+        }
+        const metadataBytes = prefix.readUInt32BE(4);
+        if (metadataBytes > MAX_FRAME_METADATA_BYTES)
+            throw new Error(`Managed archive metadata is ${metadataBytes} bytes; the safe limit is ${MAX_FRAME_METADATA_BYTES}`);
+        if (info.size < 8 + metadataBytes)
+            throw new Error("Managed archive header is truncated");
+        const metadata = Buffer.alloc(metadataBytes);
+        if (metadataBytes > 0) {
+            const result = await handle.read(metadata, 0, metadataBytes, 8);
+            if (result.bytesRead !== metadataBytes)
+                throw new Error("Managed archive header is truncated");
+        }
+        try {
+            return { envelope: JSON.parse(metadata.toString("utf8")), bodyOffset: 8 + metadataBytes, gzip: false };
+        }
+        catch {
+            throw new Error("Managed archive header is not readable JSON");
+        }
+    }
+    finally {
+        await handle.close();
+    }
+}
+/** Open one physical managed archive to plaintext with bounded RAM. */
+async function openManagedArchiveToTemp(dataDir, archiveId, destination, phrase) {
+    const { statCiphertext, pullCiphertextStream } = await import("./cloud.js");
+    const { managedRefForKey } = await import("./managed-chunks.js");
+    const { chunkObjectName, ENVELOPE_OBJECT } = await import("./chunk-store.js");
+    const scratch = `${destination}.${randomUUID()}`;
+    const downloaded = `${scratch}.managed`;
+    const ciphertext = `${scratch}.ciphertext`;
+    try {
+        let envelope = null;
+        if ((await statCiphertext(dataDir, archiveId)).exists) {
+            await downloadManagedRef(dataDir, archiveId, downloaded);
+            const framed = await framedEnvelope(downloaded);
+            if (!framed.envelope) {
+                if (!framed.gzip)
+                    throw new Error(`${archiveId} is a legacy managed object without a self-contained envelope`);
+                await pipeline(createReadStream(downloaded), createGunzip(), createWriteStream(destination, { flags: "wx", mode: 0o600 }));
+                return hashFileRange(destination);
+            }
+            envelope = framed.envelope;
+            await pipeline(createReadStream(downloaded, { start: framed.bodyOffset }), createWriteStream(ciphertext, { flags: "wx", mode: 0o600 }));
+        }
+        else {
+            const sidecarRef = managedRefForKey(`${archiveId}/${ENVELOPE_OBJECT}`);
+            if (!(await statCiphertext(dataDir, sidecarRef)).exists)
+                throw new Error(`Managed storage holds no complete archive ${archiveId}`);
+            await downloadManagedRef(dataDir, sidecarRef, downloaded);
+            const framed = await framedEnvelope(downloaded);
+            if (!framed.envelope)
+                throw new Error(`The managed sidecar for ${archiveId} carries no envelope`);
+            envelope = framed.envelope;
+            const headers = Array.isArray(envelope.chunks) ? envelope.chunks : [];
+            if (headers.length === 0)
+                throw new Error(`The managed sidecar for ${archiveId} lists no chunks`);
+            for (let index = 0; index < headers.length; index += 1) {
+                const ref = managedRefForKey(`${archiveId}/${chunkObjectName(index)}`);
+                await pipeline(await pullCiphertextStream(dataDir, ref), createWriteStream(ciphertext, { flags: index === 0 ? "wx" : "a", mode: 0o600 }));
+            }
+        }
+        const opened = await openArchiveToFile(envelope, ciphertext, destination, { phrase: canonicalPhrase(phrase) });
+        return { bytes: opened.bytes, sha256: opened.sha256 };
+    }
+    finally {
+        await rm(downloaded, { force: true }).catch(() => undefined);
+        await rm(ciphertext, { force: true }).catch(() => undefined);
+    }
+}
+/** Restore a fresh machine only from a complete verified managed delta chain. */
+async function rehydrateManaged(dataDir, target, phrase, options, knownStub = null) {
+    if (!phrase)
+        return null;
+    const { refreshIndexFromRemote, loadContentIndexForQuery } = await import("./search.js");
+    let index;
+    try {
+        await refreshIndexFromRemote(dataDir, phrase);
+        // V3 parsing streams the expanded checkpoint and keeps catalog/meta rows
+        // only. It does not materialize the token map that reached 808 MB in a
+        // production vault. No search terms are needed for an id/path lookup.
+        index = await loadContentIndexForQuery(dataDir, phrase, []);
+    }
+    catch (error) {
+        if (error instanceof Error && /No (?:local )?content index(?: this key can open)?/i.test(error.message))
+            return null;
+        throw error;
+    }
+    const wanted = target.path ? resolve(target.path) : null;
+    const allEntries = Object.entries(index.archives).flatMap(([id, row]) => {
+        const agent = row[0] ?? "";
+        const path = row[1] ?? "";
+        const meta = index.meta?.[id];
+        return (agent === "codex" || agent === "claude") && validCatalogFacts(meta?.archiveRecord)
+            ? [{ id, agent, path, meta, facts: meta.archiveRecord }]
+            : [];
+    });
+    const entries = new Map(allEntries.map((entry) => [entry.id, entry]));
+    const matches = allEntries.filter((entry) => (wanted && (resolve(entry.path) === wanted || basename(entry.path) === basename(wanted)))
+        || (target.sessionId && (entry.id.includes(target.sessionId) || basename(entry.path).includes(target.sessionId))));
+    if (matches.length === 0)
+        return null;
+    const { listCloudArchives } = await import("./cloud.js");
+    const { collapseCloudRows } = await import("./managed-chunks.js");
+    const durableRows = collapseCloudRows((await listCloudArchives(dataDir)).filter((row) => row.state === "durable"));
+    const durable = new Set(durableRows.map((row) => row.vault_ref));
+    const candidates = matches.filter((entry) => durable.has(entry.id) && (!knownStub || entry.id === knownStub.archiveId));
+    if (candidates.length === 0)
+        return null;
+    candidates.sort((a, b) => {
+        const at = a.meta?.endedAt ?? a.meta?.startedAt ?? a.facts.createdAt;
+        const bt = b.meta?.endedAt ?? b.meta?.startedAt ?? b.facts.createdAt;
+        return bt.localeCompare(at) || b.id.localeCompare(a.id);
+    });
+    const selected = candidates[0];
+    const chain = managedDeltaChain(selected, entries, durable);
+    if (!chain)
+        return null;
+    const output = wanted ?? resolve(selected.path);
+    const synthetic = {
+        version: 2, id: selected.id, createdAt: selected.facts.createdAt,
+        source: { path: output, agent: selected.agent, bytes: selected.facts.sourceBytes, sha256: selected.facts.sourceSha256 },
+        cipher: { algorithm: selected.facts.algorithm, ciphertextSha256: selected.facts.ciphertextSha256, storedBytes: selected.facts.storedBytes, chunks: selected.facts.chunks },
+        envelope: {}, objectPath: "",
+    };
+    const nativeOutput = nativeRestoreTarget(synthetic, options.home);
+    const existing = await access(nativeOutput, constants.R_OK).then(() => true, () => false);
+    const stub = knownStub ?? (existing ? await readUntouchedCodexResumeStub(nativeOutput) : null);
+    if (existing && !stub)
+        return null;
+    const originalTimes = stub ? await stat(nativeOutput).catch(() => null) : null;
+    await validateNativeRestoreOutput(nativeOutput, options.home ?? homedir(), stub ? "file" : "missing-or-file");
+    let assembled = join(dirname(nativeOutput), `.${basename(nativeOutput)}.${randomUUID()}.rehydrate-partial`);
+    try {
+        for (let index = 0; index < chain.length; index += 1) {
+            const entry = chain[index];
+            const piece = `${assembled}.${index}.piece`;
+            try {
+                const opened = await openManagedArchiveToTemp(dataDir, entry.id, piece, phrase);
+                const expectedBytes = index === 0 ? entry.facts.sourceBytes : entry.facts.sourceBytes - entry.facts.delta.baseBytes;
+                if (opened.bytes !== expectedBytes)
+                    throw new Error(`Managed archive ${entry.id} produced ${opened.bytes} bytes; its authenticated catalog requires ${expectedBytes}`);
+                await pipeline(createReadStream(piece), createWriteStream(assembled, { flags: index === 0 ? "wx" : "a", mode: 0o600 }));
+            }
+            finally {
+                await rm(piece, { force: true }).catch(() => undefined);
+            }
+        }
+        const complete = await hashFileRange(assembled);
+        if (complete.bytes !== selected.facts.sourceBytes || complete.sha256.toLowerCase() !== selected.facts.sourceSha256.toLowerCase()) {
+            throw new Error(`Managed delta chain for ${selected.id} did not reproduce its authenticated source digest`);
+        }
+        await options.operations?.beforeCommit?.({ archiveId: selected.id, output: nativeOutput, managed: true });
+        const committed = await commitRestoredTranscript(assembled, nativeOutput, stub, originalTimes);
+        assembled = null;
+        await recordAudit(dataDir, "archive.restore", "allowed", { archiveId: selected.id, path: nativeOutput, rehydrated: true, managed: true });
+        return { rehydrated: true, archiveId: selected.id, output: nativeOutput, bytes: complete.bytes, ...committed };
+    }
+    finally {
+        if (assembled)
+            await rm(assembled, { force: true }).catch(() => undefined);
+    }
+}
+/** Lenient because an absent transcript is this feature's subject. */
 export function rehydrateTargetFromPayload(stdin) {
     try {
         const payload = JSON.parse(stdin);
@@ -57,7 +353,7 @@ export function rehydrateTargetFromPayload(stdin) {
         };
         return {
             path: pick(["transcript_path", "transcriptPath", "rollout_path", "rolloutPath"]),
-            sessionId: pick(["session_id", "sessionId", "conversation_id", "thread_id"])
+            sessionId: pick(["session_id", "sessionId", "conversation_id", "thread_id"]),
         };
     }
     catch {

package/dist/src/restore.d.ts

@@ -1,11 +1,41 @@
 import type { AgentId } from "./adapters.js";
 import type { ArchiveRecord } from "./types.js";
+import { type SourceReference } from "./source-reader.js";
 export type OverwritePolicy = "refuse" | "backup" | "replace";
+export type RestoreOperations = {
+    /** Deterministic race seams for tests; production callers never provide these. */
+    beforeCommit?: (input: {
+        output: string;
+        policy: OverwritePolicy;
+    }) => void | Promise<void>;
+    beforePark?: (input: {
+        output: string;
+        parkPath: string;
+        policy: "backup" | "replace";
+    }) => void | Promise<void>;
+    afterPark?: (input: {
+        output: string;
+        parkPath: string;
+        policy: "backup" | "replace";
+    }) => void | Promise<void>;
+};
 export type RestoreOptions = {
+    source?: SourceReference;
+    project?: string;
+    projectKey?: string;
     destination?: string;
     native?: boolean;
     overwrite?: OverwritePolicy;
     home?: string;
+    /**
+     * Report what would be restored — record, byte count, destination, whether a
+     * file already sits there — without creating or modifying anything. The
+     * shared-memory context pack promises agents that recovery previews by
+     * default, so the preview path must exist as a first-class outcome.
+     */
+    dryRun?: boolean;
+    /** Internal deterministic test seam. */
+    operations?: RestoreOperations;
 };
 export type RestoreOutcome = {
     record: ArchiveRecord;
@@ -13,6 +43,8 @@ export type RestoreOutcome = {
     bytes: number;
     native: boolean;
     backupPath?: string;
+    dryRun?: boolean;
+    destinationExists?: boolean;
 };
 /**
  * Where each adapter's transcripts legitimately live. Restoring "natively" means
@@ -22,15 +54,36 @@ export type RestoreOutcome = {
 export declare const NATIVE_ROOTS: Record<AgentId, (home: string) => string[]>;
 /** Resolves the adapter-native destination, refusing anything outside the agent's roots. */
 export declare function nativeRestoreTarget(record: ArchiveRecord, home?: string): string;
+/**
+ * Build and verify a native transcript parent without following a symlinked
+ * `.codex`, `.claude`, sessions root, or date/project directory. The lexical
+ * target still remains the adapter's recorded path; canonicalisation here
+ * only makes platform aliases such as macOS `/var` -> `/private/var` safe to
+ * compare.
+ */
+export declare function validateNativeRestoreOutput(output: string, home?: string, leaf?: "file" | "missing-or-file"): Promise<void>;
 /**
  * Restores one archive atomically.
  *
- * The plaintext is streamed to a temporary file beside the destination and renamed
- * into place, so a crash never leaves a half-written transcript — and a transcript
+ * The plaintext is streamed to a temporary file beside the destination and
+ * published with create-if-absent semantics, so a crash never leaves a
+ * half-written transcript — and a transcript
  * of any size is restored without ever holding it in memory. An existing file
- * is refused by default; `backup` preserves it beside the restored copy and
- * `replace` is the only policy that discards it.
+ * is refused by default. `backup` preserves the displaced inode beside the
+ * restored copy. `replace` replaces the active destination too, but retains
+ * the displaced inode under a visible `.sealkeep-replaced-*` safety path: no
+ * portable filesystem primitive can prove that another process has stopped
+ * writing through an already-open descriptor.
  */
 export declare function restoreArchive(dataDir: string, id: string, phrase: string, options?: RestoreOptions & {
     privateKey?: string;
 }): Promise<RestoreOutcome>;
+/**
+ * Proves an archive can produce the bytes it claims, before anything is taken.
+ *
+ * Decrypts the archive to a scratch file and hashes the result. Nothing else
+ * in this file is allowed to trash a source until this has returned true —
+ * the point is to catch a codec or storage fault while the original is still
+ * on disk, which is the only moment catching it is worth anything.
+ */
+export declare function verifyArchiveReadsBack(dataDir: string, archiveId: string, phrase: string, expectedSha256: string): Promise<boolean>;