sealkeep 0.9.0 → 0.11.0

package/dist/src/index-manifest.js

@@ -0,0 +1,444 @@
+/**
+ * The segment manifest — the list of immutable sealed segments that make up
+ * the content index — and the compaction that keeps that list short.
+ *
+ * Sealed segments never change, so the index over them need not be rewritten
+ * either: appending memory means appending a segment, removing an archive
+ * means tombstoning it, and staying fast means occasionally streaming a few
+ * small segments into one bigger immutable one. The manifest itself is the
+ * only thing ever rewritten, and every rewrite is temp-write + rename under a
+ * lock, so two processes can never interleave a read-modify-write.
+ */
+import { randomUUID } from "node:crypto";
+import { mkdir, open, readFile, rename, rm, stat, unlink, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { fail } from "./errors.js";
+import { MAX_POSTINGS_PER_TOKEN, } from "./index-segment-types.js";
+function manifestPath(dataDir) {
+    return join(dataDir, "index", "segments", "manifest.json");
+}
+function emptyManifest() {
+    return { version: 1, generation: "", segments: [], tombstones: [] };
+}
+/** Atomic publication that never leaves this call's temp behind on error.
+ * Mirrors `writeIndexFile` in src/search.ts, which is not exported. */
+async function writeManifestFile(target, contents) {
+    const temp = `${target}.${randomUUID()}.tmp`;
+    try {
+        await writeFile(temp, contents, { mode: 0o600 });
+        await rename(temp, target);
+    }
+    finally {
+        // After a successful rename this is an ENOENT no-op; after a failed write
+        // or rename it removes exactly the temp this call created.
+        await rm(temp, { force: true }).catch(() => undefined);
+    }
+}
+// Mirrors the lock discipline `withContextStateLock` (src/agent-context.ts)
+// uses, with the numbers this spec calls for: a bounded wait via short
+// sleeps, and stealing only a lock whose owner is provably gone.
+const LOCK_WAIT_BUDGET_MS = 30_000;
+const LOCK_POLL_MS = 20;
+const LOCK_STALE_MS = 5 * 60_000;
+/** True if a stale lock was removed and the caller should retry the open. */
+async function stealStaleManifestLock(lockPath) {
+    try {
+        const owner = JSON.parse(await readFile(lockPath, "utf8"));
+        let alive = typeof owner.pid === "number";
+        if (alive) {
+            try {
+                process.kill(owner.pid, 0);
+            }
+            catch (signalError) {
+                alive = signalError.code === "EPERM";
+            }
+        }
+        const age = Date.now() - (typeof owner.at === "number" ? owner.at : (await stat(lockPath)).mtimeMs);
+        if (alive && age <= LOCK_STALE_MS)
+            return false;
+    }
+    catch {
+        // `open(path, "wx")` makes the lock visible before its tiny owner record
+        // is fully written. An unreadable *fresh* file is held, not stale.
+        const age = await stat(lockPath).then((file) => Date.now() - file.mtimeMs).catch(() => Infinity);
+        if (age <= LOCK_STALE_MS)
+            return false;
+    }
+    await unlink(lockPath).catch(() => undefined);
+    return true;
+}
+async function withManifestLock(path, work) {
+    const lockPath = `${path}.lock`;
+    const token = randomUUID();
+    await mkdir(dirname(lockPath), { recursive: true, mode: 0o700 });
+    const deadline = Date.now() + LOCK_WAIT_BUDGET_MS;
+    let handle;
+    for (;;) {
+        try {
+            handle = await open(lockPath, "wx", 0o600);
+            await handle.writeFile(JSON.stringify({ pid: process.pid, at: Date.now(), token }));
+            break;
+        }
+        catch (error) {
+            if (error.code !== "EEXIST")
+                throw error;
+            if (await stealStaleManifestLock(lockPath))
+                continue;
+            if (Date.now() >= deadline)
+                fail("internal", "timed out waiting for the segment manifest lock", { path: lockPath });
+            await delay(LOCK_POLL_MS);
+        }
+    }
+    try {
+        return await work();
+    }
+    finally {
+        await handle.close().catch(() => undefined);
+        try {
+            const owner = JSON.parse(await readFile(lockPath, "utf8"));
+            if (owner.token === token)
+                await unlink(lockPath);
+        }
+        catch { /* a stale-lock recovery already replaced or removed it */ }
+    }
+}
+function parseManifest(raw, path) {
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return fail("internal", "segment manifest is corrupt", { path });
+    }
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+        return fail("internal", "segment manifest is corrupt", { path });
+    }
+    const value = parsed;
+    if (!Array.isArray(value.segments) || !Array.isArray(value.tombstones)) {
+        return fail("internal", "segment manifest is corrupt", { path });
+    }
+    return {
+        version: 1,
+        generation: typeof value.generation === "string" ? value.generation : "",
+        segments: value.segments,
+        tombstones: value.tombstones.filter((id) => typeof id === "string"),
+    };
+}
+export async function readManifest(dataDir) {
+    const path = manifestPath(dataDir);
+    try {
+        return parseManifest(await readFile(path, "utf8"), path);
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return emptyManifest();
+        throw error;
+    }
+}
+/** Read-modify-write under the manifest lock. Every successful write gets a
+ * fresh `generation`, whether or not `mutate` actually changed anything. */
+export async function updateManifest(dataDir, mutate) {
+    const path = manifestPath(dataDir);
+    return withManifestLock(path, async () => {
+        const current = await readManifest(dataDir);
+        const next = await mutate(current);
+        const stamped = { ...next, version: 1, generation: randomUUID() };
+        await writeManifestFile(path, JSON.stringify(stamped, null, 2) + "\n");
+        return stamped;
+    });
+}
+export async function appendSegment(dataDir, entry) {
+    return updateManifest(dataDir, (current) => {
+        if (current.segments.some((segment) => segment.id === entry.id)) {
+            fail("invalid_argument", `segment "${entry.id}" is already present in the manifest`, { segmentId: entry.id });
+        }
+        return { ...current, segments: [...current.segments, entry] };
+    });
+}
+export async function retireArchives(dataDir, archiveIds) {
+    return updateManifest(dataDir, (current) => {
+        const tombstones = new Set(current.tombstones);
+        for (const id of archiveIds)
+            tombstones.add(id);
+        return { ...current, tombstones: [...tombstones] };
+    });
+}
+const DEFAULT_MAX_SEGMENTS = 8;
+const DEFAULT_MAX_SMALL_BYTES = 8 * 1024 * 1024;
+const MIN_SMALL_GROUP = 4;
+/**
+ * Ids to merge, or `[]`. A "remote" segment (the whole account index pulled
+ * from the cloud) is never a candidate: it never merges with a locally-built
+ * one, and it does not count toward the segment-count trigger either.
+ */
+export function planCompaction(manifest, options) {
+    const maxSegments = options?.maxSegments ?? DEFAULT_MAX_SEGMENTS;
+    const maxSmallBytes = options?.maxSmallBytes ?? DEFAULT_MAX_SMALL_BYTES;
+    const local = manifest.segments.filter((segment) => segment.origin !== "remote");
+    const ids = new Set();
+    if (local.length > maxSegments) {
+        let largest = local[0];
+        for (const segment of local)
+            if (segment.bytes > largest.bytes)
+                largest = segment;
+        for (const segment of local)
+            if (segment.id !== largest.id)
+                ids.add(segment.id);
+    }
+    const small = local.filter((segment) => segment.bytes < maxSmallBytes);
+    if (small.length >= MIN_SMALL_GROUP)
+        for (const segment of small)
+            ids.add(segment.id);
+    if (ids.size < 2)
+        return [];
+    const order = new Map(manifest.segments.map((segment, index) => [segment.id, index]));
+    return [...ids].sort((a, b) => order.get(a) - order.get(b));
+}
+// ---------------------------------------------------------------------------
+// Streaming merge
+// ---------------------------------------------------------------------------
+const ROW_YIELD_INTERVAL = 2_048;
+// A captured tool artifact, not a real session line; a real summary always
+// beats it regardless of which segment it came from.
+const JUNK_SUMMARY = /^\s*<(?:system-reminder|local-command-stdout|task-notification|command-name|command-message)/;
+function isJunkSummary(value) {
+    return typeof value !== "string" || JUNK_SUMMARY.test(value);
+}
+/** A value that isn't a captured artifact beats one that is; otherwise the
+ * later (by manifest order) segment's value wins, like every other field. */
+function betterSummary(existing, incoming) {
+    if (existing === undefined)
+        return incoming;
+    if (incoming === undefined)
+        return existing;
+    if (isJunkSummary(incoming) && !isJunkSummary(existing))
+        return existing;
+    return incoming;
+}
+function unionArrayField(existing, incoming) {
+    if (!Array.isArray(incoming))
+        return existing;
+    if (!Array.isArray(existing))
+        return incoming;
+    const seen = new Set(existing.map((item) => JSON.stringify(item)));
+    const merged = existing.slice();
+    for (const item of incoming) {
+        const key = JSON.stringify(item);
+        if (!seen.has(key)) {
+            seen.add(key);
+            merged.push(item);
+        }
+    }
+    return merged;
+}
+/** Later (by manifest order) wins field by field, except `summary` and the
+ * unioned `commits`/`files` arrays. */
+function mergeMeta(base, incoming) {
+    const merged = { ...base };
+    for (const [key, value] of Object.entries(incoming)) {
+        if (key === "summary")
+            merged[key] = betterSummary(merged[key], value);
+        else if (key === "commits" || key === "files")
+            merged[key] = unionArrayField(merged[key], value);
+        else
+            merged[key] = value;
+    }
+    return merged;
+}
+function postingArchiveId(posting) {
+    return posting.split("@")[0];
+}
+/**
+ * Ascending by the covered archive's `archivedAt`. Postings with no known
+ * date sort first (oldest), so capping to the newest never keeps a guess over
+ * a dated session; ties (including every undated posting, among themselves)
+ * keep the stable union order they arrived in.
+ */
+function chronologicalPostings(postings, archivedAt) {
+    return postings
+        .map((posting, position) => ({ posting, position, at: archivedAt.get(postingArchiveId(posting)) }))
+        .sort((left, right) => {
+        if (left.at && right.at)
+            return left.at.localeCompare(right.at) || left.position - right.position;
+        if (left.at)
+            return 1;
+        if (right.at)
+            return -1;
+        return left.position - right.position;
+    })
+        .map((entry) => entry.posting);
+}
+async function openCursor(rows) {
+    const iterator = rows[Symbol.asyncIterator]();
+    const { value, done } = await iterator.next();
+    return { iterator, current: done ? null : value };
+}
+async function advanceCursor(cursor) {
+    const { value, done } = await cursor.iterator.next();
+    cursor.current = done ? null : value;
+}
+/**
+ * The compaction merge on its own: the union/cap/tombstone/summary rules that
+ * turn several sealed segments' rows into one segment's worth, streamed so no
+ * more than one batch of archive or token rows is held at once.
+ *
+ * `readers` must be given oldest first (manifest order) — every "later wins"
+ * rule below is defined relative to that order. Meta rows are the one
+ * exception to streaming: a segment holds only a few hundred bytes of
+ * metadata per archive, so the whole merged map is kept in memory (also
+ * needed to rank token postings by `archivedAt`) and emitted last.
+ */
+export async function* mergeSegmentRows(readers, tombstones, options) {
+    let produced = 0;
+    const maybeYield = async () => {
+        produced += 1;
+        if (produced % ROW_YIELD_INTERVAL === 0)
+            await options?.onYield?.();
+    };
+    // "m": collected first (small; also feeds the "t" ranking below) but
+    // emitted last, in region order.
+    const meta = new Map();
+    for (const reader of readers) {
+        for await (const row of reader.rows("meta", { signal: options?.signal })) {
+            options?.signal?.throwIfAborted();
+            const [, archiveId, metaObject] = row;
+            if (tombstones.has(archiveId))
+                continue;
+            const existing = meta.get(archiveId);
+            meta.set(archiveId, existing ? mergeMeta(existing, metaObject) : metaObject);
+        }
+    }
+    const archivedAt = new Map();
+    for (const [archiveId, metaObject] of meta) {
+        const value = metaObject.archivedAt;
+        if (typeof value === "string")
+            archivedAt.set(archiveId, value);
+    }
+    // "a": k-way merge by archive id; the richest (longest) row wins, later
+    // segment wins ties.
+    const archiveCursors = await Promise.all(readers.map((reader) => openCursor(reader.rows("archives", { signal: options?.signal }))));
+    for (;;) {
+        options?.signal?.throwIfAborted();
+        let minId = null;
+        for (const cursor of archiveCursors) {
+            const id = cursor.current?.[1];
+            if (id !== undefined && (minId === null || id < minId))
+                minId = id;
+        }
+        if (minId === null)
+            break;
+        let winner = null;
+        for (const cursor of archiveCursors) {
+            if (!cursor.current || cursor.current[1] !== minId)
+                continue;
+            if (!winner || cursor.current.length >= winner.length)
+                winner = cursor.current;
+            await advanceCursor(cursor);
+        }
+        if (!tombstones.has(minId)) {
+            yield winner;
+            await maybeYield();
+        }
+    }
+    // "t": k-way merge by token; union the postings, drop tombstoned ones, cap
+    // to the newest MAX_POSTINGS_PER_TOKEN. A token left with none is dropped.
+    const tokenCursors = await Promise.all(readers.map((reader) => openCursor(reader.rows("tokens", { signal: options?.signal }))));
+    for (;;) {
+        options?.signal?.throwIfAborted();
+        let minToken = null;
+        for (const cursor of tokenCursors) {
+            const token = cursor.current?.[1];
+            if (token !== undefined && (minToken === null || token < minToken))
+                minToken = token;
+        }
+        if (minToken === null)
+            break;
+        const seen = new Set();
+        const union = [];
+        for (const cursor of tokenCursors) {
+            if (!cursor.current || cursor.current[1] !== minToken)
+                continue;
+            for (const posting of cursor.current[2])
+                if (!seen.has(posting)) {
+                    seen.add(posting);
+                    union.push(posting);
+                }
+            await advanceCursor(cursor);
+        }
+        const live = union.filter((posting) => !tombstones.has(postingArchiveId(posting)));
+        if (live.length) {
+            const ranked = chronologicalPostings(live, archivedAt);
+            const kept = ranked.length > MAX_POSTINGS_PER_TOKEN ? ranked.slice(-MAX_POSTINGS_PER_TOKEN) : ranked;
+            yield ["t", minToken, kept];
+            await maybeYield();
+        }
+    }
+    for (const archiveId of [...meta.keys()].sort()) {
+        yield ["m", archiveId, meta.get(archiveId)];
+        await maybeYield();
+    }
+}
+/** Replace every id in `mergedIds` with `entry`, at the position the newest
+ * (last, by manifest/newest-last order) of them held; other segments keep
+ * their relative order. */
+function replaceCompactedSegments(segments, mergedIds, entry) {
+    let newestIndex = -1;
+    segments.forEach((segment, index) => { if (mergedIds.has(segment.id))
+        newestIndex = index; });
+    const result = [];
+    segments.forEach((segment, index) => {
+        if (!mergedIds.has(segment.id)) {
+            result.push(segment);
+            return;
+        }
+        if (index === newestIndex)
+            result.push(entry);
+    });
+    return result;
+}
+export async function compactSegments(dataDir, phrase, segmentIds, tools, options) {
+    options?.signal?.throwIfAborted();
+    // Nothing to merge into: the position-of-the-newest-merged-segment
+    // placement below has no meaning, and the new segment `write` would produce
+    // could never be referenced by the manifest.
+    if (segmentIds.length === 0)
+        fail("invalid_argument", "compaction requires at least one segment id", { segmentIds });
+    const manifest = await readManifest(dataDir);
+    const position = new Map(manifest.segments.map((segment, index) => [segment.id, index]));
+    for (const id of segmentIds) {
+        if (!position.has(id))
+            fail("invalid_argument", `segment "${id}" is not in the manifest`, { segmentId: id });
+    }
+    // Merge rules ("later wins") are defined relative to manifest order, not
+    // whatever order the caller happened to list ids in.
+    const mergedIds = new Set(segmentIds);
+    const orderedIds = [...mergedIds].sort((a, b) => position.get(a) - position.get(b));
+    const tombstones = new Set(manifest.tombstones);
+    const readers = [];
+    try {
+        for (const id of orderedIds)
+            readers.push(await tools.open(dataDir, phrase, id));
+        const entry = await tools.write(dataDir, phrase, { segmentId: randomUUID(), builtAt: new Date().toISOString(), rows: mergeSegmentRows(readers, tombstones, options) }, options);
+        // Only after the manifest is durably updated do the old segments go —
+        // a crash between these two steps leaves the old segments plus one
+        // unreferenced new one, which `orphanSegmentIds` is for.
+        await updateManifest(dataDir, (current) => ({
+            ...current,
+            segments: replaceCompactedSegments(current.segments, mergedIds, entry),
+        }));
+        for (const id of orderedIds)
+            await tools.remove(dataDir, id);
+        return entry;
+    }
+    finally {
+        for (const reader of readers)
+            await reader.close();
+    }
+}
+/** Segment ids present on disk that the manifest does not list. */
+export async function orphanSegmentIds(dataDir, listed) {
+    const manifest = await readManifest(dataDir);
+    const known = new Set(manifest.segments.map((segment) => segment.id));
+    return (await listed(dataDir)).filter((id) => !known.has(id));
+}

package/dist/src/index-publication-proof.d.ts

@@ -0,0 +1,84 @@
+import type { ContentIndex } from "./search.js";
+/**
+ * The part of a content index whose meaning is committed by a publication
+ * receipt. The extraction policy is required and included in every entry;
+ * `builtAt` is intentionally absent because republishing identical search
+ * knowledge later must not invalidate every archive requirement.
+ */
+export type IndexPublicationProofSource = Pick<ContentIndex, "archives" | "tokens" | "meta"> & {
+    /** The transcript-extraction semantics used to produce these postings. */
+    extractionPolicy: number;
+};
+export type IndexPublicationProofOptions = {
+    extractionPolicy: number;
+    /** Writers use the lexical default. `wire` exists only for authenticated
+     * legacy-v3 readers which independently prove token-row uniqueness. */
+    tokenOrder?: "lexical" | "wire";
+};
+export type IndexPublicationCoverage = Record<string, string>;
+export type IndexPublicationProof = {
+    /** Every archive/catalog row committed by the generation, including bases. */
+    coverage: IndexPublicationCoverage;
+    /** Zero means catalog-only; a directly searchable head must be greater than zero. */
+    effectivePostingCounts: Record<string, number>;
+};
+export declare const INDEX_PUBLICATION_DIGEST_ALGORITHM: "sha256";
+export declare const INDEX_PUBLICATION_MAX_POSTINGS_PER_TOKEN = 256;
+export declare class IndexPublicationProofError extends Error {
+    readonly code = "INDEX_PUBLICATION_PROOF_INVALID";
+    constructor(message: string);
+}
+/**
+ * Incremental coverage for the exact rows an index checkpoint emits.
+ *
+ * The writer should call `addArchive` from its archive-row loop, `addToken`
+ * with the already chronological/capped posting array used for that exact
+ * token row, `addMetadata` from its metadata-row loop, then `finish`. Construct
+ * it with the same positive extraction policy written in the JSONL header. This
+ * avoids cloning or traversing a 600 MB index a second time, and it cannot race
+ * indexing lanes because the proof consumes the same values as the JSONL row.
+ * Token rows must be supplied in lexical order by default; enforcing that
+ * keeps new coverage independent of object insertion order. The authenticated
+ * legacy-v3 reader may explicitly preserve wire order after independently
+ * proving uniqueness with bounded state. `finish` also returns exact posting
+ * counts: zero is valid catalog coverage for a delta base, but must not be used
+ * as proof that a requested/head archive is directly searchable.
+ */
+export declare class IndexPublicationCoverageBuilder {
+    #private;
+    constructor(options: IndexPublicationProofOptions);
+    addArchive(refValue: string, row: readonly string[]): void;
+    /** Add one effective `t` row, after chronology and the 256-entry cap. */
+    addToken(token: string, postings: readonly string[]): void;
+    addMetadata(refValue: string, value: unknown): void;
+    finish(): IndexPublicationProof;
+}
+export declare function createIndexPublicationCoverageBuilder(options: IndexPublicationProofOptions): IndexPublicationCoverageBuilder;
+/**
+ * Produce the exact per-archive semantic coverage committed by a search-index
+ * generation.
+ *
+ * Each SHA-256 binds the extraction policy, archive reference, its complete
+ * archive row, matching metadata (missing and `{}` differ), and every
+ * token/posting occurrence that
+ * survives the real JSONL serializer's chronological 256-posting cap. Object
+ * key order never matters. Duplicate posting occurrences do matter. Orphan or
+ * malformed data throws, so damaged input can never become positive proof.
+ * Catalog-only delta bases remain in `coverage` with a posting count of zero;
+ * callers authorizing reclaim must demand a positive count for the logical
+ * session head and exact coverage for every base needed to restore it.
+ *
+ * Runtime is O(A log A + T log T + sum(P_t log P_t)): one traversal of the
+ * sorted archive/token maps and each sorted posting list, with O(A) hash state.
+ */
+export declare function buildIndexPublicationProof(index: IndexPublicationProofSource): IndexPublicationProof;
+/** Convenience for publishers that only need the canonical CAS coverage map. */
+export declare function buildIndexPublicationCoverage(index: IndexPublicationProofSource): IndexPublicationCoverage;
+/** Canonical, lexically sorted object JSON suitable for an opaque CAS field. */
+export declare function canonicalIndexPublicationCoverageJson(value: unknown): string;
+/**
+ * Parse only the canonical representation produced above. Byte-for-byte
+ * reserialization rejects duplicate JSON keys as well as ambiguous whitespace
+ * or ordering, instead of letting JSON.parse silently keep a last value.
+ */
+export declare function parseIndexPublicationCoverageJson(raw: string): IndexPublicationCoverage;