sealkeep 0.9.0 → 0.11.0

package/dist/src/index-scratch-cleanup.js

@@ -0,0 +1,166 @@
+import { lstat, mkdir, open, readdir, rmdir, unlink } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { fail } from "./errors.js";
+const UUID = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
+const SCRATCH_GENERATION = new RegExp(`^${UUID}$`);
+const OWNED_SCRATCH_FILE = new RegExp(`^(?:\\.content-index\\.${UUID}\\.jsonl|\\.scoped-index\\.${UUID}\\.jsonl|\\.vlbody-${UUID}`
+    + `|content-index(?:\\.[0-9a-f]{16})?\\.skindex\\.${UUID}\\.tmp)$`);
+function errorCode(error) {
+    return typeof error === "object" && error !== null && "code" in error
+        ? String(error.code)
+        : undefined;
+}
+function validateGeneration(generation) {
+    if (!SCRATCH_GENERATION.test(generation)) {
+        fail("invalid_argument", "A historical-index scratch generation must be a canonical lowercase UUID");
+    }
+}
+/**
+ * One daemon-owned historical child gets one private scratch namespace.
+ * Foreground/manual builders deliberately keep using the ordinary index
+ * directory, so recovery of a dead child can never unlink their live files.
+ */
+export function backgroundIndexScratchDirectory(dataDir, generation) {
+    validateGeneration(generation);
+    const indexDirectory = resolve(dataDir, "index");
+    const scratchDirectory = resolve(indexDirectory, `.background-index-scratch.${generation}`);
+    if (dirname(scratchDirectory) !== indexDirectory) {
+        fail("internal", "Historical-index scratch escaped its owned index directory");
+    }
+    return scratchDirectory;
+}
+function ownedByThisUser(info) {
+    return typeof process.getuid !== "function" || info.uid === process.getuid();
+}
+function privateDirectory(info) {
+    return process.platform === "win32" || (Number(info.mode) & 0o077) === 0;
+}
+function ownedRegularFile(info) {
+    return info.isFile() && !info.isSymbolicLink() && ownedByThisUser(info) && Number(info.nlink) === 1;
+}
+function concurrentlyUnlinkedOwnedFile(info) {
+    return info.isFile() && !info.isSymbolicLink() && ownedByThisUser(info) && Number(info.nlink) === 0;
+}
+async function syncDirectory(path) {
+    let handle;
+    try {
+        handle = await open(path, "r");
+        await handle.sync();
+    }
+    catch (error) {
+        // Windows does not support opening directories. POSIX must make the
+        // privacy cleanup durable before its child barrier can be retired.
+        if (process.platform !== "win32" && errorCode(error) !== "ENOENT")
+            throw error;
+    }
+    finally {
+        await handle?.close().catch(() => undefined);
+    }
+}
+/** Create or reopen only the exact namespace assigned to this worker. */
+export async function ensureBackgroundIndexScratch(dataDir, generation) {
+    const indexDirectory = resolve(dataDir, "index");
+    const scratchDirectory = backgroundIndexScratchDirectory(dataDir, generation);
+    await mkdir(indexDirectory, { recursive: true, mode: 0o700 });
+    try {
+        await mkdir(scratchDirectory, { mode: 0o700 });
+    }
+    catch (error) {
+        if (errorCode(error) !== "EEXIST")
+            throw error;
+    }
+    const info = await lstat(scratchDirectory);
+    if (!info.isDirectory() || info.isSymbolicLink() || !ownedByThisUser(info) || !privateDirectory(info)) {
+        fail("internal", "Historical-index scratch is not an owned private directory");
+    }
+    return scratchDirectory;
+}
+/**
+ * Remove scratch for one process generation after its durable child barrier
+ * has proved that exact worker dead. The caller owns that proof; this helper
+ * intentionally cannot scan legacy/global names because another foreground
+ * builder may be using them.
+ *
+ * Preflight every entry before deleting any. An unknown name, symlink or
+ * directory leaves the entire namespace intact for diagnosis. Concurrent
+ * successor contenders may safely repeat the same cleanup.
+ */
+export async function cleanupInterruptedIndexScratch(dataDir, generation) {
+    const scratchDirectory = backgroundIndexScratchDirectory(dataDir, generation);
+    let directoryInfo;
+    try {
+        directoryInfo = await lstat(scratchDirectory);
+    }
+    catch (error) {
+        if (errorCode(error) === "ENOENT") {
+            await syncDirectory(resolve(dataDir, "index"));
+            return;
+        }
+        throw error;
+    }
+    if (!directoryInfo.isDirectory() || directoryInfo.isSymbolicLink()
+        || !ownedByThisUser(directoryInfo) || !privateDirectory(directoryInfo)) {
+        fail("internal", "Refusing to clean an unowned historical-index scratch path");
+    }
+    let entries;
+    try {
+        entries = await readdir(scratchDirectory, { withFileTypes: true });
+    }
+    catch (error) {
+        // A simultaneous successor may have completed the exact cleanup after our
+        // lstat. Missing at any boundary is the intended final state.
+        if (errorCode(error) === "ENOENT") {
+            await syncDirectory(resolve(dataDir, "index"));
+            return;
+        }
+        throw error;
+    }
+    for (const entry of entries) {
+        if (!OWNED_SCRATCH_FILE.test(entry.name) || !entry.isFile() || entry.isSymbolicLink()) {
+            fail("internal", "Refusing to clean a historical-index scratch namespace containing an unknown entry");
+        }
+        try {
+            const info = await lstat(join(scratchDirectory, entry.name));
+            if (!ownedRegularFile(info) && !concurrentlyUnlinkedOwnedFile(info)) {
+                fail("internal", "Refusing to clean an unowned historical-index scratch file");
+            }
+        }
+        catch (error) {
+            // Another successor may have claimed the same dead generation. A file
+            // disappearing is success; no other lookup failure is downgraded.
+            if (errorCode(error) !== "ENOENT")
+                throw error;
+        }
+    }
+    for (const entry of entries) {
+        const path = join(scratchDirectory, entry.name);
+        try {
+            // Recheck immediately before unlink. unlink itself never follows a
+            // symlink; a replacement directory is refused by the operating system.
+            const info = await lstat(path);
+            // POSIX may return the inode we just looked up with nlink=0 when a
+            // simultaneous cleaner wins unlink between lookup and inspection. Do
+            // not unlink the pathname in that case: it may already name a
+            // replacement. The final rmdir will conservatively refuse any survivor.
+            if (concurrentlyUnlinkedOwnedFile(info)) {
+                continue;
+            }
+            if (!ownedRegularFile(info)) {
+                fail("internal", "Refusing to clean a replaced historical-index scratch file");
+            }
+            await unlink(path);
+        }
+        catch (error) {
+            if (errorCode(error) !== "ENOENT")
+                throw error;
+        }
+    }
+    try {
+        await rmdir(scratchDirectory);
+    }
+    catch (error) {
+        if (errorCode(error) !== "ENOENT")
+            throw error;
+    }
+    await syncDirectory(resolve(dataDir, "index"));
+}

package/dist/src/index-segment-types.d.ts

@@ -0,0 +1,132 @@
+/**
+ * The segmented content index — shared contracts.
+ *
+ * Sealed memories never change, so the index that describes them need not
+ * be rewritten either. The index becomes a manifest over immutable sealed
+ * SEGMENTS: each segment carries the archive rows, token postings and
+ * session metadata for a set of archives, written once. Adding memory
+ * appends a segment; a lookup opens every segment's small directory and
+ * reads only the blocks that hold its terms; compaction occasionally
+ * streams several small segments into one bigger immutable one. Nothing is
+ * ever rewritten in place, and no build holds more than one batch in memory.
+ *
+ * Row shapes are exactly the v3 index rows the reader already understands:
+ *   ["a", archiveId, ...archiveLine]      sorted by archiveId
+ *   ["t", token, [posting, ...]]          sorted by token; a posting is
+ *                                         "<archiveId>" or "<archiveId>@c<chunk>"
+ *   ["m", archiveId, metaObject]          sorted by archiveId
+ * preceded by one header row `{"version":4,"segmentId":…,"builtAt":…}`.
+ */
+/** One row of a segment body, in the order the body stores them. */
+export type SegmentRow = ["a", string, ...string[]] | ["t", string, string[]] | ["m", string, Record<string, unknown>];
+export type SegmentHeader = {
+    version: 4;
+    segmentId: string;
+    builtAt: string;
+};
+/** Byte ranges of the three regions in the segment's raw (decrypted) body. */
+export type SegmentRegions = {
+    archives: [number, number];
+    tokens: [number, number];
+    meta: [number, number];
+};
+/**
+ * The sealed sidecar that makes a segment addressable without reading it:
+ * the raw offset of every `stride`-th token row, and the region bounds. It is
+ * bound to the exact sealed body by the body envelope's archive id.
+ */
+export type SegmentDirectory = {
+    version: 1;
+    segmentId: string;
+    envelopeArchiveId: string;
+    builtAt: string;
+    stride: number;
+    regions: SegmentRegions;
+    /** [token, raw byte offset of its row], ascending by token. */
+    entries: Array<[string, number]>;
+    archives: number;
+    tokens: number;
+};
+/** What the manifest records about one segment; metadata only, never tokens. */
+export type SegmentEntry = {
+    id: string;
+    builtAt: string;
+    archives: number;
+    tokens: number;
+    /** Sealed body bytes on disk. */
+    bytes: number;
+    /**
+     * "local" (default): rows this machine derived from its own archives.
+     * "remote": the account index pulled from the cloud, converted whole; it is
+     * replaced when a newer copy arrives and is never compacted with local ones.
+     */
+    origin?: "local" | "remote";
+    /**
+     * The archive ids this machine sealed that the segment covers, so coverage
+     * is known without opening it. Only these may ever be tombstoned as "no
+     * longer here": a build forgets what this machine indexed, never what it
+     * learned from another machine.
+     */
+    archiveIds?: string[];
+    /**
+     * Rows the segment also carries for archives another machine sealed (a
+     * migrated blob had pulled them from the account index). Searchable here,
+     * never pruned here, and replaced by segment publication (step 2).
+     */
+    remoteArchiveIds?: string[];
+    /**
+     * The account machine this segment came from, for an `origin: "remote"`
+     * entry (step 2's `pullSegments`) — which machine's `<machineId>/` prefix
+     * to delete from when a peer's compaction or retirement replaces it.
+     * Absent on a "local" entry: it is unambiguously this machine's own.
+     */
+    machineId?: string;
+};
+export type SegmentManifest = {
+    version: 1;
+    /** Changes whenever the segment set changes; readers use it as a cache key. */
+    generation: string;
+    /** Newest last. A lookup consults all of them. */
+    segments: SegmentEntry[];
+    /** Archive ids removed from the vault; their rows are ignored everywhere and dropped by the next compaction. */
+    tombstones: string[];
+};
+/** The result of a lookup across one or many segments: the same shape queries already consume. */
+export type SegmentLookup = {
+    archives: Record<string, string[]>;
+    tokens: Record<string, string[]>;
+    meta: Record<string, Record<string, unknown>>;
+};
+/** A reader over one sealed segment; opened once, closed by the caller. */
+export interface SegmentReader {
+    readonly id: string;
+    readonly directory: SegmentDirectory;
+    /**
+     * Rows for the given terms: every "a" and "m" row of the segment, and the
+     * "t" rows whose token equals a term or starts with one (at most
+     * `prefixKeep` prefix matches per term, exact matches always). Only the
+     * blocks holding those rows are opened.
+     */
+    lookup(terms: readonly string[], options?: {
+        prefixKeep?: number;
+        signal?: AbortSignal;
+    }): Promise<SegmentLookup>;
+    /** Every row of one region, in stored order, streamed in bounded windows. */
+    rows(region: keyof SegmentRegions, options?: {
+        signal?: AbortSignal;
+    }): AsyncIterable<SegmentRow>;
+    close(): Promise<void>;
+}
+/** Writes one immutable segment from rows that are already sorted region by region. */
+export interface SegmentWriter {
+    (dataDir: string, phrase: string, input: {
+        segmentId: string;
+        builtAt: string;
+        rows: AsyncIterable<SegmentRow> | Iterable<SegmentRow>;
+    }, options?: {
+        signal?: AbortSignal;
+        onYield?: () => void | Promise<void>;
+    }): Promise<SegmentEntry>;
+}
+export declare const SEGMENT_DIRECTORY_STRIDE = 64;
+export declare const MAX_POSTINGS_PER_TOKEN = 256;

package/dist/src/index-segment-types.js

@@ -0,0 +1,21 @@
+/**
+ * The segmented content index — shared contracts.
+ *
+ * Sealed memories never change, so the index that describes them need not
+ * be rewritten either. The index becomes a manifest over immutable sealed
+ * SEGMENTS: each segment carries the archive rows, token postings and
+ * session metadata for a set of archives, written once. Adding memory
+ * appends a segment; a lookup opens every segment's small directory and
+ * reads only the blocks that hold its terms; compaction occasionally
+ * streams several small segments into one bigger immutable one. Nothing is
+ * ever rewritten in place, and no build holds more than one batch in memory.
+ *
+ * Row shapes are exactly the v3 index rows the reader already understands:
+ *   ["a", archiveId, ...archiveLine]      sorted by archiveId
+ *   ["t", token, [posting, ...]]          sorted by token; a posting is
+ *                                         "<archiveId>" or "<archiveId>@c<chunk>"
+ *   ["m", archiveId, metaObject]          sorted by archiveId
+ * preceded by one header row `{"version":4,"segmentId":…,"builtAt":…}`.
+ */
+export const SEGMENT_DIRECTORY_STRIDE = 64;
+export const MAX_POSTINGS_PER_TOKEN = 256;

package/dist/src/index-segments.d.ts

@@ -0,0 +1,9 @@
+import { type SegmentReader, type SegmentWriter } from "./index-segment-types.js";
+export declare function segmentPaths(dataDir: string, segmentId: string): Promise<{
+    body: string;
+    envelope: string;
+    directory: string;
+}>;
+export declare function removeSegment(dataDir: string, segmentId: string): Promise<void>;
+export declare const writeSegment: SegmentWriter;
+export declare function openSegment(dataDir: string, phrase: string, segmentId: string): Promise<SegmentReader>;