sealkeep 0.9.0 → 0.11.0

package/dist/src/index-segments.js

@@ -0,0 +1,516 @@
+/**
+ * One immutable sealed index segment: write it once, then open it and answer
+ * term lookups by reading only the blocks that hold the answer.
+ *
+ * The whole-index file this replaces made every seal rewrite the entire
+ * ciphertext and forced a lookup to decrypt the whole thing before a 64-row
+ * directory could narrow it down. A segment applies the same "seal in
+ * independent chunks, index the chunk boundaries" trick search.ts already
+ * uses for the legacy index, but at the grain of one write instead of the
+ * whole vault: nothing already sealed is ever read back, decrypted, or
+ * rewritten to add more. See docs/design/index-segments.md.
+ *
+ * File layout under `<dataDir>/index/segments/`:
+ *   `<segmentId>.skseg`       the sealed, chunk-compressed body (JSONL rows)
+ *   `<segmentId>.skseg.json`  that body's envelope, in the clear (it carries
+ *                             no plaintext — same shape as content-index.json)
+ *   `<segmentId>.skdir`       the sealed directory sidecar (byte offsets)
+ *
+ * Every file is published temp-then-rename, and a failed write removes
+ * everything this call produced — a half-written segment must never be
+ * mistaken for a real one.
+ */
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, open, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { decryptArchive as openEnvelope, encryptArchive as sealEnvelope, sealArchiveToFile, } from "../packages/sealkeep-crypto/src/index.js";
+import { chunkWindowForRange, openChunkWindow, supportsChunkAccess, unwrapWindowKey } from "../packages/sealkeep-crypto/src/chunk-access.js";
+import { fail } from "./errors.js";
+import { SEGMENT_DIRECTORY_STRIDE, } from "./index-segment-types.js";
+const SEGMENT_ID_PATTERN = /^[0-9a-f-]{36}$/;
+// Rows below this size ride the same coalesced write writeContentIndexCheckpoint
+// uses; a handful of write() syscalls beats one per row on a 600-token segment.
+const COALESCE_BYTES = 1024 * 1024;
+// ~4 MiB raw per rows() window: bounded memory regardless of segment size.
+const ROWS_WINDOW_BYTES = 4 * 1024 * 1024;
+// 8 x stride rows bounds a wide prefix the same way MAX_RANGED_WINDOWS_PER_TERM
+// bounds it for the legacy whole-index directory.
+const MAX_LOOKUP_WINDOWS_PER_TERM = 8;
+const segmentsDir = (dataDir) => join(dataDir, "index", "segments");
+const openedSegments = new Map();
+const OPENED_SEGMENTS_MAX = 64;
+const phraseDigest = (phrase) => createHash("sha256").update(phrase, "utf8").digest("hex");
+function rememberOpenedSegment(cacheKey, opened) {
+    if (openedSegments.size >= OPENED_SEGMENTS_MAX) {
+        const oldest = openedSegments.keys().next().value;
+        if (oldest !== undefined) {
+            openedSegments.get(oldest)?.archiveKey.fill(0);
+            openedSegments.delete(oldest);
+        }
+    }
+    openedSegments.set(cacheKey, opened);
+}
+function assertSegmentId(segmentId) {
+    if (!SEGMENT_ID_PATTERN.test(segmentId))
+        fail("invalid_argument", "Segment id must be a UUID", { segmentId });
+}
+function isEnoent(error) {
+    return error?.code === "ENOENT";
+}
+const isArchiveRow = (row) => row[0] === "a";
+const isTokenRow = (row) => row[0] === "t";
+const isMetaRow = (row) => row[0] === "m";
+/** The last directory entry whose token is at or before `term`, or -1. */
+function directoryPosition(entries, term) {
+    let low = 0;
+    let high = entries.length - 1;
+    let found = -1;
+    while (low <= high) {
+        const mid = (low + high) >> 1;
+        if (entries[mid][0] <= term) {
+            found = mid;
+            low = mid + 1;
+        }
+        else
+            high = mid - 1;
+    }
+    return found;
+}
+/** Normalizes the writer's row source: callers may hand over either kind of iterable. */
+async function* asRowStream(rows) {
+    if (Symbol.asyncIterator in rows)
+        yield* rows;
+    else
+        yield* rows;
+}
+/** Atomic publication that never leaves this call's temp behind on error. Same
+ * pattern as the private writeIndexFile in search.ts, reimplemented here since
+ * that helper is not exported. */
+async function writeSegmentFile(target, contents) {
+    const temp = `${target}.${randomUUID()}.tmp`;
+    try {
+        await writeFile(temp, contents, { mode: 0o600 });
+        await rename(temp, target);
+    }
+    finally {
+        await rm(temp, { force: true }).catch(() => undefined);
+    }
+}
+async function writeSealedDirectory(directoryPath, phrase, directory) {
+    const plaintext = Buffer.from(JSON.stringify(directory), "utf8");
+    let sealed;
+    try {
+        sealed = sealEnvelope(plaintext, {
+            recipients: [{ kind: "phrase", phrase }],
+            compression: "zstd",
+            adapter: { agent: "sealkeep-index-segment-directory", version: "1" },
+        });
+    }
+    finally {
+        plaintext.fill(0);
+    }
+    try {
+        await writeSegmentFile(directoryPath, `${JSON.stringify({ version: 1, envelope: sealed.envelope, ciphertext: sealed.ciphertext.toString("base64") })}\n`);
+    }
+    finally {
+        sealed.ciphertext.fill(0);
+    }
+}
+export async function segmentPaths(dataDir, segmentId) {
+    assertSegmentId(segmentId);
+    const dir = segmentsDir(dataDir);
+    return {
+        body: join(dir, `${segmentId}.skseg`),
+        envelope: join(dir, `${segmentId}.skseg.json`),
+        directory: join(dir, `${segmentId}.skdir`),
+    };
+}
+export async function removeSegment(dataDir, segmentId) {
+    const paths = await segmentPaths(dataDir, segmentId);
+    await Promise.all([
+        rm(paths.body, { force: true }),
+        rm(paths.envelope, { force: true }),
+        rm(paths.directory, { force: true }),
+    ]);
+}
+const REGION_RANK = { a: 0, t: 1, m: 2 };
+const REGION_NAME = { 0: "archives", 1: "tokens", 2: "meta" };
+export const writeSegment = async (dataDir, phrase, input, options = {}) => {
+    const paths = await segmentPaths(dataDir, input.segmentId);
+    const dir = segmentsDir(dataDir);
+    await mkdir(dir, { recursive: true, mode: 0o700 });
+    const generation = randomUUID();
+    const plaintextTemp = join(dir, `.${input.segmentId}.${generation}.jsonl`);
+    const bodyTemp = `${paths.body}.${generation}.tmp`;
+    const state = {
+        offset: 0,
+        lastRank: -1,
+        lastKey: undefined,
+        regions: { archives: [0, 0], tokens: [0, 0], meta: [0, 0] },
+        entries: [],
+        archives: 0,
+        tokens: 0,
+        tokenRowIndex: 0,
+    };
+    try {
+        const handle = await open(plaintextTemp, "wx", 0o600);
+        try {
+            const pending = [];
+            let pendingBytes = 0;
+            let rowsSeen = 0;
+            const flush = async () => {
+                if (pending.length === 0)
+                    return;
+                options.signal?.throwIfAborted();
+                await handle.writeFile(pending.join(""), { encoding: "utf8" });
+                pending.length = 0;
+                pendingBytes = 0;
+            };
+            // Written by the writer, not supplied: every reader can trust it without
+            // asking whoever produced the rows to have gotten it right.
+            const header = { version: 4, segmentId: input.segmentId, builtAt: input.builtAt };
+            const headerLine = `${JSON.stringify(header)}\n`;
+            await handle.writeFile(headerLine, { encoding: "utf8" });
+            state.offset = Buffer.byteLength(headerLine, "utf8");
+            for await (const row of asRowStream(input.rows)) {
+                options.signal?.throwIfAborted();
+                const kind = row[0];
+                const key = row[1];
+                const rank = REGION_RANK[kind];
+                if (rank < state.lastRank) {
+                    fail("invalid_argument", `Segment rows are out of order: a "${kind}" row followed a row from a later region`, { segmentId: input.segmentId });
+                }
+                if (rank !== state.lastRank) {
+                    if (state.lastRank !== -1)
+                        state.regions[REGION_NAME[state.lastRank]][1] = state.offset;
+                    state.regions[REGION_NAME[rank]] = [state.offset, state.offset];
+                    state.lastRank = rank;
+                    state.lastKey = undefined;
+                }
+                if (state.lastKey !== undefined && key <= state.lastKey) {
+                    fail("invalid_argument", `Segment rows are out of order: "${key}" did not sort after "${state.lastKey}"`, { segmentId: input.segmentId });
+                }
+                state.lastKey = key;
+                if (kind === "t") {
+                    if (state.tokenRowIndex % SEGMENT_DIRECTORY_STRIDE === 0)
+                        state.entries.push([key, state.offset]);
+                    state.tokenRowIndex += 1;
+                    state.tokens += 1;
+                }
+                else if (kind === "a") {
+                    state.archives += 1;
+                }
+                const serialized = `${JSON.stringify(row)}\n`;
+                const bytes = Buffer.byteLength(serialized, "utf8");
+                if (pendingBytes > 0 && pendingBytes + bytes > COALESCE_BYTES)
+                    await flush();
+                if (bytes > COALESCE_BYTES)
+                    await handle.writeFile(serialized, { encoding: "utf8" });
+                else {
+                    pending.push(serialized);
+                    pendingBytes += bytes;
+                }
+                state.offset += bytes;
+                rowsSeen += 1;
+                if (rowsSeen % 2_048 === 0) {
+                    if (options.onYield)
+                        await options.onYield();
+                    else
+                        await new Promise((resolve) => setImmediate(resolve));
+                }
+            }
+            if (state.lastRank !== -1)
+                state.regions[REGION_NAME[state.lastRank]][1] = state.offset;
+            await flush();
+            await handle.sync();
+        }
+        finally {
+            await handle.close();
+        }
+        const sealedFile = await sealArchiveToFile({ path: plaintextTemp }, bodyTemp, {
+            recipients: [{ kind: "phrase", phrase }],
+            // Per chunk, not whole-file: the point of a segment is that a lookup
+            // opens two chunks of it, never the whole body.
+            compression: "zstd-chunk",
+            adapter: { agent: "sealkeep-index-segment", version: "1" },
+            signal: options.signal,
+            onYield: options.onYield,
+        });
+        try {
+            await rename(bodyTemp, paths.body);
+            await writeSegmentFile(paths.envelope, `${JSON.stringify(sealedFile.envelope, null, 2)}\n`);
+            const directory = {
+                version: 1,
+                segmentId: input.segmentId,
+                envelopeArchiveId: sealedFile.envelope.archiveId,
+                builtAt: input.builtAt,
+                stride: SEGMENT_DIRECTORY_STRIDE,
+                regions: state.regions,
+                entries: state.entries,
+                archives: state.archives,
+                tokens: state.tokens,
+            };
+            await writeSealedDirectory(paths.directory, phrase, directory);
+        }
+        catch (error) {
+            await Promise.all([
+                rm(paths.body, { force: true }),
+                rm(paths.envelope, { force: true }),
+                rm(paths.directory, { force: true }),
+            ]);
+            throw error;
+        }
+        const bytes = sealedFile.envelope.chunks.reduce((total, chunk) => total + chunk.storedBytes, 0);
+        return { id: input.segmentId, builtAt: input.builtAt, archives: state.archives, tokens: state.tokens, bytes };
+    }
+    finally {
+        await rm(plaintextTemp, { force: true }).catch(() => undefined);
+        await rm(bodyTemp, { force: true }).catch(() => undefined);
+    }
+};
+async function readSegmentEnvelope(envelopePath, segmentId) {
+    let text;
+    try {
+        text = await readFile(envelopePath, "utf8");
+    }
+    catch (error) {
+        if (isEnoent(error))
+            return fail("invalid_argument", `No segment ${segmentId} envelope at ${envelopePath}`, { segmentId });
+        throw error;
+    }
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return fail("invalid_argument", `Segment ${segmentId} envelope is not valid JSON`, { segmentId });
+    }
+}
+async function readSegmentDirectory(directoryPath, phrase, segmentId) {
+    let text;
+    try {
+        text = await readFile(directoryPath, "utf8");
+    }
+    catch (error) {
+        if (isEnoent(error))
+            return fail("invalid_argument", `No segment ${segmentId} directory at ${directoryPath}`, { segmentId });
+        throw error;
+    }
+    let stored;
+    try {
+        stored = JSON.parse(text);
+    }
+    catch {
+        return fail("invalid_argument", `Segment ${segmentId} directory is not valid JSON`, { segmentId });
+    }
+    if (stored.version !== 1 || !stored.envelope || typeof stored.ciphertext !== "string") {
+        return fail("invalid_argument", `Segment ${segmentId} directory is malformed`, { segmentId });
+    }
+    // A wrong phrase and a damaged directory ciphertext cannot be told apart
+    // from outside; reporting the phrase is the actionable answer for nearly
+    // every case that reaches this branch, exactly as the legacy whole-index
+    // reader in search.ts treats any decrypt failure of its own directory.
+    let plaintext;
+    try {
+        plaintext = openEnvelope(stored.envelope, Buffer.from(stored.ciphertext, "base64"), { phrase });
+    }
+    catch {
+        return fail("recovery_phrase_mismatch", `Segment ${segmentId} directory would not open with this phrase`, { segmentId });
+    }
+    try {
+        const parsed = JSON.parse(plaintext.toString("utf8"));
+        if (parsed.version !== 1
+            || typeof parsed.segmentId !== "string"
+            || typeof parsed.envelopeArchiveId !== "string"
+            || typeof parsed.builtAt !== "string"
+            || typeof parsed.stride !== "number"
+            || !parsed.regions
+            || !Array.isArray(parsed.entries)
+            || typeof parsed.archives !== "number"
+            || typeof parsed.tokens !== "number") {
+            return fail("ciphertext_integrity_failed", `Segment ${segmentId} directory contents are invalid`, { segmentId });
+        }
+        return parsed;
+    }
+    finally {
+        plaintext.fill(0);
+    }
+}
+export async function openSegment(dataDir, phrase, segmentId) {
+    const paths = await segmentPaths(dataDir, segmentId);
+    const envelope = await readSegmentEnvelope(paths.envelope, segmentId);
+    if (!supportsChunkAccess(envelope)) {
+        return fail("ciphertext_integrity_failed", `Segment ${segmentId} body is not chunk-addressable`, { segmentId });
+    }
+    const directoryIdentity = await stat(paths.directory)
+        .then((info) => `${info.size}:${info.mtimeMs}`)
+        .catch((error) => (isEnoent(error) ? fail("invalid_argument", `No segment ${segmentId} directory at ${paths.directory}`, { segmentId }) : Promise.reject(error)));
+    const cacheKey = `${segmentId}:${phraseDigest(phrase)}`;
+    let opened = openedSegments.get(cacheKey);
+    if (opened && (opened.envelopeArchiveId !== envelope.archiveId || opened.directoryIdentity !== directoryIdentity)) {
+        opened.archiveKey.fill(0);
+        openedSegments.delete(cacheKey);
+        opened = undefined;
+    }
+    if (!opened) {
+        const directory = await readSegmentDirectory(paths.directory, phrase, segmentId);
+        // Binds the sealed sidecar to the exact sealed body: a directory copied from
+        // another segment decrypts fine under the same phrase but names a body that
+        // is not this one.
+        if (directory.envelopeArchiveId !== envelope.archiveId) {
+            return fail("ciphertext_integrity_failed", `Segment ${segmentId} directory does not match its sealed body`, { segmentId });
+        }
+        // The directory opened under this phrase, so a body that will not is a
+        // damaged envelope, not a wrong phrase.
+        let archiveKey;
+        try {
+            archiveKey = unwrapWindowKey(envelope, { phrase });
+        }
+        catch {
+            return fail("ciphertext_integrity_failed", `Segment ${segmentId} body key would not unwrap under the phrase that opened its directory`, { segmentId });
+        }
+        opened = { envelopeArchiveId: envelope.archiveId, directoryIdentity, directory, archiveKey };
+        rememberOpenedSegment(cacheKey, opened);
+    }
+    const { directory, archiveKey } = opened;
+    const expectedBytes = envelope.chunks.reduce((total, chunk) => total + chunk.storedBytes, 0);
+    let bodySize;
+    try {
+        bodySize = (await stat(paths.body)).size;
+    }
+    catch (error) {
+        if (isEnoent(error))
+            return fail("invalid_argument", `Segment ${segmentId} has no sealed body in ${dataDir}`, { segmentId });
+        throw error;
+    }
+    if (bodySize !== expectedBytes) {
+        return fail("ciphertext_integrity_failed", `Segment ${segmentId} body is ${bodySize} bytes; its envelope describes ${expectedBytes}`, { segmentId });
+    }
+    const handle = await open(paths.body, "r");
+    return new SealedSegmentReader(segmentId, directory, envelope, phrase, archiveKey, handle);
+}
+class SealedSegmentReader {
+    id;
+    directory;
+    #envelope;
+    #phrase;
+    /** Shared with the per-process cache; never zeroized by a reader. */
+    #archiveKey;
+    #handle;
+    #closed = false;
+    #fixed = null;
+    constructor(id, directory, envelope, phrase, archiveKey, handle) {
+        this.id = id;
+        this.directory = directory;
+        this.#envelope = envelope;
+        this.#phrase = phrase;
+        this.#archiveKey = archiveKey;
+        this.#handle = handle;
+    }
+    async #readRaw(start, end, signal) {
+        if (end <= start)
+            return Buffer.alloc(0);
+        signal?.throwIfAborted();
+        const window = chunkWindowForRange(this.#envelope, start, end);
+        const slice = Buffer.allocUnsafe(window.storedLength);
+        const { bytesRead } = await this.#handle.read(slice, 0, window.storedLength, window.storedOffset);
+        if (bytesRead !== window.storedLength) {
+            return fail("ciphertext_integrity_failed", `Segment ${this.id} body is shorter than its envelope describes`, { segmentId: this.id });
+        }
+        return openChunkWindow(this.#envelope, window, slice, { phrase: this.#phrase }, { archiveKey: this.#archiveKey, signal });
+    }
+    async #readRows(start, end, signal) {
+        const raw = await this.#readRaw(start, end, signal);
+        if (raw.length === 0)
+            return [];
+        const rows = [];
+        for (const line of raw.toString("utf8").split("\n"))
+            if (line)
+                rows.push(JSON.parse(line));
+        return rows;
+    }
+    async #fixedRegions(signal) {
+        if (this.#fixed)
+            return this.#fixed;
+        const [archiveRows, metaRows] = await Promise.all([
+            this.#readRows(this.directory.regions.archives[0], this.directory.regions.archives[1], signal),
+            this.#readRows(this.directory.regions.meta[0], this.directory.regions.meta[1], signal),
+        ]);
+        const archives = {};
+        for (const row of archiveRows)
+            if (isArchiveRow(row))
+                archives[row[1]] = row.slice(2);
+        const meta = {};
+        for (const row of metaRows)
+            if (isMetaRow(row))
+                meta[row[1]] = row[2];
+        this.#fixed = { archives, meta };
+        return this.#fixed;
+    }
+    async lookup(terms, options = {}) {
+        const prefixKeep = options.prefixKeep ?? 64;
+        const signal = options.signal;
+        signal?.throwIfAborted();
+        const { archives, meta } = await this.#fixedRegions(signal);
+        const tokens = {};
+        const prefixSeen = new Map();
+        const { entries } = this.directory;
+        for (const term of new Set(terms)) {
+            if (entries.length === 0)
+                continue;
+            const position = Math.max(0, directoryPosition(entries, term));
+            let windows = 0;
+            for (let k = position; k < entries.length && windows < MAX_LOOKUP_WINDOWS_PER_TERM; k += 1, windows += 1) {
+                const start = entries[k][1];
+                const end = k + 1 < entries.length ? entries[k + 1][1] : this.directory.regions.tokens[1];
+                for (const row of await this.#readRows(start, end, signal)) {
+                    if (!isTokenRow(row))
+                        continue;
+                    const token = row[1];
+                    if (token === term) {
+                        tokens[token] = row[2];
+                        continue;
+                    }
+                    if (token.startsWith(term)) {
+                        const seen = prefixSeen.get(term) ?? 0;
+                        if (seen < prefixKeep) {
+                            tokens[token] = row[2];
+                            prefixSeen.set(term, seen + 1);
+                        }
+                    }
+                }
+                // Rows are sorted: once the next stride entry's token no longer
+                // carries this prefix, nothing after it can either.
+                const next = entries[k + 1]?.[0];
+                if (next === undefined || !next.startsWith(term))
+                    break;
+            }
+        }
+        return { archives: { ...archives }, tokens, meta: { ...meta } };
+    }
+    async *rows(region, options = {}) {
+        const [regionStart, regionEnd] = this.directory.regions[region];
+        const signal = options.signal;
+        let cursor = regionStart;
+        let carry = "";
+        while (cursor < regionEnd) {
+            signal?.throwIfAborted();
+            const windowEnd = Math.min(cursor + ROWS_WINDOW_BYTES, regionEnd);
+            const raw = await this.#readRaw(cursor, windowEnd, signal);
+            cursor = windowEnd;
+            const text = carry + raw.toString("utf8");
+            const lines = text.split("\n");
+            carry = lines.pop() ?? "";
+            for (const line of lines)
+                if (line)
+                    yield JSON.parse(line);
+        }
+        if (carry)
+            yield JSON.parse(carry);
+    }
+    async close() {
+        if (this.#closed)
+            return;
+        this.#closed = true;
+        await this.#handle.close();
+    }
+}

package/dist/src/index-store.d.ts

@@ -0,0 +1,123 @@
+import { type SegmentEntry, type SegmentManifest, type SegmentRow } from "./index-segment-types.js";
+import { applyArchiveTokens, type ContentIndex } from "./search.js";
+import { type RemoteIndexStore } from "./index-sync.js";
+type IndexableRecord = Parameters<typeof applyArchiveTokens>[1];
+/** Sorted "a", then "t", then "m" rows — the region order every segment
+ * writer requires. Declared synchronous (not `async`, despite the design
+ * doc's signature): an `async function` may only return a `Promise`, and
+ * every caller here — `writeSegment`'s `rows` parameter — accepts a plain
+ * `Iterable<SegmentRow>` directly, so there is nothing for a Promise to buy. */
+export declare function segmentRowsFromIndex(index: ContentIndex): Iterable<SegmentRow>;
+/**
+ * Segments mode is a one-way door for a vault: once `sealkeep index migrate`
+ * has run, it must never silently fall back to the legacy whole-blob build
+ * (and its whole-index-in-memory cost) just because `dropContentIndex`
+ * happened to empty the manifest. So this is a FILE-existence check, not a
+ * segment-count one — `dropContentIndex` deliberately leaves behind an empty
+ * manifest (see `resetSegments`) rather than removing it, precisely so this
+ * keeps answering true.
+ */
+export declare function segmentsEnabled(dataDir: string): Promise<boolean>;
+/** The archive ids segments cover on this machine: the union of every
+ * segment's `archiveIds` minus tombstones — exactly what `coverage.json`
+ * holds, computed straight from the plaintext manifest. No segment body is
+ * opened and no phrase is needed. */
+export declare function manifestCoverage(dataDir: string): Promise<Set<string>>;
+/** Archives another machine sealed whose rows a segment here still carries — searchable, never this machine's to prune. */
+export declare function manifestRemoteCoverage(dataDir: string): Promise<Set<string>>;
+/**
+ * `coverage.json` exactly as `saveLocal` writes it in search.ts: the unsealed
+ * sidecar `index status`, doctor, and the daemon read to answer "what is
+ * searchable" without the recovery phrase. Always re-derived from the
+ * manifest, so it can never drift from what the segments actually cover.
+ */
+export declare function writeCoverageFromManifest(dataDir: string): Promise<void>;
+/**
+ * Empties a vault's segment set back to nothing while keeping it IN segments
+ * mode (the manifest file itself survives, freshly empty) — the counterpart
+ * `dropContentIndex` needs so a drop-then-rebuild never regresses to the
+ * legacy blob, and the extraction-policy bump needs so a stale segment set
+ * can be discarded wholesale rather than patched in place (segments are
+ * immutable; there is no way to re-extract one that already exists).
+ */
+export declare function resetSegments(dataDir: string): Promise<SegmentManifest>;
+/**
+ * `planCompaction` over the current manifest; when it has work, stream the
+ * merge and replace the merged segments with the single result, then bring
+ * `coverage.json` and any stray `.skseg` files left by an interrupted prior
+ * attempt back in line with the manifest.
+ */
+export declare function maybeCompact(dataDir: string, phrase: string, options?: {
+    signal?: AbortSignal;
+    onYield?: () => void | Promise<void>;
+    smallOnly?: boolean;
+}): Promise<SegmentEntry | null>;
+/** One archive, sealed at seal time: no whole-index rewrite, just a new
+ * one-archive segment. Also publishes it (step 2): a segment is exactly the
+ * unit a peer wants as soon as possible, not just at the next background
+ * build. A publish failure never fails the seal that produced the segment —
+ * it is already durable locally — so it is caught and named for the next
+ * successful publish (a build's, or this machine's next seal) to pick up. */
+export declare function appendArchiveSegment(dataDir: string, phrase: string, record: IndexableRecord, tokensByChunk: ReadonlyMap<number, ReadonlySet<string>>, options?: {
+    sync?: boolean;
+    signal?: AbortSignal;
+    remoteIdleMs?: number;
+    maxBytesPerSecond?: number;
+}): Promise<SegmentEntry>;
+/** The catch-up build's per-batch equivalent: one segment for however many
+ * archives a checkpoint's worth of indexing collected. */
+export declare function appendBatchSegment(dataDir: string, phrase: string, batch: ContentIndex, options?: {
+    signal?: AbortSignal;
+    onYield?: () => void | Promise<void>;
+}): Promise<SegmentEntry | null>;
+export declare function retireArchiveIds(dataDir: string, archiveIds: readonly string[]): Promise<void>;
+/**
+ * Open every segment, answer `terms` from each concurrently, and merge into
+ * one `ContentIndex` a query-time caller reads exactly like the legacy blob:
+ * archives keep the richer (longer) row, meta merges field by field with the
+ * later (newest-last) segment winning except a real `summary` over scaffold
+ * and unioned `commits`/`files`, and token postings union, drop anything
+ * tombstoned, and rank chronologically before the usual per-token cap.
+ */
+export declare function lookupSegments(dataDir: string, phrase: string, terms: readonly string[], options?: {
+    prefixKeep?: number;
+    signal?: AbortSignal;
+    onYield?: () => void | Promise<void>;
+}): Promise<ContentIndex>;
+/**
+ * The blob for cloud publication, produced from segments without ever
+ * holding more than the streaming merge needs: `mergeSegmentRows` folds every
+ * segment into archive/token/meta rows with tombstones already applied, and
+ * `persistContentIndex` seals and pushes exactly as the legacy build does.
+ * `maxTokens` is the background child's guard — a merge that would exceed it
+ * is abandoned (nothing written) rather than produced and then rejected.
+ *
+ * The blob-fallback publish path (step 2 addendum): a store with no generic
+ * keyed storage (managed today) cannot hold segments, so this is still how
+ * such a vault reaches the cloud at all. A successful push also records
+ * `published-blob.json`, so `publishPending` knows the manifest it merged is
+ * no longer waiting, and a peer without segments support (`pullSegments`'s
+ * own blob fallback) can tell its own publication apart from one worth
+ * converting.
+ */
+export declare function publishFromSegments(dataDir: string, phrase: string, options: {
+    store: RemoteIndexStore | null;
+    signal?: AbortSignal;
+    onYield?: () => void | Promise<void>;
+    maxTokens?: number;
+}): Promise<boolean>;
+/**
+ * One-time move of the existing whole-index blob into a single segment. The
+ * blob files are left exactly where they are — the cloud path still reads
+ * them until the next publish, and a vault with neither blob nor manifest is
+ * simply "build one" (unchanged). Accepts a raw or already-canonical phrase,
+ * like the other public entry points in search.ts (`buildContentIndex`,
+ * `loadContentIndex`): `sealkeep index migrate` hands it the CLI's raw
+ * `--recovery-phrase` value directly.
+ */
+export declare function migrateBlobToSegments(dataDir: string, rawPhrase: string): Promise<{
+    migrated: boolean;
+    archives: number;
+    tokens: number;
+}>;
+export {};