sealkeep 0.9.0 → 0.11.0

package/dist/src/index-publication-verifier.d.ts

@@ -0,0 +1,89 @@
+import { type IndexPublicationReceipt, type IndexPublicationRequirement } from "./index-publication-state.js";
+import type { RemoteIndexStore } from "./index-sync.js";
+export type IndexPublicationChainLink = {
+    archiveId: string;
+    archiveRef: string;
+};
+export type IndexPublicationVerificationReason = "invalid_request" | "managed_v2_required" | "ledger_requirement_missing" | "ledger_receipt_missing" | "ledger_receipt_mismatch" | "head_missing" | "head_changed" | "object_integrity" | "index_integrity" | "coverage_mismatch" | "head_not_searchable";
+export declare class IndexPublicationVerificationError extends Error {
+    readonly code: "INDEX_PUBLICATION_VERIFICATION_FAILED";
+    readonly reason: IndexPublicationVerificationReason;
+    readonly cause?: unknown;
+    constructor(reason: IndexPublicationVerificationReason, message: string, cause?: unknown);
+}
+export type VerifiedManagedIndexHead = {
+    kind: "managed-account-index-v2";
+    vaultId: string;
+    revision: number;
+    generationId: string;
+    committedAt: string;
+    coverageSha256: string;
+    blob: {
+        objectKey: string;
+        bytes: number;
+        sha256: string;
+    };
+    meta: {
+        objectKey: string;
+        bytes: number;
+        sha256: string;
+    };
+};
+export type VerifiedManagedIndexChainLink = IndexPublicationChainLink & {
+    entryDigest: string;
+    effectivePostingCount: number;
+    requirement: IndexPublicationRequirement;
+    receipt: IndexPublicationReceipt;
+};
+export type VerifiedManagedAccountIndexPublication = {
+    head: VerifiedManagedIndexHead;
+    requestedHead: IndexPublicationChainLink;
+    extractionPolicy: number;
+    /** Physical restore dependencies. Delta bases may be catalog-only. */
+    chain: VerifiedManagedIndexChainLink[];
+    /** Logical Codex continuation heads. Every one must have searchable postings. */
+    searchableHeads: VerifiedManagedIndexChainLink[];
+};
+export type ManagedAccountIndexVerificationOptions = {
+    dataDir: string;
+    vaultId: string;
+    phrase: string;
+    extractionPolicy: number;
+    chain: readonly IndexPublicationChainLink[];
+    requestedHead: IndexPublicationChainLink;
+    /**
+     * Logical continuation heads that must remain independently discoverable.
+     * Defaults to requestedHead for callers that have no repeated-resume chain.
+     */
+    searchableHeads?: readonly IndexPublicationChainLink[];
+    /** Omit in production to construct the managed account store directly. */
+    store?: RemoteIndexStore;
+    signal?: AbortSignal;
+    idleMs?: number;
+    maxBytesPerSecond?: number;
+    onProgress?: (bytes: number) => void;
+};
+export type ManagedAccountIndexRecheckOptions = {
+    dataDir: string;
+    verification: VerifiedManagedAccountIndexPublication;
+    /** Omit in production to construct the managed account store directly. */
+    store?: RemoteIndexStore;
+    signal?: AbortSignal;
+    idleMs?: number;
+};
+/**
+ * Full positive proof used before reclaim can become eligible.
+ *
+ * It accepts only the managed immutable/CAS lane. Local ledger receipts alone,
+ * legacy v9 slots, Google Drive, and own-bucket stores can never authorize a
+ * source deletion. The returned head token is deliberately sufficient for a
+ * cheap exact-head recheck immediately before unlink.
+ */
+export declare function verifyManagedAccountIndexPublication(options: ManagedAccountIndexVerificationOptions): Promise<VerifiedManagedAccountIndexPublication>;
+/**
+ * Last destructive-boundary check. It performs no large download: every local
+ * obligation/receipt must still be exact, and the registry must still expose
+ * the same immutable head (revision, generation, object digests, and coverage).
+ * Any advance causes the caller to repeat the full verification.
+ */
+export declare function recheckManagedAccountIndexPublicationBeforeUnlink(options: ManagedAccountIndexRecheckOptions): Promise<void>;

package/dist/src/index-publication-verifier.js

@@ -0,0 +1,341 @@
+import { createHash } from "node:crypto";
+import { canonicalIndexPublicationCoverageJson, } from "./index-publication-proof.js";
+import { getIndexPublicationReceipt, listIndexPublicationRequirements, } from "./index-publication-state.js";
+import { canonicalPhrase } from "./mnemonic.js";
+import { createProgressDeadline } from "./progress-deadline.js";
+import { openVerifiedManagedIndexV3Streaming } from "./search.js";
+const DEFAULT_VERIFY_IDLE_MS = 2 * 60_000;
+const MANAGED_PUBLISHER = "managed-cas-v2";
+export class IndexPublicationVerificationError extends Error {
+    code = "INDEX_PUBLICATION_VERIFICATION_FAILED";
+    reason;
+    cause;
+    constructor(reason, message, cause) {
+        super(message, cause === undefined ? undefined : { cause });
+        this.name = "IndexPublicationVerificationError";
+        this.reason = reason;
+        this.cause = cause;
+    }
+}
+function refuse(reason, message, cause) {
+    throw new IndexPublicationVerificationError(reason, message, cause);
+}
+function text(value, label) {
+    if (typeof value !== "string" || value.length === 0)
+        return refuse("object_integrity", `The managed index has no ${label}`);
+    return value;
+}
+function sha(value, label) {
+    const digest = text(value, label);
+    if (!/^[0-9a-f]{64}$/.test(digest))
+        return refuse("object_integrity", `The managed index has an invalid ${label}`);
+    return digest;
+}
+function integer(value, label, minimum) {
+    if (!Number.isSafeInteger(value) || Number(value) < minimum) {
+        return refuse("object_integrity", `The managed index has an invalid ${label}`);
+    }
+    return Number(value);
+}
+function sameObject(left, right) {
+    return left.objectKey === right.objectKey && left.bytes === right.bytes && left.sha256 === right.sha256;
+}
+function managedHeadToken(vaultId, receipt) {
+    if (receipt.membershipVersion !== undefined) {
+        return refuse("managed_v2_required", "A team index head cannot authorize account-session reclaim");
+    }
+    const coverage = canonicalIndexPublicationCoverageJson(receipt.coverageJson);
+    return {
+        kind: "managed-account-index-v2",
+        vaultId,
+        revision: integer(receipt.revision, "head revision", 1),
+        generationId: text(receipt.generationId, "generation id"),
+        committedAt: text(receipt.committedAt, "commit time"),
+        coverageSha256: createHash("sha256").update(coverage, "utf8").digest("hex"),
+        blob: {
+            objectKey: text(receipt.blob?.objectKey, "blob object key"),
+            bytes: integer(receipt.blob?.bytes, "blob byte count", 1),
+            sha256: sha(receipt.blob?.sha256, "blob SHA-256"),
+        },
+        meta: {
+            objectKey: text(receipt.meta?.objectKey, "metadata object key"),
+            bytes: integer(receipt.meta?.bytes, "metadata byte count", 1),
+            sha256: sha(receipt.meta?.sha256, "metadata SHA-256"),
+        },
+    };
+}
+function sameHead(left, right) {
+    return left.kind === right.kind
+        && left.vaultId === right.vaultId
+        && left.revision === right.revision
+        && left.generationId === right.generationId
+        && left.committedAt === right.committedAt
+        && left.coverageSha256 === right.coverageSha256
+        && sameObject(left.blob, right.blob)
+        && sameObject(left.meta, right.meta);
+}
+function sameReceipt(left, right) {
+    return left.requirementKey === right.requirementKey
+        && left.policyVersion === right.policyVersion
+        && left.publisherId === right.publisherId
+        && left.generationId === right.generationId
+        && left.registryReceiptId === right.registryReceiptId
+        && left.registryRevision === right.registryRevision
+        && left.objectRef === right.objectRef
+        && left.ciphertextSha256 === right.ciphertextSha256
+        && left.ciphertextBytes === right.ciphertextBytes
+        && left.committedAt === right.committedAt;
+}
+function receiptMatchesHead(receipt, head) {
+    return receipt.publisherId === MANAGED_PUBLISHER
+        && receipt.generationId === head.generationId
+        && receipt.registryReceiptId === `index-v2:${head.revision}:${head.generationId}`
+        && receipt.registryRevision === head.revision
+        && receipt.objectRef === JSON.stringify([head.blob.objectKey, head.meta.objectKey])
+        && receipt.ciphertextSha256 === head.blob.sha256
+        && receipt.ciphertextBytes === head.blob.bytes
+        && receipt.committedAt === head.committedAt;
+}
+function validateLinks(links, label) {
+    const validated = links.map((link) => {
+        if (!link || typeof link.archiveId !== "string" || !link.archiveId
+            || typeof link.archiveRef !== "string" || !link.archiveRef) {
+            return refuse("invalid_request", `Every ${label} link needs an archive id and reference`);
+        }
+        return { archiveId: link.archiveId, archiveRef: link.archiveRef };
+    });
+    if (new Set(validated.map((link) => link.archiveId)).size !== validated.length
+        || new Set(validated.map((link) => link.archiveRef)).size !== validated.length) {
+        return refuse("invalid_request", `The ${label} contains duplicate archive identities`);
+    }
+    return validated;
+}
+function validateRequest(options) {
+    if (!options.dataDir || !options.vaultId || !options.phrase) {
+        return refuse("invalid_request", "Index publication verification requires a data directory, vault id, and phrase");
+    }
+    if (!Number.isSafeInteger(options.extractionPolicy) || options.extractionPolicy < 1) {
+        return refuse("invalid_request", "Index publication verification requires a positive extraction policy");
+    }
+    if (!Array.isArray(options.chain) || options.chain.length === 0) {
+        return refuse("invalid_request", "Index publication verification requires an archive chain");
+    }
+    const chain = validateLinks(options.chain, "index publication chain");
+    if (!options.requestedHead
+        || !chain.some((link) => link.archiveId === options.requestedHead.archiveId
+            && link.archiveRef === options.requestedHead.archiveRef)) {
+        return refuse("invalid_request", "The requested session head is not in its archive chain");
+    }
+    const searchableHeads = validateLinks(options.searchableHeads ?? [options.requestedHead], "searchable continuation set");
+    if (searchableHeads.length === 0
+        || !searchableHeads.some((link) => link.archiveId === options.requestedHead.archiveId
+            && link.archiveRef === options.requestedHead.archiveRef)) {
+        return refuse("invalid_request", "The requested session head is not in its searchable continuation set");
+    }
+    const obligations = new Map();
+    for (const link of [...chain, ...searchableHeads]) {
+        const sameArchive = [...obligations.values()].find((candidate) => candidate.archiveId === link.archiveId);
+        const sameRef = [...obligations.values()].find((candidate) => candidate.archiveRef === link.archiveRef);
+        if ((sameArchive && sameArchive.archiveRef !== link.archiveRef)
+            || (sameRef && sameRef.archiveId !== link.archiveId)) {
+            return refuse("invalid_request", "An archive obligation has conflicting ids or references");
+        }
+        obligations.set(`${link.archiveId}\u0000${link.archiveRef}`, link);
+    }
+    return { chain, searchableHeads, obligations: [...obligations.values()] };
+}
+async function accountStore(dataDir, injected, signal) {
+    if (injected)
+        return injected;
+    const { managedIndexStore } = await import("./index-sync.js");
+    return managedIndexStore(dataDir, { signal });
+}
+function verifyObjectBytes(body, expected, label) {
+    if (!Buffer.isBuffer(body)
+        || body.length !== expected.bytes
+        || createHash("sha256").update(body).digest("hex") !== expected.sha256) {
+        refuse("object_integrity", `The managed index ${label} does not match its immutable head receipt`);
+    }
+}
+function envelopeBytes(envelope) {
+    try {
+        const raw = JSON.stringify(envelope);
+        if (!raw)
+            return refuse("object_integrity", "The managed index metadata is not serializable");
+        return Buffer.from(raw, "utf8");
+    }
+    catch (error) {
+        return refuse("object_integrity", "The managed index metadata is not serializable", error);
+    }
+}
+/**
+ * Full positive proof used before reclaim can become eligible.
+ *
+ * It accepts only the managed immutable/CAS lane. Local ledger receipts alone,
+ * legacy v9 slots, Google Drive, and own-bucket stores can never authorize a
+ * source deletion. The returned head token is deliberately sufficient for a
+ * cheap exact-head recheck immediately before unlink.
+ */
+export async function verifyManagedAccountIndexPublication(options) {
+    options.signal?.throwIfAborted();
+    const { chain, searchableHeads, obligations: requestedObligations } = validateRequest(options);
+    const statuses = await listIndexPublicationRequirements(options.dataDir, { vaultId: options.vaultId });
+    options.signal?.throwIfAborted();
+    const obligations = requestedObligations.map((link) => {
+        const matches = statuses.filter((status) => status.requirement.scope.kind === "account"
+            && status.requirement.scope.vaultId === options.vaultId
+            && status.requirement.archiveId === link.archiveId
+            && status.requirement.archiveRef === link.archiveRef);
+        if (matches.length !== 1) {
+            return refuse("ledger_requirement_missing", `No exact current account-index requirement exists for ${link.archiveId}`);
+        }
+        const status = matches[0];
+        if (!status.receipt) {
+            return refuse("ledger_receipt_missing", `No positive managed-index receipt exists for ${link.archiveId}`);
+        }
+        return { link, requirement: status.requirement, receipt: status.receipt };
+    });
+    const store = await accountStore(options.dataDir, options.store, options.signal);
+    if (!store.cas)
+        return refuse("managed_v2_required", "This storage target has no immutable managed index protocol");
+    const deadline = createProgressDeadline({
+        operation: "Verifying the managed search index",
+        idleMs: options.idleMs ?? DEFAULT_VERIFY_IDLE_MS,
+        signal: options.signal,
+    });
+    try {
+        const pulled = await deadline.wait(store.cas.pullSnapshot({
+            signal: deadline.signal,
+            maxBytesPerSecond: options.maxBytesPerSecond,
+            onProgress: (bytes) => {
+                deadline.progress();
+                options.onProgress?.(bytes);
+            },
+        }));
+        deadline.progress();
+        deadline.signal.throwIfAborted();
+        if (pulled.status === "unsupported") {
+            return refuse("managed_v2_required", "The managed service does not support immutable index generations");
+        }
+        if (!pulled.snapshot)
+            return refuse("head_missing", "No committed managed index generation exists");
+        const snapshot = pulled.snapshot;
+        const head = managedHeadToken(options.vaultId, snapshot.receipt);
+        const nestedHead = managedHeadToken(options.vaultId, snapshot.head.receipt);
+        if (!sameHead(head, nestedHead)) {
+            return refuse("head_changed", "The downloaded index snapshot is not bound to one exact registry head");
+        }
+        verifyObjectBytes(snapshot.ciphertext, head.blob, "ciphertext");
+        verifyObjectBytes(envelopeBytes(snapshot.envelope), head.meta, "metadata");
+        if (snapshot.envelope.manifest?.adapter?.agent !== "sealkeep-index") {
+            return refuse("index_integrity", "The committed object is not a Sealkeep search index");
+        }
+        let proof;
+        try {
+            proof = await openVerifiedManagedIndexV3Streaming(snapshot.envelope, snapshot.ciphertext, { phrase: canonicalPhrase(options.phrase) }, snapshot.receipt.coverageJson, {
+                signal: deadline.signal,
+                onYield: () => { deadline.progress(); },
+            });
+            deadline.progress();
+            deadline.signal.throwIfAborted();
+            if (proof.extractionPolicy !== options.extractionPolicy) {
+                return refuse("index_integrity", "The committed search index uses a different extraction policy");
+            }
+        }
+        catch (error) {
+            if (error instanceof IndexPublicationVerificationError)
+                throw error;
+            if (error instanceof Error
+                && error.message === "The committed search index does not contain the coverage recorded by its head") {
+                return refuse("coverage_mismatch", "The registry coverage does not describe the authenticated index bytes", error);
+            }
+            return refuse("index_integrity", "The committed search index could not be authenticated and parsed", error);
+        }
+        const recomputedCoverage = canonicalIndexPublicationCoverageJson(proof.coverageJson);
+        const advertisedCoverage = canonicalIndexPublicationCoverageJson(snapshot.receipt.coverageJson);
+        if (recomputedCoverage !== advertisedCoverage) {
+            return refuse("coverage_mismatch", "The registry coverage does not describe the authenticated index bytes");
+        }
+        const verifiedObligations = obligations.map(({ link, requirement, receipt }) => {
+            if (!receiptMatchesHead(receipt, head)) {
+                return refuse("ledger_receipt_mismatch", `The local publication receipt for ${link.archiveId} is not for the current managed head`);
+            }
+            if (proof.coverageJson[link.archiveRef] !== requirement.entryDigest) {
+                return refuse("coverage_mismatch", `The current managed index does not contain the exact entry for ${link.archiveId}`);
+            }
+            return {
+                ...link,
+                entryDigest: requirement.entryDigest,
+                effectivePostingCount: proof.effectivePostingCounts[link.archiveRef] ?? 0,
+                requirement: structuredClone(requirement),
+                receipt: structuredClone(receipt),
+            };
+        });
+        const verifiedByIdentity = new Map(verifiedObligations.map((link) => [
+            `${link.archiveId}\u0000${link.archiveRef}`,
+            link,
+        ]));
+        const verifiedChain = chain.map((link) => verifiedByIdentity.get(`${link.archiveId}\u0000${link.archiveRef}`));
+        const verifiedSearchableHeads = searchableHeads.map((link) => verifiedByIdentity.get(`${link.archiveId}\u0000${link.archiveRef}`));
+        for (const searchable of verifiedSearchableHeads) {
+            if (searchable.effectivePostingCount <= 0) {
+                return refuse("head_not_searchable", `The searchable continuation ${searchable.archiveId} has no effective searchable postings`);
+            }
+        }
+        return {
+            head,
+            requestedHead: { ...options.requestedHead },
+            extractionPolicy: options.extractionPolicy,
+            chain: verifiedChain,
+            searchableHeads: verifiedSearchableHeads,
+        };
+    }
+    finally {
+        deadline.close();
+    }
+}
+/**
+ * Last destructive-boundary check. It performs no large download: every local
+ * obligation/receipt must still be exact, and the registry must still expose
+ * the same immutable head (revision, generation, object digests, and coverage).
+ * Any advance causes the caller to repeat the full verification.
+ */
+export async function recheckManagedAccountIndexPublicationBeforeUnlink(options) {
+    options.signal?.throwIfAborted();
+    const verification = options.verification;
+    const obligations = new Map();
+    for (const link of [...verification.chain, ...(verification.searchableHeads ?? [])]) {
+        obligations.set(`${link.archiveId}\u0000${link.archiveRef}`, link);
+    }
+    for (const link of obligations.values()) {
+        const receipt = await getIndexPublicationReceipt(options.dataDir, { vaultId: verification.head.vaultId }, link.requirement);
+        if (!receipt || !sameReceipt(receipt, link.receipt)) {
+            return refuse("ledger_receipt_mismatch", `The publication receipt changed before unlink for ${link.archiveId}`);
+        }
+    }
+    options.signal?.throwIfAborted();
+    const store = await accountStore(options.dataDir, options.store, options.signal);
+    if (!store.cas)
+        return refuse("managed_v2_required", "This storage target has no immutable managed index protocol");
+    const deadline = createProgressDeadline({
+        operation: "Rechecking the managed search index head",
+        idleMs: options.idleMs ?? DEFAULT_VERIFY_IDLE_MS,
+        signal: options.signal,
+    });
+    try {
+        const current = await deadline.wait(store.cas.head({ signal: deadline.signal }));
+        deadline.progress();
+        if (current.status === "unsupported") {
+            return refuse("managed_v2_required", "The managed service no longer supports immutable index generations");
+        }
+        if (!current.head)
+            return refuse("head_missing", "The managed search index head disappeared before unlink");
+        const observed = managedHeadToken(verification.head.vaultId, current.head.receipt);
+        if (!sameHead(observed, verification.head)) {
+            return refuse("head_changed", "The managed search index head changed before unlink; full verification must run again");
+        }
+    }
+    finally {
+        deadline.close();
+    }
+}

package/dist/src/index-publish.d.ts

@@ -0,0 +1,62 @@
+import type { RemoteIndexStore } from "./index-sync.js";
+export type PublishedBlobState = {
+    version: 1;
+    generation: string;
+    builtAt: string;
+};
+/** Called by `publishFromSegments` (src/index-store.ts) after a successful
+ * push, so `publishPending` and the blob-fallback `pullSegments` both know
+ * which generation and builtAt reached the store last. */
+export declare function writePublishedBlobState(dataDir: string, state: PublishedBlobState): Promise<void>;
+/**
+ * Uploads every local segment this machine has not yet published, deletes the
+ * remote copies of ids compaction or retirement has since replaced locally,
+ * republishes this machine's `head.json`, and re-registers it in the shared
+ * `heads.json`. Idempotent by construction: `head.json` is written only after
+ * every object it lists has actually landed, so a crash between any two steps
+ * is simply repaired by the next call, and nothing is re-uploaded because
+ * `published.json` is the last thing written.
+ */
+export declare function publishSegments(dataDir: string, phrase: string, store: RemoteIndexStore | null, options?: {
+    signal?: AbortSignal;
+    onYield?: () => void | Promise<void>;
+}): Promise<{
+    published: number;
+    removed: number;
+    skipped?: true;
+}>;
+/**
+ * True when a segments-mode vault has local segment content the account
+ * store has not seen yet — the daemon's cue to run a catch-up pass even when
+ * nothing is left to index. `published.json` (a segment store) and
+ * `published-blob.json` (the blob fallback) each name the manifest
+ * generation their own kind of publish last reached; either matching the
+ * current manifest counts as "nothing pending", since a vault only ever uses
+ * one or the other depending on what the account store supports. A blob-mode
+ * vault (not segments) always answers false: `indexArchiveTokens` already
+ * publishes inline at seal time there, so the daemon needs no extra nudge.
+ */
+export declare function publishPending(dataDir: string): Promise<boolean>;
+/**
+ * Reads the shared `heads.json`, then for every other machine whose
+ * generation has moved since this machine last reconciled it: fetches that
+ * machine's `head.json`, downloads whichever listed segments are not already
+ * local, verifies each, and replaces that machine's `origin: "remote"`
+ * entries in the local manifest with the current set in one `updateManifest`
+ * — dropping local files for ids it no longer lists only after the manifest
+ * no longer references them. A machine whose `head.json` has vanished (404)
+ * loses its remote entries entirely.
+ *
+ * A store with no generic keyed storage (managed today) has no `<machineId>`
+ * objects to read at all; `pullFromBlobFallback` converts its single legacy
+ * blob slot instead (step 2 addendum — see docs/design/index-publish.md).
+ */
+export declare function pullSegments(dataDir: string, phrase: string, store: RemoteIndexStore | null, options?: {
+    signal?: AbortSignal;
+    onYield?: () => void | Promise<void>;
+}): Promise<{
+    machines: number;
+    added: number;
+    removed: number;
+    skipped?: true;
+}>;