primitive-admin 1.1.0-alpha.82 → 1.1.0-alpha.83

package/dist/src/commands/documents.js

@@ -1,15 +1,56 @@
+import { snapshotBuildRows } from "../lib/snapshot-build-rows.js";
 import { ApiClient } from "../lib/api-client.js";
 import { resolveAppId } from "../lib/config.js";
-import { success, error, info, formatTable, formatId, formatDate, json, keyValue, result as printResult, warn, } from "../lib/output.js";
+import { success, error, info, formatTable, formatId, formatDate, json, keyValue, result as printResult, warn, flushOutput, } from "../lib/output.js";
 import { confirmPrompt } from "../lib/confirm-prompt.js";
 import { parseDataOption } from "../lib/data-input.js";
 import { parseFilterOptions } from "../lib/record-filter.js";
 import { buildPermissionsExport } from "../lib/document-export-permissions.js";
 import { chunkAddressOf, renumberManifestForExport, } from "../lib/snapshot-manifest-layout.js";
 import { validateSnapshotManifest } from "js-bao";
+import { auditSnapshotChain, } from "../lib/snapshot-audit.js";
+import { createAuditApiSource } from "../lib/snapshot-audit-source.js";
+import { createAuditLocalStore, createChainFingerprintReader, } from "../lib/snapshot-audit-store.js";
+import { discoverIngestInput, } from "../lib/document-ingest-input.js";
+import { buildIngestArtifact, ingestSchemaFromIntrospection, } from "../lib/document-ingest-artifact.js";
+import { driveDocumentIngest, summariseIngestPlan, assertIngestConfirmable, } from "../lib/document-ingest.js";
 import { ulid } from "ulid";
 import * as fs from "fs";
 import * as path from "path";
+/**
+ * One bulk-load session as rows (#3434, #3435).
+ *
+ * Shared by `documents ingests get` and by `documents ingest`'s own terminal
+ * report, so an operator reads the same thing however they arrived at it.
+ */
+function printIngestSession(session) {
+    printResult("Session", session.sessionId);
+    printResult("Document", session.documentId);
+    printResult("State", session.state);
+    printResult("Created by", session.createdBy);
+    printResult("Created", formatDate(session.createdAt));
+    printResult("Updated", formatDate(session.updatedAt));
+    printResult("Completed", session.completedAt ? formatDate(session.completedAt) : "—");
+    printResult("Chunks", session.chunks);
+    printResult("Rows", session.rows);
+    printResult("Bytes", session.bytes);
+    const progress = session.progress ?? {};
+    printResult("Progress", `copied ${progress.rowsCopied ?? 0} rows, applied ` +
+        `${progress.chunksApplied ?? 0} chunks / ${progress.rowsApplied ?? 0} rows, ` +
+        `reconciled ${progress.recordsReconciled ?? 0} records`);
+    printResult("Copy point", session.copyStartedAt ? formatDate(session.copyStartedAt) : "—");
+    printResult("Discontinuity epoch", session.discontinuityEpoch ?? "—");
+    printResult("Build", session.buildId ?? "—");
+    printResult("Expires", formatDate(session.expiresAt));
+    // Why it stopped is the reason an operator opened this at all.
+    if (session.failure) {
+        printResult("Failure", session.failure.code);
+        printResult("Reason", session.failure.reason);
+        for (const row of session.failure.rows ?? []) {
+            printResult("  Row", typeof row === "string" ? row : `${row.model} ${row.id}`);
+        }
+    }
+}
 /**
  * Report a document's format (#2816) — but only when it is a large document.
  *
@@ -900,6 +941,195 @@ Permissions (enforced by the server, not the CLI):
             process.exit(1);
         }
     });
+    // The action verb (#3435, criterion 9): a directory of records becomes a
+    // chunked, digested, uploaded artifact and one atomic swap of the
+    // document's tables. `documents ingests list|get` below is how an operator
+    // watches one afterwards.
+    //
+    // Destructive by nature — a bulk load deletes and overwrites records in a
+    // live document with no undo — so it confirms unless `-y`, which is what
+    // `docs/cli-design.md` reserves the destructive default for.
+    documents
+        .command("ingest")
+        .description("Bulk-load records into a large document from a directory")
+        .argument("<document-id>", "Document ID")
+        .requiredOption("--input <dir>", "Directory of <model>.ndjson[.gz] files, or a `documents export` output")
+        .option("--app <app-id>", "App ID")
+        .option("-y, --yes", "Skip the confirmation prompt")
+        .option("--no-wait", "Exit once the session is committed")
+        .option("--timeout <seconds>", "Give up waiting after this many seconds")
+        .option("--json", "Output as JSON")
+        .action(async (documentId, options) => {
+        const resolvedAppId = resolveAppId(undefined, options);
+        const client = new ApiClient();
+        let plan = null;
+        // The exit code is carried out of here rather than taken inside: a
+        // `process.exit` does not unwind, so ending the process in the try block
+        // would skip the cleanup in the finally and leave the compressed
+        // artifact — up to the whole input, gzipped — in the OS temp directory
+        // on every run, successful or not (finding 3435-FROZEN-05).
+        const run = async () => {
+            // The preflight, in this order and before a byte is read from disk:
+            // the schema read says whether this caller may see the document's
+            // declarations at all and hands over the ones the lines are checked
+            // against, and the session reader says whether this is a large
+            // document. Both are READS on the arm the ingest routes admit, so a
+            // caller who would be refused halfway is refused at once, and neither
+            // opens a session.
+            const introspection = await client.getDocumentSchema(resolvedAppId, documentId);
+            await client.listDocumentIngests(resolvedAppId, documentId, { limit: 1 });
+            const schema = ingestSchemaFromIntrospection(introspection);
+            const input = discoverIngestInput(options.input, documentId);
+            for (const ignored of input.ignored) {
+                info(`Ignoring ${ignored}: a bulk load reads the snapshot only.`);
+            }
+            plan = await buildIngestArtifact({ input, schema });
+            for (const line of summariseIngestPlan(plan))
+                info(line);
+            assertIngestConfirmable({
+                yes: Boolean(options.yes),
+                isTTY: Boolean(process.stdin.isTTY),
+            });
+            if (!options.yes) {
+                const confirmed = await confirmPrompt(`Bulk-load ${plan.rows} records into ${documentId}? ` +
+                    "This replaces records in a live document and cannot be undone.");
+                if (!confirmed) {
+                    info("Nothing was uploaded.");
+                    return 1;
+                }
+            }
+            // Ctrl-C is a decision the driver makes differently depending on
+            // where it lands, so it is handed the signal rather than killing the
+            // process: mid-upload the session is aborted, mid-poll it is left
+            // running, because by then the document is already being changed.
+            const interrupt = new AbortController();
+            const onSignal = () => interrupt.abort();
+            process.on("SIGINT", onSignal);
+            let outcome;
+            try {
+                outcome = await driveDocumentIngest({
+                    client: {
+                        createIngest: (id) => client.createDocumentIngest(resolvedAppId, id),
+                        uploadIngestChunk: (id, sessionId, chunk) => client.uploadDocumentIngestChunk(resolvedAppId, id, sessionId, chunk),
+                        commitIngest: (id, sessionId) => client.commitDocumentIngest(resolvedAppId, id, sessionId),
+                        abortIngest: (id, sessionId) => client.abortDocumentIngest(resolvedAppId, id, sessionId),
+                        getIngest: (id, sessionId) => client.getDocumentIngest(resolvedAppId, id, sessionId),
+                    },
+                    documentId,
+                    plan,
+                    wait: options.wait !== false,
+                    timeoutMs: options.timeout ? Number(options.timeout) * 1000 : null,
+                    signal: interrupt.signal,
+                    onProgress: (line) => info(line),
+                });
+            }
+            finally {
+                process.off("SIGINT", onSignal);
+            }
+            if (outcome.message)
+                error(outcome.message);
+            if (options.json) {
+                json(outcome.session ?? { sessionId: outcome.sessionId });
+            }
+            else if (outcome.session) {
+                printIngestSession(outcome.session);
+            }
+            else if (outcome.sessionId) {
+                printResult("Session", outcome.sessionId);
+            }
+            return outcome.exitCode;
+        };
+        let code = 1;
+        try {
+            code = await run();
+        }
+        catch (err) {
+            error(err.message);
+            code = 1;
+        }
+        finally {
+            await plan?.cleanup().catch(() => undefined);
+        }
+        // The result has to reach the pipe before the process ends: stdout to a
+        // pipe is asynchronous and `process.exit` does not wait for it.
+        await flushOutput();
+        process.exit(code);
+    });
+    // Bulk-load sessions of a large document (#3434, criterion 9): a data noun
+    // with `list` and `get` readers, per docs/cli-design.md, beside `snapshots`
+    // below. The ACTION verb `documents ingest` is #3435's; these two are how an
+    // operator sees what a session did and why one stopped.
+    const ingests = documents
+        .command("ingests")
+        .description("Inspect a large document's bulk-load sessions");
+    ingests
+        .command("list")
+        .description("List a large document's bulk-load sessions, newest first")
+        .argument("<document-id>", "Document ID")
+        .option("--app <app-id>", "App ID")
+        .option("--limit <n>", "Page size (1–100)", "20")
+        .option("--cursor <cursor>", "Continue from a previous page's nextCursor")
+        .option("--json", "Output as JSON")
+        .action(async (documentId, options) => {
+        const resolvedAppId = resolveAppId(undefined, options);
+        const client = new ApiClient();
+        try {
+            const page = await client.listDocumentIngests(resolvedAppId, documentId, {
+                limit: Number(options.limit),
+                ...(options.cursor ? { cursor: options.cursor } : {}),
+            });
+            if (options.json) {
+                json(page);
+                return;
+            }
+            if (page.items.length === 0) {
+                info("No ingest sessions yet.");
+                return;
+            }
+            console.log(formatTable(page.items, [
+                { header: "SESSION", key: "sessionId" },
+                { header: "STATE", key: "state" },
+                { header: "CREATED", key: "createdAt", format: formatDate },
+                { header: "CHUNKS", key: "chunks" },
+                { header: "ROWS", key: "rows" },
+                {
+                    header: "COMPLETED",
+                    key: "completedAt",
+                    format: (at) => (at ? formatDate(at) : "—"),
+                },
+            ]));
+            if (page.hasMore && page.nextCursor) {
+                info(`More sessions: --cursor ${page.nextCursor}`);
+            }
+        }
+        catch (err) {
+            error(err.message);
+            process.exit(1);
+        }
+    });
+    ingests
+        .command("get")
+        .description("Show one bulk-load session, its progress and why it stopped")
+        .argument("<document-id>", "Document ID")
+        .argument("<session-id>", "Session ID")
+        .option("--app <app-id>", "App ID")
+        .option("--json", "Output as JSON")
+        .action(async (documentId, sessionId, options) => {
+        const resolvedAppId = resolveAppId(undefined, options);
+        const client = new ApiClient();
+        try {
+            const session = await client.getDocumentIngest(resolvedAppId, documentId, sessionId);
+            if (options.json) {
+                json(session);
+                return;
+            }
+            printIngestSession(session);
+        }
+        catch (err) {
+            error(err.message);
+            process.exit(1);
+        }
+    });
     // Snapshot builds of a large document (#3432, criterion 9): a data noun
     // with `list` and `get` readers, per docs/cli-design.md, over the app API's
     // read-only inspection routes.
@@ -974,47 +1204,11 @@ Permissions (enforced by the server, not the CLI):
                 json(build);
                 return;
             }
-            printResult("Build", build.buildId);
-            printResult("Epoch", build.epoch);
-            printResult("State", build.state);
-            printResult("Source", build.source);
-            printResult("Manifest version", build.manifestVersion);
-            printResult("Started", formatDate(build.startedAt));
-            printResult("Completed", build.completedAt ? formatDate(build.completedAt) : "—");
-            printResult("Rows", build.rows);
-            printResult("Bytes", build.bytes);
-            printResult("Chunks", build.chunks);
-            const verification = build.verification;
-            printResult("Verification", verification ? verification.state : "not started");
-            if (verification) {
-                printResult("Checked at", formatDate(verification.checkedAt));
-                printResult("Chunks checked", verification.chunksChecked);
-                if (verification.code)
-                    printResult("Code", verification.code);
-                if (verification.reason)
-                    printResult("Reason", verification.reason);
-            }
-            if (build.failure) {
-                printResult("Failed at", formatDate(build.failure.at));
-                if (build.failure.chunk) {
-                    printResult("Failed on", `${build.failure.chunk.model} chunk ${build.failure.chunk.ordinal}`);
-                }
-            }
-            const timings = build.timings ?? { ticks: 0, totals: {} };
-            printResult("Ticks", timings.ticks);
-            const stages = [
-                ["Read (ms)", "readMs"],
-                ["Encode (ms)", "encodeMs"],
-                ["Compress (ms)", "compressMs"],
-                ["Hash (ms)", "hashMs"],
-                ["Upload (ms)", "uploadMs"],
-                ["Manifest (ms)", "manifestMs"],
-                ["Verify read (ms)", "verifyReadMs"],
-                ["Verify digest (ms)", "verifyDigestMs"],
-                ["Verify coverage (ms)", "verifyCoverageMs"],
-            ];
-            for (const [label, key] of stages) {
-                printResult(label, timings.totals?.[key] ?? 0);
+            // The rows are composed by a pure function (#3435), so what a
+            // terminal shows is checkable without spawning one; the command still
+            // owns the printing.
+            for (const [label, value] of snapshotBuildRows(build)) {
+                printResult(label, value);
             }
         }
         catch (err) {
@@ -1022,6 +1216,74 @@ Permissions (enforced by the server, not the CLI):
             process.exit(1);
         }
     });
+    // #3433, criterion 10 — the operator audit: `snapshot ⊕ overlays == table`,
+    // recomputed on this side with the CLIENT's loader and fold and compared
+    // with the authoritative table page by page. The one check that can catch a
+    // DO/client fold divergence, which per-build verification runs on one side
+    // of and therefore cannot see.
+    snapshots
+        .command("audit")
+        .description("Recompute a large document's snapshot chain locally and compare it with the table")
+        .argument("<document-id>", "Document ID")
+        .option("--app <app-id>", "App ID")
+        .option("--retries <n>", "Attempts before giving up on a document that keeps changing", "3")
+        .option("--json", "Output as JSON")
+        .action(async (documentId, options) => {
+        const resolvedAppId = resolveAppId(undefined, options);
+        const client = new ApiClient();
+        const report = await auditDocumentChain(client, resolvedAppId, documentId, Number(options.retries));
+        if (options.json) {
+            json(report);
+        }
+        else if (report.verdict === "none") {
+            // No verdict is not a clean bill of health, so nothing that could be
+            // read as one is printed: the reason, and nothing else.
+            error(`No verdict: ${report.reason}`);
+        }
+        else {
+            printResult("Document", documentId);
+            printResult("Base build", report.buildId);
+            printResult("Base epoch", report.epoch);
+            printResult("Attempts", report.attempts);
+            printResult("Verdict", report.verdict === "match"
+                ? "match — the table is the chain"
+                : "DIVERGENCE");
+            console.log(formatTable(report.models, [
+                { header: "MODEL", key: "model" },
+                { header: "COMPARED", key: "compared" },
+                {
+                    header: "ONLY IN CHAIN",
+                    key: "onlyRecomputed",
+                    format: (ids) => (ids.length ? ids.join(", ") : "-"),
+                },
+                {
+                    header: "ONLY IN TABLE",
+                    key: "onlyTable",
+                    format: (ids) => (ids.length ? ids.join(", ") : "-"),
+                },
+                {
+                    header: "DIFFERING",
+                    key: "differing",
+                    format: (rows) => rows.length
+                        ? rows.map((row) => `${row.id} (${row.fields.join(", ")})`).join("; ")
+                        : "-",
+                },
+            ]));
+            if (report.models.some((model) => model.truncated)) {
+                info("Some lists were truncated; re-run with --json for the full report.");
+            }
+            if (report.verdict === "match")
+                success("snapshot ⊕ overlays == table");
+        }
+        // The report has to reach the pipe before the process ends: stdout to a
+        // pipe is asynchronous and `process.exit` does not wait for it, so a
+        // large report piped into anything that is not reading as fast as this
+        // writes used to arrive cut off mid-document (3433-C12).
+        await flushOutput();
+        // 0 match, 1 divergence, 2 no verdict — so the validation child can
+        // script on "no verdict" without reading the text.
+        process.exit(report.verdict === "match" ? 0 : report.verdict === "divergence" ? 1 : 2);
+    });
     // Transfer document ownership
     documents
         .command("transfer-owner")
@@ -2360,4 +2622,33 @@ async function importRootDocument(client, appId, docDir, metadata, plan, overwri
             `root document ${targetRootDocId}`);
     }
 }
+/**
+ * Wire the audit to the app API (#3433, criterion 10).
+ *
+ * Two permissions, not one (3433-C08): the chain export takes the app-admin
+ * arm, and the schema and records reads it is compared against take the
+ * console arm or a read grant on the document — the documents surface grants
+ * no app-role bypass on a read (#3279, D3279-002). The schema is read first,
+ * so a caller holding only the first is refused before anything is exported.
+ */
+async function auditDocumentChain(client, appId, documentId, retries) {
+    // One page's worth of records is the audit's whole working set on the
+    // server side; everything else lives in the local store.
+    const AUDIT_PAGE_SIZE = 200;
+    const fingerprint = await createChainFingerprintReader();
+    // Nothing the server answers is carried from one attempt to the next: the
+    // attempt a moved document discards is exactly the one whose schema has
+    // changed under it (3433-C09).
+    const source = createAuditApiSource(client, {
+        appId,
+        documentId,
+        pageSize: AUDIT_PAGE_SIZE,
+    });
+    return await auditSnapshotChain(source, {
+        store: () => createAuditLocalStore(documentId),
+        fingerprint,
+        retries,
+        pageSize: AUDIT_PAGE_SIZE,
+    });
+}
 //# sourceMappingURL=documents.js.map