primitive-admin 1.1.0-alpha.81 → 1.1.0-alpha.83

package/assets/skill/skills/primitive-platform/SKILL.md

@@ -279,6 +279,18 @@ primitive integrations archive <id>     # the ID column of `integrations list`
 primitive prompts archive <id>          # the ID column of `prompts list`
 primitive functions archive <id>        # the ID column of `functions list`
 
+# Running a server function (#3448)
+primitive functions invoke <key> --input '{"n":1}'   # request mode: run it, print the result
+primitive functions start <key> --wait               # task mode: start a run and wait for it
+primitive functions runs wait <function-id> <run-id> # wait for a run already started
+primitive functions invoke <key> --user <user-id>    # as that app user (admin/owner; token revoked after)
+primitive functions invoke <key> --as system         # no caller: ctx.user null, a manual trigger
+primitive functions logs <function-id> --invocation <id>   # the record an invoke's id names
+primitive functions logs <function-id> --run <run-id>      # one task run's records
+# invoke/start take the KEY; runs, wait, steps, terminate and logs take the ID.
+# Exit codes: 0 completed, 1 failed or refused, 124 the wait gave up, 130 Ctrl-C.
+# The wrong verb is REFUSED, never converted — on both sides.
+
 # Common operations
 primitive apps list                # List apps on the active env's server
 primitive apps create "Name"       # Create an app (does NOT auto-bind to an env;

package/dist/src/commands/documents.js

@@ -1,13 +1,56 @@
+import { snapshotBuildRows } from "../lib/snapshot-build-rows.js";
 import { ApiClient } from "../lib/api-client.js";
 import { resolveAppId } from "../lib/config.js";
-import { success, error, info, formatTable, formatId, formatDate, json, keyValue, result as printResult, warn, } from "../lib/output.js";
+import { success, error, info, formatTable, formatId, formatDate, json, keyValue, result as printResult, warn, flushOutput, } from "../lib/output.js";
 import { confirmPrompt } from "../lib/confirm-prompt.js";
 import { parseDataOption } from "../lib/data-input.js";
 import { parseFilterOptions } from "../lib/record-filter.js";
 import { buildPermissionsExport } from "../lib/document-export-permissions.js";
+import { chunkAddressOf, renumberManifestForExport, } from "../lib/snapshot-manifest-layout.js";
+import { validateSnapshotManifest } from "js-bao";
+import { auditSnapshotChain, } from "../lib/snapshot-audit.js";
+import { createAuditApiSource } from "../lib/snapshot-audit-source.js";
+import { createAuditLocalStore, createChainFingerprintReader, } from "../lib/snapshot-audit-store.js";
+import { discoverIngestInput, } from "../lib/document-ingest-input.js";
+import { buildIngestArtifact, ingestSchemaFromIntrospection, } from "../lib/document-ingest-artifact.js";
+import { driveDocumentIngest, summariseIngestPlan, assertIngestConfirmable, } from "../lib/document-ingest.js";
 import { ulid } from "ulid";
 import * as fs from "fs";
 import * as path from "path";
+/**
+ * One bulk-load session as rows (#3434, #3435).
+ *
+ * Shared by `documents ingests get` and by `documents ingest`'s own terminal
+ * report, so an operator reads the same thing however they arrived at it.
+ */
+function printIngestSession(session) {
+    printResult("Session", session.sessionId);
+    printResult("Document", session.documentId);
+    printResult("State", session.state);
+    printResult("Created by", session.createdBy);
+    printResult("Created", formatDate(session.createdAt));
+    printResult("Updated", formatDate(session.updatedAt));
+    printResult("Completed", session.completedAt ? formatDate(session.completedAt) : "—");
+    printResult("Chunks", session.chunks);
+    printResult("Rows", session.rows);
+    printResult("Bytes", session.bytes);
+    const progress = session.progress ?? {};
+    printResult("Progress", `copied ${progress.rowsCopied ?? 0} rows, applied ` +
+        `${progress.chunksApplied ?? 0} chunks / ${progress.rowsApplied ?? 0} rows, ` +
+        `reconciled ${progress.recordsReconciled ?? 0} records`);
+    printResult("Copy point", session.copyStartedAt ? formatDate(session.copyStartedAt) : "—");
+    printResult("Discontinuity epoch", session.discontinuityEpoch ?? "—");
+    printResult("Build", session.buildId ?? "—");
+    printResult("Expires", formatDate(session.expiresAt));
+    // Why it stopped is the reason an operator opened this at all.
+    if (session.failure) {
+        printResult("Failure", session.failure.code);
+        printResult("Reason", session.failure.reason);
+        for (const row of session.failure.rows ?? []) {
+            printResult("  Row", typeof row === "string" ? row : `${row.model} ${row.id}`);
+        }
+    }
+}
 /**
  * Report a document's format (#2816) — but only when it is a large document.
  *
@@ -898,6 +941,349 @@ Permissions (enforced by the server, not the CLI):
             process.exit(1);
         }
     });
+    // The action verb (#3435, criterion 9): a directory of records becomes a
+    // chunked, digested, uploaded artifact and one atomic swap of the
+    // document's tables. `documents ingests list|get` below is how an operator
+    // watches one afterwards.
+    //
+    // Destructive by nature — a bulk load deletes and overwrites records in a
+    // live document with no undo — so it confirms unless `-y`, which is what
+    // `docs/cli-design.md` reserves the destructive default for.
+    documents
+        .command("ingest")
+        .description("Bulk-load records into a large document from a directory")
+        .argument("<document-id>", "Document ID")
+        .requiredOption("--input <dir>", "Directory of <model>.ndjson[.gz] files, or a `documents export` output")
+        .option("--app <app-id>", "App ID")
+        .option("-y, --yes", "Skip the confirmation prompt")
+        .option("--no-wait", "Exit once the session is committed")
+        .option("--timeout <seconds>", "Give up waiting after this many seconds")
+        .option("--json", "Output as JSON")
+        .action(async (documentId, options) => {
+        const resolvedAppId = resolveAppId(undefined, options);
+        const client = new ApiClient();
+        let plan = null;
+        // The exit code is carried out of here rather than taken inside: a
+        // `process.exit` does not unwind, so ending the process in the try block
+        // would skip the cleanup in the finally and leave the compressed
+        // artifact — up to the whole input, gzipped — in the OS temp directory
+        // on every run, successful or not (finding 3435-FROZEN-05).
+        const run = async () => {
+            // The preflight, in this order and before a byte is read from disk:
+            // the schema read says whether this caller may see the document's
+            // declarations at all and hands over the ones the lines are checked
+            // against, and the session reader says whether this is a large
+            // document. Both are READS on the arm the ingest routes admit, so a
+            // caller who would be refused halfway is refused at once, and neither
+            // opens a session.
+            const introspection = await client.getDocumentSchema(resolvedAppId, documentId);
+            await client.listDocumentIngests(resolvedAppId, documentId, { limit: 1 });
+            const schema = ingestSchemaFromIntrospection(introspection);
+            const input = discoverIngestInput(options.input, documentId);
+            for (const ignored of input.ignored) {
+                info(`Ignoring ${ignored}: a bulk load reads the snapshot only.`);
+            }
+            plan = await buildIngestArtifact({ input, schema });
+            for (const line of summariseIngestPlan(plan))
+                info(line);
+            assertIngestConfirmable({
+                yes: Boolean(options.yes),
+                isTTY: Boolean(process.stdin.isTTY),
+            });
+            if (!options.yes) {
+                const confirmed = await confirmPrompt(`Bulk-load ${plan.rows} records into ${documentId}? ` +
+                    "This replaces records in a live document and cannot be undone.");
+                if (!confirmed) {
+                    info("Nothing was uploaded.");
+                    return 1;
+                }
+            }
+            // Ctrl-C is a decision the driver makes differently depending on
+            // where it lands, so it is handed the signal rather than killing the
+            // process: mid-upload the session is aborted, mid-poll it is left
+            // running, because by then the document is already being changed.
+            const interrupt = new AbortController();
+            const onSignal = () => interrupt.abort();
+            process.on("SIGINT", onSignal);
+            let outcome;
+            try {
+                outcome = await driveDocumentIngest({
+                    client: {
+                        createIngest: (id) => client.createDocumentIngest(resolvedAppId, id),
+                        uploadIngestChunk: (id, sessionId, chunk) => client.uploadDocumentIngestChunk(resolvedAppId, id, sessionId, chunk),
+                        commitIngest: (id, sessionId) => client.commitDocumentIngest(resolvedAppId, id, sessionId),
+                        abortIngest: (id, sessionId) => client.abortDocumentIngest(resolvedAppId, id, sessionId),
+                        getIngest: (id, sessionId) => client.getDocumentIngest(resolvedAppId, id, sessionId),
+                    },
+                    documentId,
+                    plan,
+                    wait: options.wait !== false,
+                    timeoutMs: options.timeout ? Number(options.timeout) * 1000 : null,
+                    signal: interrupt.signal,
+                    onProgress: (line) => info(line),
+                });
+            }
+            finally {
+                process.off("SIGINT", onSignal);
+            }
+            if (outcome.message)
+                error(outcome.message);
+            if (options.json) {
+                json(outcome.session ?? { sessionId: outcome.sessionId });
+            }
+            else if (outcome.session) {
+                printIngestSession(outcome.session);
+            }
+            else if (outcome.sessionId) {
+                printResult("Session", outcome.sessionId);
+            }
+            return outcome.exitCode;
+        };
+        let code = 1;
+        try {
+            code = await run();
+        }
+        catch (err) {
+            error(err.message);
+            code = 1;
+        }
+        finally {
+            await plan?.cleanup().catch(() => undefined);
+        }
+        // The result has to reach the pipe before the process ends: stdout to a
+        // pipe is asynchronous and `process.exit` does not wait for it.
+        await flushOutput();
+        process.exit(code);
+    });
+    // Bulk-load sessions of a large document (#3434, criterion 9): a data noun
+    // with `list` and `get` readers, per docs/cli-design.md, beside `snapshots`
+    // below. The ACTION verb `documents ingest` is #3435's; these two are how an
+    // operator sees what a session did and why one stopped.
+    const ingests = documents
+        .command("ingests")
+        .description("Inspect a large document's bulk-load sessions");
+    ingests
+        .command("list")
+        .description("List a large document's bulk-load sessions, newest first")
+        .argument("<document-id>", "Document ID")
+        .option("--app <app-id>", "App ID")
+        .option("--limit <n>", "Page size (1–100)", "20")
+        .option("--cursor <cursor>", "Continue from a previous page's nextCursor")
+        .option("--json", "Output as JSON")
+        .action(async (documentId, options) => {
+        const resolvedAppId = resolveAppId(undefined, options);
+        const client = new ApiClient();
+        try {
+            const page = await client.listDocumentIngests(resolvedAppId, documentId, {
+                limit: Number(options.limit),
+                ...(options.cursor ? { cursor: options.cursor } : {}),
+            });
+            if (options.json) {
+                json(page);
+                return;
+            }
+            if (page.items.length === 0) {
+                info("No ingest sessions yet.");
+                return;
+            }
+            console.log(formatTable(page.items, [
+                { header: "SESSION", key: "sessionId" },
+                { header: "STATE", key: "state" },
+                { header: "CREATED", key: "createdAt", format: formatDate },
+                { header: "CHUNKS", key: "chunks" },
+                { header: "ROWS", key: "rows" },
+                {
+                    header: "COMPLETED",
+                    key: "completedAt",
+                    format: (at) => (at ? formatDate(at) : "—"),
+                },
+            ]));
+            if (page.hasMore && page.nextCursor) {
+                info(`More sessions: --cursor ${page.nextCursor}`);
+            }
+        }
+        catch (err) {
+            error(err.message);
+            process.exit(1);
+        }
+    });
+    ingests
+        .command("get")
+        .description("Show one bulk-load session, its progress and why it stopped")
+        .argument("<document-id>", "Document ID")
+        .argument("<session-id>", "Session ID")
+        .option("--app <app-id>", "App ID")
+        .option("--json", "Output as JSON")
+        .action(async (documentId, sessionId, options) => {
+        const resolvedAppId = resolveAppId(undefined, options);
+        const client = new ApiClient();
+        try {
+            const session = await client.getDocumentIngest(resolvedAppId, documentId, sessionId);
+            if (options.json) {
+                json(session);
+                return;
+            }
+            printIngestSession(session);
+        }
+        catch (err) {
+            error(err.message);
+            process.exit(1);
+        }
+    });
+    // Snapshot builds of a large document (#3432, criterion 9): a data noun
+    // with `list` and `get` readers, per docs/cli-design.md, over the app API's
+    // read-only inspection routes.
+    const snapshots = documents
+        .command("snapshots")
+        .description("Inspect a large document's snapshot builds and their verification");
+    snapshots
+        .command("list")
+        .description("List a large document's snapshot builds, newest first")
+        .argument("<document-id>", "Document ID")
+        .option("--app <app-id>", "App ID")
+        .option("--limit <n>", "Page size (1–100)", "20")
+        .option("--cursor <cursor>", "Continue from a previous page's nextCursor")
+        .option("--json", "Output as JSON")
+        .action(async (documentId, options) => {
+        const resolvedAppId = resolveAppId(undefined, options);
+        const client = new ApiClient();
+        try {
+            const page = await client.listDocumentSnapshots(resolvedAppId, documentId, {
+                limit: Number(options.limit),
+                ...(options.cursor ? { cursor: options.cursor } : {}),
+            });
+            if (options.json) {
+                json(page);
+                return;
+            }
+            if (page.items.length === 0) {
+                info("No snapshot builds yet.");
+                return;
+            }
+            console.log(formatTable(page.items, [
+                { header: "BUILD", key: "buildId" },
+                { header: "EPOCH", key: "epoch" },
+                { header: "STATE", key: "state" },
+                { header: "SOURCE", key: "source" },
+                {
+                    header: "VERIFICATION",
+                    key: "verification",
+                    format: (verification) => verification
+                        ? verification.code
+                            ? `${verification.state} (${verification.code})`
+                            : verification.state
+                        : "-",
+                },
+                { header: "ROWS", key: "rows" },
+                { header: "CHUNKS", key: "chunks" },
+                { header: "STARTED", key: "startedAt", format: formatDate },
+                { header: "COMPLETED", key: "completedAt", format: formatDate },
+            ]));
+            if (page.hasMore && page.nextCursor) {
+                info(`More builds: --cursor ${page.nextCursor}`);
+            }
+        }
+        catch (err) {
+            error(err.message);
+            process.exit(1);
+        }
+    });
+    snapshots
+        .command("get")
+        .description("Show one snapshot build, its verification result and where its time went")
+        .argument("<document-id>", "Document ID")
+        .argument("<build-id>", "Build ID")
+        .option("--app <app-id>", "App ID")
+        .option("--json", "Output as JSON")
+        .action(async (documentId, buildId, options) => {
+        const resolvedAppId = resolveAppId(undefined, options);
+        const client = new ApiClient();
+        try {
+            const build = await client.getDocumentSnapshot(resolvedAppId, documentId, buildId);
+            if (options.json) {
+                json(build);
+                return;
+            }
+            // The rows are composed by a pure function (#3435), so what a
+            // terminal shows is checkable without spawning one; the command still
+            // owns the printing.
+            for (const [label, value] of snapshotBuildRows(build)) {
+                printResult(label, value);
+            }
+        }
+        catch (err) {
+            error(err.message);
+            process.exit(1);
+        }
+    });
+    // #3433, criterion 10 — the operator audit: `snapshot ⊕ overlays == table`,
+    // recomputed on this side with the CLIENT's loader and fold and compared
+    // with the authoritative table page by page. The one check that can catch a
+    // DO/client fold divergence, which per-build verification runs on one side
+    // of and therefore cannot see.
+    snapshots
+        .command("audit")
+        .description("Recompute a large document's snapshot chain locally and compare it with the table")
+        .argument("<document-id>", "Document ID")
+        .option("--app <app-id>", "App ID")
+        .option("--retries <n>", "Attempts before giving up on a document that keeps changing", "3")
+        .option("--json", "Output as JSON")
+        .action(async (documentId, options) => {
+        const resolvedAppId = resolveAppId(undefined, options);
+        const client = new ApiClient();
+        const report = await auditDocumentChain(client, resolvedAppId, documentId, Number(options.retries));
+        if (options.json) {
+            json(report);
+        }
+        else if (report.verdict === "none") {
+            // No verdict is not a clean bill of health, so nothing that could be
+            // read as one is printed: the reason, and nothing else.
+            error(`No verdict: ${report.reason}`);
+        }
+        else {
+            printResult("Document", documentId);
+            printResult("Base build", report.buildId);
+            printResult("Base epoch", report.epoch);
+            printResult("Attempts", report.attempts);
+            printResult("Verdict", report.verdict === "match"
+                ? "match — the table is the chain"
+                : "DIVERGENCE");
+            console.log(formatTable(report.models, [
+                { header: "MODEL", key: "model" },
+                { header: "COMPARED", key: "compared" },
+                {
+                    header: "ONLY IN CHAIN",
+                    key: "onlyRecomputed",
+                    format: (ids) => (ids.length ? ids.join(", ") : "-"),
+                },
+                {
+                    header: "ONLY IN TABLE",
+                    key: "onlyTable",
+                    format: (ids) => (ids.length ? ids.join(", ") : "-"),
+                },
+                {
+                    header: "DIFFERING",
+                    key: "differing",
+                    format: (rows) => rows.length
+                        ? rows.map((row) => `${row.id} (${row.fields.join(", ")})`).join("; ")
+                        : "-",
+                },
+            ]));
+            if (report.models.some((model) => model.truncated)) {
+                info("Some lists were truncated; re-run with --json for the full report.");
+            }
+            if (report.verdict === "match")
+                success("snapshot ⊕ overlays == table");
+        }
+        // The report has to reach the pipe before the process ends: stdout to a
+        // pipe is asynchronous and `process.exit` does not wait for it, so a
+        // large report piped into anything that is not reading as fast as this
+        // writes used to arrive cut off mid-document (3433-C12).
+        await flushOutput();
+        // 0 match, 1 divergence, 2 no verdict — so the validation child can
+        // script on "no verdict" without reading the text.
+        process.exit(report.verdict === "match" ? 0 : report.verdict === "divergence" ? 1 : 2);
+    });
     // Transfer document ownership
     documents
         .command("transfer-owner")
@@ -1429,17 +1815,21 @@ async function exportLargeDocumentChain(client, appId, documentId, docDir) {
         const snapshotDir = path.join(docDir, "snapshot");
         fs.mkdirSync(snapshotDir, { recursive: true });
         const manifestBytes = await client.downloadDocumentArtifact(appId, chain.base.download.path);
-        fs.writeFileSync(path.join(snapshotDir, "manifest.json"), manifestBytes);
-        const manifest = JSON.parse(manifestBytes.toString("utf-8"));
-        for (const chunk of manifest.chunks ?? []) {
-            // Addressed within the granted build — `{model}/{n}` — never by key.
-            const index = chunkIndexFromKey(chunk.key);
-            const body = await client.downloadDocumentArtifact(appId, `${chain.base.download.path}/${chunk.model}/${index}`);
-            const modelDir = path.join(snapshotDir, chunk.model);
-            fs.mkdirSync(modelDir, { recursive: true });
-            fs.writeFileSync(path.join(modelDir, `${index}.ndjson.gz`), body);
+        const downloaded = JSON.parse(manifestBytes.toString("utf-8"));
+        // Addressed within the granted build by each entry's `path` — the
+        // own-build `{model}/{n}` or the `{epoch}-{buildId}/{model}/{n}` of a
+        // chunk reused from an earlier build — never by key (#3432). On disk the
+        // chunks are renumbered per model into the own-build form and the stored
+        // manifest says so, which is the one layout `documents import` reads.
+        const { manifest, files } = renumberManifestForExport(downloaded);
+        for (const file of files) {
+            const body = await client.downloadDocumentArtifact(appId, `${chain.base.download.path}/${file.download}`);
+            const target = path.join(snapshotDir, file.file);
+            fs.mkdirSync(path.dirname(target), { recursive: true });
+            fs.writeFileSync(target, body);
             bytes += body.byteLength;
         }
+        fs.writeFileSync(path.join(snapshotDir, "manifest.json"), JSON.stringify(manifest, null, 2));
         base = {
             epoch: chain.base.epoch,
             buildId: chain.base.buildId,
@@ -1459,13 +1849,16 @@ async function exportLargeDocumentChain(client, appId, documentId, docDir) {
     }, null, 2));
     return bytes;
 }
-/** The chunk number out of a manifest entry's key (`…/{model}/{n}.ndjson.gz`). */
-function chunkIndexFromKey(key) {
-    const match = /\/(\d+)\.ndjson\.gz$/.exec(String(key ?? ""));
-    if (!match) {
-        throw new Error(`Snapshot manifest names a chunk this CLI cannot address: ${key}`);
-    }
-    return Number(match[1]);
+/**
+ * Where a stored manifest entry's chunk is on disk: `snapshot/{model}/{n}`.
+ *
+ * An exported manifest is in the own-build form (renumbered per model), so
+ * the entry's `path` — or, for an artifact written before the field existed,
+ * its key — names the file directly.
+ */
+function chunkFileOf(chunk) {
+    const address = chunkAddressOf(chunk);
+    return { model: address.model, index: address.index };
 }
 async function exportSingleDocument(client, appId, documentId, outputDir, includeBlobs, jsonOutput) {
     const docDir = path.join(outputDir, "documents", documentId);
@@ -1682,9 +2075,19 @@ function verifyChainArtifact(docDir, documentId, chain) {
             refuse(`has a base snapshot manifest that is not readable ` +
                 `(${parseError.message})`);
         }
+        // Validated where it enters (#3432): the fields the server keys on — an
+        // entry's ordinal, its path, its range — fail silently rather than loudly
+        // if they are wrong, and the server refuses the same manifest later with
+        // the same message.
+        try {
+            validateSnapshotManifest(manifest);
+        }
+        catch (invalid) {
+            refuse(`has a base snapshot manifest that is not valid (${invalid.message})`);
+        }
         for (const chunk of manifest.chunks ?? []) {
-            const index = chunkIndexFromKey(String(chunk.key));
-            require(path.join(docDir, "snapshot", String(chunk.model), `${index}.ndjson.gz`), `chunk ${index} of ${chunk.model}`);
+            const { model, index } = chunkFileOf(chunk);
+            require(path.join(docDir, "snapshot", model, `${index}.ndjson.gz`), `chunk ${index} of ${model}`);
         }
     }
     for (const overlay of chain.overlays) {
@@ -1722,14 +2125,15 @@ async function importLargeDocumentChain(client, appId, docDir, documentId) {
         }, manifestBytes);
         const manifest = JSON.parse(manifestBytes.toString("utf-8"));
         for (const chunk of manifest.chunks ?? []) {
-            const index = chunkIndexFromKey(chunk.key);
+            // Addressed by the entry's `path` (#3432); the server installs by it too.
+            const { model, index } = chunkFileOf(chunk);
             await client.uploadDocumentChainArtifact(appId, documentId, {
                 kind: "chunk",
                 epoch: chain.base.epoch,
                 buildId: chain.base.buildId,
-                model: chunk.model,
+                model,
                 index,
-            }, fs.readFileSync(path.join(docDir, "snapshot", chunk.model, `${index}.ndjson.gz`)));
+            }, fs.readFileSync(path.join(docDir, "snapshot", model, `${index}.ndjson.gz`)));
         }
     }
     // The sealed overlays, oldest first, then the open epoch's — which travels
@@ -2218,4 +2622,33 @@ async function importRootDocument(client, appId, docDir, metadata, plan, overwri
             `root document ${targetRootDocId}`);
     }
 }
+/**
+ * Wire the audit to the app API (#3433, criterion 10).
+ *
+ * Two permissions, not one (3433-C08): the chain export takes the app-admin
+ * arm, and the schema and records reads it is compared against take the
+ * console arm or a read grant on the document — the documents surface grants
+ * no app-role bypass on a read (#3279, D3279-002). The schema is read first,
+ * so a caller holding only the first is refused before anything is exported.
+ */
+async function auditDocumentChain(client, appId, documentId, retries) {
+    // One page's worth of records is the audit's whole working set on the
+    // server side; everything else lives in the local store.
+    const AUDIT_PAGE_SIZE = 200;
+    const fingerprint = await createChainFingerprintReader();
+    // Nothing the server answers is carried from one attempt to the next: the
+    // attempt a moved document discards is exactly the one whose schema has
+    // changed under it (3433-C09).
+    const source = createAuditApiSource(client, {
+        appId,
+        documentId,
+        pageSize: AUDIT_PAGE_SIZE,
+    });
+    return await auditSnapshotChain(source, {
+        store: () => createAuditLocalStore(documentId),
+        fingerprint,
+        retries,
+        pageSize: AUDIT_PAGE_SIZE,
+    });
+}
 //# sourceMappingURL=documents.js.map