primitive-admin 1.1.0-alpha.9 → 1.1.0

package/dist/src/lib/document-ingest-input.js

@@ -0,0 +1,132 @@
+/**
+ * Which files `primitive documents ingest` reads, and in which order.
+ *
+ * Two layouts, and the directory itself says which one it is:
+ *
+ * - **plain** — `<dir>/<model>.ndjson` or `<model>.ndjson.gz`, one record per
+ *   line. Nothing on disk promises the lines ascend, so they are sorted.
+ * - **export** — what `primitive documents export` wrote: a
+ *   `snapshot/manifest.json` and its chunks, at the directory itself or under
+ *   `documents/<document-id>/`. A snapshot's chunks ascend and are disjoint
+ *   per model in manifest order, so an export needs no sort — it is read in
+ *   that order and streamed straight through.
+ *
+ * A directory that is neither, or that is both, is refused naming what it
+ * held. Every refusal here happens before a session exists: an ingest replaces
+ * a live document's tables, and half of one is not something to discover from
+ * the server's side.
+ */
+import fs from "fs";
+import path from "path";
+import { chunkAddressOf } from "./snapshot-manifest-layout.js";
+/** A directory that is not an ingest input, said as what was found. */
+export class IngestInputError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "IngestInputError";
+    }
+}
+/** Files and directories an export carries that a bulk load does not read. */
+const EXPORT_IGNORED = [
+    "epochs",
+    "current.yjs",
+    "chain.json",
+    "document.yjs",
+    "blobs",
+];
+const LINE_FILE = /^(.+)\.ndjson(\.gz)?$/;
+/**
+ * Decide what `dir` is and list the files to read, in order.
+ *
+ * `documentId` matters only for the export layout: an export directory may
+ * hold several documents, and re-ingesting one document's rows into another
+ * is not a thing to guess at.
+ */
+export function discoverIngestInput(dir, documentId) {
+    if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) {
+        throw new IngestInputError(`Not a directory: ${dir}`);
+    }
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+    const lineFiles = entries.filter((entry) => entry.isFile() && LINE_FILE.test(entry.name));
+    const hasOwnSnapshot = fs.existsSync(path.join(dir, "snapshot", "manifest.json"));
+    const documentsDir = path.join(dir, "documents");
+    const hasDocuments = fs.existsSync(documentsDir) && fs.statSync(documentsDir).isDirectory();
+    if (hasOwnSnapshot && lineFiles.length > 0) {
+        throw new IngestInputError(`${dir} holds both layouts: a snapshot/manifest.json and ` +
+            `${lineFiles.map((entry) => entry.name).join(", ")}. ` +
+            "An ingest reads one directory as one layout; split them.");
+    }
+    if (hasOwnSnapshot)
+        return exportInput(dir, dir);
+    if (hasDocuments && lineFiles.length === 0) {
+        const mine = path.join(documentsDir, documentId);
+        if (fs.existsSync(path.join(mine, "snapshot", "manifest.json"))) {
+            return exportInput(dir, mine);
+        }
+        const others = fs
+            .readdirSync(documentsDir, { withFileTypes: true })
+            .filter((entry) => entry.isDirectory())
+            .map((entry) => entry.name);
+        throw new IngestInputError(`This export holds ${others.length === 0 ? "no documents" : others.join(", ")}` +
+            ` and nothing for ${documentId}. Point --input at the export of the ` +
+            "document being ingested, or at its documents/<document-id> directory.");
+    }
+    if (lineFiles.length === 0) {
+        const found = entries.map((entry) => entry.name);
+        if (found.length === 0) {
+            throw new IngestInputError(`${dir} is empty. An ingest reads <model>.ndjson[.gz] files, or an ` +
+                "export's snapshot/manifest.json.");
+        }
+        throw new IngestInputError(`${dir} holds nothing an ingest can read: ${found.join(", ")}. ` +
+            "Expected <model>.ndjson[.gz] files, or an export's snapshot/manifest.json.");
+    }
+    // A plain directory with a stray file in it is still ambiguous: the stray
+    // may be the model file somebody meant to name, so it is named rather than
+    // quietly skipped.
+    const strays = entries
+        .filter((entry) => !(entry.isFile() && LINE_FILE.test(entry.name)))
+        .map((entry) => entry.name);
+    if (strays.length > 0) {
+        throw new IngestInputError(`${dir} holds files an ingest cannot read: ${strays.join(", ")}. ` +
+            "A plain input directory holds <model>.ndjson[.gz] files and nothing else.");
+    }
+    const sources = lineFiles
+        .map((entry) => {
+        const match = LINE_FILE.exec(entry.name);
+        return {
+            model: match[1],
+            file: path.join(dir, entry.name),
+            gzip: Boolean(match[2]),
+            label: entry.name,
+        };
+    })
+        .sort((left, right) => left.model.localeCompare(right.model));
+    return { layout: "plain", root: dir, sources, ignored: [], presorted: false };
+}
+/** The sources of an export rooted at `root`, in manifest order. */
+function exportInput(dir, root) {
+    const manifestPath = path.join(root, "snapshot", "manifest.json");
+    let manifest;
+    try {
+        manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+    }
+    catch (error) {
+        throw new IngestInputError(`${path.relative(dir, manifestPath) || manifestPath} is not readable JSON: ${error.message}`);
+    }
+    const chunks = Array.isArray(manifest?.chunks) ? manifest.chunks : null;
+    if (!chunks) {
+        throw new IngestInputError(`${path.relative(dir, manifestPath) || manifestPath} names no chunks.`);
+    }
+    const sources = chunks.map((chunk) => {
+        const address = chunkAddressOf(chunk);
+        const label = `snapshot/${address.model}/${address.index}.ndjson.gz`;
+        const file = path.join(root, "snapshot", address.model, `${address.index}.ndjson.gz`);
+        if (!fs.existsSync(file)) {
+            throw new IngestInputError(`The export's manifest names ${label}, which is not on disk.`);
+        }
+        return { model: address.model, file, gzip: true, label };
+    });
+    const ignored = EXPORT_IGNORED.filter((name) => fs.existsSync(path.join(root, name)));
+    return { layout: "export", root, sources, ignored, presorted: true };
+}
+//# sourceMappingURL=document-ingest-input.js.map

package/dist/src/lib/document-ingest-input.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-ingest-input.js","sourceRoot":"","sources":["../../../src/lib/document-ingest-input.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAE/D,uEAAuE;AACvE,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IACzC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAyBD,8EAA8E;AAC9E,MAAM,cAAc,GAAG;IACrB,QAAQ;IACR,aAAa;IACb,YAAY;IACZ,cAAc;IACd,OAAO;CACR,CAAC;AAEF,MAAM,SAAS,GAAG,uBAAuB,CAAC;AAE1C;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CACjC,GAAW,EACX,UAAkB;IAElB,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QAC3D,MAAM,IAAI,gBAAgB,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAC9B,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CACxD,CAAC;IACF,MAAM,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,eAAe,CAAC,CAAC,CAAC;IAClF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IACjD,MAAM,YAAY,GAChB,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC;IAEzE,IAAI,cAAc,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,gBAAgB,CACxB,GAAG,GAAG,oDAAoD;YACxD,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YACtD,0DAA0D,CAC7D,CAAC;IACJ,CAAC;IAED,IAAI,cAAc;QAAE,OAAO,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAEjD,IAAI,YAAY,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;QACjD,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,eAAe,CAAC,CAAC,EAAE,CAAC;YAChE,OAAO,WAAW,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAChC,CAAC;QACD,MAAM,MAAM,GAAG,EAAE;aACd,WAAW,CAAC,YAAY,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;aAClD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;aACtC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,IAAI,gBAAgB,CACxB,qBAAqB,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAC7E,oBAAoB,UAAU,uCAAuC;YACrE,uEAAuE,CAC1E,CAAC;IACJ,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,gBAAgB,CACxB,GAAG,GAAG,8DAA8D;gBAClE,kCAAkC,CACrC,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,gBAAgB,CACxB,GAAG,GAAG,sCAAsC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YAC9D,4EAA4E,CAC/E,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,mBAAmB;IACnB,MAAM,MAAM,GAAG,OAAO;SACnB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;SAClE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,gBAAgB,CACxB,GAAG,GAAG,uCAAuC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YAChE,2EAA2E,CAC9E,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,SAAS;SACtB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACb,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;YACf,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC;YAChC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,KAAK,EAAE,KAAK,CAAC,IAAI;SAClB,CAAC;IACJ,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAEhE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AAChF,CAAC;AAED,oEAAoE;AACpE,SAAS,WAAW,CAAC,GAAW,EAAE,IAAY;IAC5C,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,eAAe,CAAC,CAAC;IAClE,IAAI,QAAa,CAAC;IAClB,IAAI,CAAC;QACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;IAC/D,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,MAAM,IAAI,gBAAgB,CACxB,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC,IAAI,YAAY,0BAA0B,KAAK,CAAC,OAAO,EAAE,CAC7F,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IACxE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,gBAAgB,CACxB,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC,IAAI,YAAY,mBAAmB,CACvE,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAmB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAU,EAAE,EAAE;QACxD,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,YAAY,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,YAAY,CAAC;QACrE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,OAAO,CAAC,KAAK,YAAY,CAAC,CAAC;QACtF,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,gBAAgB,CACxB,+BAA+B,KAAK,yBAAyB,CAC9D,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IAC3D,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAC7C,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CACrC,CAAC;IAEF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AACvE,CAAC"}

package/dist/src/lib/document-ingest-rows.d.ts

@@ -0,0 +1,137 @@
+/**
+ * What `documents ingests get` shows an operator, as data (#3598).
+ *
+ * The rows were composed inline in the command's action, which made the only
+ * way to check one a spawned CLI against a live server with a session in
+ * exactly the right state — and #3598 adds a block of rows nobody could check
+ * that way without a slow load to look at. They are a pure function of the
+ * payload now, as `snapshot-build-rows` is; the command still owns the
+ * printing, and the live suite still owns "a real terminal really shows this".
+ *
+ * Label and value, in the order they print. Everything past the first block is
+ * optional here, because a session written before its field existed carries
+ * none of it and an operator inspecting an old session should see the session
+ * rather than a crash.
+ */
+/** One printed row: its label and the value beside it. */
+export type IngestSessionRow = [string, unknown];
+/** Where one tick's time went, as the server publishes it. */
+export interface IngestTickTotals {
+    manifestMs?: number;
+    readMs?: number;
+    decodeMs?: number;
+    copyMs?: number;
+    applyMs?: number;
+    reconcileMs?: number;
+    swapMs?: number;
+    sealMs?: number;
+    registerMs?: number;
+    totalMs?: number;
+    /**
+     * Ticks whose stage timers were bracketed by a clock settle (#3638).
+     *
+     * Equal to `ticks` when every tick managed it. Less means the stage
+     * milliseconds above are, for those ticks, the old un-bracketed figures: 0
+     * for every synchronous stage.
+     */
+    bracketedTicks?: number;
+    /** What each stage's SQL did, without asking a clock (#3638). */
+    counters?: Record<string, IngestStageCounters>;
+}
+/** One stage's clock-free counters, as the server publishes them (#3638). */
+export interface IngestStageCounters {
+    statements?: number;
+    /** From the SQL cursor's own counter; `null` where the host has none. */
+    rowsRead?: number | null;
+    /** From the SQL cursor's own counter; `null` where the host has none. */
+    rowsWritten?: number | null;
+    rowsReturned?: number;
+    byTarget?: Record<string, IngestStageCounters>;
+}
+/** The payload `GET .../ingest/{sessionId}` answers with. */
+export interface IngestSessionPayload {
+    sessionId: string;
+    documentId: string;
+    state: string;
+    createdBy?: string;
+    createdAt?: string | null;
+    updatedAt?: string | null;
+    completedAt?: string | null;
+    expiresAt?: string | null;
+    chunks: number;
+    rows: number;
+    bytes: number;
+    /** #3598 — decompressed bytes; null when a swept session cannot say. */
+    rawBytes?: number | null;
+    stateEnteredAt?: string | null;
+    progress?: {
+        rowsCopied?: number;
+        chunksApplied?: number;
+        rowsApplied?: number;
+        recordsReconciled?: number;
+        /** #3618 — moved-aside rows removed; the counter that moves in `registering`. */
+        oldRowsRemoved?: number;
+    };
+    copyStartedAt?: string | null;
+    discontinuityEpoch?: number | null;
+    buildId?: string | null;
+    failure?: {
+        code: string;
+        reason: string;
+        rows?: Array<{
+            model: string;
+            id: string;
+        } | string>;
+    } | null;
+    timings?: {
+        since?: string | null;
+        ticks?: number;
+        totals?: IngestTickTotals;
+        states?: Record<string, {
+            ms?: number;
+            ticks?: number;
+            intervalMs?: number;
+        }>;
+        longestInterval?: {
+            intervalMs: number;
+            state: string;
+        } | null;
+    } | null;
+    throughput?: {
+        pipelineMs: number;
+        rowsPerSecond: number;
+        storedBytesPerSecond: number;
+        rawBytesPerSecond: number | null;
+    } | null;
+}
+export declare function ingestSessionRows(session: IngestSessionPayload): IngestSessionRow[];
+/**
+ * What the client itself spent, beside what the server did (#3598).
+ *
+ * Every phase past the build is OPTIONAL, and absent means the run never
+ * finished it — an upload that failed halfway, a commit that was never
+ * reached. A 0 there would read as a measurement of no time at all, which is
+ * the same mistake `rawBytes` and `throughput` refuse to make on the server
+ * side (finding 3598-FROZEN-02).
+ */
+export interface IngestClientTimings {
+    /** Reading, checking, sorting, cutting and gzipping the artifact. */
+    buildMs: number;
+    createMs?: number;
+    uploadMs?: number;
+    /** Stored bytes put, from the plan's own chunk sizes. */
+    uploadedBytes?: number;
+    commitMs?: number;
+    /** Time spent polling after the commit; 0 under `--no-wait`. */
+    waitMs?: number;
+}
+/**
+ * The client's phases as rows, so the two sides read side by side.
+ *
+ * The wall-clock number `documents ingest` used to print covered the artifact
+ * build, the upload AND everything the platform then did, and could be blamed
+ * on either. These say which. A phase the run never finished prints `—`,
+ * which here means what it means everywhere else in this file: not a
+ * measurement.
+ */
+export declare function ingestClientRows(timings: IngestClientTimings): IngestSessionRow[];

package/dist/src/lib/document-ingest-rows.js

@@ -0,0 +1,181 @@
+/**
+ * What `documents ingests get` shows an operator, as data (#3598).
+ *
+ * The rows were composed inline in the command's action, which made the only
+ * way to check one a spawned CLI against a live server with a session in
+ * exactly the right state — and #3598 adds a block of rows nobody could check
+ * that way without a slow load to look at. They are a pure function of the
+ * payload now, as `snapshot-build-rows` is; the command still owns the
+ * printing, and the live suite still owns "a real terminal really shows this".
+ *
+ * Label and value, in the order they print. Everything past the first block is
+ * optional here, because a session written before its field existed carries
+ * none of it and an operator inspecting an old session should see the session
+ * rather than a crash.
+ */
+import { formatDate } from "./output.js";
+/** The states a session passes through, in the order it passes through them. */
+const STATES = [
+    "uploading",
+    "committed",
+    "validating",
+    "staging",
+    "applying",
+    "finalizing",
+    "registering",
+];
+/** The per-stage rows, in the order a tick spends them. */
+const STAGES = [
+    ["Manifest (ms)", "manifestMs"],
+    ["Read (ms)", "readMs"],
+    ["Decode (ms)", "decodeMs"],
+    ["Copy (ms)", "copyMs"],
+    ["Apply (ms)", "applyMs"],
+    ["Reconcile (ms)", "reconcileMs"],
+    ["Swap (ms)", "swapMs"],
+    ["Seal (ms)", "sealMs"],
+    ["Register (ms)", "registerMs"],
+];
+const DASH = "—";
+const title = (state) => state.charAt(0).toUpperCase() + state.slice(1);
+/** Megabytes per second, at one decimal — the unit a load is discussed in. */
+const mbPerSecond = (bytesPerSecond) => `${(bytesPerSecond / 1_000_000).toFixed(2)} MB/s`;
+export function ingestSessionRows(session) {
+    const progress = session.progress ?? {};
+    const rows = [
+        ["Session", session.sessionId],
+        ["Document", session.documentId],
+        ["State", session.state],
+        ["Created by", session.createdBy],
+        ["Created", formatDate(session.createdAt)],
+        ["Updated", formatDate(session.updatedAt)],
+        [
+            "Completed",
+            session.completedAt
+                ? formatDate(session.completedAt)
+                : DASH,
+        ],
+        ["Chunks", session.chunks],
+        ["Rows", session.rows],
+        ["Bytes", session.bytes],
+        // #3598 — what the pipeline actually parsed, which is the numerator of
+        // any megabytes-per-second figure. `—` rather than 0 when a completed
+        // session's chunk descriptors were swept before timings existed.
+        ["Raw bytes", session.rawBytes ?? DASH],
+        [
+            "Progress",
+            `copied ${progress.rowsCopied ?? 0} rows, applied ` +
+                `${progress.chunksApplied ?? 0} chunks / ${progress.rowsApplied ?? 0} rows, ` +
+                `reconciled ${progress.recordsReconciled ?? 0} records, ` +
+                `removed ${progress.oldRowsRemoved ?? 0} moved-aside rows`,
+        ],
+        [
+            "Copy point",
+            session.copyStartedAt
+                ? formatDate(session.copyStartedAt)
+                : DASH,
+        ],
+        ["Discontinuity epoch", session.discontinuityEpoch ?? DASH],
+        ["Build", session.buildId ?? DASH],
+        ["Expires", formatDate(session.expiresAt)],
+    ];
+    // Why it stopped is the reason an operator opened this at all, so it comes
+    // before the profile rather than after it.
+    if (session.failure) {
+        rows.push(["Failure", session.failure.code]);
+        rows.push(["Reason", session.failure.reason]);
+        for (const row of session.failure.rows ?? []) {
+            rows.push([
+                "  Row",
+                typeof row === "string" ? row : `${row.model} ${row.id}`,
+            ]);
+        }
+    }
+    const timings = session.timings;
+    rows.push(["Ticks", timings?.ticks ?? DASH]);
+    // One row per state the session actually recorded. A state it never entered
+    // gets none: a block an operator learns is usually blank is a block they
+    // stop reading.
+    for (const state of STATES) {
+        const window = timings?.states?.[state];
+        if (!window)
+            continue;
+        rows.push([
+            title(state),
+            `${window.ms ?? 0} ms / ${window.ticks ?? 0} ticks / ` +
+                `${window.intervalMs ?? 0} ms between ticks`,
+        ]);
+    }
+    for (const [label, key] of STAGES) {
+        rows.push([label, timings?.totals?.[key] ?? DASH]);
+    }
+    // #3638 — the clock-free half, and the flag that says whether the
+    // milliseconds above measure anything. A stage row reads
+    // `statements / rows read / rows written`, with `—` where the host had no
+    // cursor counter to read: an unknown, never a measured zero.
+    const counters = timings?.totals?.counters;
+    rows.push([
+        "Bracketed ticks",
+        timings?.totals?.bracketedTicks ?? DASH,
+    ]);
+    for (const [label, key] of STAGES) {
+        const bucket = counters?.[key.replace(/Ms$/, "")];
+        if (!bucket)
+            continue;
+        rows.push([
+            `${label.replace(" (ms)", "")} SQL`,
+            `${bucket.statements ?? 0} statements / ` +
+                `${bucket.rowsRead ?? DASH} rows read / ` +
+                `${bucket.rowsWritten ?? DASH} rows written`,
+        ]);
+    }
+    const longest = timings?.longestInterval;
+    rows.push([
+        "Longest interval",
+        longest ? `${longest.intervalMs} ms in ${longest.state}` : DASH,
+    ]);
+    // Stated only when the server stated it — which it does only when the
+    // window was measured from `uploading`. `—` here means "not a measurement",
+    // and that is the honest thing for it to mean.
+    const throughput = session.throughput;
+    rows.push([
+        "Throughput",
+        throughput
+            ? `${throughput.rowsPerSecond.toFixed(1)} rows/s, ` +
+                `${mbPerSecond(throughput.storedBytesPerSecond)} stored` +
+                (throughput.rawBytesPerSecond === null
+                    ? ""
+                    : `, ${mbPerSecond(throughput.rawBytesPerSecond)} raw`) +
+                ` over ${(throughput.pipelineMs / 1000).toFixed(1)}s`
+            : DASH,
+    ]);
+    return rows;
+}
+/**
+ * The client's phases as rows, so the two sides read side by side.
+ *
+ * The wall-clock number `documents ingest` used to print covered the artifact
+ * build, the upload AND everything the platform then did, and could be blamed
+ * on either. These say which. A phase the run never finished prints `—`,
+ * which here means what it means everywhere else in this file: not a
+ * measurement.
+ */
+export function ingestClientRows(timings) {
+    const rows = [
+        ["Client build (ms)", timings.buildMs],
+        ["Client create (ms)", timings.createMs ?? DASH],
+        ["Client upload (ms)", timings.uploadMs ?? DASH],
+        ["Client uploaded bytes", timings.uploadedBytes ?? DASH],
+        ["Client commit (ms)", timings.commitMs ?? DASH],
+        ["Client wait (ms)", timings.waitMs ?? DASH],
+    ];
+    const { uploadMs, uploadedBytes } = timings;
+    if (uploadMs !== undefined && uploadMs > 0 && (uploadedBytes ?? 0) > 0) {
+        rows.push([
+            "Client upload rate",
+            mbPerSecond((uploadedBytes / uploadMs) * 1000),
+        ]);
+    }
+    return rows;
+}
+//# sourceMappingURL=document-ingest-rows.js.map

package/dist/src/lib/document-ingest-rows.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-ingest-rows.js","sourceRoot":"","sources":["../../../src/lib/document-ingest-rows.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AA0FzC,gFAAgF;AAChF,MAAM,MAAM,GAAG;IACb,WAAW;IACX,WAAW;IACX,YAAY;IACZ,SAAS;IACT,UAAU;IACV,YAAY;IACZ,aAAa;CACL,CAAC;AAEX,2DAA2D;AAC3D,MAAM,MAAM,GAA4C;IACtD,CAAC,eAAe,EAAE,YAAY,CAAC;IAC/B,CAAC,WAAW,EAAE,QAAQ,CAAC;IACvB,CAAC,aAAa,EAAE,UAAU,CAAC;IAC3B,CAAC,WAAW,EAAE,QAAQ,CAAC;IACvB,CAAC,YAAY,EAAE,SAAS,CAAC;IACzB,CAAC,gBAAgB,EAAE,aAAa,CAAC;IACjC,CAAC,WAAW,EAAE,QAAQ,CAAC;IACvB,CAAC,WAAW,EAAE,QAAQ,CAAC;IACvB,CAAC,eAAe,EAAE,YAAY,CAAC;CAChC,CAAC;AAEF,MAAM,IAAI,GAAG,GAAG,CAAC;AAEjB,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAEhF,8EAA8E;AAC9E,MAAM,WAAW,GAAG,CAAC,cAAsB,EAAE,EAAE,CAC7C,GAAG,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;AAEpD,MAAM,UAAU,iBAAiB,CAC/B,OAA6B;IAE7B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;IACxC,MAAM,IAAI,GAAuB;QAC/B,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC;QAC9B,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC;QAChC,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC;QACxB,CAAC,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC;QACjC,CAAC,SAAS,EAAE,UAAU,CAAC,OAAO,CAAC,SAA4B,CAAC,CAAC;QAC7D,CAAC,SAAS,EAAE,UAAU,CAAC,OAAO,CAAC,SAA4B,CAAC,CAAC;QAC7D;YACE,WAAW;YACX,OAAO,CAAC,WAAW;gBACjB,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,WAA8B,CAAC;gBACpD,CAAC,CAAC,IAAI;SACT;QACD,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC;QAC1B,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC;QACxB,uEAAuE;QACvE,sEAAsE;QACtE,iEAAiE;QACjE,CAAC,WAAW,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;QACvC;YACE,UAAU;YACV,UAAU,QAAQ,CAAC,UAAU,IAAI,CAAC,iBAAiB;gBACjD,GAAG,QAAQ,CAAC,aAAa,IAAI,CAAC,aAAa,QAAQ,CAAC,WAAW,IAAI,CAAC,SAAS;gBAC7E,cAAc,QAAQ,CAAC,iBAAiB,IAAI,CAAC,YAAY;gBACzD,WAAW,QAAQ,CAAC,cAAc,IAAI,CAAC,mBAAmB;SAC7D;QACD;YACE,YAAY;YACZ,OAAO,CAAC,aAAa;gBACnB,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,aAAgC,CAAC;gBACtD,CAAC,CAAC,IAAI;SACT;QACD,CAAC,qBAAqB,EAAE,OAAO,CAAC,kBAAkB,IAAI,IAAI,CAAC;QAC3D,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC;QAClC,CAAC,SAAS,EAAE,UAAU,CAAC,OAAO,CAAC,SAA4B,CAAC,CAAC;KAC9D,CAAC;IAEF,2EAA2E;IAC3E,2CAA2C;IAC3C,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9C,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;YAC7C,IAAI,CAAC,IAAI,CAAC;gBACR,OAAO;gBACP,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,EAAE,EAAE;aACzD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAChC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC;IAE7C,4EAA4E;IAC5E,yEAAyE;IACzE,gBAAgB;IAChB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,IAAI,CAAC,IAAI,CAAC;YACR,KAAK,CAAC,KAAK,CAAC;YACZ,GAAG,MAAM,CAAC,EAAE,IAAI,CAAC,SAAS,MAAM,CAAC,KAAK,IAAI,CAAC,WAAW;gBACpD,GAAG,MAAM,CAAC,UAAU,IAAI,CAAC,mBAAmB;SAC/C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,MAAM,EAAE,CAAC;QAClC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,kEAAkE;IAClE,yDAAyD;IACzD,0EAA0E;IAC1E,6DAA6D;IAC7D,MAAM,QAAQ,GAAG,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC;IAC3C,IAAI,CAAC,IAAI,CAAC;QACR,iBAAiB;QACjB,OAAO,EAAE,MAAM,EAAE,cAAc,IAAI,IAAI;KACxC,CAAC,CAAC;IACH,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,MAAM,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,QAAQ,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,IAAI,CAAC,IAAI,CAAC;YACR,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM;YACnC,GAAG,MAAM,CAAC,UAAU,IAAI,CAAC,gBAAgB;gBACvC,GAAG,MAAM,CAAC,QAAQ,IAAI,IAAI,eAAe;gBACzC,GAAG,MAAM,CAAC,WAAW,IAAI,IAAI,eAAe;SAC/C,CAAC,CAAC;IACL,CAAC;IAED,MAAM,OAAO,GAAG,OAAO,EAAE,eAAe,CAAC;IACzC,IAAI,CAAC,IAAI,CAAC;QACR,kBAAkB;QAClB,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,UAAU,UAAU,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;KAChE,CAAC,CAAC;IAEH,sEAAsE;IACtE,4EAA4E;IAC5E,+CAA+C;IAC/C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;IACtC,IAAI,CAAC,IAAI,CAAC;QACR,YAAY;QACZ,UAAU;YACR,CAAC,CAAC,GAAG,UAAU,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW;gBACjD,GAAG,WAAW,CAAC,UAAU,CAAC,oBAAoB,CAAC,SAAS;gBACxD,CAAC,UAAU,CAAC,iBAAiB,KAAK,IAAI;oBACpC,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAC,KAAK,WAAW,CAAC,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC;gBACzD,SAAS,CAAC,UAAU,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YACvD,CAAC,CAAC,IAAI;KACT,CAAC,CAAC;IAEH,OAAO,IAAI,CAAC;AACd,CAAC;AAuBD;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAC9B,OAA4B;IAE5B,MAAM,IAAI,GAAuB;QAC/B,CAAC,mBAAmB,EAAE,OAAO,CAAC,OAAO,CAAC;QACtC,CAAC,oBAAoB,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;QAChD,CAAC,oBAAoB,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;QAChD,CAAC,uBAAuB,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC;QACxD,CAAC,oBAAoB,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;QAChD,CAAC,kBAAkB,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC;KAC7C,CAAC;IACF,MAAM,EAAE,QAAQ,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;IAC5C,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,GAAG,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC;QACvE,IAAI,CAAC,IAAI,CAAC;YACR,oBAAoB;YACpB,WAAW,CAAC,CAAE,aAAwB,GAAG,QAAQ,CAAC,GAAG,IAAI,CAAC;SAC3D,CAAC,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/src/lib/document-ingest.d.ts

@@ -0,0 +1,101 @@
+/**
+ * Driving one bulk-load session: create, upload, commit, poll.
+ *
+ * The artifact is already built and checked by the time anything here runs
+ * (`document-ingest-artifact`), so every refusal this module reports comes
+ * from the server. What it decides is what to do about one:
+ *
+ * - a create the server refuses uploads nothing;
+ * - an upload that fails ABORTS the session rather than retrying a slot whose
+ *   reservation this side cannot see — the artifact is deterministic and a
+ *   re-run is cheap;
+ * - Ctrl-C during the uploads aborts too, because the session it was filling
+ *   is not going to be committed; Ctrl-C while POLLING leaves the session
+ *   alone, because by then the document is already being changed and giving up
+ *   on watching is not giving up on the load;
+ * - a session READ that fails while polling is retried on a bounded schedule
+ *   (#3597), for the same reason: the read failing says nothing about the load,
+ *   and a multi-hour load will meet transient reads. When the bound is spent
+ *   the wait stops and says the session is still running — it never reports it
+ *   as cancelled or failed.
+ *
+ * Exit codes follow `functions runs wait`/`invoke`, as `docs/cli-design.md`
+ * records them: 0 completed, 1 failed or refused, 124 a wait that gave up,
+ * 130 Ctrl-C. Both ways of giving up on the WATCHING are 124 and both say the
+ * session keeps running; only the load itself failing is 1.
+ */
+import type { IngestArtifactPlan, IngestChunkDescriptor } from "./document-ingest-artifact.js";
+import type { IngestClientTimings } from "./document-ingest-rows.js";
+/** A plan the operator did not confirm, or cannot be asked about. */
+export declare class IngestRefusedError extends Error {
+    constructor(message: string);
+}
+/** The six calls driving a session; `ApiClient` supplies them in the verb. */
+export interface IngestDriverClient {
+    createIngest(documentId: string): Promise<any>;
+    uploadIngestChunk(documentId: string, sessionId: string, chunk: IngestChunkDescriptor & {
+        body: Buffer;
+    }): Promise<any>;
+    commitIngest(documentId: string, sessionId: string): Promise<any>;
+    abortIngest(documentId: string, sessionId: string): Promise<any>;
+    getIngest(documentId: string, sessionId: string): Promise<any>;
+}
+export interface DriveIngestOptions {
+    client: IngestDriverClient;
+    documentId: string;
+    plan: IngestArtifactPlan;
+    /** Uploads in flight at once. */
+    concurrency?: number;
+    /** Poll until the session settles; `--no-wait` sets this false. */
+    wait?: boolean;
+    /** `--timeout`, in milliseconds; null waits indefinitely. */
+    timeoutMs?: number | null;
+    /** Ctrl-C. */
+    signal?: AbortSignal;
+    /** State changes and progress; stderr in the verb. */
+    onProgress?: (line: string) => void;
+    /** Where a chunk's bytes come from; the plan's own files by default. */
+    readChunk?: (chunk: IngestChunkDescriptor) => Promise<Buffer>;
+    now?: () => number;
+    sleep?: (ms: number) => Promise<void>;
+}
+/** What the run settled on. */
+export interface IngestOutcome {
+    /** 0 complete, 1 failed or refused, 124 timed out, 130 interrupted. */
+    exitCode: number;
+    sessionId: string | null;
+    /** The terminal session view, when one was read. */
+    session: any | null;
+    /** The reason, for stderr. */
+    message?: string;
+    /**
+     * What THIS side spent, phase by phase (#3598).
+     *
+     * A bulk load's wall-clock time used to be one number covering the artifact
+     * build, the upload and everything the platform then did, which meant it
+     * could be blamed on either side. A phase the run never finished is
+     * ABSENT rather than 0 (finding 3598-FROZEN-02): an upload that died
+     * halfway has no rate, and a commit that was never reached did not take no
+     * time. The one exception is the wait, which is real elapsed time whether
+     * or not the session settled, so a poll that fails after minutes reports
+     * those minutes (finding 3598-FROZEN-01).
+     *
+     * `buildMs` is the verb's, not the driver's — the artifact is already built
+     * by the time anything here runs — and is filled in by the caller.
+     */
+    timings: Omit<IngestClientTimings, "buildMs">;
+}
+/** The plan summary an operator confirms, one line per row. */
+export declare function summariseIngestPlan(plan: IngestArtifactPlan): string[];
+/**
+ * Refuse a run that cannot ask, before it asks the server anything.
+ *
+ * A bulk load deletes and overwrites a live document's records with no undo,
+ * which is what the CLI design's destructive default is for. A pipeline that
+ * meant to run one says so with `-y`.
+ */
+export declare function assertIngestConfirmable(options: {
+    yes: boolean;
+    isTTY: boolean;
+}): void;
+export declare function driveDocumentIngest(options: DriveIngestOptions): Promise<IngestOutcome>;