primitive-admin 1.2.0-alpha.1 → 1.2.0-alpha.10

package/dist/src/lib/document-export-lookups.d.ts

@@ -0,0 +1,31 @@
+/**
+ * The side reads `primitive documents export` makes for each document (#4149):
+ * its permissions, its pending invitations, its aliases and its blob list.
+ *
+ * Each one used to sit in a bare `catch {}`, so a 403, a 429 or a 5xx wrote an
+ * empty list and the export reported success. Aliases and blobs are what an
+ * import restores, so the loss carried straight into the migrated app. Only a
+ * documented "none" answer reads as empty here — a 404, or whatever extra
+ * statuses the caller names — exactly as the import's own existence check
+ * reads only a 404 as absence. A transient failure is retried through the
+ * CLI's shared retry; anything else fails the document.
+ */
+/** Options for {@link readExportLookup}. */
+export interface ExportLookupOptions {
+    /** Statuses that mean "the document has none". Defaults to `[404]`. */
+    absentStatuses?: readonly number[];
+    /** The wait between tries; injectable so a test need not spend it. */
+    sleep?: (ms: number) => Promise<void>;
+}
+/**
+ * Read one of a document's export lookups, or `null` when it has none.
+ *
+ * Throws when the read fails with anything other than an absent status, after
+ * retrying a transient failure. The error names the document and the lookup,
+ * so a caller exporting many documents can report which one is incomplete.
+ *
+ * @param documentId - The document being exported, named in a failure.
+ * @param lookup - What is being read, as an operator reads it ("aliases").
+ * @param read - The read itself.
+ */
+export declare function readExportLookup<T>(documentId: string, lookup: string, read: () => Promise<T>, options?: ExportLookupOptions): Promise<T | null>;

package/dist/src/lib/document-export-lookups.js

@@ -0,0 +1,53 @@
+/**
+ * The side reads `primitive documents export` makes for each document (#4149):
+ * its permissions, its pending invitations, its aliases and its blob list.
+ *
+ * Each one used to sit in a bare `catch {}`, so a 403, a 429 or a 5xx wrote an
+ * empty list and the export reported success. Aliases and blobs are what an
+ * import restores, so the loss carried straight into the migrated app. Only a
+ * documented "none" answer reads as empty here — a 404, or whatever extra
+ * statuses the caller names — exactly as the import's own existence check
+ * reads only a 404 as absence. A transient failure is retried through the
+ * CLI's shared retry; anything else fails the document.
+ */
+import { failureStatus, withTransientRetry } from "./transient-retry.js";
+/** An absent status, carried out of the retry so it is not retried. */
+const ABSENT = Symbol("absent");
+/**
+ * Read one of a document's export lookups, or `null` when it has none.
+ *
+ * Throws when the read fails with anything other than an absent status, after
+ * retrying a transient failure. The error names the document and the lookup,
+ * so a caller exporting many documents can report which one is incomplete.
+ *
+ * @param documentId - The document being exported, named in a failure.
+ * @param lookup - What is being read, as an operator reads it ("aliases").
+ * @param read - The read itself.
+ */
+export async function readExportLookup(documentId, lookup, read, options = {}) {
+    const absent = options.absentStatuses ?? [404];
+    let attempts = 0;
+    try {
+        const result = await withTransientRetry(async () => {
+            attempts++;
+            try {
+                return await read();
+            }
+            catch (error) {
+                const status = failureStatus(error);
+                if (status !== undefined && absent.includes(status))
+                    return ABSENT;
+                throw error;
+            }
+        }, undefined, undefined, options.sleep);
+        return result === ABSENT ? null : result;
+    }
+    catch (error) {
+        const status = failureStatus(error);
+        const reason = `${error?.message || "no reason given"}${status !== undefined ? ` (${status})` : ""}`;
+        const tries = attempts > 1 ? ` after ${attempts} attempts` : "";
+        throw new Error(`Document ${documentId}: reading its ${lookup} failed${tries} — ` +
+            `${reason}. Its export would be incomplete, so it was not exported.`);
+    }
+}
+//# sourceMappingURL=document-export-lookups.js.map

package/dist/src/lib/document-export-lookups.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-export-lookups.js","sourceRoot":"","sources":["../../../src/lib/document-export-lookups.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAUzE,uEAAuE;AACvE,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;AAEhC;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,UAAkB,EAClB,MAAc,EACd,IAAsB,EACtB,UAA+B,EAAE;IAEjC,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/C,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CACrC,KAAK,IAAI,EAAE;YACT,QAAQ,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,EAAE,CAAC;YACtB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBACpC,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAAE,OAAO,MAAM,CAAC;gBACnE,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC,EACD,SAAS,EACT,SAAS,EACT,OAAO,CAAC,KAAK,CACd,CAAC;QACF,OAAO,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IAC3C,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,GAAG,KAAK,EAAE,OAAO,IAAI,iBAAiB,GACnD,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC,CAAC,EAC1C,EAAE,CAAC;QACH,MAAM,KAAK,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,QAAQ,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CACb,YAAY,UAAU,iBAAiB,MAAM,UAAU,KAAK,KAAK;YAC/D,GAAG,MAAM,2DAA2D,CACvE,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/src/lib/document-export-permissions.d.ts

@@ -9,10 +9,12 @@
  * which returns a `{ items }` envelope of deferred grants the server has
  * already filtered down to unresolved, unexpired, still-invited ones.
  *
- * The export tolerates the pending read failing (an older server answers 403
- * or 404): the caller swallows the error and passes `null`, and the export
- * still writes the permission rows.
+ * An older server answers the pending read with 403 or 404; the caller reads
+ * either as "no pending invitations" and passes `null`, and the export still
+ * writes the permission rows. Any other failure fails the export (#4149).
  */
+/** The pending-invitations statuses that mean "none": an older server's 403 or 404. */
+export declare const PENDING_INVITATIONS_ABSENT_STATUSES: readonly number[];
 export interface ExportedPermissionRow {
     email: string;
     permission: string;
@@ -24,7 +26,7 @@ export interface ExportedPermissionRow {
  * Merge a document's permission rows and its pending deferred grants into the
  * single email-keyed list `permissions.json` carries.
  *
- * @param permissionsResponse - `GET /documents/:id/permissions`, or `null` when the read failed.
- * @param pendingResponse - `GET /documents/:id/pending-invitations`, or `null` when the read failed.
+ * @param permissionsResponse - `GET /documents/:id/permissions`, or `null` when the document has none.
+ * @param pendingResponse - `GET /documents/:id/pending-invitations`, or `null` when the server has none to report.
  */
 export declare function buildPermissionsExport(permissionsResponse: unknown, pendingResponse: unknown): ExportedPermissionRow[];

package/dist/src/lib/document-export-permissions.js

@@ -9,10 +9,12 @@
  * which returns a `{ items }` envelope of deferred grants the server has
  * already filtered down to unresolved, unexpired, still-invited ones.
  *
- * The export tolerates the pending read failing (an older server answers 403
- * or 404): the caller swallows the error and passes `null`, and the export
- * still writes the permission rows.
+ * An older server answers the pending read with 403 or 404; the caller reads
+ * either as "no pending invitations" and passes `null`, and the export still
+ * writes the permission rows. Any other failure fails the export (#4149).
  */
+/** The pending-invitations statuses that mean "none": an older server's 403 or 404. */
+export const PENDING_INVITATIONS_ABSENT_STATUSES = [403, 404];
 /** Unwrap a list that may arrive bare, or under `items`, or under a named key. */
 function rows(value, key) {
     if (Array.isArray(value))
@@ -30,8 +32,8 @@ function rows(value, key) {
  * Merge a document's permission rows and its pending deferred grants into the
  * single email-keyed list `permissions.json` carries.
  *
- * @param permissionsResponse - `GET /documents/:id/permissions`, or `null` when the read failed.
- * @param pendingResponse - `GET /documents/:id/pending-invitations`, or `null` when the read failed.
+ * @param permissionsResponse - `GET /documents/:id/permissions`, or `null` when the document has none.
+ * @param pendingResponse - `GET /documents/:id/pending-invitations`, or `null` when the server has none to report.
  */
 export function buildPermissionsExport(permissionsResponse, pendingResponse) {
     const granted = rows(permissionsResponse, "permissions")

package/dist/src/lib/document-export-permissions.js.map

@@ -1 +1 @@
-{"version":3,"file":"document-export-permissions.js","sourceRoot":"","sources":["../../../src/lib/document-export-permissions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAUH,kFAAkF;AAClF,SAAS,IAAI,CAAC,KAAc,EAAE,GAAW;IACvC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,KAAgC,CAAC;QAClD,KAAK,MAAM,SAAS,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC;YACvC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBAAE,OAAO,QAAQ,CAAC,SAAS,CAAU,CAAC;QAC9E,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CACpC,mBAA4B,EAC5B,eAAwB;IAExB,MAAM,OAAO,GAAG,IAAI,CAAC,mBAAmB,EAAE,aAAa,CAAC;SACrD,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC;SAC5B,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;QAChB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,SAAS,EAAE,CAAC,CAAC,SAAS;KACvB,CAAC,CAAC,CAAC;IAEN,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,CAAC;SACjD,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC;SAC5B,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;QAChB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,MAAM,EAAE,YAAqB;QAC7B,MAAM,EAAE,SAAkB;KAC3B,CAAC,CAAC,CAAC;IAEN,OAAO,CAAC,GAAG,OAAO,EAAE,GAAG,OAAO,CAAC,CAAC;AAClC,CAAC"}
+{"version":3,"file":"document-export-permissions.js","sourceRoot":"","sources":["../../../src/lib/document-export-permissions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,uFAAuF;AACvF,MAAM,CAAC,MAAM,mCAAmC,GAAsB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AAUjF,kFAAkF;AAClF,SAAS,IAAI,CAAC,KAAc,EAAE,GAAW;IACvC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACvC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,KAAgC,CAAC;QAClD,KAAK,MAAM,SAAS,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC;YACvC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBAAE,OAAO,QAAQ,CAAC,SAAS,CAAU,CAAC;QAC9E,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CACpC,mBAA4B,EAC5B,eAAwB;IAExB,MAAM,OAAO,GAAG,IAAI,CAAC,mBAAmB,EAAE,aAAa,CAAC;SACrD,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC;SAC5B,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;QAChB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,SAAS,EAAE,CAAC,CAAC,SAAS;KACvB,CAAC,CAAC,CAAC;IAEN,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,EAAE,aAAa,CAAC;SACjD,MAAM,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC;SAC5B,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;QAChB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,MAAM,EAAE,YAAqB;QAC7B,MAAM,EAAE,SAAkB;KAC3B,CAAC,CAAC,CAAC;IAEN,OAAO,CAAC,GAAG,OAAO,EAAE,GAAG,OAAO,CAAC,CAAC;AAClC,CAAC"}

package/dist/src/lib/document-export-record-log.d.ts

@@ -0,0 +1,57 @@
+/**
+ * `documents export` of a record-log (format 3) document (#4310, behavior 24).
+ *
+ * The document's base is a segment: its row chunks were written across
+ * alarms while writes continued, so they are no image of the document at any
+ * one log number; its corrections chunk holds every row and tombstone written
+ * between the export's start `B` and the read `C`. This module requests a
+ * segment, waits for it, downloads the chunks and the corrections through the
+ * chain export's grant, verifies each, merges the corrections into the rows
+ * (a changed row replaces its line, a tombstone removes it, a new row joins
+ * in id order) and re-cuts the merged rows into snapshot-layout chunks — so
+ * the directory is the document at `C`, which `documents ingest` loads into
+ * another format-3 document.
+ *
+ * Nothing is written under `snapshot/` until every download has verified.
+ */
+/** The calls an export makes; `ApiClient` supplies them in the verb. */
+export interface RecordLogExportClient {
+    requestSegment(): Promise<any>;
+    getSegment(segmentId: string): Promise<any>;
+    listSegments(): Promise<any>;
+    exportChain(): Promise<any>;
+    /** Read one object through a `/doc-artifact/…` path. */
+    download(artifactPath: string): Promise<Buffer>;
+}
+/** What the export wrote. */
+export interface RecordLogExportResult {
+    /** Bytes of the segment's chunks and corrections, as downloaded. */
+    bytes: number;
+    /** `C`: the log number the directory is an image of. */
+    lsn: number;
+    segment: {
+        segmentId: string;
+        lsn: number;
+        completedLsn: number;
+        rows: number;
+        chunks: number;
+        corrections: number;
+    };
+}
+/** Rows per re-cut chunk. */
+export declare const RECORD_LOG_EXPORT_CHUNK_ROWS = 10000;
+/**
+ * Export a format-3 document into `docDir`: `snapshot/manifest.json`, its
+ * re-cut chunks `snapshot/{model}/{n}.ndjson.gz`, and `chain.json`
+ * (`{version: 3, documentFormat: 3, documentId, lsn: C, segment}`, with no
+ * grant in it). The caller writes `permissions.json`, `metadata.json` and the
+ * blobs, as for every format.
+ */
+export declare function exportRecordLogDocument(options: {
+    client: RecordLogExportClient;
+    documentId: string;
+    docDir: string;
+    chunkRows?: number;
+    sleep?: (ms: number) => Promise<void>;
+    onProgress?: (line: string) => void;
+}): Promise<RecordLogExportResult>;

package/dist/src/lib/document-export-record-log.js

@@ -0,0 +1,215 @@
+/**
+ * `documents export` of a record-log (format 3) document (#4310, behavior 24).
+ *
+ * The document's base is a segment: its row chunks were written across
+ * alarms while writes continued, so they are no image of the document at any
+ * one log number; its corrections chunk holds every row and tombstone written
+ * between the export's start `B` and the read `C`. This module requests a
+ * segment, waits for it, downloads the chunks and the corrections through the
+ * chain export's grant, verifies each, merges the corrections into the rows
+ * (a changed row replaces its line, a tombstone removes it, a new row joins
+ * in id order) and re-cuts the merged rows into snapshot-layout chunks — so
+ * the directory is the document at `C`, which `documents ingest` loads into
+ * another format-3 document.
+ *
+ * Nothing is written under `snapshot/` until every download has verified.
+ */
+import fs from "fs";
+import path from "path";
+import zlib from "zlib";
+import { createHash } from "crypto";
+import { compareRecordIds, readSnapshotChunk } from "js-bao";
+/** Rows per re-cut chunk. */
+export const RECORD_LOG_EXPORT_CHUNK_ROWS = 10_000;
+const POLL_MS = 500;
+/**
+ * Request a segment and wait for it: one already exporting is waited for
+ * rather than refused. Answers the complete segment's view; a failed one
+ * throws with its reason.
+ */
+async function awaitSegment(client, options) {
+    let segmentId;
+    try {
+        segmentId = (await client.requestSegment()).segmentId;
+    }
+    catch (error) {
+        if (error?.code !== "SEGMENT_EXPORT_ACTIVE")
+            throw error;
+        const listed = await client.listSegments();
+        const active = (listed?.items ?? []).find((s) => s.state === "exporting");
+        if (!active)
+            throw error;
+        segmentId = active.segmentId;
+    }
+    options.onProgress?.(`Exporting segment ${segmentId}...`);
+    for (;;) {
+        const view = await client.getSegment(segmentId);
+        if (view.state === "complete")
+            return view;
+        if (view.state === "failed" || view.state === "superseded") {
+            const failure = view.failure ? `${view.failure.code}: ${view.failure.reason}` : view.state;
+            throw new Error(`The segment export ${segmentId} failed (${failure}); nothing was written.`);
+        }
+        await options.sleep(POLL_MS);
+    }
+}
+/** The lines of a downloaded corrections chunk, verified against the manifest's entry. */
+function readCorrections(body, entry) {
+    if (!entry || body.byteLength !== entry.bytes || createHash("sha256").update(body).digest("hex") !== entry.sha256) {
+        throw new Error(`The segment's corrections chunk does not match its manifest (expected ${entry?.bytes} bytes and sha256 ${entry?.sha256}); nothing was written.`);
+    }
+    const text = zlib.gunzipSync(body).toString("utf8");
+    const out = [];
+    for (const raw of text.split("\n")) {
+        if (!raw)
+            continue;
+        const tab = raw.indexOf("\t");
+        if (tab < 0)
+            throw new Error("The segment's corrections chunk holds a line with no id; nothing was written.");
+        const line = JSON.parse(raw.slice(tab + 1));
+        if (!line || typeof line !== "object" || Array.isArray(line)) {
+            throw new Error("The segment's corrections chunk holds a line that is not an object; nothing was written.");
+        }
+        out.push({ id: raw.slice(0, tab), line });
+    }
+    if (out.length !== entry.rows) {
+        throw new Error(`The segment's corrections chunk holds ${out.length} lines, not ${entry.rows}; nothing was written.`);
+    }
+    return out;
+}
+/**
+ * Whether a corrections line is a tombstone: exactly `{_type, _deleted: true}`,
+ * as the room writes one. A live row's line carries its fields beside `_type`,
+ * so a field of the record named `_deleted` does not remove it.
+ */
+function isTombstoneLine(line) {
+    const keys = Object.keys(line);
+    return line._deleted === true && keys.length === 2 && keys.includes("_type");
+}
+/**
+ * Refuse a model whose directory would not be a plain child of `snapshotDir`:
+ * an empty name, `.` or `..`, a path separator, or a NUL. Model names are any
+ * string a writer sent, so they are checked before they become paths.
+ */
+function assertModelDirectory(snapshotDir, model) {
+    const plain = model !== "" && model !== "." && model !== ".." && !/[\\/\0]/.test(model);
+    const resolved = path.resolve(snapshotDir, model);
+    if (!plain || path.dirname(resolved) !== path.resolve(snapshotDir)) {
+        throw new Error(`The document holds a model named ${JSON.stringify(model)}, which cannot be written as a directory of the export; nothing was written.`);
+    }
+}
+/** Gzip one chunk's rows in the snapshot layout, `⟨id⟩\t⟨data⟩` per line, and describe it. */
+function encodeChunk(model, index, rows) {
+    const raw = Buffer.from(rows.map(([id, data]) => `${id}\t${data}\n`).join(""), "utf8");
+    const body = zlib.gzipSync(raw, { level: 9 });
+    return {
+        body,
+        entry: {
+            key: `snapshot/${model}/${index}.ndjson.gz`,
+            path: `${model}/${index}`,
+            model,
+            bytes: body.byteLength,
+            rawBytes: raw.byteLength,
+            rows: rows.length,
+            firstId: rows[0][0],
+            lastId: rows[rows.length - 1][0],
+            sha256: createHash("sha256").update(body).digest("hex"),
+        },
+    };
+}
+/**
+ * Export a format-3 document into `docDir`: `snapshot/manifest.json`, its
+ * re-cut chunks `snapshot/{model}/{n}.ndjson.gz`, and `chain.json`
+ * (`{version: 3, documentFormat: 3, documentId, lsn: C, segment}`, with no
+ * grant in it). The caller writes `permissions.json`, `metadata.json` and the
+ * blobs, as for every format.
+ */
+export async function exportRecordLogDocument(options) {
+    const { client, documentId, docDir } = options;
+    const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    await awaitSegment(client, { sleep, onProgress: options.onProgress });
+    const chain = await client.exportChain();
+    const grant = chain.segment.download.path;
+    const manifest = JSON.parse((await client.download(grant)).toString("utf8"));
+    let bytes = 0;
+    // Every row of every chunk, verified, by model and id.
+    const tables = new Map();
+    const table = (model) => {
+        let rows = tables.get(model);
+        if (!rows)
+            tables.set(model, (rows = new Map()));
+        return rows;
+    };
+    for (const entry of manifest.chunks ?? []) {
+        const body = await client.download(`${grant}/${entry.path}`);
+        bytes += body.byteLength;
+        let rows;
+        try {
+            rows = await readSnapshotChunk(new Uint8Array(body), { ...entry, key: entry.path });
+        }
+        catch (error) {
+            throw new Error(`The segment's chunk ${entry.path} failed verification (${error?.message ?? error}); nothing was written.`);
+        }
+        for (const row of rows)
+            table(row._type).set(row._id, row._data);
+    }
+    const correctionsBody = await client.download(`${grant}/${manifest.corrections?.path ?? "corrections"}`);
+    bytes += correctionsBody.byteLength;
+    for (const { id, line } of readCorrections(correctionsBody, manifest.corrections)) {
+        const { _type, ...fields } = line;
+        if (typeof _type !== "string") {
+            throw new Error(`The segment's corrections chunk holds a line for ${id} with no model; nothing was written.`);
+        }
+        if (isTombstoneLine(line))
+            table(_type).delete(id);
+        else
+            table(_type).set(id, JSON.stringify(fields));
+    }
+    // Re-cut, model by model in id order. A model is a directory of the export,
+    // so one that is not a plain name refuses the export before anything is written.
+    const chunkRows = options.chunkRows ?? RECORD_LOG_EXPORT_CHUNK_ROWS;
+    const snapshotDir = path.join(docDir, "snapshot");
+    for (const model of tables.keys())
+        assertModelDirectory(snapshotDir, model);
+    fs.rmSync(snapshotDir, { recursive: true, force: true });
+    const entries = [];
+    let rows = 0;
+    let written = 0;
+    for (const model of [...tables.keys()].sort(compareRecordIds)) {
+        const sorted = [...tables.get(model)].sort(([a], [b]) => compareRecordIds(a, b));
+        for (let at = 0, index = 0; at < sorted.length; at += chunkRows, index++) {
+            const { body, entry } = encodeChunk(model, index, sorted.slice(at, at + chunkRows));
+            const file = path.join(snapshotDir, model, `${index}.ndjson.gz`);
+            fs.mkdirSync(path.dirname(file), { recursive: true });
+            fs.writeFileSync(file, body);
+            entries.push({ ...entry, ordinal: entries.length });
+            rows += entry.rows;
+            written += entry.bytes;
+        }
+    }
+    const completedLsn = Number(chain.completedLsn);
+    fs.mkdirSync(snapshotDir, { recursive: true });
+    fs.writeFileSync(path.join(snapshotDir, "manifest.json"), JSON.stringify({
+        kind: "segment-export",
+        version: 1,
+        documentFormat: 3,
+        documentId,
+        lsn: completedLsn,
+        schema: manifest.schema ?? {},
+        chunks: entries,
+        totals: { rows, chunks: entries.length, bytes: written },
+    }, null, 2));
+    const segment = {
+        segmentId: String(chain.segment.segmentId),
+        lsn: Number(chain.lsn),
+        completedLsn,
+        rows: Number(chain.segment.rows),
+        chunks: Number(chain.segment.chunks),
+        corrections: Number(chain.segment.corrections),
+    };
+    // Last, and without the grant: a signature is this run's read of the
+    // document, and an artifact that outlives it must not carry one.
+    fs.writeFileSync(path.join(docDir, "chain.json"), JSON.stringify({ version: 3, documentFormat: 3, documentId, lsn: completedLsn, segment }, null, 2));
+    return { bytes, lsn: completedLsn, segment };
+}
+//# sourceMappingURL=document-export-record-log.js.map

package/dist/src/lib/document-export-record-log.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"document-export-record-log.js","sourceRoot":"","sources":["../../../src/lib/document-export-record-log.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,QAAQ,CAAC;AAqB7D,6BAA6B;AAC7B,MAAM,CAAC,MAAM,4BAA4B,GAAG,MAAM,CAAC;AAEnD,MAAM,OAAO,GAAG,GAAG,CAAC;AAEpB;;;;GAIG;AACH,KAAK,UAAU,YAAY,CACzB,MAA6B,EAC7B,OAAsF;IAEtF,IAAI,SAAiB,CAAC;IACtB,IAAI,CAAC;QACH,SAAS,GAAG,CAAC,MAAM,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,SAAS,CAAC;IACxD,CAAC;IAAC,OAAO,KAAU,EAAE,CAAC;QACpB,IAAI,KAAK,EAAE,IAAI,KAAK,uBAAuB;YAAE,MAAM,KAAK,CAAC;QACzD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC;QAC/E,IAAI,CAAC,MAAM;YAAE,MAAM,KAAK,CAAC;QACzB,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;IAC/B,CAAC;IACD,OAAO,CAAC,UAAU,EAAE,CAAC,qBAAqB,SAAS,KAAK,CAAC,CAAC;IAC1D,SAAS,CAAC;QACR,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAChD,IAAI,IAAI,CAAC,KAAK,KAAK,UAAU;YAAE,OAAO,IAAI,CAAC;QAC3C,IAAI,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,IAAI,CAAC,KAAK,KAAK,YAAY,EAAE,CAAC;YAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YAC3F,MAAM,IAAI,KAAK,CAAC,sBAAsB,SAAS,YAAY,OAAO,yBAAyB,CAAC,CAAC;QAC/F,CAAC;QACD,MAAM,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC;AAED,0FAA0F;AAC1F,SAAS,eAAe,CAAC,IAAY,EAAE,KAAU;IAC/C,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,UAAU,KAAK,KAAK,CAAC,KAAK,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAClH,MAAM,IAAI,KAAK,CACb,yEAAyE,KAAK,EAAE,KAAK,qBAAqB,KAAK,EAAE,MAAM,yBAAyB,CACjJ,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACpD,MAAM,GAAG,GAAyD,EAAE,CAAC;IACrE,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,MAAM,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC9B,IAAI,GAAG,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,+EAA+E,CAAC,CAAC;QAC9G,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7D,MAAM,IAAI,KAAK,CAAC,0FAA0F,CAAC,CAAC;QAC9G,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,CAAC,MAAM,eAAe,KAAK,CAAC,IAAI,wBAAwB,CAAC,CAAC;IACxH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,IAA6B;IACpD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,OAAO,IAAI,CAAC,QAAQ,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AAC/E,CAAC;AAED;;;;GAIG;AACH,SAAS,oBAAoB,CAAC,WAAmB,EAAE,KAAa;IAC9D,MAAM,KAAK,GAAG,KAAK,KAAK,EAAE,IAAI,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxF,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IAClD,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;QACnE,MAAM,IAAI,KAAK,CAAC,oCAAoC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAC;IAC3J,CAAC;AACH,CAAC;AAED,8FAA8F;AAC9F,SAAS,WAAW,CAAC,KAAa,EAAE,KAAa,EAAE,IAA6B;IAC9E,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;IACvF,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC9C,OAAO;QACL,IAAI;QACJ,KAAK,EAAE;YACL,GAAG,EAAE,YAAY,KAAK,IAAI,KAAK,YAAY;YAC3C,IAAI,EAAE,GAAG,KAAK,IAAI,KAAK,EAAE;YACzB,KAAK;YACL,KAAK,EAAE,IAAI,CAAC,UAAU;YACtB,QAAQ,EAAE,GAAG,CAAC,UAAU;YACxB,IAAI,EAAE,IAAI,CAAC,MAAM;YACjB,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnB,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SACxD;KACF,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAO7C;IACC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACzG,MAAM,YAAY,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IAEtE,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,KAAK,GAAW,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAC7E,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,uDAAuD;IACvD,MAAM,MAAM,GAAG,IAAI,GAAG,EAA+B,CAAC;IACtD,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,EAAE;QAC9B,IAAI,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC7B,IAAI,CAAC,IAAI;YAAE,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC;QACzB,IAAI,IAAI,CAAC;QACT,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,iBAAiB,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACtF,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,uBAAuB,KAAK,CAAC,IAAI,yBAAyB,KAAK,EAAE,OAAO,IAAI,KAAK,yBAAyB,CAAC,CAAC;QAC9H,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,IAAI;YAAE,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IACnE,CAAC;IACD,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,GAAG,KAAK,IAAI,QAAQ,CAAC,WAAW,EAAE,IAAI,IAAI,aAAa,EAAE,CAAC,CAAC;IACzG,KAAK,IAAI,eAAe,CAAC,UAAU,CAAC;IACpC,KAAK,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,eAAe,CAAC,eAAe,EAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAClF,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,EAAE,GAAG,IAA2B,CAAC;QACzD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,oDAAoD,EAAE,sCAAsC,CAAC,CAAC;QAChH,CAAC;QACD,IAAI,eAAe,CAAC,IAAI,CAAC;YAAE,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;;YAC9C,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,4EAA4E;IAC5E,iFAAiF;IACjF,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,4BAA4B,CAAC;IACpE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAClD,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,EAAE;QAAE,oBAAoB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;IAC5E,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,OAAO,GAAU,EAAE,CAAC;IAC1B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC;QAC9D,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAClF,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,EAAE,IAAI,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC;YACzE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,WAAW,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,GAAG,SAAS,CAAC,CAAC,CAAC;YACpF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,GAAG,KAAK,YAAY,CAAC,CAAC;YACjE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtD,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YACpD,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC;YACnB,OAAO,IAAI,KAAK,CAAC,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAChD,EAAE,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,eAAe,CAAC,EACvC,IAAI,CAAC,SAAS,CACZ;QACE,IAAI,EAAE,gBAAgB;QACtB,OAAO,EAAE,CAAC;QACV,cAAc,EAAE,CAAC;QACjB,UAAU;QACV,GAAG,EAAE,YAAY;QACjB,MAAM,EAAE,QAAQ,CAAC,MAAM,IAAI,EAAE;QAC7B,MAAM,EAAE,OAAO;QACf,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE;KACzD,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;IACF,MAAM,OAAO,GAAG;QACd,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;QAC1C,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;QACtB,YAAY;QACZ,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAChC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QACpC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC;KAC/C,CAAC;IACF,qEAAqE;IACrE,iEAAiE;IACjE,EAAE,CAAC,aAAa,CACd,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAC/B,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CACnG,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;AAC/C,CAAC"}

package/dist/src/lib/document-ingest-artifact.d.ts

@@ -14,8 +14,8 @@
  * falls back to a bounded external sort.
  *
  * The local check is the server's, said on this side: the rules it mirrors are
- * `planLine`'s in `src/large-documents/ingest-pipeline.ts`, and the artifact's
- * own rules are in `docs/large-document-ingest.md`.
+ * `planLine`'s in `src/rooms/record-room/load.ts`, and the artifact's own
+ * rules are in `docs/large-document-ingest.md`.
  */
 import type { DiscoveredIngestInput } from "./document-ingest-input.js";
 /** A line, a file or a plan this document's schema does not accept. */
@@ -26,6 +26,12 @@ export declare class IngestArtifactError extends Error {
 export interface IngestModelSchema {
     fields: Set<string>;
     stringSets: Set<string>;
+    /**
+     * A record-log (format 3) document takes fields and models it does not
+     * declare, as its writes do; the room refuses a row that breaks a declared
+     * type, row by row.
+     */
+    open?: boolean;
 }
 /** Every model the document has a declaration for. */
 export interface IngestSchema {
@@ -62,10 +68,23 @@ export interface IngestModelSummary {
     rawBytes: number;
     deletes: number;
 }
+/** A row of a format-2 export left out of the artifact, and why. */
+export interface IngestRefusedRow {
+    model: string;
+    id: string;
+    /** `file:line`. */
+    where: string;
+    reason: string;
+}
 /** The artifact, ready to upload. */
 export interface IngestArtifactPlan {
     models: IngestModelSummary[];
     chunks: IngestChunkDescriptor[];
+    /**
+     * Rows of a format-2 export that cannot be sent as one record, left out
+     * before upload. Every other input either builds whole or is refused.
+     */
+    refused: IngestRefusedRow[];
     rows: number;
     bytes: number;
     rawBytes: number;
@@ -79,6 +98,11 @@ export interface IngestArtifactPlan {
 export interface BuildIngestArtifactOptions {
     input: DiscoveredIngestInput;
     schema: IngestSchema;
+    /**
+     * The document is a record-log (format 3) document: a model or a field it
+     * does not declare is taken as it is, as its writes take them.
+     */
+    open?: boolean;
     /** Where chunk bodies and sort runs are written. */
     tmpDir?: string;
     /** Cut a chunk at this many decompressed bytes. */

package/dist/src/lib/document-ingest-artifact.js

@@ -14,8 +14,8 @@
  * falls back to a bounded external sort.
  *
  * The local check is the server's, said on this side: the rules it mirrors are
- * `planLine`'s in `src/large-documents/ingest-pipeline.ts`, and the artifact's
- * own rules are in `docs/large-document-ingest.md`.
+ * `planLine`'s in `src/rooms/record-room/load.ts`, and the artifact's own
+ * rules are in `docs/large-document-ingest.md`.
  */
 import fs from "fs";
 import path from "path";
@@ -25,6 +25,7 @@ import zlib from "zlib";
 import { createHash } from "crypto";
 import { StringDecoder } from "string_decoder";
 import { compareRecordIds, INGEST_CHUNK_LIMITS } from "js-bao";
+import { flattenFormat2Row } from "./document-ingest-format2.js";
 /** A line, a file or a plan this document's schema does not accept. */
 export class IngestArtifactError extends Error {
     constructor(message) {
@@ -64,6 +65,8 @@ export async function buildIngestArtifact(options) {
     const { input, schema, cutRawBytes = INGEST_CUT_RAW_BYTES, cutRows = INGEST_CUT_ROWS, runBytes = INGEST_SORT_RUN_BYTES, maxChunkBytes = INGEST_CHUNK_LIMITS.maxChunkBytes, maxChunkRawBytes = INGEST_CHUNK_LIMITS.maxChunkRawBytes, maxChunkRows = INGEST_CHUNK_LIMITS.maxChunkRows, maxChunks = INGEST_CHUNK_LIMITS.maxChunks, maxTotalBytes = INGEST_CHUNK_LIMITS.maxTotalBytes, } = options;
     const scratch = fs.mkdtempSync(path.join(options.tmpDir ?? os.tmpdir(), "primitive-ingest-"));
     const written = [];
+    const refused = [];
+    const read = (source, onRefused) => readRows(source, Boolean(input.format2), onRefused);
     const cleanup = async () => {
         fs.rmSync(scratch, { recursive: true, force: true });
     };
@@ -79,7 +82,15 @@ export async function buildIngestArtifact(options) {
             entry.sources.push(source);
         }
         for (const [model, entry] of models) {
-            const declared = schema.models.get(model);
+            // A format-2 export's converted row into a record-log document is held
+            // to the line's structure only: a value its declaration refuses (an
+            // array that is not all strings) is the room's to refuse, row by row
+            // and naming the field, not a reason to stop the whole load.
+            const declared = options.open
+                ? input.format2
+                    ? { fields: new Set(), stringSets: new Set(), open: true }
+                    : { ...(schema.models.get(model) ?? { fields: new Set(), stringSets: new Set() }), open: true }
+                : schema.models.get(model);
             if (!declared) {
                 throw new IngestArtifactError(`This document declares no model ${model}. An ingest applies the ` +
                     "document's existing schema; declare the model through your app's " +
@@ -87,7 +98,7 @@ export async function buildIngestArtifact(options) {
             }
             let previous = null;
             for (const source of entry.sources) {
-                for await (const line of readSource(source)) {
+                for await (const line of read(source, (row) => refused.push(row))) {
                     const parsed = checkIngestLine(line.where, model, line.text, declared);
                     entry.rows += 1;
                     if (parsed.deleted)
@@ -124,7 +135,7 @@ export async function buildIngestArtifact(options) {
             if (entry.sorted) {
                 let previousId = null;
                 for (const source of entry.sources) {
-                    for await (const line of readSource(source)) {
+                    for await (const line of read(source)) {
                         const parsed = parseIngestLine(line.where, line.text);
                         assertNoDuplicate(previousId, parsed, line.where);
                         previousId = parsed.id;
@@ -133,7 +144,7 @@ export async function buildIngestArtifact(options) {
                 }
             }
             else {
-                const sorted = await externalSort(entry.sources, scratch, runBytes);
+                const sorted = await externalSort(entry.sources, read, scratch, runBytes);
                 sortRuns += sorted.runs;
                 let previous = null;
                 for (const line of sorted.lines()) {
@@ -172,6 +183,7 @@ export async function buildIngestArtifact(options) {
         return {
             models: summaries,
             chunks,
+            refused,
             rows: summaries.reduce((sum, model) => sum + model.rows, 0),
             bytes: totalBytes,
             rawBytes: summaries.reduce((sum, model) => sum + model.rawBytes, 0),
@@ -211,6 +223,26 @@ async function* readSource(source) {
         stream.destroy?.();
     }
 }
+/**
+ * Every row of one source, as {@link readSource} reads it, or as a record-log
+ * row when the source is a format-2 export's. A row that cannot be converted
+ * is left out and handed to `onRefused`.
+ */
+async function* readRows(source, format2, onRefused) {
+    for await (const line of readSource(source)) {
+        if (!format2) {
+            yield line;
+            continue;
+        }
+        const parsed = parseIngestLine(line.where, line.text);
+        const converted = flattenFormat2Row(source.model, parsed.id, JSON.parse(parsed.patch));
+        if ("refused" in converted) {
+            onRefused?.({ model: source.model, id: parsed.id, where: line.where, reason: converted.refused });
+            continue;
+        }
+        yield { where: line.where, text: `${parsed.id}\t${JSON.stringify(converted.fields)}` };
+    }
+}
 /** One line's id and patch, whichever of the two forms it is written in. */
 export function parseIngestLine(where, text) {
     let id;
@@ -299,6 +331,8 @@ export function checkIngestLine(where, model, text, declared) {
             continue;
         }
         if (!declared.fields.has(key)) {
+            if (declared.open)
+                continue;
             throw new IngestArtifactError(`${where}: the model ${model} does not declare the field ${key}`);
         }
         if (Array.isArray(value)) {
@@ -314,7 +348,7 @@ export function checkIngestLine(where, model, text, declared) {
  * writes each sorted run out and merges them back k-way, so the peak cost is
  * the run size plus one line per run whatever the input's size.
  */
-async function externalSort(sources, scratch, runBytes) {
+async function externalSort(sources, read, scratch, runBytes) {
     const runFiles = [];
     let buffer = [];
     let buffered = 0;
@@ -327,7 +361,7 @@ async function externalSort(sources, scratch, runBytes) {
         buffered = 0;
     };
     for (const source of sources) {
-        for await (const raw of readSource(source)) {
+        for await (const raw of read(source)) {
             const parsed = parseIngestLine(raw.where, raw.text);
             buffer.push({ ...parsed, where: raw.where });
             buffered += parsed.id.length + parsed.patch.length + raw.where.length + 2;