rastack 0.0.46 → 0.0.47

package/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.0.47](https://github.com/theserverkid/reactapistack/compare/v0.0.46...v0.0.47) (2026-07-13)
+
 ### [0.0.46](https://github.com/theserverkid/reactapistack/compare/v0.0.45...v0.0.46) (2026-07-13)
 
 ### [0.0.45](https://github.com/theserverkid/reactapistack/compare/v0.0.44...v0.0.45) (2026-07-13)

package/dist/import/index.d.ts

@@ -0,0 +1,18 @@
+/**
+ * `rastack import` — load external data (CSV, Excel, PDF) and validate it
+ * against a resource's constraint state machine.
+ *
+ * `parseTabular` dispatches on the file extension to the format parsers, all
+ * of which produce the same {@link TabularData}; `validateDataset` (in
+ * `tabular.ts`) is the shared, pure validation core.
+ */
+import type { TabularData } from "./tabular";
+export * from "./tabular";
+export { parseXlsx } from "./xlsx";
+export { parsePdfTable } from "./pdf";
+/** The source formats `rastack import` understands. */
+export type TabularFormat = "csv" | "xlsx" | "pdf";
+/** Infer the source format from a filename, or `null` if unsupported. */
+export declare function formatOf(fileName: string): TabularFormat | null;
+/** Parse a source file's bytes into headers + rows, by format. */
+export declare function parseTabular(format: TabularFormat, data: Buffer): TabularData;

package/dist/import/index.js

@@ -0,0 +1,57 @@
+"use strict";
+/**
+ * `rastack import` — load external data (CSV, Excel, PDF) and validate it
+ * against a resource's constraint state machine.
+ *
+ * `parseTabular` dispatches on the file extension to the format parsers, all
+ * of which produce the same {@link TabularData}; `validateDataset` (in
+ * `tabular.ts`) is the shared, pure validation core.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parsePdfTable = exports.parseXlsx = void 0;
+exports.formatOf = formatOf;
+exports.parseTabular = parseTabular;
+const csv_schema_1 = require("../csv-schema");
+const xlsx_1 = require("./xlsx");
+const pdf_1 = require("./pdf");
+__exportStar(require("./tabular"), exports);
+var xlsx_2 = require("./xlsx");
+Object.defineProperty(exports, "parseXlsx", { enumerable: true, get: function () { return xlsx_2.parseXlsx; } });
+var pdf_2 = require("./pdf");
+Object.defineProperty(exports, "parsePdfTable", { enumerable: true, get: function () { return pdf_2.parsePdfTable; } });
+/** Infer the source format from a filename, or `null` if unsupported. */
+function formatOf(fileName) {
+    const ext = fileName.toLowerCase().split(".").pop();
+    if (ext === "csv")
+        return "csv";
+    if (ext === "xlsx" || ext === "xlsm")
+        return "xlsx";
+    if (ext === "pdf")
+        return "pdf";
+    return null;
+}
+/** Parse a source file's bytes into headers + rows, by format. */
+function parseTabular(format, data) {
+    switch (format) {
+        case "csv":
+            return (0, csv_schema_1.parseCsv)(data.toString("utf-8"));
+        case "xlsx":
+            return (0, xlsx_1.parseXlsx)(data);
+        case "pdf":
+            return (0, pdf_1.parsePdfTable)(data);
+    }
+}

package/dist/import/pdf.d.ts

@@ -0,0 +1,25 @@
+/**
+ * A tiny, dependency-free PDF *table* extractor for `rastack import`.
+ *
+ * PDFs carry no table structure — only positioned text — so this is
+ * deliberately a best-effort extractor for the common case of a machine-
+ * generated tabular report: it decodes every content stream (FlateDecode via
+ * node's zlib, plus raw streams), replays the text-positioning operators
+ * (`Tm`/`Td`/`TD`/`T*`) to give each shown string (`Tj`/`TJ`/`'`) an (x, y),
+ * clusters strings that share a baseline into rows, and orders each row's
+ * cells left-to-right. The first row becomes the header.
+ *
+ * Anything a text-line model can't represent — merged cells, wrapped cell
+ * text, scanned images — won't survive; for those, export the source data to
+ * CSV/XLSX instead. Every extracted value still goes through the constraint
+ * state machine, so a misread cell is *rejected with a located error*, never
+ * silently imported.
+ */
+import type { TabularData } from "./tabular";
+/**
+ * Parse a PDF buffer into headers + rows: text chunks sharing a baseline form
+ * a row (top of the page first), ordered left-to-right; the first row is the
+ * header. Rows with a different cell count than the header are padded or
+ * truncated — the state machine will flag what doesn't validate.
+ */
+export declare function parsePdfTable(buf: Buffer): TabularData;

package/dist/import/pdf.js

@@ -0,0 +1,326 @@
+"use strict";
+/**
+ * A tiny, dependency-free PDF *table* extractor for `rastack import`.
+ *
+ * PDFs carry no table structure — only positioned text — so this is
+ * deliberately a best-effort extractor for the common case of a machine-
+ * generated tabular report: it decodes every content stream (FlateDecode via
+ * node's zlib, plus raw streams), replays the text-positioning operators
+ * (`Tm`/`Td`/`TD`/`T*`) to give each shown string (`Tj`/`TJ`/`'`) an (x, y),
+ * clusters strings that share a baseline into rows, and orders each row's
+ * cells left-to-right. The first row becomes the header.
+ *
+ * Anything a text-line model can't represent — merged cells, wrapped cell
+ * text, scanned images — won't survive; for those, export the source data to
+ * CSV/XLSX instead. Every extracted value still goes through the constraint
+ * state machine, so a misread cell is *rejected with a located error*, never
+ * silently imported.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parsePdfTable = parsePdfTable;
+const zlib_1 = require("zlib");
+/** How close two baselines must be (in text-space units) to share a row. */
+const ROW_TOLERANCE = 2;
+// ---------------------------------------------------------------------------
+// Stream extraction
+// ---------------------------------------------------------------------------
+/** Decode every content stream in the document to latin1 text. */
+function contentStreams(buf) {
+    const raw = buf.toString("latin1");
+    const streams = [];
+    const re = /stream\r?\n/g;
+    let m;
+    while ((m = re.exec(raw)) !== null) {
+        const start = m.index + m[0].length;
+        const end = raw.indexOf("endstream", start);
+        if (end === -1)
+            break;
+        // The stream dictionary sits just before the `stream` keyword.
+        const dictStart = raw.lastIndexOf("<<", m.index);
+        const dict = dictStart === -1 ? "" : raw.slice(dictStart, m.index);
+        let data = buf.subarray(start, end);
+        // Trim the EOL PDF writers place before `endstream`.
+        while (data.length &&
+            (data[data.length - 1] === 0x0a || data[data.length - 1] === 0x0d)) {
+            data = data.subarray(0, data.length - 1);
+        }
+        if (/\/FlateDecode/.test(dict)) {
+            try {
+                streams.push((0, zlib_1.inflateSync)(data).toString("latin1"));
+            }
+            catch {
+                // Not a decodable text stream (e.g. an image) — skip it.
+            }
+        }
+        else if (!/\/Filter/.test(dict)) {
+            streams.push(data.toString("latin1"));
+        }
+        re.lastIndex = end;
+    }
+    return streams;
+}
+// ---------------------------------------------------------------------------
+// Content-stream text replay
+// ---------------------------------------------------------------------------
+/** Unescape a PDF literal string body: `\(`, `\)`, `\\`, `\n`, `\t`, `\ddd`. */
+function unescapePdfString(body) {
+    let out = "";
+    for (let i = 0; i < body.length; i++) {
+        const ch = body[i];
+        if (ch !== "\\") {
+            out += ch;
+            continue;
+        }
+        const next = body[++i];
+        switch (next) {
+            case "n":
+                out += "\n";
+                break;
+            case "r":
+                out += "\r";
+                break;
+            case "t":
+                out += "\t";
+                break;
+            case "b":
+                out += "\b";
+                break;
+            case "f":
+                out += "\f";
+                break;
+            case "(":
+                out += "(";
+                break;
+            case ")":
+                out += ")";
+                break;
+            case "\\":
+                out += "\\";
+                break;
+            default: {
+                // Octal escape \ddd (1–3 digits)
+                if (next >= "0" && next <= "7") {
+                    let digits = next;
+                    while (digits.length < 3 &&
+                        body[i + 1] >= "0" &&
+                        body[i + 1] <= "7") {
+                        digits += body[++i];
+                    }
+                    out += String.fromCharCode(parseInt(digits, 8));
+                }
+                else if (next !== undefined) {
+                    out += next;
+                }
+            }
+        }
+    }
+    return out;
+}
+/**
+ * Tokenise a content stream into PDF operands/operators — enough of the
+ * grammar for text extraction: literal strings, numbers, names, arrays and
+ * bare operators.
+ */
+function tokenize(stream) {
+    const tokens = [];
+    const stack = [];
+    let current = tokens;
+    let i = 0;
+    while (i < stream.length) {
+        const ch = stream[i];
+        if (/\s/.test(ch)) {
+            i++;
+            continue;
+        }
+        if (ch === "(") {
+            // Literal string — respects nesting and escapes.
+            let depth = 1;
+            let j = i + 1;
+            let body = "";
+            while (j < stream.length && depth > 0) {
+                const c = stream[j];
+                if (c === "\\") {
+                    body += c + (stream[j + 1] ?? "");
+                    j += 2;
+                    continue;
+                }
+                if (c === "(")
+                    depth++;
+                else if (c === ")") {
+                    depth--;
+                    if (depth === 0)
+                        break;
+                }
+                body += c;
+                j++;
+            }
+            current.push({ str: unescapePdfString(body) });
+            i = j + 1;
+        }
+        else if (ch === "<" && stream[i + 1] !== "<") {
+            // Hex string
+            const end = stream.indexOf(">", i);
+            const hex = stream.slice(i + 1, end === -1 ? stream.length : end).replace(/\s/g, "");
+            let text = "";
+            for (let k = 0; k + 1 < hex.length; k += 2) {
+                text += String.fromCharCode(parseInt(hex.slice(k, k + 2), 16));
+            }
+            current.push({ str: text });
+            i = (end === -1 ? stream.length : end) + 1;
+        }
+        else if (ch === "[") {
+            const arr = [];
+            current.push(arr);
+            stack.push(current);
+            current = arr;
+            i++;
+        }
+        else if (ch === "]") {
+            current = stack.pop() ?? tokens;
+            i++;
+        }
+        else if (ch === "<" && stream[i + 1] === "<") {
+            // Inline dictionary — skip to the matching >>
+            let depth = 1;
+            let j = i + 2;
+            while (j < stream.length && depth > 0) {
+                if (stream.startsWith("<<", j)) {
+                    depth++;
+                    j += 2;
+                }
+                else if (stream.startsWith(">>", j)) {
+                    depth--;
+                    j += 2;
+                }
+                else
+                    j++;
+            }
+            i = j;
+        }
+        else if (/[-+.\d]/.test(ch)) {
+            const m = /^[-+]?[\d.]+/.exec(stream.slice(i));
+            current.push(parseFloat(m[0]));
+            i += m[0].length;
+        }
+        else if (ch === "/") {
+            const m = /^\/[^\s()<>[\]{}/%]*/.exec(stream.slice(i));
+            current.push(m[0]);
+            i += m[0].length;
+        }
+        else {
+            const m = /^[^\s()<>[\]{}/%]+/.exec(stream.slice(i));
+            if (!m) {
+                i++;
+                continue;
+            }
+            current.push(m[0]);
+            i += m[0].length;
+        }
+    }
+    return tokens;
+}
+/** Replay text operators, collecting each shown string with its position. */
+function extractChunks(stream) {
+    const chunks = [];
+    const tokens = tokenize(stream);
+    let x = 0;
+    let y = 0;
+    let leading = 0;
+    const operands = [];
+    const shown = (text) => {
+        if (text !== "")
+            chunks.push({ x, y, text });
+    };
+    const num = (v) => (typeof v === "number" ? v : 0);
+    for (const token of tokens) {
+        if (typeof token !== "string" || token.startsWith("/")) {
+            operands.push(token);
+            continue;
+        }
+        switch (token) {
+            case "Tm":
+                x = num(operands[operands.length - 2]);
+                y = num(operands[operands.length - 1]);
+                break;
+            case "Td":
+                x += num(operands[operands.length - 2]);
+                y += num(operands[operands.length - 1]);
+                break;
+            case "TD":
+                leading = -num(operands[operands.length - 1]);
+                x += num(operands[operands.length - 2]);
+                y += num(operands[operands.length - 1]);
+                break;
+            case "TL":
+                leading = num(operands[operands.length - 1]);
+                break;
+            case "T*":
+                y -= leading;
+                break;
+            case "Tj":
+            case "'": {
+                if (token === "'")
+                    y -= leading;
+                const s = operands[operands.length - 1];
+                if (s && typeof s === "object" && "str" in s)
+                    shown(s.str);
+                break;
+            }
+            case "TJ": {
+                const arr = operands[operands.length - 1];
+                if (Array.isArray(arr)) {
+                    const text = arr
+                        .filter((el) => Boolean(el && typeof el === "object" && "str" in el))
+                        .map((el) => el.str)
+                        .join("");
+                    shown(text);
+                }
+                break;
+            }
+            case "BT":
+                x = 0;
+                y = 0;
+                break;
+        }
+        operands.length = 0;
+    }
+    return chunks;
+}
+// ---------------------------------------------------------------------------
+// Row clustering
+// ---------------------------------------------------------------------------
+/**
+ * Parse a PDF buffer into headers + rows: text chunks sharing a baseline form
+ * a row (top of the page first), ordered left-to-right; the first row is the
+ * header. Rows with a different cell count than the header are padded or
+ * truncated — the state machine will flag what doesn't validate.
+ */
+function parsePdfTable(buf) {
+    const chunks = contentStreams(buf).flatMap(extractChunks);
+    if (chunks.length === 0)
+        return { headers: [], rows: [] };
+    // Cluster by baseline (y), newest tolerance-merge wins.
+    const lines = [];
+    for (const chunk of chunks) {
+        const line = lines.find((l) => Math.abs(l.y - chunk.y) <= ROW_TOLERANCE);
+        if (line)
+            line.chunks.push(chunk);
+        else
+            lines.push({ y: chunk.y, chunks: [chunk] });
+    }
+    // Page order: highest y first (PDF's origin is bottom-left).
+    lines.sort((a, b) => b.y - a.y);
+    const grid = lines.map((line) => line.chunks
+        .sort((a, b) => a.x - b.x)
+        .map((c) => c.text.trim()));
+    const [headers = [], ...rows] = grid;
+    return {
+        headers,
+        rows: rows.map((row) => {
+            const cells = row.slice(0, headers.length);
+            while (cells.length < headers.length)
+                cells.push("");
+            return cells;
+        }),
+    };
+}

package/dist/import/tabular.d.ts

@@ -0,0 +1,76 @@
+/**
+ * The pure core of `rastack import`: map a tabular source (CSV/XLSX/PDF —
+ * already parsed to headers + string rows) onto a resource's fields, then run
+ * every row through the resource's **constraint state machine**
+ * (`rastack/validate`) with a dataset-level context, so `unique` sees
+ * duplicates across the file and `resolved` checks foreign keys against
+ * supplied related ids.
+ *
+ * Everything here is data-in/data-out (no filesystem), tested in
+ * `tools/test/import.spec.ts`; the parsers and the CLI are the IO shell.
+ */
+import { type FieldSpec, type FieldState, type RecordRun, type SpecRelation } from "../validate";
+/** A parsed tabular source: one header row + string cell rows. */
+export interface TabularData {
+    headers: string[];
+    rows: string[][];
+}
+/** How one source column mapped onto the resource (or didn't). */
+export interface ColumnMapping {
+    header: string;
+    field: string | null;
+}
+/** One constraint failure, located by row (0-based, excluding the header). */
+export interface ImportIssue {
+    row: number;
+    field: string;
+    gate: FieldState;
+    constraint: string;
+    message: string;
+}
+/** The full validation report for one imported file. */
+export interface ImportReport {
+    total: number;
+    valid: number;
+    invalid: number;
+    /** Source-column → field mapping (null = column ignored). */
+    columns: ColumnMapping[];
+    /** Resource fields no source column covered (they'll fail `present` if required). */
+    missingFields: string[];
+    /** Every constraint failure, in row order. */
+    issues: ImportIssue[];
+    /** The coerced records of the *valid* rows — ready to POST or seed. */
+    records: Record<string, unknown>[];
+    /** Per-row machine runs, index-aligned with the source rows. */
+    rows: RecordRun[];
+}
+/**
+ * Map source headers onto field specs by normalised name (case, `_`, `-` and
+ * spaces are insignificant). Unmatched headers are carried as ignored columns;
+ * unmatched fields are reported so a missing required column reads as a
+ * mapping problem, not just N× `present` failures.
+ */
+export declare function mapHeaders(headers: string[], specs: FieldSpec[]): {
+    columns: ColumnMapping[];
+    missingFields: string[];
+};
+/** Optional dataset-level inputs for FK resolution. */
+export interface ValidateDatasetOptions {
+    /** Known ids per `"app.model"` — engages the `resolved` gate. */
+    relatedIds?: Record<string, Set<string>>;
+    /** Custom FK resolver (wins over `relatedIds`). */
+    resolveRelation?: (relation: SpecRelation, value: unknown) => boolean;
+}
+/**
+ * Run a whole tabular source through a resource's record machine. Each row is
+ * projected onto the mapped fields and validated; the dataset context makes
+ * the `unique` gate reject in-file duplicates and (when related ids are
+ * given) the `resolved` gate reject dangling foreign keys.
+ */
+export declare function validateDataset(specs: FieldSpec[], data: TabularData, options?: ValidateDatasetOptions): ImportReport;
+/**
+ * Pull the id set out of an already-parsed related table (its `id` column,
+ * else its first column) — the ids `validateDataset` resolves foreign keys
+ * against when importing a dependent table alongside its target.
+ */
+export declare function idsFromTabular(data: TabularData): Set<string>;

package/dist/import/tabular.js

@@ -0,0 +1,98 @@
+"use strict";
+/**
+ * The pure core of `rastack import`: map a tabular source (CSV/XLSX/PDF —
+ * already parsed to headers + string rows) onto a resource's fields, then run
+ * every row through the resource's **constraint state machine**
+ * (`rastack/validate`) with a dataset-level context, so `unique` sees
+ * duplicates across the file and `resolved` checks foreign keys against
+ * supplied related ids.
+ *
+ * Everything here is data-in/data-out (no filesystem), tested in
+ * `tools/test/import.spec.ts`; the parsers and the CLI are the IO shell.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapHeaders = mapHeaders;
+exports.validateDataset = validateDataset;
+exports.idsFromTabular = idsFromTabular;
+const validate_1 = require("../validate");
+/** `"Airport Code"`, `airport_code`, `airportCode` → `airportcode`. */
+function normalise(name) {
+    return name.toLowerCase().replace(/[^a-z0-9]/g, "");
+}
+/**
+ * Map source headers onto field specs by normalised name (case, `_`, `-` and
+ * spaces are insignificant). Unmatched headers are carried as ignored columns;
+ * unmatched fields are reported so a missing required column reads as a
+ * mapping problem, not just N× `present` failures.
+ */
+function mapHeaders(headers, specs) {
+    const byNormalised = new Map(specs.map((s) => [normalise(s.name), s.name]));
+    const used = new Set();
+    const columns = headers.map((header) => {
+        const field = byNormalised.get(normalise(header)) ?? null;
+        if (field)
+            used.add(field);
+        return { header, field };
+    });
+    return {
+        columns,
+        missingFields: specs.map((s) => s.name).filter((n) => !used.has(n)),
+    };
+}
+/**
+ * Run a whole tabular source through a resource's record machine. Each row is
+ * projected onto the mapped fields and validated; the dataset context makes
+ * the `unique` gate reject in-file duplicates and (when related ids are
+ * given) the `resolved` gate reject dangling foreign keys.
+ */
+function validateDataset(specs, data, options = {}) {
+    const { columns, missingFields } = mapHeaders(data.headers, specs);
+    const machine = (0, validate_1.compileRecordMachine)(specs);
+    const ctx = (0, validate_1.createDatasetContext)(machine, options);
+    const rows = [];
+    const issues = [];
+    const records = [];
+    data.rows.forEach((cells, index) => {
+        const record = {};
+        columns.forEach((col, i) => {
+            if (col.field !== null)
+                record[col.field] = cells[i] ?? "";
+        });
+        const run = (0, validate_1.runRecordMachine)(machine, record, ctx);
+        rows.push(run);
+        if (run.state === "valid") {
+            records.push(run.values);
+        }
+        else {
+            for (const error of run.errors)
+                issues.push({ row: index, ...error });
+        }
+    });
+    return {
+        total: data.rows.length,
+        valid: records.length,
+        invalid: data.rows.length - records.length,
+        columns,
+        missingFields,
+        issues,
+        records,
+        rows,
+    };
+}
+/**
+ * Pull the id set out of an already-parsed related table (its `id` column,
+ * else its first column) — the ids `validateDataset` resolves foreign keys
+ * against when importing a dependent table alongside its target.
+ */
+function idsFromTabular(data) {
+    let idColumn = data.headers.findIndex((h) => normalise(h) === "id");
+    if (idColumn === -1)
+        idColumn = 0;
+    const ids = new Set();
+    for (const row of data.rows) {
+        const value = (row[idColumn] ?? "").trim();
+        if (value !== "")
+            ids.add(value);
+    }
+    return ids;
+}

package/dist/import/xlsx.d.ts

@@ -0,0 +1,18 @@
+/**
+ * A tiny, dependency-free XLSX reader for `rastack import`.
+ *
+ * Why hand-roll it: the CLI stays dependency-light (the same reasoning as the
+ * hand-rolled ZIP *writer* in `deploy/zip.ts`), and an `.xlsx` workbook is
+ * just a ZIP of small XML parts — the reader below understands exactly the
+ * subset a tabular import needs: the ZIP central directory (STORE and DEFLATE
+ * entries, inflated via node's zlib), `xl/sharedStrings.xml`, and the cells of
+ * the first worksheet. Formulas are read through their cached `<v>` results;
+ * merged cells, styles and charts are ignored — an import wants values.
+ */
+import type { TabularData } from "./tabular";
+/**
+ * Parse an `.xlsx` workbook buffer into headers + rows (first worksheet, first
+ * row as the header). Cell values come back as strings — the constraint state
+ * machine's `typed` gate owns coercion, same as for CSV.
+ */
+export declare function parseXlsx(buf: Buffer): TabularData;