rastack 0.0.46 → 0.0.48

package/src/import/pdf.ts

@@ -0,0 +1,304 @@
+/**
+ * A tiny, dependency-free PDF *table* extractor for `rastack import`.
+ *
+ * PDFs carry no table structure — only positioned text — so this is
+ * deliberately a best-effort extractor for the common case of a machine-
+ * generated tabular report: it decodes every content stream (FlateDecode via
+ * node's zlib, plus raw streams), replays the text-positioning operators
+ * (`Tm`/`Td`/`TD`/`T*`) to give each shown string (`Tj`/`TJ`/`'`) an (x, y),
+ * clusters strings that share a baseline into rows, and orders each row's
+ * cells left-to-right. The first row becomes the header.
+ *
+ * Anything a text-line model can't represent — merged cells, wrapped cell
+ * text, scanned images — won't survive; for those, export the source data to
+ * CSV/XLSX instead. Every extracted value still goes through the constraint
+ * state machine, so a misread cell is *rejected with a located error*, never
+ * silently imported.
+ */
+
+import { inflateSync } from "zlib";
+import type { TabularData } from "./tabular";
+
+interface TextChunk {
+  x: number;
+  y: number;
+  text: string;
+}
+
+/** How close two baselines must be (in text-space units) to share a row. */
+const ROW_TOLERANCE = 2;
+
+// ---------------------------------------------------------------------------
+// Stream extraction
+// ---------------------------------------------------------------------------
+
+/** Decode every content stream in the document to latin1 text. */
+function contentStreams(buf: Buffer): string[] {
+  const raw = buf.toString("latin1");
+  const streams: string[] = [];
+  const re = /stream\r?\n/g;
+  let m: RegExpExecArray | null;
+
+  while ((m = re.exec(raw)) !== null) {
+    const start = m.index + m[0].length;
+    const end = raw.indexOf("endstream", start);
+    if (end === -1) break;
+
+    // The stream dictionary sits just before the `stream` keyword.
+    const dictStart = raw.lastIndexOf("<<", m.index);
+    const dict = dictStart === -1 ? "" : raw.slice(dictStart, m.index);
+
+    let data = buf.subarray(start, end);
+    // Trim the EOL PDF writers place before `endstream`.
+    while (
+      data.length &&
+      (data[data.length - 1] === 0x0a || data[data.length - 1] === 0x0d)
+    ) {
+      data = data.subarray(0, data.length - 1);
+    }
+
+    if (/\/FlateDecode/.test(dict)) {
+      try {
+        streams.push(inflateSync(data).toString("latin1"));
+      } catch {
+        // Not a decodable text stream (e.g. an image) — skip it.
+      }
+    } else if (!/\/Filter/.test(dict)) {
+      streams.push(data.toString("latin1"));
+    }
+    re.lastIndex = end;
+  }
+  return streams;
+}
+
+// ---------------------------------------------------------------------------
+// Content-stream text replay
+// ---------------------------------------------------------------------------
+
+/** Unescape a PDF literal string body: `\(`, `\)`, `\\`, `\n`, `\t`, `\ddd`. */
+function unescapePdfString(body: string): string {
+  let out = "";
+  for (let i = 0; i < body.length; i++) {
+    const ch = body[i];
+    if (ch !== "\\") {
+      out += ch;
+      continue;
+    }
+    const next = body[++i];
+    switch (next) {
+      case "n": out += "\n"; break;
+      case "r": out += "\r"; break;
+      case "t": out += "\t"; break;
+      case "b": out += "\b"; break;
+      case "f": out += "\f"; break;
+      case "(": out += "("; break;
+      case ")": out += ")"; break;
+      case "\\": out += "\\"; break;
+      default: {
+        // Octal escape \ddd (1–3 digits)
+        if (next >= "0" && next <= "7") {
+          let digits = next;
+          while (
+            digits.length < 3 &&
+            body[i + 1] >= "0" &&
+            body[i + 1] <= "7"
+          ) {
+            digits += body[++i];
+          }
+          out += String.fromCharCode(parseInt(digits, 8));
+        } else if (next !== undefined) {
+          out += next;
+        }
+      }
+    }
+  }
+  return out;
+}
+
+/**
+ * Tokenise a content stream into PDF operands/operators — enough of the
+ * grammar for text extraction: literal strings, numbers, names, arrays and
+ * bare operators.
+ */
+function tokenize(stream: string): Array<string | number | { str: string } | unknown[]> {
+  const tokens: Array<string | number | { str: string } | unknown[]> = [];
+  const stack: Array<Array<string | number | { str: string } | unknown[]>> = [];
+  let current = tokens;
+  let i = 0;
+
+  while (i < stream.length) {
+    const ch = stream[i];
+    if (/\s/.test(ch)) { i++; continue; }
+
+    if (ch === "(") {
+      // Literal string — respects nesting and escapes.
+      let depth = 1;
+      let j = i + 1;
+      let body = "";
+      while (j < stream.length && depth > 0) {
+        const c = stream[j];
+        if (c === "\\") { body += c + (stream[j + 1] ?? ""); j += 2; continue; }
+        if (c === "(") depth++;
+        else if (c === ")") { depth--; if (depth === 0) break; }
+        body += c;
+        j++;
+      }
+      current.push({ str: unescapePdfString(body) });
+      i = j + 1;
+    } else if (ch === "<" && stream[i + 1] !== "<") {
+      // Hex string
+      const end = stream.indexOf(">", i);
+      const hex = stream.slice(i + 1, end === -1 ? stream.length : end).replace(/\s/g, "");
+      let text = "";
+      for (let k = 0; k + 1 < hex.length; k += 2) {
+        text += String.fromCharCode(parseInt(hex.slice(k, k + 2), 16));
+      }
+      current.push({ str: text });
+      i = (end === -1 ? stream.length : end) + 1;
+    } else if (ch === "[") {
+      const arr: unknown[] = [];
+      current.push(arr);
+      stack.push(current);
+      current = arr as typeof current;
+      i++;
+    } else if (ch === "]") {
+      current = stack.pop() ?? tokens;
+      i++;
+    } else if (ch === "<" && stream[i + 1] === "<") {
+      // Inline dictionary — skip to the matching >>
+      let depth = 1;
+      let j = i + 2;
+      while (j < stream.length && depth > 0) {
+        if (stream.startsWith("<<", j)) { depth++; j += 2; }
+        else if (stream.startsWith(">>", j)) { depth--; j += 2; }
+        else j++;
+      }
+      i = j;
+    } else if (/[-+.\d]/.test(ch)) {
+      const m = /^[-+]?[\d.]+/.exec(stream.slice(i))!;
+      current.push(parseFloat(m[0]));
+      i += m[0].length;
+    } else if (ch === "/") {
+      const m = /^\/[^\s()<>[\]{}/%]*/.exec(stream.slice(i))!;
+      current.push(m[0]);
+      i += m[0].length;
+    } else {
+      const m = /^[^\s()<>[\]{}/%]+/.exec(stream.slice(i));
+      if (!m) { i++; continue; }
+      current.push(m[0]);
+      i += m[0].length;
+    }
+  }
+  return tokens;
+}
+
+/** Replay text operators, collecting each shown string with its position. */
+function extractChunks(stream: string): TextChunk[] {
+  const chunks: TextChunk[] = [];
+  const tokens = tokenize(stream);
+
+  let x = 0;
+  let y = 0;
+  let leading = 0;
+  const operands: Array<string | number | { str: string } | unknown[]> = [];
+
+  const shown = (text: string) => {
+    if (text !== "") chunks.push({ x, y, text });
+  };
+  const num = (v: unknown): number => (typeof v === "number" ? v : 0);
+
+  for (const token of tokens) {
+    if (typeof token !== "string" || token.startsWith("/")) {
+      operands.push(token);
+      continue;
+    }
+    switch (token) {
+      case "Tm":
+        x = num(operands[operands.length - 2]);
+        y = num(operands[operands.length - 1]);
+        break;
+      case "Td":
+        x += num(operands[operands.length - 2]);
+        y += num(operands[operands.length - 1]);
+        break;
+      case "TD":
+        leading = -num(operands[operands.length - 1]);
+        x += num(operands[operands.length - 2]);
+        y += num(operands[operands.length - 1]);
+        break;
+      case "TL":
+        leading = num(operands[operands.length - 1]);
+        break;
+      case "T*":
+        y -= leading;
+        break;
+      case "Tj":
+      case "'": {
+        if (token === "'") y -= leading;
+        const s = operands[operands.length - 1];
+        if (s && typeof s === "object" && "str" in s) shown((s as { str: string }).str);
+        break;
+      }
+      case "TJ": {
+        const arr = operands[operands.length - 1];
+        if (Array.isArray(arr)) {
+          const text = arr
+            .filter((el): el is { str: string } =>
+              Boolean(el && typeof el === "object" && "str" in el),
+            )
+            .map((el) => el.str)
+            .join("");
+          shown(text);
+        }
+        break;
+      }
+      case "BT":
+        x = 0;
+        y = 0;
+        break;
+    }
+    operands.length = 0;
+  }
+  return chunks;
+}
+
+// ---------------------------------------------------------------------------
+// Row clustering
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse a PDF buffer into headers + rows: text chunks sharing a baseline form
+ * a row (top of the page first), ordered left-to-right; the first row is the
+ * header. Rows with a different cell count than the header are padded or
+ * truncated — the state machine will flag what doesn't validate.
+ */
+export function parsePdfTable(buf: Buffer): TabularData {
+  const chunks = contentStreams(buf).flatMap(extractChunks);
+  if (chunks.length === 0) return { headers: [], rows: [] };
+
+  // Cluster by baseline (y), newest tolerance-merge wins.
+  const lines: Array<{ y: number; chunks: TextChunk[] }> = [];
+  for (const chunk of chunks) {
+    const line = lines.find((l) => Math.abs(l.y - chunk.y) <= ROW_TOLERANCE);
+    if (line) line.chunks.push(chunk);
+    else lines.push({ y: chunk.y, chunks: [chunk] });
+  }
+
+  // Page order: highest y first (PDF's origin is bottom-left).
+  lines.sort((a, b) => b.y - a.y);
+  const grid = lines.map((line) =>
+    line.chunks
+      .sort((a, b) => a.x - b.x)
+      .map((c) => c.text.trim()),
+  );
+
+  const [headers = [], ...rows] = grid;
+  return {
+    headers,
+    rows: rows.map((row) => {
+      const cells = row.slice(0, headers.length);
+      while (cells.length < headers.length) cells.push("");
+      return cells;
+    }),
+  };
+}

package/src/import/tabular.ts

@@ -0,0 +1,159 @@
+/**
+ * The pure core of `rastack import`: map a tabular source (CSV/XLSX/PDF —
+ * already parsed to headers + string rows) onto a resource's fields, then run
+ * every row through the resource's **constraint state machine**
+ * (`rastack/validate`) with a dataset-level context, so `unique` sees
+ * duplicates across the file and `resolved` checks foreign keys against
+ * supplied related ids.
+ *
+ * Everything here is data-in/data-out (no filesystem), tested in
+ * `tools/test/import.spec.ts`; the parsers and the CLI are the IO shell.
+ */
+
+import {
+  compileRecordMachine,
+  createDatasetContext,
+  runRecordMachine,
+  type FieldSpec,
+  type FieldState,
+  type RecordRun,
+  type SpecRelation,
+} from "../validate";
+
+/** A parsed tabular source: one header row + string cell rows. */
+export interface TabularData {
+  headers: string[];
+  rows: string[][];
+}
+
+/** How one source column mapped onto the resource (or didn't). */
+export interface ColumnMapping {
+  header: string;
+  field: string | null;
+}
+
+/** One constraint failure, located by row (0-based, excluding the header). */
+export interface ImportIssue {
+  row: number;
+  field: string;
+  gate: FieldState;
+  constraint: string;
+  message: string;
+}
+
+/** The full validation report for one imported file. */
+export interface ImportReport {
+  total: number;
+  valid: number;
+  invalid: number;
+  /** Source-column → field mapping (null = column ignored). */
+  columns: ColumnMapping[];
+  /** Resource fields no source column covered (they'll fail `present` if required). */
+  missingFields: string[];
+  /** Every constraint failure, in row order. */
+  issues: ImportIssue[];
+  /** The coerced records of the *valid* rows — ready to POST or seed. */
+  records: Record<string, unknown>[];
+  /** Per-row machine runs, index-aligned with the source rows. */
+  rows: RecordRun[];
+}
+
+/** `"Airport Code"`, `airport_code`, `airportCode` → `airportcode`. */
+function normalise(name: string): string {
+  return name.toLowerCase().replace(/[^a-z0-9]/g, "");
+}
+
+/**
+ * Map source headers onto field specs by normalised name (case, `_`, `-` and
+ * spaces are insignificant). Unmatched headers are carried as ignored columns;
+ * unmatched fields are reported so a missing required column reads as a
+ * mapping problem, not just N× `present` failures.
+ */
+export function mapHeaders(
+  headers: string[],
+  specs: FieldSpec[],
+): { columns: ColumnMapping[]; missingFields: string[] } {
+  const byNormalised = new Map(specs.map((s) => [normalise(s.name), s.name]));
+  const used = new Set<string>();
+
+  const columns = headers.map((header) => {
+    const field = byNormalised.get(normalise(header)) ?? null;
+    if (field) used.add(field);
+    return { header, field };
+  });
+
+  return {
+    columns,
+    missingFields: specs.map((s) => s.name).filter((n) => !used.has(n)),
+  };
+}
+
+/** Optional dataset-level inputs for FK resolution. */
+export interface ValidateDatasetOptions {
+  /** Known ids per `"app.model"` — engages the `resolved` gate. */
+  relatedIds?: Record<string, Set<string>>;
+  /** Custom FK resolver (wins over `relatedIds`). */
+  resolveRelation?: (relation: SpecRelation, value: unknown) => boolean;
+}
+
+/**
+ * Run a whole tabular source through a resource's record machine. Each row is
+ * projected onto the mapped fields and validated; the dataset context makes
+ * the `unique` gate reject in-file duplicates and (when related ids are
+ * given) the `resolved` gate reject dangling foreign keys.
+ */
+export function validateDataset(
+  specs: FieldSpec[],
+  data: TabularData,
+  options: ValidateDatasetOptions = {},
+): ImportReport {
+  const { columns, missingFields } = mapHeaders(data.headers, specs);
+  const machine = compileRecordMachine(specs);
+  const ctx = createDatasetContext(machine, options);
+
+  const rows: RecordRun[] = [];
+  const issues: ImportIssue[] = [];
+  const records: Record<string, unknown>[] = [];
+
+  data.rows.forEach((cells, index) => {
+    const record: Record<string, unknown> = {};
+    columns.forEach((col, i) => {
+      if (col.field !== null) record[col.field] = cells[i] ?? "";
+    });
+
+    const run = runRecordMachine(machine, record, ctx);
+    rows.push(run);
+    if (run.state === "valid") {
+      records.push(run.values);
+    } else {
+      for (const error of run.errors) issues.push({ row: index, ...error });
+    }
+  });
+
+  return {
+    total: data.rows.length,
+    valid: records.length,
+    invalid: data.rows.length - records.length,
+    columns,
+    missingFields,
+    issues,
+    records,
+    rows,
+  };
+}
+
+/**
+ * Pull the id set out of an already-parsed related table (its `id` column,
+ * else its first column) — the ids `validateDataset` resolves foreign keys
+ * against when importing a dependent table alongside its target.
+ */
+export function idsFromTabular(data: TabularData): Set<string> {
+  let idColumn = data.headers.findIndex((h) => normalise(h) === "id");
+  if (idColumn === -1) idColumn = 0;
+  const ids = new Set<string>();
+  for (const row of data.rows) {
+    const value = (row[idColumn] ?? "").trim();
+    if (value !== "") ids.add(value);
+  }
+  return ids;
+}

package/src/import/xlsx.ts

@@ -0,0 +1,186 @@
+/**
+ * A tiny, dependency-free XLSX reader for `rastack import`.
+ *
+ * Why hand-roll it: the CLI stays dependency-light (the same reasoning as the
+ * hand-rolled ZIP *writer* in `deploy/zip.ts`), and an `.xlsx` workbook is
+ * just a ZIP of small XML parts — the reader below understands exactly the
+ * subset a tabular import needs: the ZIP central directory (STORE and DEFLATE
+ * entries, inflated via node's zlib), `xl/sharedStrings.xml`, and the cells of
+ * the first worksheet. Formulas are read through their cached `<v>` results;
+ * merged cells, styles and charts are ignored — an import wants values.
+ */
+
+import { inflateRawSync } from "zlib";
+import type { TabularData } from "./tabular";
+
+// ---------------------------------------------------------------------------
+// ZIP reading
+// ---------------------------------------------------------------------------
+
+interface ZipEntryRecord {
+  name: string;
+  method: number;
+  compressedSize: number;
+  localHeaderOffset: number;
+}
+
+/** Locate the end-of-central-directory record (scan back over the comment). */
+function findEocd(buf: Buffer): number {
+  for (let i = buf.length - 22; i >= 0; i--) {
+    if (buf.readUInt32LE(i) === 0x06054b50) return i;
+  }
+  throw new Error("not a ZIP archive (no end-of-central-directory record)");
+}
+
+function readCentralDirectory(buf: Buffer): ZipEntryRecord[] {
+  const eocd = findEocd(buf);
+  const count = buf.readUInt16LE(eocd + 10);
+  let offset = buf.readUInt32LE(eocd + 16);
+
+  const entries: ZipEntryRecord[] = [];
+  for (let i = 0; i < count; i++) {
+    if (buf.readUInt32LE(offset) !== 0x02014b50) {
+      throw new Error("corrupt ZIP central directory");
+    }
+    const method = buf.readUInt16LE(offset + 10);
+    const compressedSize = buf.readUInt32LE(offset + 20);
+    const nameLength = buf.readUInt16LE(offset + 28);
+    const extraLength = buf.readUInt16LE(offset + 30);
+    const commentLength = buf.readUInt16LE(offset + 32);
+    const localHeaderOffset = buf.readUInt32LE(offset + 42);
+    const name = buf
+      .subarray(offset + 46, offset + 46 + nameLength)
+      .toString("utf-8");
+    entries.push({ name, method, compressedSize, localHeaderOffset });
+    offset += 46 + nameLength + extraLength + commentLength;
+  }
+  return entries;
+}
+
+function readEntry(buf: Buffer, entry: ZipEntryRecord): Buffer {
+  const at = entry.localHeaderOffset;
+  if (buf.readUInt32LE(at) !== 0x04034b50) {
+    throw new Error(`corrupt ZIP local header for ${entry.name}`);
+  }
+  const nameLength = buf.readUInt16LE(at + 26);
+  const extraLength = buf.readUInt16LE(at + 28);
+  const start = at + 30 + nameLength + extraLength;
+  const data = buf.subarray(start, start + entry.compressedSize);
+
+  if (entry.method === 0) return Buffer.from(data); // STORE
+  if (entry.method === 8) return inflateRawSync(data); // DEFLATE
+  throw new Error(`unsupported ZIP compression method ${entry.method}`);
+}
+
+// ---------------------------------------------------------------------------
+// Minimal XML helpers (the sheet parts are machine-written, flat XML)
+// ---------------------------------------------------------------------------
+
+function unescapeXml(text: string): string {
+  return text
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)))
+    .replace(/&amp;/g, "&");
+}
+
+/** All matches of `<tag ...>inner</tag>` (non-nested — true of sheet XML). */
+function elements(xml: string, tag: string): string[] {
+  const re = new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)</${tag}>`, "g");
+  const out: string[] = [];
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(xml)) !== null) out.push(m[1]);
+  return out;
+}
+
+/** `"BC7"` → 0-based column index (54). */
+function columnIndex(cellRef: string): number {
+  let index = 0;
+  for (const ch of cellRef) {
+    if (ch < "A" || ch > "Z") break;
+    index = index * 26 + (ch.charCodeAt(0) - 64);
+  }
+  return index - 1;
+}
+
+// ---------------------------------------------------------------------------
+// Workbook parsing
+// ---------------------------------------------------------------------------
+
+function parseSharedStrings(xml: string | undefined): string[] {
+  if (!xml) return [];
+  // Each <si> may hold one <t> or several rich-text runs — concatenate them.
+  return elements(xml, "si").map((si) =>
+    elements(si, "t").map(unescapeXml).join(""),
+  );
+}
+
+/**
+ * Parse an `.xlsx` workbook buffer into headers + rows (first worksheet, first
+ * row as the header). Cell values come back as strings — the constraint state
+ * machine's `typed` gate owns coercion, same as for CSV.
+ */
+export function parseXlsx(buf: Buffer): TabularData {
+  const entries = readCentralDirectory(buf);
+  const byName = new Map(entries.map((e) => [e.name, e]));
+
+  const sheetEntry =
+    byName.get("xl/worksheets/sheet1.xml") ??
+    entries
+      .filter((e) => /^xl\/worksheets\/[^/]+\.xml$/.test(e.name))
+      .sort((a, b) => a.name.localeCompare(b.name))[0];
+  if (!sheetEntry) throw new Error("no worksheet found in workbook");
+
+  const sharedEntry = byName.get("xl/sharedStrings.xml");
+  const shared = parseSharedStrings(
+    sharedEntry ? readEntry(buf, sharedEntry).toString("utf-8") : undefined,
+  );
+
+  const sheetXml = readEntry(buf, sheetEntry).toString("utf-8");
+
+  const cellRe =
+    /<c(?:\s([^>]*?))?(?:\/>|>([\s\S]*?)<\/c>)/g;
+  const attr = (attrs: string | undefined, name: string): string | undefined =>
+    attrs?.match(new RegExp(`${name}="([^"]*)"`))?.[1];
+
+  const grid: string[][] = [];
+  for (const rowXml of elements(sheetXml, "row")) {
+    const cells: string[] = [];
+    let m: RegExpExecArray | null;
+    let fallbackColumn = 0;
+    cellRe.lastIndex = 0;
+    while ((m = cellRe.exec(rowXml)) !== null) {
+      const [, attrs, inner = ""] = m;
+      const ref = attr(attrs, "r");
+      const column = ref ? columnIndex(ref) : fallbackColumn;
+      fallbackColumn = column + 1;
+
+      const type = attr(attrs, "t");
+      let value = "";
+      if (type === "inlineStr") {
+        value = elements(inner, "t").map(unescapeXml).join("");
+      } else {
+        const v = elements(inner, "v")[0];
+        value = v === undefined ? "" : unescapeXml(v);
+        if (type === "s") value = shared[Number(value)] ?? "";
+      }
+
+      while (cells.length < column) cells.push("");
+      cells[column] = value;
+    }
+    grid.push(cells);
+  }
+
+  const [headers = [], ...rows] = grid;
+  const width = headers.length;
+  return {
+    headers,
+    rows: rows.map((r) => {
+      const row = r.slice(0, Math.max(width, r.length));
+      while (row.length < width) row.push("");
+      return row;
+    }),
+  };
+}