rastack 0.0.45 → 0.0.47

package/test/import.spec.ts

@@ -0,0 +1,241 @@
+/**
+ * The external-data import pipeline: format parsers (CSV via `parseCsv`, the
+ * dependency-free XLSX and PDF readers) and the pure validation core that
+ * runs a whole tabular source through a resource's constraint state machine.
+ *
+ * The XLSX fixture is built with the repo's own `zipStore` (deploy/zip.ts) —
+ * the reader handles STORE entries, which is exactly what the writer emits.
+ * The PDF fixtures are hand-written minimal documents (one raw content
+ * stream, one FlateDecode) exercising the text operators the extractor
+ * replays.
+ */
+
+import { deflateSync } from "zlib";
+import { zipStore } from "../src/deploy/zip";
+import {
+  formatOf,
+  idsFromTabular,
+  mapHeaders,
+  parsePdfTable,
+  parseTabular,
+  parseXlsx,
+  validateDataset,
+} from "../src/import";
+import { specsFromResource } from "../src/validate";
+import type { FieldModel } from "../src/compile/model";
+
+// The airline fixtures, as `rastack compile` writes them to the manifest.
+const AIRPORT_FIELDS: FieldModel[] = [
+  { name: "code", type: "string", maxLength: 3, unique: true },
+  { name: "name", type: "string", maxLength: 120 },
+];
+const TERMINAL_FIELDS: FieldModel[] = [
+  {
+    name: "airport",
+    type: "fk",
+    relation: { app: "airports", model: "airport" },
+  },
+  { name: "label", type: "string", maxLength: 20 },
+];
+
+describe("header mapping", () => {
+  it("matches case-, space- and separator-insensitively", () => {
+    const specs = specsFromResource({
+      fields: [
+        { name: "owner_id", type: "int" },
+        { name: "code", type: "string" },
+      ] as FieldModel[],
+    });
+    const { columns, missingFields } = mapHeaders(
+      ["Owner ID", "CODE", "junk"],
+      specs,
+    );
+    expect(columns).toEqual([
+      { header: "Owner ID", field: "owner_id" },
+      { header: "CODE", field: "code" },
+      { header: "junk", field: null },
+    ]);
+    expect(missingFields).toEqual([]);
+  });
+
+  it("reports fields no column covers", () => {
+    const specs = specsFromResource({ fields: AIRPORT_FIELDS });
+    const { missingFields } = mapHeaders(["code"], specs);
+    expect(missingFields).toEqual(["name"]);
+  });
+});
+
+describe("dataset validation through the state machine", () => {
+  const airportSpecs = specsFromResource({ fields: AIRPORT_FIELDS });
+
+  it("validates a CSV row-by-row, locating each failed gate", () => {
+    const csv = Buffer.from(
+      "code,name\n" +
+        "LHR,Heathrow\n" +
+        "LHRX,Too Long A Code\n" +
+        "LHR,Duplicate Of Row One\n" +
+        ",Missing Code\n",
+    );
+    const report = validateDataset(airportSpecs, parseTabular("csv", csv));
+
+    expect(report.total).toBe(4);
+    expect(report.valid).toBe(1);
+    expect(report.invalid).toBe(3);
+    expect(report.records).toEqual([{ code: "LHR", name: "Heathrow" }]);
+    expect(report.issues).toMatchObject([
+      { row: 1, field: "code", gate: "bounded" },
+      { row: 2, field: "code", gate: "unique" },
+      { row: 3, field: "code", gate: "present" },
+    ]);
+  });
+
+  it("resolves foreign keys against a related file's ids", () => {
+    const airports = parseTabular(
+      "csv",
+      Buffer.from("id,code,name\n1,LHR,Heathrow\n2,JFK,Kennedy\n"),
+    );
+    const terminals = parseTabular(
+      "csv",
+      Buffer.from("airport,label\n1,Terminal 5\n9,Ghost Terminal\n"),
+    );
+
+    const report = validateDataset(
+      specsFromResource({ fields: TERMINAL_FIELDS }),
+      terminals,
+      { relatedIds: { "airports.airport": idsFromTabular(airports) } },
+    );
+
+    expect(report.valid).toBe(1);
+    // FK ids coerce to numbers on the way through the `typed` gate.
+    expect(report.records).toEqual([{ airport: 1, label: "Terminal 5" }]);
+    expect(report.issues).toMatchObject([
+      { row: 1, field: "airport", gate: "resolved" },
+    ]);
+  });
+
+  it("infers the source format from the file name", () => {
+    expect(formatOf("airports.csv")).toBe("csv");
+    expect(formatOf("Airports.XLSX")).toBe("xlsx");
+    expect(formatOf("report.pdf")).toBe("pdf");
+    expect(formatOf("notes.txt")).toBeNull();
+  });
+});
+
+describe("XLSX reader", () => {
+  const SHARED_STRINGS =
+    `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>` +
+    `<sst xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" count="4" uniqueCount="4">` +
+    `<si><t>code</t></si><si><t>name</t></si><si><t>LHR</t></si>` +
+    `<si><r><t>Heath</t></r><r><t>row &amp; Co</t></r></si>` +
+    `</sst>`;
+
+  const SHEET =
+    `<?xml version="1.0" encoding="UTF-8" standalone="yes"?>` +
+    `<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>` +
+    `<row r="1"><c r="A1" t="s"><v>0</v></c><c r="B1" t="s"><v>1</v></c></row>` +
+    `<row r="2"><c r="A2" t="s"><v>2</v></c><c r="B2" t="s"><v>3</v></c></row>` +
+    `<row r="3"><c r="A3" t="inlineStr"><is><t>JFK</t></is></c><c r="B3"><v>42</v></c></row>` +
+    // A row with only column B populated — column A must come back as "".
+    `<row r="4"><c r="B4" t="inlineStr"><is><t>Orphan</t></is></c></row>` +
+    `</sheetData></worksheet>`;
+
+  const workbook = () =>
+    zipStore([
+      { name: "xl/sharedStrings.xml", data: Buffer.from(SHARED_STRINGS) },
+      { name: "xl/worksheets/sheet1.xml", data: Buffer.from(SHEET) },
+    ]);
+
+  it("reads shared strings (incl. rich-text runs), inline strings and numbers", () => {
+    const { headers, rows } = parseXlsx(workbook());
+    expect(headers).toEqual(["code", "name"]);
+    expect(rows).toEqual([
+      ["LHR", "Heathrow & Co"],
+      ["JFK", "42"],
+      ["", "Orphan"],
+    ]);
+  });
+
+  it("feeds straight into the state machine like any other source", () => {
+    const report = validateDataset(
+      specsFromResource({ fields: AIRPORT_FIELDS }),
+      parseXlsx(workbook()),
+    );
+    expect(report.valid).toBe(2);
+    // Row 3 has no `code` — required, so it halts at `present`.
+    expect(report.issues).toMatchObject([
+      { row: 2, field: "code", gate: "present" },
+    ]);
+  });
+});
+
+describe("PDF table extractor", () => {
+  const CONTENT =
+    "BT\n" +
+    "/F1 12 Tf\n" +
+    "72 720 Td\n" +
+    "(code) Tj\n" +
+    "150 0 Td\n" +
+    "(name) Tj\n" +
+    "ET\n" +
+    "BT\n" +
+    "72 700 Td\n" +
+    "(LHR) Tj\n" +
+    "150 0 Td\n" +
+    "(Heathrow \\(west\\)) Tj\n" +
+    "ET\n" +
+    "BT\n" +
+    "72 680 Td\n" +
+    "[(J) 20 (FK)] TJ\n" +
+    "150 0 Td\n" +
+    "(Kennedy) Tj\n" +
+    "ET\n";
+
+  function rawPdf(content: string): Buffer {
+    return Buffer.from(
+      `%PDF-1.4\n1 0 obj\n<< /Length ${content.length} >>\nstream\n` +
+        content +
+        `\nendstream\nendobj\n%%EOF\n`,
+      "latin1",
+    );
+  }
+
+  function flatePdf(content: string): Buffer {
+    const compressed = deflateSync(Buffer.from(content, "latin1"));
+    return Buffer.concat([
+      Buffer.from(
+        `%PDF-1.4\n1 0 obj\n<< /Length ${compressed.length} /Filter /FlateDecode >>\nstream\n`,
+        "latin1",
+      ),
+      compressed,
+      Buffer.from(`\nendstream\nendobj\n%%EOF\n`, "latin1"),
+    ]);
+  }
+
+  const expected = {
+    headers: ["code", "name"],
+    rows: [
+      ["LHR", "Heathrow (west)"],
+      ["JFK", "Kennedy"],
+    ],
+  };
+
+  it("clusters positioned text into rows (raw stream)", () => {
+    expect(parsePdfTable(rawPdf(CONTENT))).toEqual(expected);
+  });
+
+  it("decodes FlateDecode content streams", () => {
+    expect(parsePdfTable(flatePdf(CONTENT))).toEqual(expected);
+  });
+
+  it("feeds straight into the state machine like any other source", () => {
+    const report = validateDataset(
+      specsFromResource({ fields: AIRPORT_FIELDS }),
+      parsePdfTable(rawPdf(CONTENT)),
+    );
+    expect(report.valid).toBe(2);
+    expect(report.records).toEqual([
+      { code: "LHR", name: "Heathrow (west)" },
+      { code: "JFK", name: "Kennedy" },
+    ]);
+  });
+});

package/test/validate.spec.ts

@@ -0,0 +1,319 @@
+/**
+ * The constraint state machine (`rastack/validate`): compilation from both
+ * adapters (manifest field, write JSON Schema property), gate-by-gate
+ * execution, dataset-level `unique`/`resolved` gates, record machines, and
+ * the form-hook integration (`getFieldMachine`).
+ */
+
+import {
+  compileFieldMachine,
+  compileRecordMachine,
+  createDatasetContext,
+  describeMachine,
+  runFieldMachine,
+  runRecordMachine,
+  specFromManifestField,
+  specFromSchemaProperty,
+  specsFromResource,
+  type FieldSpec,
+} from "../src/validate";
+import { getFieldMachine } from "../hooks/form/structure";
+import type { FieldModel } from "../src/compile/model";
+
+// The Airport.code field, as `rastack compile` writes it to the manifest.
+const CODE_FIELD: FieldModel = {
+  name: "code",
+  type: "string",
+  maxLength: 3,
+  unique: true,
+};
+
+describe("constraint state machine — compilation", () => {
+  it("builds one gate per constraint, in canonical order", () => {
+    const machine = compileFieldMachine(specFromManifestField(CODE_FIELD));
+    expect(machine.states).toEqual([
+      "pristine",
+      "present",
+      "typed",
+      "bounded",
+      "unique",
+      "valid",
+    ]);
+  });
+
+  it("emits no gate for an absent constraint", () => {
+    const machine = compileFieldMachine(
+      specFromManifestField({ name: "note", type: "string", null: true }),
+    );
+    expect(machine.states).toEqual(["pristine", "present", "typed", "valid"]);
+  });
+
+  it("describes the constraint chain for UI hints", () => {
+    const machine = compileFieldMachine(specFromManifestField(CODE_FIELD));
+    expect(describeMachine(machine)).toEqual([
+      "required",
+      "string",
+      "maxLength ≤ 3",
+      "unique",
+    ]);
+  });
+});
+
+describe("constraint state machine — execution", () => {
+  const codeMachine = compileFieldMachine(specFromManifestField(CODE_FIELD));
+
+  it("accepts a conforming value, recording the full path", () => {
+    const run = runFieldMachine(codeMachine, "LHR");
+    expect(run.state).toBe("valid");
+    expect(run.value).toBe("LHR");
+    expect(run.path).toEqual([
+      "pristine",
+      "present",
+      "typed",
+      "bounded",
+      "unique",
+      "valid",
+    ]);
+  });
+
+  it("halts at `present` for a missing required value", () => {
+    const run = runFieldMachine(codeMachine, "");
+    expect(run.state).toBe("invalid");
+    expect(run.failed?.gate).toBe("present");
+    expect(run.path).toEqual(["pristine", "invalid"]);
+  });
+
+  it("halts at `bounded` when maxLength is exceeded", () => {
+    const run = runFieldMachine(codeMachine, "LHRX");
+    expect(run.state).toBe("invalid");
+    expect(run.failed).toMatchObject({
+      gate: "bounded",
+      constraint: "maxLength ≤ 3",
+    });
+    // The path shows exactly how far the value got.
+    expect(run.path).toEqual(["pristine", "present", "typed", "invalid"]);
+  });
+
+  it("short-circuits an empty optional field to valid with its default", () => {
+    const machine = compileFieldMachine(
+      specFromManifestField({
+        name: "status",
+        type: "string",
+        maxLength: 20,
+        default: "scheduled",
+      }),
+    );
+    const run = runFieldMachine(machine, "");
+    expect(run.state).toBe("valid");
+    expect(run.value).toBe("scheduled");
+    expect(run.path).toEqual(["pristine", "present", "valid"]);
+  });
+
+  it("coerces at the `typed` gate (int strings become ints)", () => {
+    const machine = compileFieldMachine(
+      specFromManifestField({ name: "owner_id", type: "int" }),
+    );
+    expect(runFieldMachine(machine, "12").value).toBe(12);
+    expect(runFieldMachine(machine, 12).value).toBe(12);
+
+    const bad = runFieldMachine(machine, "12.5");
+    expect(bad.state).toBe("invalid");
+    expect(bad.failed?.gate).toBe("typed");
+  });
+
+  it("coerces booleans and datetimes", () => {
+    const bool = compileFieldMachine(
+      specFromManifestField({ name: "active", type: "bool" }),
+    );
+    expect(runFieldMachine(bool, "yes").value).toBe(true);
+    expect(runFieldMachine(bool, "0").value).toBe(false);
+    expect(runFieldMachine(bool, "maybe").state).toBe("invalid");
+
+    const dt = compileFieldMachine(
+      specFromManifestField({ name: "departs_at", type: "datetime" }),
+    );
+    expect(runFieldMachine(dt, "2026-07-13T10:30:00Z").state).toBe("valid");
+    expect(runFieldMachine(dt, "not a date").state).toBe("invalid");
+  });
+
+  it("enforces enum membership at the `member` gate", () => {
+    const spec: FieldSpec = {
+      name: "status",
+      type: "string",
+      required: true,
+      nullable: false,
+      options: ["scheduled", "boarding", "departed"],
+    };
+    const machine = compileFieldMachine(spec);
+    expect(runFieldMachine(machine, "boarding").state).toBe("valid");
+    const bad = runFieldMachine(machine, "cancelled");
+    expect(bad.failed?.gate).toBe("member");
+  });
+
+  it("leaves the `unique` gate inert without a dataset context", () => {
+    expect(runFieldMachine(codeMachine, "LHR").state).toBe("valid");
+    expect(runFieldMachine(codeMachine, "LHR").state).toBe("valid");
+  });
+
+  it("rejects duplicates through a dataset context", () => {
+    const record = compileRecordMachine([specFromManifestField(CODE_FIELD)]);
+    const ctx = createDatasetContext(record);
+    expect(runRecordMachine(record, { code: "LHR" }, ctx).state).toBe("valid");
+    const dup = runRecordMachine(record, { code: "LHR" }, ctx);
+    expect(dup.state).toBe("invalid");
+    expect(dup.errors[0]).toMatchObject({ field: "code", gate: "unique" });
+  });
+
+  it("resolves foreign keys through the context, inert without one", () => {
+    const fk: FieldModel = {
+      name: "airport",
+      type: "fk",
+      relation: { app: "airports", model: "airport" },
+    };
+    const machine = compileFieldMachine(specFromManifestField(fk));
+    expect(machine.states).toContain("resolved");
+
+    // No resolver — the gate passes (FK existence is a dataset/server check).
+    expect(runFieldMachine(machine, "7").state).toBe("valid");
+
+    const record = compileRecordMachine([specFromManifestField(fk)]);
+    const ctx = createDatasetContext(record, {
+      relatedIds: { "airports.airport": new Set(["1", "2"]) },
+    });
+    expect(runRecordMachine(record, { airport: "2" }, ctx).state).toBe("valid");
+    const dangling = runRecordMachine(record, { airport: "7" }, ctx);
+    expect(dangling.errors[0]).toMatchObject({
+      field: "airport",
+      gate: "resolved",
+    });
+  });
+});
+
+describe("spec adapters", () => {
+  it("manifest: required iff not nullable, no default, not a primary key", () => {
+    expect(
+      specFromManifestField({ name: "code", type: "string" }).required,
+    ).toBe(true);
+    expect(
+      specFromManifestField({ name: "code", type: "string", null: true })
+        .required,
+    ).toBe(false);
+    expect(
+      specFromManifestField({ name: "code", type: "string", default: "x" })
+        .required,
+    ).toBe(false);
+    expect(
+      specFromManifestField({ name: "id", type: "int", primaryKey: true })
+        .required,
+    ).toBe(false);
+  });
+
+  it("manifest: primary keys are excluded from a resource's input specs", () => {
+    const specs = specsFromResource({
+      fields: [
+        { name: "id", type: "int", primaryKey: true },
+        { name: "code", type: "string", maxLength: 3 },
+      ],
+    });
+    expect(specs.map((s) => s.name)).toEqual(["code"]);
+  });
+
+  it("JSON Schema: recovers type, nullability, maxLength, enum and the FK relation", () => {
+    const schema = {
+      type: "object",
+      required: ["code", "airport"],
+      properties: {
+        code: { type: "string", maxLength: 3 },
+        seats: { type: ["integer", "null"] },
+        status: { type: "string", enum: ["scheduled", "departed"] },
+        contact: { type: "string", format: "email" },
+        airport: {
+          type: "integer",
+          "x-rastack-relation": { app: "airports", model: "airport" },
+        },
+      },
+    };
+
+    expect(specFromSchemaProperty(schema, "code")).toMatchObject({
+      type: "string",
+      required: true,
+      maxLength: 3,
+    });
+    expect(specFromSchemaProperty(schema, "seats")).toMatchObject({
+      type: "int",
+      required: false,
+      nullable: true,
+    });
+    expect(specFromSchemaProperty(schema, "status")?.options).toEqual([
+      "scheduled",
+      "departed",
+    ]);
+    expect(specFromSchemaProperty(schema, "contact")).toMatchObject({
+      type: "string",
+      format: "email",
+    });
+    expect(specFromSchemaProperty(schema, "airport")).toMatchObject({
+      type: "fk",
+      relation: { app: "airports", model: "airport" },
+    });
+    expect(specFromSchemaProperty(schema, "missing")).toBeUndefined();
+  });
+
+  it("both adapters compile machines that agree on a value", () => {
+    const fromManifest = compileFieldMachine(specFromManifestField(CODE_FIELD));
+    const fromSchema = compileFieldMachine(
+      specFromSchemaProperty(
+        {
+          type: "object",
+          required: ["code"],
+          properties: { code: { type: "string", maxLength: 3 } },
+        },
+        "code",
+      )!,
+    );
+
+    for (const value of ["LHR", "LHRX", "", 42]) {
+      const a = runFieldMachine(fromManifest, value);
+      const b = runFieldMachine(fromSchema, value);
+      expect(b.state).toBe(a.state);
+      expect(b.failed?.gate).toBe(a.failed?.gate);
+    }
+  });
+});
+
+describe("form hook integration — getFieldMachine", () => {
+  const writeSchema = {
+    type: "object",
+    required: ["code"],
+    properties: {
+      code: { type: "string", maxLength: 3 },
+      contact: { type: "string", format: "email", nullable: true },
+    },
+  };
+
+  it("compiles a machine straight from the write schema the form validates against", () => {
+    const machine = getFieldMachine(writeSchema, "code")!;
+    expect(machine.states).toEqual([
+      "pristine",
+      "present",
+      "typed",
+      "bounded",
+      "valid",
+    ]);
+    expect(runFieldMachine(machine, "LHRX").failed?.gate).toBe("bounded");
+  });
+
+  it("routes string formats through the `typed` gate", () => {
+    const machine = getFieldMachine(writeSchema, "contact")!;
+    expect(runFieldMachine(machine, "ops@example.com").state).toBe("valid");
+    const bad = runFieldMachine(machine, "not-an-email");
+    expect(bad.failed?.gate).toBe("typed");
+    // Optional: empty is fine.
+    expect(runFieldMachine(machine, "").state).toBe("valid");
+  });
+
+  it("returns undefined for a field the schema does not describe", () => {
+    expect(getFieldMachine(writeSchema, "nope")).toBeUndefined();
+    expect(getFieldMachine(undefined, "code")).toBeUndefined();
+  });
+});

package/validate.ts

@@ -0,0 +1,10 @@
+/**
+ * `rastack/validate` — the constraint state machine.
+ *
+ * The database generator's constraints compile into a deterministic finite
+ * state machine (`pristine → present → typed → bounded → member → unique →
+ * resolved → valid`); the same machine validates form fields in the browser
+ * (via the write JSON Schema) and imported datasets on the CLI (via
+ * `schema.rastack.json`). Filesystem-free, so it runs everywhere the hooks do.
+ */
+export * from "./src/validate";