rastack 0.0.45 → 0.0.47

package/src/rastack-import.ts

@@ -0,0 +1,203 @@
+#!/usr/bin/env node
+
+/**
+ * `rastack import` — load external data (CSV, Excel, PDF) into the shape of a
+ * compiled resource and run every row through the resource's constraint state
+ * machine.
+ *
+ *   rastack import <file> --resource app.model
+ *       [--schema .rastack/schema.rastack.json]   the compiled manifest
+ *       [--related app.model=<file>]              ids to resolve FKs against (repeatable)
+ *       [--out cleaned.json]                      write the coerced valid records
+ *       [--report report.json]                    write the full validation report
+ *       [--max-errors 20]                         issues to print before summarising
+ *       [--allow-invalid]                         exit 0 even when rows fail
+ *
+ * The validation is the same machine the form hooks run per keystroke and the
+ * API enforces on writes: `present → typed → bounded → member → unique →
+ * resolved`. Here it runs with the full dataset context, so in-file duplicate
+ * `unique` values and dangling foreign keys are rejected with row-located
+ * errors. Exit status is non-zero when any row fails (like the seed PII gate)
+ * unless `--allow-invalid` is passed.
+ */
+
+import fs from "fs";
+import path from "path";
+import type { Manifest, ResourceModel } from "./compile/model";
+import { specsFromResource } from "./validate";
+import {
+  formatOf,
+  idsFromTabular,
+  parseTabular,
+  validateDataset,
+  type ImportReport,
+  type TabularData,
+} from "./import";
+
+// ---------------------------------------------------------------------------
+// Args
+// ---------------------------------------------------------------------------
+
+const args = process.argv.slice(2);
+
+function getArg(flag: string): string | null {
+  const i = args.indexOf(flag);
+  return i !== -1 && args[i + 1] ? args[i + 1] : null;
+}
+
+function getArgs(flag: string): string[] {
+  const values: string[] = [];
+  for (let i = 0; i < args.length - 1; i++) {
+    if (args[i] === flag) values.push(args[i + 1]);
+  }
+  return values;
+}
+
+const USAGE =
+  "Usage: rastack import <file.csv|.xlsx|.pdf> --resource app.model\n" +
+  "         [--schema .rastack/schema.rastack.json] [--related app.model=<file>]\n" +
+  "         [--out cleaned.json] [--report report.json] [--max-errors n] [--allow-invalid]";
+
+function fail(message: string): never {
+  console.error(`✗ ${message}\n\n${USAGE}`);
+  process.exit(1);
+}
+
+// ---------------------------------------------------------------------------
+// Load
+// ---------------------------------------------------------------------------
+
+function loadFile(filePath: string): TabularData {
+  const format = formatOf(filePath);
+  if (!format) fail(`unsupported file type: ${filePath} (use .csv, .xlsx or .pdf)`);
+  if (!fs.existsSync(filePath)) fail(`file not found: ${filePath}`);
+  return parseTabular(format, fs.readFileSync(filePath));
+}
+
+function loadManifest(schemaPath: string): Manifest {
+  if (!fs.existsSync(schemaPath)) {
+    fail(
+      `manifest not found at ${schemaPath} — run \`rastack compile\` first, ` +
+        `or point --schema at a schema.rastack.json`,
+    );
+  }
+  return JSON.parse(fs.readFileSync(schemaPath, "utf-8")) as Manifest;
+}
+
+function findResource(manifest: Manifest, name: string): ResourceModel {
+  const resource = manifest.resources.find(
+    (r) => `${r.app}.${r.model}` === name,
+  );
+  if (!resource) {
+    fail(
+      `unknown resource "${name}" — available: ` +
+        manifest.resources.map((r) => `${r.app}.${r.model}`).join(", "),
+    );
+  }
+  return resource;
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+const VALUE_FLAGS = new Set([
+  "--resource",
+  "--schema",
+  "--related",
+  "--out",
+  "--report",
+  "--max-errors",
+]);
+
+/** The first token that is neither a flag nor a flag's value. */
+function positionalArg(): string | null {
+  for (let i = 0; i < args.length; i++) {
+    if (VALUE_FLAGS.has(args[i])) {
+      i++; // skip the flag's value
+    } else if (!args[i].startsWith("--")) {
+      return args[i];
+    }
+  }
+  return null;
+}
+
+function main(): void {
+  const file = positionalArg();
+  const resourceName = getArg("--resource");
+  if (!file || !resourceName) fail("a source file and --resource are required");
+
+  const schemaPath =
+    getArg("--schema") ??
+    path.join(process.cwd(), ".rastack", "schema.rastack.json");
+  const manifest = loadManifest(schemaPath);
+  const resource = findResource(manifest, resourceName);
+  const specs = specsFromResource(resource);
+
+  // Related id sets for FK resolution: --related airports.airport=airports.csv
+  const relatedIds: Record<string, Set<string>> = {};
+  for (const spec of getArgs("--related")) {
+    const eq = spec.indexOf("=");
+    if (eq === -1) fail(`--related expects app.model=<file>, got "${spec}"`);
+    relatedIds[spec.slice(0, eq)] = idsFromTabular(loadFile(spec.slice(eq + 1)));
+  }
+
+  const data = loadFile(file);
+  console.log(
+    `\nrastack import  ${path.basename(file)} → ${resourceName}  ` +
+      `[${data.headers.length} column(s), ${data.rows.length} row(s)]\n`,
+  );
+
+  const report = validateDataset(specs, data, { relatedIds });
+  printReport(report);
+
+  const outPath = getArg("--out");
+  if (outPath) {
+    fs.writeFileSync(outPath, JSON.stringify(report.records, null, 2) + "\n");
+    console.log(`✓ ${report.records.length} valid record(s) → ${outPath}`);
+  }
+  const reportPath = getArg("--report");
+  if (reportPath) {
+    // Keep the on-disk report data-shaped: drop the per-row machine runs.
+    const { rows: _rows, ...summary } = report;
+    fs.writeFileSync(reportPath, JSON.stringify(summary, null, 2) + "\n");
+    console.log(`✓ validation report → ${reportPath}`);
+  }
+
+  if (report.invalid > 0 && !args.includes("--allow-invalid")) {
+    process.exit(1);
+  }
+}
+
+function printReport(report: ImportReport): void {
+  for (const col of report.columns) {
+    console.log(
+      col.field
+        ? `  ${col.header} → ${col.field}`
+        : `  ${col.header} → (ignored — no matching field)`,
+    );
+  }
+  for (const missing of report.missingFields) {
+    console.log(`  (no column) → ${missing}`);
+  }
+  console.log("");
+
+  const maxErrors = Number(getArg("--max-errors") ?? 20);
+  for (const issue of report.issues.slice(0, maxErrors)) {
+    console.log(
+      `  ✗ row ${issue.row + 1}  ${issue.field}: ${issue.message}` +
+        `  [failed at "${issue.gate}" — ${issue.constraint}]`,
+    );
+  }
+  if (report.issues.length > maxErrors) {
+    console.log(`  … and ${report.issues.length - maxErrors} more issue(s)`);
+  }
+  if (report.issues.length) console.log("");
+
+  console.log(
+    `${report.invalid === 0 ? "✓" : "✗"} ${report.valid}/${report.total} row(s) valid` +
+      (report.invalid ? `, ${report.invalid} rejected` : ""),
+  );
+}
+
+main();

package/src/rastack.ts

@@ -13,6 +13,7 @@
  *   rastack design  [input] [--port n]          Live design-system studio (showcase + edit)
  *   rastack dev     [--port n] [--warehouse d]  Local dev: run the app + admin in-browser over WASM
  *   rastack admin   [--warehouse dir] [--port n] Full admin console — React UI (rastack/components) over the native server
+ *   rastack import <file> --resource app.model  Load CSV/XLSX/PDF data and validate it (state machine)
  *   rastack scan   [files...] [--fail-on-pii]   Scan CSVs for PII and schema info
  *   rastack seed   [--env dev] [--source gs://] Load CSVs into Iceberg namespace
  *   rastack bootstrap [--owner o] [--repo r]    Stand the tier-0 AWS account stack up (once per account+region)
@@ -90,6 +91,11 @@ switch (command) {
     // Live design-system studio (showcase + inspect + edit).
     run("rastack-design.js", rest);
     break;
+  case "import":
+    // Load external data (CSV / Excel / PDF) and validate it row-by-row
+    // through the resource's constraint state machine.
+    run("rastack-import.js", rest);
+    break;
   case "scan":
     run("scan.js");
     break;
@@ -135,6 +141,7 @@ switch (command) {
         `  rastack design  [input] [--port n]             Live design-system studio (showcase + edit)\n` +
         `  rastack dev     [--port n] [--warehouse dir] [--no-open]  Run the app + admin in-browser over WASM (local dev)\n` +
         `  rastack admin   [--warehouse dir] [--port n] [--no-open]  Full admin console (React UI over the native server; needs a built rastack-server)\n` +
+        `  rastack import <file.csv|.xlsx|.pdf> --resource app.model [--related app.model=<file>] [--out cleaned.json]\n` +
         `  rastack scan   [files...] [--fail-on-pii]\n` +
         `  rastack seed   [--env dev|staging] [--source gs://bucket/ | ./myapp-dev/]\n` +
         `  rastack bootstrap [--owner o] [--repo r] [--region r] [--prefix p] [--reuse-oidc] [--print]  Stand the AWS account stack up\n` +

package/src/validate/adapters.ts

@@ -0,0 +1,118 @@
+/**
+ * The two adapters that lower a field definition into a {@link FieldSpec} —
+ * the normalised constraint set the state machine is compiled from.
+ *
+ * - {@link specFromManifestField} reads `schema.rastack.json` (the compiler's
+ *   canonical manifest) — what the data-import pipeline uses. It sees the
+ *   *full* constraint set, including `unique` and the FK relation.
+ * - {@link specFromSchemaProperty} reads a write JSON Schema property (the
+ *   dereferenced `Deref{Name}Request` the form hooks validate against) — what
+ *   `getFormField` uses in the browser. It sees whatever the OpenAPI emitter
+ *   carries: type/format/maxLength/enum/nullable/`x-rastack-relation`.
+ *
+ * Both lower to the same `FieldSpec`, so the *same* compiled machine (same
+ * states, same gate order, same messages) validates a keystroke in a form and
+ * a row in an imported spreadsheet.
+ */
+
+import type { FieldModel } from "../compile/model";
+import type { FieldSpec, SpecType } from "./machine";
+
+/**
+ * Lower a manifest field (`schema.rastack.json`) to a field spec. Mirrors the
+ * OpenAPI emitter's semantics: a field is required iff it is not nullable and
+ * carries no default; primary keys are server-assigned and never validated on
+ * input.
+ */
+export function specFromManifestField(field: FieldModel): FieldSpec {
+  const spec: FieldSpec = {
+    name: field.name,
+    type: (field.type as SpecType) ?? "string",
+    required: !field.null && field.default === undefined && !field.primaryKey,
+    nullable: field.null === true,
+  };
+  if (field.maxLength !== undefined) spec.maxLength = field.maxLength;
+  if (field.unique) spec.unique = true;
+  if (field.default !== undefined) spec.default = field.default;
+  if (field.type === "fk" && field.relation) spec.relation = field.relation;
+  return spec;
+}
+
+/**
+ * Lower every input-validated field of a manifest resource — primary keys are
+ * skipped, matching the write schema the OpenAPI emitter produces.
+ */
+export function specsFromResource(resource: {
+  fields: FieldModel[];
+}): FieldSpec[] {
+  return resource.fields
+    .filter((f) => !f.primaryKey)
+    .map(specFromManifestField);
+}
+
+/** Map a JSON-Schema scalar (`type` + `format`) back to a manifest scalar kind. */
+function schemaScalar(typeList: string[], format?: string): SpecType {
+  if (typeList.includes("integer")) return "int";
+  if (typeList.includes("number")) return "float";
+  if (typeList.includes("boolean")) return "bool";
+  if (format === "date-time") return "datetime";
+  if (format === "uuid") return "uuid";
+  return "string";
+}
+
+/**
+ * Lower one property of a write JSON Schema to a field spec. `schema` is the
+ * object schema (so requiredness can be read from its `required` list);
+ * returns `undefined` when the property does not exist.
+ */
+export function specFromSchemaProperty(
+  schema:
+    | {
+        properties?: Record<string, any>;
+        required?: unknown;
+        [key: string]: unknown;
+      }
+    | undefined,
+  fieldName: string,
+): FieldSpec | undefined {
+  const property: any = schema?.properties?.[fieldName];
+  if (!property) return undefined;
+
+  const rawType = property.type;
+  const typeList: string[] =
+    typeof rawType === "string"
+      ? [rawType]
+      : Array.isArray(rawType)
+        ? rawType
+        : [];
+  const nullable = typeList.includes("null") || property.nullable === true;
+  const required =
+    Array.isArray(schema?.required) && schema!.required!.includes(fieldName);
+
+  const relation = property["x-rastack-relation"];
+  const format = typeof property.format === "string" ? property.format : undefined;
+
+  const spec: FieldSpec = {
+    name: fieldName,
+    type:
+      relation && typeof relation === "object"
+        ? "fk"
+        : schemaScalar(typeList, format),
+    required,
+    nullable,
+  };
+
+  if (relation && typeof relation === "object") spec.relation = relation;
+  // `format` refines strings (`email`/`url`/`date`/`date-time`/`uuid`); the
+  // scalar kinds already encode datetime/uuid, so only keep it for strings.
+  if (spec.type === "string" && format) spec.format = format;
+  if (typeof property.maxLength === "number") spec.maxLength = property.maxLength;
+  if (Array.isArray(property.enum) && property.enum.length > 0) {
+    spec.options = property.enum.filter(
+      (v: unknown): v is string | number => v !== null && v !== undefined,
+    );
+  }
+  if (property.default !== undefined) spec.default = property.default;
+
+  return spec;
+}

package/src/validate/index.ts

@@ -0,0 +1,2 @@
+export * from "./machine";
+export * from "./adapters";