rastack 0.0.45 → 0.0.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/dist/import/index.d.ts +18 -0
- package/dist/import/index.js +57 -0
- package/dist/import/pdf.d.ts +25 -0
- package/dist/import/pdf.js +326 -0
- package/dist/import/tabular.d.ts +76 -0
- package/dist/import/tabular.js +98 -0
- package/dist/import/xlsx.d.ts +18 -0
- package/dist/import/xlsx.js +160 -0
- package/dist/rastack-import.d.ts +22 -0
- package/dist/rastack-import.js +165 -0
- package/dist/rastack.d.ts +1 -0
- package/dist/rastack.js +7 -0
- package/dist/validate/adapters.d.ts +42 -0
- package/dist/validate/adapters.js +109 -0
- package/dist/validate/index.d.ts +2 -0
- package/dist/validate/index.js +18 -0
- package/dist/validate/machine.d.ts +175 -0
- package/dist/validate/machine.js +347 -0
- package/dist/wasm/rastack_wasm_bg.wasm +0 -0
- package/hooks/form/form.ts +16 -5
- package/hooks/form/interfaces.ts +15 -0
- package/hooks/form/structure.ts +28 -0
- package/package.json +1 -1
- package/src/import/index.ts +41 -0
- package/src/import/pdf.ts +304 -0
- package/src/import/tabular.ts +159 -0
- package/src/import/xlsx.ts +186 -0
- package/src/rastack-import.ts +203 -0
- package/src/rastack.ts +7 -0
- package/src/validate/adapters.ts +118 -0
- package/src/validate/index.ts +2 -0
- package/src/validate/machine.ts +525 -0
- package/test/import.spec.ts +241 -0
- package/test/validate.spec.ts +319 -0
- package/validate.ts +10 -0
- package/wasm/rastack_wasm_bg.wasm +0 -0
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* A tiny, dependency-free XLSX reader for `rastack import`.
|
|
4
|
+
*
|
|
5
|
+
* Why hand-roll it: the CLI stays dependency-light (the same reasoning as the
|
|
6
|
+
* hand-rolled ZIP *writer* in `deploy/zip.ts`), and an `.xlsx` workbook is
|
|
7
|
+
* just a ZIP of small XML parts — the reader below understands exactly the
|
|
8
|
+
* subset a tabular import needs: the ZIP central directory (STORE and DEFLATE
|
|
9
|
+
* entries, inflated via node's zlib), `xl/sharedStrings.xml`, and the cells of
|
|
10
|
+
* the first worksheet. Formulas are read through their cached `<v>` results;
|
|
11
|
+
* merged cells, styles and charts are ignored — an import wants values.
|
|
12
|
+
*/
|
|
13
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
|
+
exports.parseXlsx = parseXlsx;
|
|
15
|
+
const zlib_1 = require("zlib");
|
|
16
|
+
/** Locate the end-of-central-directory record (scan back over the comment). */
|
|
17
|
+
function findEocd(buf) {
|
|
18
|
+
for (let i = buf.length - 22; i >= 0; i--) {
|
|
19
|
+
if (buf.readUInt32LE(i) === 0x06054b50)
|
|
20
|
+
return i;
|
|
21
|
+
}
|
|
22
|
+
throw new Error("not a ZIP archive (no end-of-central-directory record)");
|
|
23
|
+
}
|
|
24
|
+
function readCentralDirectory(buf) {
|
|
25
|
+
const eocd = findEocd(buf);
|
|
26
|
+
const count = buf.readUInt16LE(eocd + 10);
|
|
27
|
+
let offset = buf.readUInt32LE(eocd + 16);
|
|
28
|
+
const entries = [];
|
|
29
|
+
for (let i = 0; i < count; i++) {
|
|
30
|
+
if (buf.readUInt32LE(offset) !== 0x02014b50) {
|
|
31
|
+
throw new Error("corrupt ZIP central directory");
|
|
32
|
+
}
|
|
33
|
+
const method = buf.readUInt16LE(offset + 10);
|
|
34
|
+
const compressedSize = buf.readUInt32LE(offset + 20);
|
|
35
|
+
const nameLength = buf.readUInt16LE(offset + 28);
|
|
36
|
+
const extraLength = buf.readUInt16LE(offset + 30);
|
|
37
|
+
const commentLength = buf.readUInt16LE(offset + 32);
|
|
38
|
+
const localHeaderOffset = buf.readUInt32LE(offset + 42);
|
|
39
|
+
const name = buf
|
|
40
|
+
.subarray(offset + 46, offset + 46 + nameLength)
|
|
41
|
+
.toString("utf-8");
|
|
42
|
+
entries.push({ name, method, compressedSize, localHeaderOffset });
|
|
43
|
+
offset += 46 + nameLength + extraLength + commentLength;
|
|
44
|
+
}
|
|
45
|
+
return entries;
|
|
46
|
+
}
|
|
47
|
+
function readEntry(buf, entry) {
|
|
48
|
+
const at = entry.localHeaderOffset;
|
|
49
|
+
if (buf.readUInt32LE(at) !== 0x04034b50) {
|
|
50
|
+
throw new Error(`corrupt ZIP local header for ${entry.name}`);
|
|
51
|
+
}
|
|
52
|
+
const nameLength = buf.readUInt16LE(at + 26);
|
|
53
|
+
const extraLength = buf.readUInt16LE(at + 28);
|
|
54
|
+
const start = at + 30 + nameLength + extraLength;
|
|
55
|
+
const data = buf.subarray(start, start + entry.compressedSize);
|
|
56
|
+
if (entry.method === 0)
|
|
57
|
+
return Buffer.from(data); // STORE
|
|
58
|
+
if (entry.method === 8)
|
|
59
|
+
return (0, zlib_1.inflateRawSync)(data); // DEFLATE
|
|
60
|
+
throw new Error(`unsupported ZIP compression method ${entry.method}`);
|
|
61
|
+
}
|
|
62
|
+
// ---------------------------------------------------------------------------
|
|
63
|
+
// Minimal XML helpers (the sheet parts are machine-written, flat XML)
|
|
64
|
+
// ---------------------------------------------------------------------------
|
|
65
|
+
function unescapeXml(text) {
|
|
66
|
+
return text
|
|
67
|
+
.replace(/</g, "<")
|
|
68
|
+
.replace(/>/g, ">")
|
|
69
|
+
.replace(/"/g, '"')
|
|
70
|
+
.replace(/'/g, "'")
|
|
71
|
+
.replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)))
|
|
72
|
+
.replace(/&/g, "&");
|
|
73
|
+
}
|
|
74
|
+
/** All matches of `<tag ...>inner</tag>` (non-nested — true of sheet XML). */
|
|
75
|
+
function elements(xml, tag) {
|
|
76
|
+
const re = new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)</${tag}>`, "g");
|
|
77
|
+
const out = [];
|
|
78
|
+
let m;
|
|
79
|
+
while ((m = re.exec(xml)) !== null)
|
|
80
|
+
out.push(m[1]);
|
|
81
|
+
return out;
|
|
82
|
+
}
|
|
83
|
+
/** `"BC7"` → 0-based column index (54). */
|
|
84
|
+
function columnIndex(cellRef) {
|
|
85
|
+
let index = 0;
|
|
86
|
+
for (const ch of cellRef) {
|
|
87
|
+
if (ch < "A" || ch > "Z")
|
|
88
|
+
break;
|
|
89
|
+
index = index * 26 + (ch.charCodeAt(0) - 64);
|
|
90
|
+
}
|
|
91
|
+
return index - 1;
|
|
92
|
+
}
|
|
93
|
+
// ---------------------------------------------------------------------------
|
|
94
|
+
// Workbook parsing
|
|
95
|
+
// ---------------------------------------------------------------------------
|
|
96
|
+
function parseSharedStrings(xml) {
|
|
97
|
+
if (!xml)
|
|
98
|
+
return [];
|
|
99
|
+
// Each <si> may hold one <t> or several rich-text runs — concatenate them.
|
|
100
|
+
return elements(xml, "si").map((si) => elements(si, "t").map(unescapeXml).join(""));
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Parse an `.xlsx` workbook buffer into headers + rows (first worksheet, first
|
|
104
|
+
* row as the header). Cell values come back as strings — the constraint state
|
|
105
|
+
* machine's `typed` gate owns coercion, same as for CSV.
|
|
106
|
+
*/
|
|
107
|
+
function parseXlsx(buf) {
|
|
108
|
+
const entries = readCentralDirectory(buf);
|
|
109
|
+
const byName = new Map(entries.map((e) => [e.name, e]));
|
|
110
|
+
const sheetEntry = byName.get("xl/worksheets/sheet1.xml") ??
|
|
111
|
+
entries
|
|
112
|
+
.filter((e) => /^xl\/worksheets\/[^/]+\.xml$/.test(e.name))
|
|
113
|
+
.sort((a, b) => a.name.localeCompare(b.name))[0];
|
|
114
|
+
if (!sheetEntry)
|
|
115
|
+
throw new Error("no worksheet found in workbook");
|
|
116
|
+
const sharedEntry = byName.get("xl/sharedStrings.xml");
|
|
117
|
+
const shared = parseSharedStrings(sharedEntry ? readEntry(buf, sharedEntry).toString("utf-8") : undefined);
|
|
118
|
+
const sheetXml = readEntry(buf, sheetEntry).toString("utf-8");
|
|
119
|
+
const cellRe = /<c(?:\s([^>]*?))?(?:\/>|>([\s\S]*?)<\/c>)/g;
|
|
120
|
+
const attr = (attrs, name) => attrs?.match(new RegExp(`${name}="([^"]*)"`))?.[1];
|
|
121
|
+
const grid = [];
|
|
122
|
+
for (const rowXml of elements(sheetXml, "row")) {
|
|
123
|
+
const cells = [];
|
|
124
|
+
let m;
|
|
125
|
+
let fallbackColumn = 0;
|
|
126
|
+
cellRe.lastIndex = 0;
|
|
127
|
+
while ((m = cellRe.exec(rowXml)) !== null) {
|
|
128
|
+
const [, attrs, inner = ""] = m;
|
|
129
|
+
const ref = attr(attrs, "r");
|
|
130
|
+
const column = ref ? columnIndex(ref) : fallbackColumn;
|
|
131
|
+
fallbackColumn = column + 1;
|
|
132
|
+
const type = attr(attrs, "t");
|
|
133
|
+
let value = "";
|
|
134
|
+
if (type === "inlineStr") {
|
|
135
|
+
value = elements(inner, "t").map(unescapeXml).join("");
|
|
136
|
+
}
|
|
137
|
+
else {
|
|
138
|
+
const v = elements(inner, "v")[0];
|
|
139
|
+
value = v === undefined ? "" : unescapeXml(v);
|
|
140
|
+
if (type === "s")
|
|
141
|
+
value = shared[Number(value)] ?? "";
|
|
142
|
+
}
|
|
143
|
+
while (cells.length < column)
|
|
144
|
+
cells.push("");
|
|
145
|
+
cells[column] = value;
|
|
146
|
+
}
|
|
147
|
+
grid.push(cells);
|
|
148
|
+
}
|
|
149
|
+
const [headers = [], ...rows] = grid;
|
|
150
|
+
const width = headers.length;
|
|
151
|
+
return {
|
|
152
|
+
headers,
|
|
153
|
+
rows: rows.map((r) => {
|
|
154
|
+
const row = r.slice(0, Math.max(width, r.length));
|
|
155
|
+
while (row.length < width)
|
|
156
|
+
row.push("");
|
|
157
|
+
return row;
|
|
158
|
+
}),
|
|
159
|
+
};
|
|
160
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* `rastack import` — load external data (CSV, Excel, PDF) into the shape of a
|
|
4
|
+
* compiled resource and run every row through the resource's constraint state
|
|
5
|
+
* machine.
|
|
6
|
+
*
|
|
7
|
+
* rastack import <file> --resource app.model
|
|
8
|
+
* [--schema .rastack/schema.rastack.json] the compiled manifest
|
|
9
|
+
* [--related app.model=<file>] ids to resolve FKs against (repeatable)
|
|
10
|
+
* [--out cleaned.json] write the coerced valid records
|
|
11
|
+
* [--report report.json] write the full validation report
|
|
12
|
+
* [--max-errors 20] issues to print before summarising
|
|
13
|
+
* [--allow-invalid] exit 0 even when rows fail
|
|
14
|
+
*
|
|
15
|
+
* The validation is the same machine the form hooks run per keystroke and the
|
|
16
|
+
* API enforces on writes: `present → typed → bounded → member → unique →
|
|
17
|
+
* resolved`. Here it runs with the full dataset context, so in-file duplicate
|
|
18
|
+
* `unique` values and dangling foreign keys are rejected with row-located
|
|
19
|
+
* errors. Exit status is non-zero when any row fails (like the seed PII gate)
|
|
20
|
+
* unless `--allow-invalid` is passed.
|
|
21
|
+
*/
|
|
22
|
+
export {};
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
/**
|
|
4
|
+
* `rastack import` — load external data (CSV, Excel, PDF) into the shape of a
|
|
5
|
+
* compiled resource and run every row through the resource's constraint state
|
|
6
|
+
* machine.
|
|
7
|
+
*
|
|
8
|
+
* rastack import <file> --resource app.model
|
|
9
|
+
* [--schema .rastack/schema.rastack.json] the compiled manifest
|
|
10
|
+
* [--related app.model=<file>] ids to resolve FKs against (repeatable)
|
|
11
|
+
* [--out cleaned.json] write the coerced valid records
|
|
12
|
+
* [--report report.json] write the full validation report
|
|
13
|
+
* [--max-errors 20] issues to print before summarising
|
|
14
|
+
* [--allow-invalid] exit 0 even when rows fail
|
|
15
|
+
*
|
|
16
|
+
* The validation is the same machine the form hooks run per keystroke and the
|
|
17
|
+
* API enforces on writes: `present → typed → bounded → member → unique →
|
|
18
|
+
* resolved`. Here it runs with the full dataset context, so in-file duplicate
|
|
19
|
+
* `unique` values and dangling foreign keys are rejected with row-located
|
|
20
|
+
* errors. Exit status is non-zero when any row fails (like the seed PII gate)
|
|
21
|
+
* unless `--allow-invalid` is passed.
|
|
22
|
+
*/
|
|
23
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
24
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
25
|
+
};
|
|
26
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
27
|
+
const fs_1 = __importDefault(require("fs"));
|
|
28
|
+
const path_1 = __importDefault(require("path"));
|
|
29
|
+
const validate_1 = require("./validate");
|
|
30
|
+
const import_1 = require("./import");
|
|
31
|
+
// ---------------------------------------------------------------------------
|
|
32
|
+
// Args
|
|
33
|
+
// ---------------------------------------------------------------------------
|
|
34
|
+
const args = process.argv.slice(2);
|
|
35
|
+
function getArg(flag) {
|
|
36
|
+
const i = args.indexOf(flag);
|
|
37
|
+
return i !== -1 && args[i + 1] ? args[i + 1] : null;
|
|
38
|
+
}
|
|
39
|
+
function getArgs(flag) {
|
|
40
|
+
const values = [];
|
|
41
|
+
for (let i = 0; i < args.length - 1; i++) {
|
|
42
|
+
if (args[i] === flag)
|
|
43
|
+
values.push(args[i + 1]);
|
|
44
|
+
}
|
|
45
|
+
return values;
|
|
46
|
+
}
|
|
47
|
+
const USAGE = "Usage: rastack import <file.csv|.xlsx|.pdf> --resource app.model\n" +
|
|
48
|
+
" [--schema .rastack/schema.rastack.json] [--related app.model=<file>]\n" +
|
|
49
|
+
" [--out cleaned.json] [--report report.json] [--max-errors n] [--allow-invalid]";
|
|
50
|
+
function fail(message) {
|
|
51
|
+
console.error(`✗ ${message}\n\n${USAGE}`);
|
|
52
|
+
process.exit(1);
|
|
53
|
+
}
|
|
54
|
+
// ---------------------------------------------------------------------------
|
|
55
|
+
// Load
|
|
56
|
+
// ---------------------------------------------------------------------------
|
|
57
|
+
function loadFile(filePath) {
|
|
58
|
+
const format = (0, import_1.formatOf)(filePath);
|
|
59
|
+
if (!format)
|
|
60
|
+
fail(`unsupported file type: ${filePath} (use .csv, .xlsx or .pdf)`);
|
|
61
|
+
if (!fs_1.default.existsSync(filePath))
|
|
62
|
+
fail(`file not found: ${filePath}`);
|
|
63
|
+
return (0, import_1.parseTabular)(format, fs_1.default.readFileSync(filePath));
|
|
64
|
+
}
|
|
65
|
+
function loadManifest(schemaPath) {
|
|
66
|
+
if (!fs_1.default.existsSync(schemaPath)) {
|
|
67
|
+
fail(`manifest not found at ${schemaPath} — run \`rastack compile\` first, ` +
|
|
68
|
+
`or point --schema at a schema.rastack.json`);
|
|
69
|
+
}
|
|
70
|
+
return JSON.parse(fs_1.default.readFileSync(schemaPath, "utf-8"));
|
|
71
|
+
}
|
|
72
|
+
function findResource(manifest, name) {
|
|
73
|
+
const resource = manifest.resources.find((r) => `${r.app}.${r.model}` === name);
|
|
74
|
+
if (!resource) {
|
|
75
|
+
fail(`unknown resource "${name}" — available: ` +
|
|
76
|
+
manifest.resources.map((r) => `${r.app}.${r.model}`).join(", "));
|
|
77
|
+
}
|
|
78
|
+
return resource;
|
|
79
|
+
}
|
|
80
|
+
// ---------------------------------------------------------------------------
|
|
81
|
+
// Main
|
|
82
|
+
// ---------------------------------------------------------------------------
|
|
83
|
+
const VALUE_FLAGS = new Set([
|
|
84
|
+
"--resource",
|
|
85
|
+
"--schema",
|
|
86
|
+
"--related",
|
|
87
|
+
"--out",
|
|
88
|
+
"--report",
|
|
89
|
+
"--max-errors",
|
|
90
|
+
]);
|
|
91
|
+
/** The first token that is neither a flag nor a flag's value. */
|
|
92
|
+
function positionalArg() {
|
|
93
|
+
for (let i = 0; i < args.length; i++) {
|
|
94
|
+
if (VALUE_FLAGS.has(args[i])) {
|
|
95
|
+
i++; // skip the flag's value
|
|
96
|
+
}
|
|
97
|
+
else if (!args[i].startsWith("--")) {
|
|
98
|
+
return args[i];
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
function main() {
|
|
104
|
+
const file = positionalArg();
|
|
105
|
+
const resourceName = getArg("--resource");
|
|
106
|
+
if (!file || !resourceName)
|
|
107
|
+
fail("a source file and --resource are required");
|
|
108
|
+
const schemaPath = getArg("--schema") ??
|
|
109
|
+
path_1.default.join(process.cwd(), ".rastack", "schema.rastack.json");
|
|
110
|
+
const manifest = loadManifest(schemaPath);
|
|
111
|
+
const resource = findResource(manifest, resourceName);
|
|
112
|
+
const specs = (0, validate_1.specsFromResource)(resource);
|
|
113
|
+
// Related id sets for FK resolution: --related airports.airport=airports.csv
|
|
114
|
+
const relatedIds = {};
|
|
115
|
+
for (const spec of getArgs("--related")) {
|
|
116
|
+
const eq = spec.indexOf("=");
|
|
117
|
+
if (eq === -1)
|
|
118
|
+
fail(`--related expects app.model=<file>, got "${spec}"`);
|
|
119
|
+
relatedIds[spec.slice(0, eq)] = (0, import_1.idsFromTabular)(loadFile(spec.slice(eq + 1)));
|
|
120
|
+
}
|
|
121
|
+
const data = loadFile(file);
|
|
122
|
+
console.log(`\nrastack import ${path_1.default.basename(file)} → ${resourceName} ` +
|
|
123
|
+
`[${data.headers.length} column(s), ${data.rows.length} row(s)]\n`);
|
|
124
|
+
const report = (0, import_1.validateDataset)(specs, data, { relatedIds });
|
|
125
|
+
printReport(report);
|
|
126
|
+
const outPath = getArg("--out");
|
|
127
|
+
if (outPath) {
|
|
128
|
+
fs_1.default.writeFileSync(outPath, JSON.stringify(report.records, null, 2) + "\n");
|
|
129
|
+
console.log(`✓ ${report.records.length} valid record(s) → ${outPath}`);
|
|
130
|
+
}
|
|
131
|
+
const reportPath = getArg("--report");
|
|
132
|
+
if (reportPath) {
|
|
133
|
+
// Keep the on-disk report data-shaped: drop the per-row machine runs.
|
|
134
|
+
const { rows: _rows, ...summary } = report;
|
|
135
|
+
fs_1.default.writeFileSync(reportPath, JSON.stringify(summary, null, 2) + "\n");
|
|
136
|
+
console.log(`✓ validation report → ${reportPath}`);
|
|
137
|
+
}
|
|
138
|
+
if (report.invalid > 0 && !args.includes("--allow-invalid")) {
|
|
139
|
+
process.exit(1);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
function printReport(report) {
|
|
143
|
+
for (const col of report.columns) {
|
|
144
|
+
console.log(col.field
|
|
145
|
+
? ` ${col.header} → ${col.field}`
|
|
146
|
+
: ` ${col.header} → (ignored — no matching field)`);
|
|
147
|
+
}
|
|
148
|
+
for (const missing of report.missingFields) {
|
|
149
|
+
console.log(` (no column) → ${missing}`);
|
|
150
|
+
}
|
|
151
|
+
console.log("");
|
|
152
|
+
const maxErrors = Number(getArg("--max-errors") ?? 20);
|
|
153
|
+
for (const issue of report.issues.slice(0, maxErrors)) {
|
|
154
|
+
console.log(` ✗ row ${issue.row + 1} ${issue.field}: ${issue.message}` +
|
|
155
|
+
` [failed at "${issue.gate}" — ${issue.constraint}]`);
|
|
156
|
+
}
|
|
157
|
+
if (report.issues.length > maxErrors) {
|
|
158
|
+
console.log(` … and ${report.issues.length - maxErrors} more issue(s)`);
|
|
159
|
+
}
|
|
160
|
+
if (report.issues.length)
|
|
161
|
+
console.log("");
|
|
162
|
+
console.log(`${report.invalid === 0 ? "✓" : "✗"} ${report.valid}/${report.total} row(s) valid` +
|
|
163
|
+
(report.invalid ? `, ${report.invalid} rejected` : ""));
|
|
164
|
+
}
|
|
165
|
+
main();
|
package/dist/rastack.d.ts
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
* rastack design [input] [--port n] Live design-system studio (showcase + edit)
|
|
13
13
|
* rastack dev [--port n] [--warehouse d] Local dev: run the app + admin in-browser over WASM
|
|
14
14
|
* rastack admin [--warehouse dir] [--port n] Full admin console — React UI (rastack/components) over the native server
|
|
15
|
+
* rastack import <file> --resource app.model Load CSV/XLSX/PDF data and validate it (state machine)
|
|
15
16
|
* rastack scan [files...] [--fail-on-pii] Scan CSVs for PII and schema info
|
|
16
17
|
* rastack seed [--env dev] [--source gs://] Load CSVs into Iceberg namespace
|
|
17
18
|
* rastack bootstrap [--owner o] [--repo r] Stand the tier-0 AWS account stack up (once per account+region)
|
package/dist/rastack.js
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* rastack design [input] [--port n] Live design-system studio (showcase + edit)
|
|
14
14
|
* rastack dev [--port n] [--warehouse d] Local dev: run the app + admin in-browser over WASM
|
|
15
15
|
* rastack admin [--warehouse dir] [--port n] Full admin console — React UI (rastack/components) over the native server
|
|
16
|
+
* rastack import <file> --resource app.model Load CSV/XLSX/PDF data and validate it (state machine)
|
|
16
17
|
* rastack scan [files...] [--fail-on-pii] Scan CSVs for PII and schema info
|
|
17
18
|
* rastack seed [--env dev] [--source gs://] Load CSVs into Iceberg namespace
|
|
18
19
|
* rastack bootstrap [--owner o] [--repo r] Stand the tier-0 AWS account stack up (once per account+region)
|
|
@@ -89,6 +90,11 @@ switch (command) {
|
|
|
89
90
|
// Live design-system studio (showcase + inspect + edit).
|
|
90
91
|
run("rastack-design.js", rest);
|
|
91
92
|
break;
|
|
93
|
+
case "import":
|
|
94
|
+
// Load external data (CSV / Excel / PDF) and validate it row-by-row
|
|
95
|
+
// through the resource's constraint state machine.
|
|
96
|
+
run("rastack-import.js", rest);
|
|
97
|
+
break;
|
|
92
98
|
case "scan":
|
|
93
99
|
run("scan.js");
|
|
94
100
|
break;
|
|
@@ -133,6 +139,7 @@ switch (command) {
|
|
|
133
139
|
` rastack design [input] [--port n] Live design-system studio (showcase + edit)\n` +
|
|
134
140
|
` rastack dev [--port n] [--warehouse dir] [--no-open] Run the app + admin in-browser over WASM (local dev)\n` +
|
|
135
141
|
` rastack admin [--warehouse dir] [--port n] [--no-open] Full admin console (React UI over the native server; needs a built rastack-server)\n` +
|
|
142
|
+
` rastack import <file.csv|.xlsx|.pdf> --resource app.model [--related app.model=<file>] [--out cleaned.json]\n` +
|
|
136
143
|
` rastack scan [files...] [--fail-on-pii]\n` +
|
|
137
144
|
` rastack seed [--env dev|staging] [--source gs://bucket/ | ./myapp-dev/]\n` +
|
|
138
145
|
` rastack bootstrap [--owner o] [--repo r] [--region r] [--prefix p] [--reuse-oidc] [--print] Stand the AWS account stack up\n` +
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two adapters that lower a field definition into a {@link FieldSpec} —
|
|
3
|
+
* the normalised constraint set the state machine is compiled from.
|
|
4
|
+
*
|
|
5
|
+
* - {@link specFromManifestField} reads `schema.rastack.json` (the compiler's
|
|
6
|
+
* canonical manifest) — what the data-import pipeline uses. It sees the
|
|
7
|
+
* *full* constraint set, including `unique` and the FK relation.
|
|
8
|
+
* - {@link specFromSchemaProperty} reads a write JSON Schema property (the
|
|
9
|
+
* dereferenced `Deref{Name}Request` the form hooks validate against) — what
|
|
10
|
+
* `getFormField` uses in the browser. It sees whatever the OpenAPI emitter
|
|
11
|
+
* carries: type/format/maxLength/enum/nullable/`x-rastack-relation`.
|
|
12
|
+
*
|
|
13
|
+
* Both lower to the same `FieldSpec`, so the *same* compiled machine (same
|
|
14
|
+
* states, same gate order, same messages) validates a keystroke in a form and
|
|
15
|
+
* a row in an imported spreadsheet.
|
|
16
|
+
*/
|
|
17
|
+
import type { FieldModel } from "../compile/model";
|
|
18
|
+
import type { FieldSpec } from "./machine";
|
|
19
|
+
/**
|
|
20
|
+
* Lower a manifest field (`schema.rastack.json`) to a field spec. Mirrors the
|
|
21
|
+
* OpenAPI emitter's semantics: a field is required iff it is not nullable and
|
|
22
|
+
* carries no default; primary keys are server-assigned and never validated on
|
|
23
|
+
* input.
|
|
24
|
+
*/
|
|
25
|
+
export declare function specFromManifestField(field: FieldModel): FieldSpec;
|
|
26
|
+
/**
|
|
27
|
+
* Lower every input-validated field of a manifest resource — primary keys are
|
|
28
|
+
* skipped, matching the write schema the OpenAPI emitter produces.
|
|
29
|
+
*/
|
|
30
|
+
export declare function specsFromResource(resource: {
|
|
31
|
+
fields: FieldModel[];
|
|
32
|
+
}): FieldSpec[];
|
|
33
|
+
/**
|
|
34
|
+
* Lower one property of a write JSON Schema to a field spec. `schema` is the
|
|
35
|
+
* object schema (so requiredness can be read from its `required` list);
|
|
36
|
+
* returns `undefined` when the property does not exist.
|
|
37
|
+
*/
|
|
38
|
+
export declare function specFromSchemaProperty(schema: {
|
|
39
|
+
properties?: Record<string, any>;
|
|
40
|
+
required?: unknown;
|
|
41
|
+
[key: string]: unknown;
|
|
42
|
+
} | undefined, fieldName: string): FieldSpec | undefined;
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* The two adapters that lower a field definition into a {@link FieldSpec} —
|
|
4
|
+
* the normalised constraint set the state machine is compiled from.
|
|
5
|
+
*
|
|
6
|
+
* - {@link specFromManifestField} reads `schema.rastack.json` (the compiler's
|
|
7
|
+
* canonical manifest) — what the data-import pipeline uses. It sees the
|
|
8
|
+
* *full* constraint set, including `unique` and the FK relation.
|
|
9
|
+
* - {@link specFromSchemaProperty} reads a write JSON Schema property (the
|
|
10
|
+
* dereferenced `Deref{Name}Request` the form hooks validate against) — what
|
|
11
|
+
* `getFormField` uses in the browser. It sees whatever the OpenAPI emitter
|
|
12
|
+
* carries: type/format/maxLength/enum/nullable/`x-rastack-relation`.
|
|
13
|
+
*
|
|
14
|
+
* Both lower to the same `FieldSpec`, so the *same* compiled machine (same
|
|
15
|
+
* states, same gate order, same messages) validates a keystroke in a form and
|
|
16
|
+
* a row in an imported spreadsheet.
|
|
17
|
+
*/
|
|
18
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.specFromManifestField = specFromManifestField;
|
|
20
|
+
exports.specsFromResource = specsFromResource;
|
|
21
|
+
exports.specFromSchemaProperty = specFromSchemaProperty;
|
|
22
|
+
/**
|
|
23
|
+
* Lower a manifest field (`schema.rastack.json`) to a field spec. Mirrors the
|
|
24
|
+
* OpenAPI emitter's semantics: a field is required iff it is not nullable and
|
|
25
|
+
* carries no default; primary keys are server-assigned and never validated on
|
|
26
|
+
* input.
|
|
27
|
+
*/
|
|
28
|
+
function specFromManifestField(field) {
|
|
29
|
+
const spec = {
|
|
30
|
+
name: field.name,
|
|
31
|
+
type: field.type ?? "string",
|
|
32
|
+
required: !field.null && field.default === undefined && !field.primaryKey,
|
|
33
|
+
nullable: field.null === true,
|
|
34
|
+
};
|
|
35
|
+
if (field.maxLength !== undefined)
|
|
36
|
+
spec.maxLength = field.maxLength;
|
|
37
|
+
if (field.unique)
|
|
38
|
+
spec.unique = true;
|
|
39
|
+
if (field.default !== undefined)
|
|
40
|
+
spec.default = field.default;
|
|
41
|
+
if (field.type === "fk" && field.relation)
|
|
42
|
+
spec.relation = field.relation;
|
|
43
|
+
return spec;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Lower every input-validated field of a manifest resource — primary keys are
|
|
47
|
+
* skipped, matching the write schema the OpenAPI emitter produces.
|
|
48
|
+
*/
|
|
49
|
+
function specsFromResource(resource) {
|
|
50
|
+
return resource.fields
|
|
51
|
+
.filter((f) => !f.primaryKey)
|
|
52
|
+
.map(specFromManifestField);
|
|
53
|
+
}
|
|
54
|
+
/** Map a JSON-Schema scalar (`type` + `format`) back to a manifest scalar kind. */
|
|
55
|
+
function schemaScalar(typeList, format) {
|
|
56
|
+
if (typeList.includes("integer"))
|
|
57
|
+
return "int";
|
|
58
|
+
if (typeList.includes("number"))
|
|
59
|
+
return "float";
|
|
60
|
+
if (typeList.includes("boolean"))
|
|
61
|
+
return "bool";
|
|
62
|
+
if (format === "date-time")
|
|
63
|
+
return "datetime";
|
|
64
|
+
if (format === "uuid")
|
|
65
|
+
return "uuid";
|
|
66
|
+
return "string";
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Lower one property of a write JSON Schema to a field spec. `schema` is the
|
|
70
|
+
* object schema (so requiredness can be read from its `required` list);
|
|
71
|
+
* returns `undefined` when the property does not exist.
|
|
72
|
+
*/
|
|
73
|
+
function specFromSchemaProperty(schema, fieldName) {
|
|
74
|
+
const property = schema?.properties?.[fieldName];
|
|
75
|
+
if (!property)
|
|
76
|
+
return undefined;
|
|
77
|
+
const rawType = property.type;
|
|
78
|
+
const typeList = typeof rawType === "string"
|
|
79
|
+
? [rawType]
|
|
80
|
+
: Array.isArray(rawType)
|
|
81
|
+
? rawType
|
|
82
|
+
: [];
|
|
83
|
+
const nullable = typeList.includes("null") || property.nullable === true;
|
|
84
|
+
const required = Array.isArray(schema?.required) && schema.required.includes(fieldName);
|
|
85
|
+
const relation = property["x-rastack-relation"];
|
|
86
|
+
const format = typeof property.format === "string" ? property.format : undefined;
|
|
87
|
+
const spec = {
|
|
88
|
+
name: fieldName,
|
|
89
|
+
type: relation && typeof relation === "object"
|
|
90
|
+
? "fk"
|
|
91
|
+
: schemaScalar(typeList, format),
|
|
92
|
+
required,
|
|
93
|
+
nullable,
|
|
94
|
+
};
|
|
95
|
+
if (relation && typeof relation === "object")
|
|
96
|
+
spec.relation = relation;
|
|
97
|
+
// `format` refines strings (`email`/`url`/`date`/`date-time`/`uuid`); the
|
|
98
|
+
// scalar kinds already encode datetime/uuid, so only keep it for strings.
|
|
99
|
+
if (spec.type === "string" && format)
|
|
100
|
+
spec.format = format;
|
|
101
|
+
if (typeof property.maxLength === "number")
|
|
102
|
+
spec.maxLength = property.maxLength;
|
|
103
|
+
if (Array.isArray(property.enum) && property.enum.length > 0) {
|
|
104
|
+
spec.options = property.enum.filter((v) => v !== null && v !== undefined);
|
|
105
|
+
}
|
|
106
|
+
if (property.default !== undefined)
|
|
107
|
+
spec.default = property.default;
|
|
108
|
+
return spec;
|
|
109
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./machine"), exports);
|
|
18
|
+
__exportStar(require("./adapters"), exports);
|