rastack 0.0.16 → 0.0.20

package/src/compile/program.ts

@@ -1,40 +1,40 @@
-import * as path from "path";
-import * as ts from "typescript";
-
-/**
- * Build a `ts.Program` (and thus a `TypeChecker`) over a set of resource
- * definition files. The DSL package (`rad-tools/define`) is mapped to the
- * in-tree/installed sources via `paths` so both the shipped package name and
- * the local demo/tests resolve without extra linking.
- */
-export function createProgram(fileNames: string[]): ts.Program {
-  const defineDir = path.resolve(__dirname, "../define");
-
-  const options: ts.CompilerOptions = {
-    target: ts.ScriptTarget.ES2019,
-    module: ts.ModuleKind.CommonJS,
-    moduleResolution: ts.ModuleResolutionKind.Node10,
-    strict: true,
-    skipLibCheck: true,
-    esModuleInterop: true,
-    noEmit: true,
-    baseUrl: process.cwd(),
-    paths: {
-      "rad-tools/define": [defineDir],
-      "rad-tools/define/*": [defineDir + "/*"],
-    },
-  };
-
-  return ts.createProgram(fileNames, options);
-}
-
-/** Every non-declaration source file the program was rooted at. */
-export function resourceSourceFiles(
-  program: ts.Program,
-  fileNames: string[],
-): ts.SourceFile[] {
-  const wanted = new Set(fileNames.map((f) => path.resolve(f)));
-  return program
-    .getSourceFiles()
-    .filter((sf) => !sf.isDeclarationFile && wanted.has(path.resolve(sf.fileName)));
-}
+import * as path from "path";
+import * as ts from "typescript";
+
+/**
+ * Build a `ts.Program` (and thus a `TypeChecker`) over a set of resource
+ * definition files. The DSL package (`rad-tools/define`) is mapped to the
+ * in-tree/installed sources via `paths` so both the shipped package name and
+ * the local demo/tests resolve without extra linking.
+ */
+export function createProgram(fileNames: string[]): ts.Program {
+  const defineDir = path.resolve(__dirname, "../define");
+
+  const options: ts.CompilerOptions = {
+    target: ts.ScriptTarget.ES2019,
+    module: ts.ModuleKind.CommonJS,
+    moduleResolution: ts.ModuleResolutionKind.Node10,
+    strict: true,
+    skipLibCheck: true,
+    esModuleInterop: true,
+    noEmit: true,
+    baseUrl: process.cwd(),
+    paths: {
+      "rad-tools/define": [defineDir],
+      "rad-tools/define/*": [defineDir + "/*"],
+    },
+  };
+
+  return ts.createProgram(fileNames, options);
+}
+
+/** Every non-declaration source file the program was rooted at. */
+export function resourceSourceFiles(
+  program: ts.Program,
+  fileNames: string[],
+): ts.SourceFile[] {
+  const wanted = new Set(fileNames.map((f) => path.resolve(f)));
+  return program
+    .getSourceFiles()
+    .filter((sf) => !sf.isDeclarationFile && wanted.has(path.resolve(sf.fileName)));
+}

package/src/csv-schema.ts

@@ -1,187 +1,187 @@
-import fs from "fs";
-import path from "path";
-import { minimatch } from "minimatch";
-import yaml from "js-yaml";
-
-export interface ColumnSchema {
-  name: string;
-  type: "integer" | "float" | "boolean" | "date" | "datetime" | "string";
-  nullable: boolean;
-  unique: boolean;
-  primaryKey: boolean;
-  pii: boolean;
-  piiReason?: string;
-}
-
-export interface TableSchema {
-  tableName: string;
-  sourceFile: string;
-  columns: ColumnSchema[];
-  rowCount: number;
-  fingerprint: string;
-}
-
-interface PiiConfig {
-  patterns: string[];
-  luhn_check: boolean;
-  require_policy: boolean;
-}
-
-const DEFAULT_PII_PATTERNS = [
-  "*passport*",
-  "*_no",
-  "*_number",
-  "ssn",
-  "national_id",
-  "date_of_birth",
-  "dob",
-  "email",
-  "*phone*",
-  "*credit_card*",
-  "*card_number*",
-  "ip_address",
-  "latitude",
-  "longitude",
-  "*salary*",
-  "*income*",
-];
-
-function loadPiiConfig(repoRoot: string): PiiConfig {
-  const configPath = path.join(repoRoot, ".rad", "pii-patterns.yml");
-  if (!fs.existsSync(configPath)) {
-    return { patterns: DEFAULT_PII_PATTERNS, luhn_check: true, require_policy: true };
-  }
-  return yaml.load(fs.readFileSync(configPath, "utf-8")) as PiiConfig;
-}
-
-function isPiiColumn(name: string, config: PiiConfig): string | null {
-  const lower = name.toLowerCase();
-  for (const pattern of config.patterns) {
-    if (minimatch(lower, pattern.toLowerCase())) {
-      return `matches PII pattern "${pattern}"`;
-    }
-  }
-  return null;
-}
-
-function luhnCheck(value: string): boolean {
-  const digits = value.replace(/\D/g, "");
-  if (digits.length < 13) return false;
-  let sum = 0;
-  let alternate = false;
-  for (let i = digits.length - 1; i >= 0; i--) {
-    let n = parseInt(digits[i], 10);
-    if (alternate) {
-      n *= 2;
-      if (n > 9) n -= 9;
-    }
-    sum += n;
-    alternate = !alternate;
-  }
-  return sum % 10 === 0;
-}
-
-function inferType(values: string[]): ColumnSchema["type"] {
-  const nonEmpty = values.filter((v) => v.trim() !== "" && v.toLowerCase() !== "null");
-  if (nonEmpty.length === 0) return "string";
-
-  if (nonEmpty.every((v) => /^\d+$/.test(v.trim()))) return "integer";
-  if (nonEmpty.every((v) => /^-?\d+(\.\d+)?$/.test(v.trim()))) return "float";
-  if (nonEmpty.every((v) => /^(true|false|yes|no|1|0)$/i.test(v.trim()))) return "boolean";
-  if (nonEmpty.every((v) => /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/.test(v.trim()))) return "datetime";
-  if (nonEmpty.every((v) => /^\d{4}-\d{2}-\d{2}$/.test(v.trim()))) return "date";
-  return "string";
-}
-
-export function parseCsv(content: string): { headers: string[]; rows: string[][] } {
-  const lines = content.split(/\r?\n/).filter((l) => l.trim() !== "");
-  if (lines.length === 0) return { headers: [], rows: [] };
-
-  const splitLine = (line: string): string[] => {
-    const result: string[] = [];
-    let current = "";
-    let inQuotes = false;
-    for (let i = 0; i < line.length; i++) {
-      if (line[i] === '"') {
-        inQuotes = !inQuotes;
-      } else if (line[i] === "," && !inQuotes) {
-        result.push(current.trim());
-        current = "";
-      } else {
-        current += line[i];
-      }
-    }
-    result.push(current.trim());
-    return result;
-  };
-
-  const headers = splitLine(lines[0]);
-  const rows = lines.slice(1).map(splitLine);
-  return { headers, rows };
-}
-
-function fingerprint(schema: Omit<TableSchema, "fingerprint">): string {
-  const sig = schema.columns.map((c) => `${c.name}:${c.type}:${c.pii}`).join("|");
-  // Simple hash — not crypto, just change-detection
-  let h = 0;
-  for (let i = 0; i < sig.length; i++) {
-    h = (Math.imul(31, h) + sig.charCodeAt(i)) | 0;
-  }
-  return Math.abs(h).toString(16).padStart(8, "0");
-}
-
-export function inferSchema(
-  csvPath: string,
-  repoRoot: string,
-): { schema: TableSchema; piiViolations: { column: string; reason: string }[] } {
-  const content = fs.readFileSync(csvPath, "utf-8");
-  const { headers, rows } = parseCsv(content);
-  const piiConfig = loadPiiConfig(repoRoot);
-  const tableName = path.basename(csvPath, ".csv");
-
-  const piiViolations: { column: string; reason: string }[] = [];
-
-  const columns: ColumnSchema[] = headers.map((name, i) => {
-    const values = rows.map((r) => r[i] ?? "");
-    const type = inferType(values);
-    const nullable = values.some((v) => v.trim() === "" || v.toLowerCase() === "null");
-    const nonEmpty = values.filter((v) => v.trim() !== "");
-    const unique = new Set(nonEmpty).size === nonEmpty.length;
-    const isPrimary = name.toLowerCase() === "id";
-
-    // PII: column name pattern match
-    let pii = false;
-    let piiReason: string | undefined;
-    const nameReason = isPiiColumn(name, piiConfig);
-    if (nameReason) {
-      pii = true;
-      piiReason = nameReason;
-    }
-
-    // PII: Luhn check on string columns that look numeric
-    if (!pii && piiConfig.luhn_check && type === "string") {
-      const luhnHits = values.filter((v) => v.trim() !== "" && luhnCheck(v));
-      if (luhnHits.length > values.length * 0.5) {
-        pii = true;
-        piiReason = "majority of values pass Luhn check (possible card numbers)";
-      }
-    }
-
-    if (pii && piiReason) {
-      piiViolations.push({ column: name, reason: piiReason });
-    }
-
-    return { name, type, nullable, unique, primaryKey: isPrimary, pii, piiReason };
-  });
-
-  const base = { tableName, sourceFile: csvPath, columns, rowCount: rows.length };
-  return { schema: { ...base, fingerprint: fingerprint(base) }, piiViolations };
-}
-
-export function discoverCsvFiles(dir: string): string[] {
-  if (!fs.existsSync(dir)) return [];
-  return fs
-    .readdirSync(dir)
-    .filter((f) => f.endsWith(".csv"))
-    .map((f) => path.join(dir, f));
-}
+import fs from "fs";
+import path from "path";
+import { minimatch } from "minimatch";
+import yaml from "js-yaml";
+
+export interface ColumnSchema {
+  name: string;
+  type: "integer" | "float" | "boolean" | "date" | "datetime" | "string";
+  nullable: boolean;
+  unique: boolean;
+  primaryKey: boolean;
+  pii: boolean;
+  piiReason?: string;
+}
+
+export interface TableSchema {
+  tableName: string;
+  sourceFile: string;
+  columns: ColumnSchema[];
+  rowCount: number;
+  fingerprint: string;
+}
+
+interface PiiConfig {
+  patterns: string[];
+  luhn_check: boolean;
+  require_policy: boolean;
+}
+
+const DEFAULT_PII_PATTERNS = [
+  "*passport*",
+  "*_no",
+  "*_number",
+  "ssn",
+  "national_id",
+  "date_of_birth",
+  "dob",
+  "email",
+  "*phone*",
+  "*credit_card*",
+  "*card_number*",
+  "ip_address",
+  "latitude",
+  "longitude",
+  "*salary*",
+  "*income*",
+];
+
+function loadPiiConfig(repoRoot: string): PiiConfig {
+  const configPath = path.join(repoRoot, ".rad", "pii-patterns.yml");
+  if (!fs.existsSync(configPath)) {
+    return { patterns: DEFAULT_PII_PATTERNS, luhn_check: true, require_policy: true };
+  }
+  return yaml.load(fs.readFileSync(configPath, "utf-8")) as PiiConfig;
+}
+
+function isPiiColumn(name: string, config: PiiConfig): string | null {
+  const lower = name.toLowerCase();
+  for (const pattern of config.patterns) {
+    if (minimatch(lower, pattern.toLowerCase())) {
+      return `matches PII pattern "${pattern}"`;
+    }
+  }
+  return null;
+}
+
+function luhnCheck(value: string): boolean {
+  const digits = value.replace(/\D/g, "");
+  if (digits.length < 13) return false;
+  let sum = 0;
+  let alternate = false;
+  for (let i = digits.length - 1; i >= 0; i--) {
+    let n = parseInt(digits[i], 10);
+    if (alternate) {
+      n *= 2;
+      if (n > 9) n -= 9;
+    }
+    sum += n;
+    alternate = !alternate;
+  }
+  return sum % 10 === 0;
+}
+
+function inferType(values: string[]): ColumnSchema["type"] {
+  const nonEmpty = values.filter((v) => v.trim() !== "" && v.toLowerCase() !== "null");
+  if (nonEmpty.length === 0) return "string";
+
+  if (nonEmpty.every((v) => /^\d+$/.test(v.trim()))) return "integer";
+  if (nonEmpty.every((v) => /^-?\d+(\.\d+)?$/.test(v.trim()))) return "float";
+  if (nonEmpty.every((v) => /^(true|false|yes|no|1|0)$/i.test(v.trim()))) return "boolean";
+  if (nonEmpty.every((v) => /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}/.test(v.trim()))) return "datetime";
+  if (nonEmpty.every((v) => /^\d{4}-\d{2}-\d{2}$/.test(v.trim()))) return "date";
+  return "string";
+}
+
+export function parseCsv(content: string): { headers: string[]; rows: string[][] } {
+  const lines = content.split(/\r?\n/).filter((l) => l.trim() !== "");
+  if (lines.length === 0) return { headers: [], rows: [] };
+
+  const splitLine = (line: string): string[] => {
+    const result: string[] = [];
+    let current = "";
+    let inQuotes = false;
+    for (let i = 0; i < line.length; i++) {
+      if (line[i] === '"') {
+        inQuotes = !inQuotes;
+      } else if (line[i] === "," && !inQuotes) {
+        result.push(current.trim());
+        current = "";
+      } else {
+        current += line[i];
+      }
+    }
+    result.push(current.trim());
+    return result;
+  };
+
+  const headers = splitLine(lines[0]);
+  const rows = lines.slice(1).map(splitLine);
+  return { headers, rows };
+}
+
+function fingerprint(schema: Omit<TableSchema, "fingerprint">): string {
+  const sig = schema.columns.map((c) => `${c.name}:${c.type}:${c.pii}`).join("|");
+  // Simple hash — not crypto, just change-detection
+  let h = 0;
+  for (let i = 0; i < sig.length; i++) {
+    h = (Math.imul(31, h) + sig.charCodeAt(i)) | 0;
+  }
+  return Math.abs(h).toString(16).padStart(8, "0");
+}
+
+export function inferSchema(
+  csvPath: string,
+  repoRoot: string,
+): { schema: TableSchema; piiViolations: { column: string; reason: string }[] } {
+  const content = fs.readFileSync(csvPath, "utf-8");
+  const { headers, rows } = parseCsv(content);
+  const piiConfig = loadPiiConfig(repoRoot);
+  const tableName = path.basename(csvPath, ".csv");
+
+  const piiViolations: { column: string; reason: string }[] = [];
+
+  const columns: ColumnSchema[] = headers.map((name, i) => {
+    const values = rows.map((r) => r[i] ?? "");
+    const type = inferType(values);
+    const nullable = values.some((v) => v.trim() === "" || v.toLowerCase() === "null");
+    const nonEmpty = values.filter((v) => v.trim() !== "");
+    const unique = new Set(nonEmpty).size === nonEmpty.length;
+    const isPrimary = name.toLowerCase() === "id";
+
+    // PII: column name pattern match
+    let pii = false;
+    let piiReason: string | undefined;
+    const nameReason = isPiiColumn(name, piiConfig);
+    if (nameReason) {
+      pii = true;
+      piiReason = nameReason;
+    }
+
+    // PII: Luhn check on string columns that look numeric
+    if (!pii && piiConfig.luhn_check && type === "string") {
+      const luhnHits = values.filter((v) => v.trim() !== "" && luhnCheck(v));
+      if (luhnHits.length > values.length * 0.5) {
+        pii = true;
+        piiReason = "majority of values pass Luhn check (possible card numbers)";
+      }
+    }
+
+    if (pii && piiReason) {
+      piiViolations.push({ column: name, reason: piiReason });
+    }
+
+    return { name, type, nullable, unique, primaryKey: isPrimary, pii, piiReason };
+  });
+
+  const base = { tableName, sourceFile: csvPath, columns, rowCount: rows.length };
+  return { schema: { ...base, fingerprint: fingerprint(base) }, piiViolations };
+}
+
+export function discoverCsvFiles(dir: string): string[] {
+  if (!fs.existsSync(dir)) return [];
+  return fs
+    .readdirSync(dir)
+    .filter((f) => f.endsWith(".csv"))
+    .map((f) => path.join(dir, f));
+}