rastack 0.0.16 → 0.0.20

package/test/csv-schema.spec.ts

@@ -1,143 +1,143 @@
-import path from "path";
-import os from "os";
-import fs from "fs";
-import { parseCsv, inferSchema, discoverCsvFiles } from "../src/csv-schema";
-
-const REPO_ROOT = path.join(__dirname, "..");
-
-function tmpCsv(content: string): string {
-  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rad-test-"));
-  const file = path.join(dir, "test.csv");
-  fs.writeFileSync(file, content);
-  return file;
-}
-
-describe("parseCsv", () => {
-  it("parses headers and rows", () => {
-    const { headers, rows } = parseCsv("id,name,age\n1,Alice,30\n2,Bob,25");
-    expect(headers).toEqual(["id", "name", "age"]);
-    expect(rows).toHaveLength(2);
-    expect(rows[0]).toEqual(["1", "Alice", "30"]);
-  });
-
-  it("handles quoted fields with commas", () => {
-    const { headers, rows } = parseCsv(`name,bio\nAlice,"Engineer, Pilot"\nBob,Writer`);
-    expect(rows[0][1]).toBe("Engineer, Pilot");
-  });
-
-  it("returns empty for blank input", () => {
-    const { headers, rows } = parseCsv("");
-    expect(headers).toHaveLength(0);
-    expect(rows).toHaveLength(0);
-  });
-});
-
-describe("inferSchema — type inference", () => {
-  it("infers integer columns", () => {
-    const csv = tmpCsv("id,score\n1,100\n2,200\n3,300");
-    const { schema } = inferSchema(csv, REPO_ROOT);
-    const id = schema.columns.find((c) => c.name === "id")!;
-    const score = schema.columns.find((c) => c.name === "score")!;
-    expect(id.type).toBe("integer");
-    expect(score.type).toBe("integer");
-  });
-
-  it("infers float columns", () => {
-    const csv = tmpCsv("price\n1.99\n3.50\n0.01");
-    const { schema } = inferSchema(csv, REPO_ROOT);
-    expect(schema.columns[0].type).toBe("float");
-  });
-
-  it("infers boolean columns", () => {
-    const csv = tmpCsv("active\ntrue\nfalse\ntrue");
-    const { schema } = inferSchema(csv, REPO_ROOT);
-    expect(schema.columns[0].type).toBe("boolean");
-  });
-
-  it("infers date columns", () => {
-    const csv = tmpCsv("dob\n1990-01-01\n1985-06-15");
-    const { schema } = inferSchema(csv, REPO_ROOT);
-    // dob matches PII but type should still be date
-    expect(schema.columns[0].type).toBe("date");
-  });
-
-  it("infers datetime columns", () => {
-    const csv = tmpCsv("created_at\n2024-01-01T10:00:00Z\n2024-02-01T12:30:00Z");
-    const { schema } = inferSchema(csv, REPO_ROOT);
-    expect(schema.columns[0].type).toBe("datetime");
-  });
-
-  it("marks id column as primaryKey", () => {
-    const csv = tmpCsv("id,name\n1,Alice");
-    const { schema } = inferSchema(csv, REPO_ROOT);
-    const id = schema.columns.find((c) => c.name === "id")!;
-    expect(id.primaryKey).toBe(true);
-  });
-
-  it("detects nullable columns", () => {
-    const csv = tmpCsv("id,note\n1,hello\n2,\n3,world");
-    const { schema } = inferSchema(csv, REPO_ROOT);
-    const note = schema.columns.find((c) => c.name === "note")!;
-    expect(note.nullable).toBe(true);
-  });
-});
-
-describe("inferSchema — PII detection", () => {
-  it("flags passport_no as PII", () => {
-    const csv = tmpCsv("id,passport_no\n1,AB123456\n2,CD789012");
-    const { schema, piiViolations } = inferSchema(csv, REPO_ROOT);
-    const col = schema.columns.find((c) => c.name === "passport_no")!;
-    expect(col.pii).toBe(true);
-    expect(piiViolations).toHaveLength(1);
-    expect(piiViolations[0].column).toBe("passport_no");
-  });
-
-  it("flags email as PII", () => {
-    const csv = tmpCsv("id,email\n1,alice@example.com");
-    const { piiViolations } = inferSchema(csv, REPO_ROOT);
-    expect(piiViolations.some((v) => v.column === "email")).toBe(true);
-  });
-
-  it("flags dob as PII", () => {
-    const csv = tmpCsv("id,dob\n1,1990-01-01");
-    const { piiViolations } = inferSchema(csv, REPO_ROOT);
-    expect(piiViolations.some((v) => v.column === "dob")).toBe(true);
-  });
-
-  it("does not flag non-PII columns", () => {
-    const csv = tmpCsv("id,flight_code,origin,destination\n1,SA101,JNB,LHR");
-    const { piiViolations } = inferSchema(csv, REPO_ROOT);
-    expect(piiViolations).toHaveLength(0);
-  });
-
-  it("generates a stable fingerprint", () => {
-    const csv = tmpCsv("id,name\n1,Alice\n2,Bob");
-    const { schema: a } = inferSchema(csv, REPO_ROOT);
-    const { schema: b } = inferSchema(csv, REPO_ROOT);
-    expect(a.fingerprint).toBe(b.fingerprint);
-  });
-
-  it("generates different fingerprints for different schemas", () => {
-    const csvA = tmpCsv("id,name\n1,Alice");
-    const csvB = tmpCsv("id,name,email\n1,Alice,a@b.com");
-    const { schema: a } = inferSchema(csvA, REPO_ROOT);
-    const { schema: b } = inferSchema(csvB, REPO_ROOT);
-    expect(a.fingerprint).not.toBe(b.fingerprint);
-  });
-});
-
-describe("discoverCsvFiles", () => {
-  it("finds all csv files in a directory", () => {
-    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rad-test-"));
-    fs.writeFileSync(path.join(dir, "a.csv"), "id\n1");
-    fs.writeFileSync(path.join(dir, "b.csv"), "id\n2");
-    fs.writeFileSync(path.join(dir, "readme.txt"), "not a csv");
-    const files = discoverCsvFiles(dir);
-    expect(files).toHaveLength(2);
-    expect(files.every((f) => f.endsWith(".csv"))).toBe(true);
-  });
-
-  it("returns empty for missing directory", () => {
-    expect(discoverCsvFiles("/nonexistent/path")).toHaveLength(0);
-  });
-});
+import path from "path";
+import os from "os";
+import fs from "fs";
+import { parseCsv, inferSchema, discoverCsvFiles } from "../src/csv-schema";
+
+const REPO_ROOT = path.join(__dirname, "..");
+
+function tmpCsv(content: string): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rad-test-"));
+  const file = path.join(dir, "test.csv");
+  fs.writeFileSync(file, content);
+  return file;
+}
+
+describe("parseCsv", () => {
+  it("parses headers and rows", () => {
+    const { headers, rows } = parseCsv("id,name,age\n1,Alice,30\n2,Bob,25");
+    expect(headers).toEqual(["id", "name", "age"]);
+    expect(rows).toHaveLength(2);
+    expect(rows[0]).toEqual(["1", "Alice", "30"]);
+  });
+
+  it("handles quoted fields with commas", () => {
+    const { headers, rows } = parseCsv(`name,bio\nAlice,"Engineer, Pilot"\nBob,Writer`);
+    expect(rows[0][1]).toBe("Engineer, Pilot");
+  });
+
+  it("returns empty for blank input", () => {
+    const { headers, rows } = parseCsv("");
+    expect(headers).toHaveLength(0);
+    expect(rows).toHaveLength(0);
+  });
+});
+
+describe("inferSchema — type inference", () => {
+  it("infers integer columns", () => {
+    const csv = tmpCsv("id,score\n1,100\n2,200\n3,300");
+    const { schema } = inferSchema(csv, REPO_ROOT);
+    const id = schema.columns.find((c) => c.name === "id")!;
+    const score = schema.columns.find((c) => c.name === "score")!;
+    expect(id.type).toBe("integer");
+    expect(score.type).toBe("integer");
+  });
+
+  it("infers float columns", () => {
+    const csv = tmpCsv("price\n1.99\n3.50\n0.01");
+    const { schema } = inferSchema(csv, REPO_ROOT);
+    expect(schema.columns[0].type).toBe("float");
+  });
+
+  it("infers boolean columns", () => {
+    const csv = tmpCsv("active\ntrue\nfalse\ntrue");
+    const { schema } = inferSchema(csv, REPO_ROOT);
+    expect(schema.columns[0].type).toBe("boolean");
+  });
+
+  it("infers date columns", () => {
+    const csv = tmpCsv("dob\n1990-01-01\n1985-06-15");
+    const { schema } = inferSchema(csv, REPO_ROOT);
+    // dob matches PII but type should still be date
+    expect(schema.columns[0].type).toBe("date");
+  });
+
+  it("infers datetime columns", () => {
+    const csv = tmpCsv("created_at\n2024-01-01T10:00:00Z\n2024-02-01T12:30:00Z");
+    const { schema } = inferSchema(csv, REPO_ROOT);
+    expect(schema.columns[0].type).toBe("datetime");
+  });
+
+  it("marks id column as primaryKey", () => {
+    const csv = tmpCsv("id,name\n1,Alice");
+    const { schema } = inferSchema(csv, REPO_ROOT);
+    const id = schema.columns.find((c) => c.name === "id")!;
+    expect(id.primaryKey).toBe(true);
+  });
+
+  it("detects nullable columns", () => {
+    const csv = tmpCsv("id,note\n1,hello\n2,\n3,world");
+    const { schema } = inferSchema(csv, REPO_ROOT);
+    const note = schema.columns.find((c) => c.name === "note")!;
+    expect(note.nullable).toBe(true);
+  });
+});
+
+describe("inferSchema — PII detection", () => {
+  it("flags passport_no as PII", () => {
+    const csv = tmpCsv("id,passport_no\n1,AB123456\n2,CD789012");
+    const { schema, piiViolations } = inferSchema(csv, REPO_ROOT);
+    const col = schema.columns.find((c) => c.name === "passport_no")!;
+    expect(col.pii).toBe(true);
+    expect(piiViolations).toHaveLength(1);
+    expect(piiViolations[0].column).toBe("passport_no");
+  });
+
+  it("flags email as PII", () => {
+    const csv = tmpCsv("id,email\n1,alice@example.com");
+    const { piiViolations } = inferSchema(csv, REPO_ROOT);
+    expect(piiViolations.some((v) => v.column === "email")).toBe(true);
+  });
+
+  it("flags dob as PII", () => {
+    const csv = tmpCsv("id,dob\n1,1990-01-01");
+    const { piiViolations } = inferSchema(csv, REPO_ROOT);
+    expect(piiViolations.some((v) => v.column === "dob")).toBe(true);
+  });
+
+  it("does not flag non-PII columns", () => {
+    const csv = tmpCsv("id,flight_code,origin,destination\n1,SA101,JNB,LHR");
+    const { piiViolations } = inferSchema(csv, REPO_ROOT);
+    expect(piiViolations).toHaveLength(0);
+  });
+
+  it("generates a stable fingerprint", () => {
+    const csv = tmpCsv("id,name\n1,Alice\n2,Bob");
+    const { schema: a } = inferSchema(csv, REPO_ROOT);
+    const { schema: b } = inferSchema(csv, REPO_ROOT);
+    expect(a.fingerprint).toBe(b.fingerprint);
+  });
+
+  it("generates different fingerprints for different schemas", () => {
+    const csvA = tmpCsv("id,name\n1,Alice");
+    const csvB = tmpCsv("id,name,email\n1,Alice,a@b.com");
+    const { schema: a } = inferSchema(csvA, REPO_ROOT);
+    const { schema: b } = inferSchema(csvB, REPO_ROOT);
+    expect(a.fingerprint).not.toBe(b.fingerprint);
+  });
+});
+
+describe("discoverCsvFiles", () => {
+  it("finds all csv files in a directory", () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rad-test-"));
+    fs.writeFileSync(path.join(dir, "a.csv"), "id\n1");
+    fs.writeFileSync(path.join(dir, "b.csv"), "id\n2");
+    fs.writeFileSync(path.join(dir, "readme.txt"), "not a csv");
+    const files = discoverCsvFiles(dir);
+    expect(files).toHaveLength(2);
+    expect(files.every((f) => f.endsWith(".csv"))).toBe(true);
+  });
+
+  it("returns empty for missing directory", () => {
+    expect(discoverCsvFiles("/nonexistent/path")).toHaveLength(0);
+  });
+});