rastack 0.0.16 → 0.0.19

package/src/seed.ts

@@ -1,297 +1,297 @@
-#!/usr/bin/env node
-
-/**
- * rad seed
- *
- * Loads CSV files from a local directory or GCS bucket into the target
- * Iceberg namespace. Performs schema inference, PII check, and idempotent
- * table creation before writing rows.
- *
- * Usage:
- *   rad seed [--env dev|staging] [--source gs://bucket/ | --source ./myapp-dev/]
- *
- * Environment variables (set via CI secrets or .env):
- *   GCP_PROJECT             GCP project ID
- *   ICEBERG_WAREHOUSE       gs://... path to Iceberg warehouse root
- *   ICEBERG_CATALOG_URI     sqlite:///... or http://... catalog URI
- */
-
-import fs from "fs";
-import path from "path";
-import { execSync } from "child_process";
-import {
-  discoverCsvFiles,
-  inferSchema,
-  parseCsv,
-  TableSchema,
-} from "./csv-schema";
-
-// ---------------------------------------------------------------------------
-// Args
-// ---------------------------------------------------------------------------
-const args = process.argv.slice(2);
-
-function getArg(flag: string): string | null {
-  const i = args.indexOf(flag);
-  return i !== -1 && args[i + 1] ? args[i + 1] : null;
-}
-
-const env = getArg("--env") ?? "dev";
-const source = getArg("--source") ?? path.join(process.cwd(), "myapp-dev");
-
-const repoRoot = process.cwd();
-
-// ---------------------------------------------------------------------------
-// Step 1: resolve CSV files
-// ---------------------------------------------------------------------------
-function resolveFiles(): string[] {
-  if (source.startsWith("gs://")) {
-    return downloadFromGcs(source);
-  }
-  return discoverCsvFiles(source);
-}
-
-function downloadFromGcs(gcsUri: string): string[] {
-  const tmpDir = path.join(repoRoot, ".rad", "tmp", "seed");
-  fs.mkdirSync(tmpDir, { recursive: true });
-
-  console.log(`Downloading CSVs from ${gcsUri} ...`);
-  // `rad seed` is a local/CI developer tool; `gcsUri` is an operator-supplied
-  // --source argument (a gs:// bucket the operator controls), not untrusted
-  // network input, so there is no command-injection surface here.
-  try {
-    // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process
-    execSync(`gcloud storage cp "${gcsUri}*.csv" "${tmpDir}/" --quiet`, {
-      stdio: "inherit",
-    });
-  } catch {
-    // gcloud storage cp uses gsutil-style globs — fall back
-    // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process
-    execSync(`gsutil -m cp "${gcsUri}*.csv" "${tmpDir}/"`, {
-      stdio: "inherit",
-    });
-  }
-
-  return discoverCsvFiles(tmpDir);
-}
-
-// ---------------------------------------------------------------------------
-// Step 2: validate schemas (PII check — hard fail in CI)
-// ---------------------------------------------------------------------------
-function validateSchema(csvPath: string): TableSchema {
-  const { schema, piiViolations } = inferSchema(csvPath, repoRoot);
-
-  if (piiViolations.length > 0) {
-    const list = piiViolations
-      .map((v) => `  ${v.column}: ${v.reason}`)
-      .join("\n");
-    throw new Error(
-      `PII detected in ${path.basename(csvPath)}:\n${list}\n` +
-        `Remove PII data before seeding. Run "rad scan" to review.`,
-    );
-  }
-
-  return schema;
-}
-
-// ---------------------------------------------------------------------------
-// Step 3: emit Iceberg DDL for a table
-// ---------------------------------------------------------------------------
-function icebergType(col: TableSchema["columns"][number]["type"]): string {
-  switch (col) {
-    case "integer":
-      return "long";
-    case "float":
-      return "double";
-    case "boolean":
-      return "boolean";
-    case "date":
-      return "date";
-    case "datetime":
-      return "timestamp";
-    default:
-      return "string";
-  }
-}
-
-function buildCreateTableSql(schema: TableSchema, namespace: string): string {
-  const cols = schema.columns
-    .map((c) => {
-      const type = icebergType(c.type);
-      const nullable = c.nullable ? "" : " NOT NULL";
-      return `  ${c.name} ${type}${nullable}`;
-    })
-    .join(",\n");
-
-  return (
-    `CREATE TABLE IF NOT EXISTS ${namespace}.${schema.tableName} (\n` +
-    `${cols}\n` +
-    `) USING iceberg\n` +
-    `TBLPROPERTIES (\n` +
-    `  'write.format.default'='parquet',\n` +
-    `  'write.metadata.fingerprint'='${schema.fingerprint}'\n` +
-    `);`
-  );
-}
-
-// ---------------------------------------------------------------------------
-// Step 4: write schema fingerprints to registry
-// ---------------------------------------------------------------------------
-function writeFingerprint(schema: TableSchema): void {
-  const registryDir = path.join(repoRoot, ".rad", "registry");
-  fs.mkdirSync(registryDir, { recursive: true });
-  const registryPath = path.join(registryDir, `${schema.tableName}.json`);
-
-  let existing: Record<string, unknown> = {};
-  if (fs.existsSync(registryPath)) {
-    existing = JSON.parse(fs.readFileSync(registryPath, "utf-8"));
-  }
-
-  const prev = (existing as { fingerprint?: string }).fingerprint;
-  const changed = prev && prev !== schema.fingerprint;
-
-  const record = {
-    tableName: schema.tableName,
-    fingerprint: schema.fingerprint,
-    columns: schema.columns,
-    rowCount: schema.rowCount,
-    seededAt: new Date().toISOString(),
-    env,
-    previousFingerprint: prev ?? null,
-  };
-
-  fs.writeFileSync(registryPath, JSON.stringify(record, null, 2));
-
-  if (changed) {
-    console.warn(
-      `  ⚠  Schema change detected for "${schema.tableName}": ` +
-        `${prev} → ${schema.fingerprint}. A migration may be required.`,
-    );
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Step 5: build a Python seed script and run it via django-iceberg
-// ---------------------------------------------------------------------------
-function buildPythonSeedScript(
-  schemas: TableSchema[],
-  csvPaths: string[],
-): string {
-  const namespace = env;
-  const warehouse = process.env.ICEBERG_WAREHOUSE ?? "data/warehouse";
-  const catalogUri =
-    process.env.ICEBERG_CATALOG_URI ?? "sqlite:///data/catalog.db";
-
-  const tableBlocks = schemas
-    .map((schema, i) => {
-      const ddl = buildCreateTableSql(schema, namespace).replace(/`/g, "\\`");
-      const csvPathEscaped = csvPaths[i].replace(/\\/g, "/");
-      return `
-    # --- ${schema.tableName} ---
-    ddl = """${ddl}"""
-    cat.execute(ddl)
-    df = pl.read_csv("${csvPathEscaped}", infer_schema_length=1000)
-    tbl = cat.load_table("${namespace}.${schema.tableName}")
-    tbl.overwrite(df.to_arrow())
-    print(f"  Seeded ${schema.tableName}: ${schema.rowCount} rows")
-`;
-    })
-    .join("\n");
-
-  return `
-import polars as pl
-from pyiceberg.catalog import load_catalog
-
-cat = load_catalog(
-    "default",
-    **{
-        "uri": "${catalogUri}",
-        "warehouse": "${warehouse}",
-    }
-)
-
-try:
-    cat.create_namespace("${namespace}")
-except Exception:
-    pass  # namespace already exists
-
-${tableBlocks}
-
-print("Seed complete.")
-`.trim();
-}
-
-// ---------------------------------------------------------------------------
-// Main
-// ---------------------------------------------------------------------------
-async function main() {
-  console.log(`\nrad seed  env=${env}  source=${source}\n`);
-
-  const files = resolveFiles();
-  if (files.length === 0) {
-    console.log("No CSV files found. Nothing to seed.");
-    process.exit(0);
-  }
-
-  console.log(`Found ${files.length} CSV file(s):\n`);
-
-  const schemas: TableSchema[] = [];
-  for (const csvPath of files) {
-    const rel = path.relative(repoRoot, csvPath);
-    process.stdout.write(`  Validating ${rel} ... `);
-    try {
-      const schema = validateSchema(csvPath);
-      schemas.push(schema);
-      console.log(
-        `OK  [${schema.columns.length} cols, ${schema.rowCount} rows]`,
-      );
-    } catch (err) {
-      console.error(`FAIL\n${(err as Error).message}`);
-      process.exit(1);
-    }
-  }
-
-  // Write fingerprints before seeding so CI can detect schema drift
-  for (const schema of schemas) {
-    writeFingerprint(schema);
-  }
-
-  // Emit DDL summary
-  const ddlDir = path.join(repoRoot, ".rad", "ddl");
-  fs.mkdirSync(ddlDir, { recursive: true });
-  for (const schema of schemas) {
-    const ddl = buildCreateTableSql(schema, env);
-    fs.writeFileSync(path.join(ddlDir, `${schema.tableName}.sql`), ddl + "\n");
-  }
-
-  console.log(`\nDDL written to .rad/ddl/`);
-
-  // Run Python seed via pyiceberg if available
-  const seedScript = buildPythonSeedScript(schemas, files);
-  const scriptPath = path.join(repoRoot, ".rad", "tmp", "seed.py");
-  fs.mkdirSync(path.dirname(scriptPath), { recursive: true });
-  fs.writeFileSync(scriptPath, seedScript);
-
-  const pythonBin = process.env.PYTHON ?? "python";
-  try {
-    console.log(`\nSeeding Iceberg namespace "${env}" ...\n`);
-    // Runs an internally-generated seed script with the operator-configured
-    // python binary; no untrusted input, so no command-injection surface.
-    // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process
-    execSync(`${pythonBin} "${scriptPath}"`, { stdio: "inherit" });
-  } catch {
-    console.error(
-      "\nPython seed step failed. Ensure pyiceberg and polars are installed:\n" +
-        "  pip install pyiceberg polars pyarrow\n\n" +
-        `Generated seed script: ${scriptPath}`,
-    );
-    process.exit(1);
-  }
-
-  console.log("\nDone.");
-}
-
-main().catch((err) => {
-  console.error(err);
-  process.exit(1);
-});
+#!/usr/bin/env node
+
+/**
+ * rad seed
+ *
+ * Loads CSV files from a local directory or GCS bucket into the target
+ * Iceberg namespace. Performs schema inference, PII check, and idempotent
+ * table creation before writing rows.
+ *
+ * Usage:
+ *   rad seed [--env dev|staging] [--source gs://bucket/ | --source ./myapp-dev/]
+ *
+ * Environment variables (set via CI secrets or .env):
+ *   GCP_PROJECT             GCP project ID
+ *   ICEBERG_WAREHOUSE       gs://... path to Iceberg warehouse root
+ *   ICEBERG_CATALOG_URI     sqlite:///... or http://... catalog URI
+ */
+
+import fs from "fs";
+import path from "path";
+import { execSync } from "child_process";
+import {
+  discoverCsvFiles,
+  inferSchema,
+  parseCsv,
+  TableSchema,
+} from "./csv-schema";
+
+// ---------------------------------------------------------------------------
+// Args
+// ---------------------------------------------------------------------------
+const args = process.argv.slice(2);
+
+function getArg(flag: string): string | null {
+  const i = args.indexOf(flag);
+  return i !== -1 && args[i + 1] ? args[i + 1] : null;
+}
+
+const env = getArg("--env") ?? "dev";
+const source = getArg("--source") ?? path.join(process.cwd(), "myapp-dev");
+
+const repoRoot = process.cwd();
+
+// ---------------------------------------------------------------------------
+// Step 1: resolve CSV files
+// ---------------------------------------------------------------------------
+function resolveFiles(): string[] {
+  if (source.startsWith("gs://")) {
+    return downloadFromGcs(source);
+  }
+  return discoverCsvFiles(source);
+}
+
+function downloadFromGcs(gcsUri: string): string[] {
+  const tmpDir = path.join(repoRoot, ".rad", "tmp", "seed");
+  fs.mkdirSync(tmpDir, { recursive: true });
+
+  console.log(`Downloading CSVs from ${gcsUri} ...`);
+  // `rad seed` is a local/CI developer tool; `gcsUri` is an operator-supplied
+  // --source argument (a gs:// bucket the operator controls), not untrusted
+  // network input, so there is no command-injection surface here.
+  try {
+    // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process
+    execSync(`gcloud storage cp "${gcsUri}*.csv" "${tmpDir}/" --quiet`, {
+      stdio: "inherit",
+    });
+  } catch {
+    // gcloud storage cp uses gsutil-style globs — fall back
+    // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process
+    execSync(`gsutil -m cp "${gcsUri}*.csv" "${tmpDir}/"`, {
+      stdio: "inherit",
+    });
+  }
+
+  return discoverCsvFiles(tmpDir);
+}
+
+// ---------------------------------------------------------------------------
+// Step 2: validate schemas (PII check — hard fail in CI)
+// ---------------------------------------------------------------------------
+function validateSchema(csvPath: string): TableSchema {
+  const { schema, piiViolations } = inferSchema(csvPath, repoRoot);
+
+  if (piiViolations.length > 0) {
+    const list = piiViolations
+      .map((v) => `  ${v.column}: ${v.reason}`)
+      .join("\n");
+    throw new Error(
+      `PII detected in ${path.basename(csvPath)}:\n${list}\n` +
+        `Remove PII data before seeding. Run "rad scan" to review.`,
+    );
+  }
+
+  return schema;
+}
+
+// ---------------------------------------------------------------------------
+// Step 3: emit Iceberg DDL for a table
+// ---------------------------------------------------------------------------
+function icebergType(col: TableSchema["columns"][number]["type"]): string {
+  switch (col) {
+    case "integer":
+      return "long";
+    case "float":
+      return "double";
+    case "boolean":
+      return "boolean";
+    case "date":
+      return "date";
+    case "datetime":
+      return "timestamp";
+    default:
+      return "string";
+  }
+}
+
+function buildCreateTableSql(schema: TableSchema, namespace: string): string {
+  const cols = schema.columns
+    .map((c) => {
+      const type = icebergType(c.type);
+      const nullable = c.nullable ? "" : " NOT NULL";
+      return `  ${c.name} ${type}${nullable}`;
+    })
+    .join(",\n");
+
+  return (
+    `CREATE TABLE IF NOT EXISTS ${namespace}.${schema.tableName} (\n` +
+    `${cols}\n` +
+    `) USING iceberg\n` +
+    `TBLPROPERTIES (\n` +
+    `  'write.format.default'='parquet',\n` +
+    `  'write.metadata.fingerprint'='${schema.fingerprint}'\n` +
+    `);`
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Step 4: write schema fingerprints to registry
+// ---------------------------------------------------------------------------
+function writeFingerprint(schema: TableSchema): void {
+  const registryDir = path.join(repoRoot, ".rad", "registry");
+  fs.mkdirSync(registryDir, { recursive: true });
+  const registryPath = path.join(registryDir, `${schema.tableName}.json`);
+
+  let existing: Record<string, unknown> = {};
+  if (fs.existsSync(registryPath)) {
+    existing = JSON.parse(fs.readFileSync(registryPath, "utf-8"));
+  }
+
+  const prev = (existing as { fingerprint?: string }).fingerprint;
+  const changed = prev && prev !== schema.fingerprint;
+
+  const record = {
+    tableName: schema.tableName,
+    fingerprint: schema.fingerprint,
+    columns: schema.columns,
+    rowCount: schema.rowCount,
+    seededAt: new Date().toISOString(),
+    env,
+    previousFingerprint: prev ?? null,
+  };
+
+  fs.writeFileSync(registryPath, JSON.stringify(record, null, 2));
+
+  if (changed) {
+    console.warn(
+      `  ⚠  Schema change detected for "${schema.tableName}": ` +
+        `${prev} → ${schema.fingerprint}. A migration may be required.`,
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Step 5: build a Python seed script and run it via django-iceberg
+// ---------------------------------------------------------------------------
+function buildPythonSeedScript(
+  schemas: TableSchema[],
+  csvPaths: string[],
+): string {
+  const namespace = env;
+  const warehouse = process.env.ICEBERG_WAREHOUSE ?? "data/warehouse";
+  const catalogUri =
+    process.env.ICEBERG_CATALOG_URI ?? "sqlite:///data/catalog.db";
+
+  const tableBlocks = schemas
+    .map((schema, i) => {
+      const ddl = buildCreateTableSql(schema, namespace).replace(/`/g, "\\`");
+      const csvPathEscaped = csvPaths[i].replace(/\\/g, "/");
+      return `
+    # --- ${schema.tableName} ---
+    ddl = """${ddl}"""
+    cat.execute(ddl)
+    df = pl.read_csv("${csvPathEscaped}", infer_schema_length=1000)
+    tbl = cat.load_table("${namespace}.${schema.tableName}")
+    tbl.overwrite(df.to_arrow())
+    print(f"  Seeded ${schema.tableName}: ${schema.rowCount} rows")
+`;
+    })
+    .join("\n");
+
+  return `
+import polars as pl
+from pyiceberg.catalog import load_catalog
+
+cat = load_catalog(
+    "default",
+    **{
+        "uri": "${catalogUri}",
+        "warehouse": "${warehouse}",
+    }
+)
+
+try:
+    cat.create_namespace("${namespace}")
+except Exception:
+    pass  # namespace already exists
+
+${tableBlocks}
+
+print("Seed complete.")
+`.trim();
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+async function main() {
+  console.log(`\nrad seed  env=${env}  source=${source}\n`);
+
+  const files = resolveFiles();
+  if (files.length === 0) {
+    console.log("No CSV files found. Nothing to seed.");
+    process.exit(0);
+  }
+
+  console.log(`Found ${files.length} CSV file(s):\n`);
+
+  const schemas: TableSchema[] = [];
+  for (const csvPath of files) {
+    const rel = path.relative(repoRoot, csvPath);
+    process.stdout.write(`  Validating ${rel} ... `);
+    try {
+      const schema = validateSchema(csvPath);
+      schemas.push(schema);
+      console.log(
+        `OK  [${schema.columns.length} cols, ${schema.rowCount} rows]`,
+      );
+    } catch (err) {
+      console.error(`FAIL\n${(err as Error).message}`);
+      process.exit(1);
+    }
+  }
+
+  // Write fingerprints before seeding so CI can detect schema drift
+  for (const schema of schemas) {
+    writeFingerprint(schema);
+  }
+
+  // Emit DDL summary
+  const ddlDir = path.join(repoRoot, ".rad", "ddl");
+  fs.mkdirSync(ddlDir, { recursive: true });
+  for (const schema of schemas) {
+    const ddl = buildCreateTableSql(schema, env);
+    fs.writeFileSync(path.join(ddlDir, `${schema.tableName}.sql`), ddl + "\n");
+  }
+
+  console.log(`\nDDL written to .rad/ddl/`);
+
+  // Run Python seed via pyiceberg if available
+  const seedScript = buildPythonSeedScript(schemas, files);
+  const scriptPath = path.join(repoRoot, ".rad", "tmp", "seed.py");
+  fs.mkdirSync(path.dirname(scriptPath), { recursive: true });
+  fs.writeFileSync(scriptPath, seedScript);
+
+  const pythonBin = process.env.PYTHON ?? "python";
+  try {
+    console.log(`\nSeeding Iceberg namespace "${env}" ...\n`);
+    // Runs an internally-generated seed script with the operator-configured
+    // python binary; no untrusted input, so no command-injection surface.
+    // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process
+    execSync(`${pythonBin} "${scriptPath}"`, { stdio: "inherit" });
+  } catch {
+    console.error(
+      "\nPython seed step failed. Ensure pyiceberg and polars are installed:\n" +
+        "  pip install pyiceberg polars pyarrow\n\n" +
+        `Generated seed script: ${scriptPath}`,
+    );
+    process.exit(1);
+  }
+
+  console.log("\nDone.");
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});