rastack 0.0.16

package/src/seed.ts

@@ -0,0 +1,297 @@
+#!/usr/bin/env node
+
+/**
+ * rad seed
+ *
+ * Loads CSV files from a local directory or GCS bucket into the target
+ * Iceberg namespace. Performs schema inference, PII check, and idempotent
+ * table creation before writing rows.
+ *
+ * Usage:
+ *   rad seed [--env dev|staging] [--source gs://bucket/ | --source ./myapp-dev/]
+ *
+ * Environment variables (set via CI secrets or .env):
+ *   GCP_PROJECT             GCP project ID
+ *   ICEBERG_WAREHOUSE       gs://... path to Iceberg warehouse root
+ *   ICEBERG_CATALOG_URI     sqlite:///... or http://... catalog URI
+ */
+
+import fs from "fs";
+import path from "path";
+import { execSync } from "child_process";
+import {
+  discoverCsvFiles,
+  inferSchema,
+  parseCsv,
+  TableSchema,
+} from "./csv-schema";
+
+// ---------------------------------------------------------------------------
+// Args
+// ---------------------------------------------------------------------------
+const args = process.argv.slice(2);
+
+function getArg(flag: string): string | null {
+  const i = args.indexOf(flag);
+  return i !== -1 && args[i + 1] ? args[i + 1] : null;
+}
+
+const env = getArg("--env") ?? "dev";
+const source = getArg("--source") ?? path.join(process.cwd(), "myapp-dev");
+
+const repoRoot = process.cwd();
+
+// ---------------------------------------------------------------------------
+// Step 1: resolve CSV files
+// ---------------------------------------------------------------------------
+function resolveFiles(): string[] {
+  if (source.startsWith("gs://")) {
+    return downloadFromGcs(source);
+  }
+  return discoverCsvFiles(source);
+}
+
+function downloadFromGcs(gcsUri: string): string[] {
+  const tmpDir = path.join(repoRoot, ".rad", "tmp", "seed");
+  fs.mkdirSync(tmpDir, { recursive: true });
+
+  console.log(`Downloading CSVs from ${gcsUri} ...`);
+  // `rad seed` is a local/CI developer tool; `gcsUri` is an operator-supplied
+  // --source argument (a gs:// bucket the operator controls), not untrusted
+  // network input, so there is no command-injection surface here.
+  try {
+    // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process
+    execSync(`gcloud storage cp "${gcsUri}*.csv" "${tmpDir}/" --quiet`, {
+      stdio: "inherit",
+    });
+  } catch {
+    // gcloud storage cp uses gsutil-style globs — fall back
+    // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process
+    execSync(`gsutil -m cp "${gcsUri}*.csv" "${tmpDir}/"`, {
+      stdio: "inherit",
+    });
+  }
+
+  return discoverCsvFiles(tmpDir);
+}
+
+// ---------------------------------------------------------------------------
+// Step 2: validate schemas (PII check — hard fail in CI)
+// ---------------------------------------------------------------------------
+function validateSchema(csvPath: string): TableSchema {
+  const { schema, piiViolations } = inferSchema(csvPath, repoRoot);
+
+  if (piiViolations.length > 0) {
+    const list = piiViolations
+      .map((v) => `  ${v.column}: ${v.reason}`)
+      .join("\n");
+    throw new Error(
+      `PII detected in ${path.basename(csvPath)}:\n${list}\n` +
+        `Remove PII data before seeding. Run "rad scan" to review.`,
+    );
+  }
+
+  return schema;
+}
+
+// ---------------------------------------------------------------------------
+// Step 3: emit Iceberg DDL for a table
+// ---------------------------------------------------------------------------
+function icebergType(col: TableSchema["columns"][number]["type"]): string {
+  switch (col) {
+    case "integer":
+      return "long";
+    case "float":
+      return "double";
+    case "boolean":
+      return "boolean";
+    case "date":
+      return "date";
+    case "datetime":
+      return "timestamp";
+    default:
+      return "string";
+  }
+}
+
+function buildCreateTableSql(schema: TableSchema, namespace: string): string {
+  const cols = schema.columns
+    .map((c) => {
+      const type = icebergType(c.type);
+      const nullable = c.nullable ? "" : " NOT NULL";
+      return `  ${c.name} ${type}${nullable}`;
+    })
+    .join(",\n");
+
+  return (
+    `CREATE TABLE IF NOT EXISTS ${namespace}.${schema.tableName} (\n` +
+    `${cols}\n` +
+    `) USING iceberg\n` +
+    `TBLPROPERTIES (\n` +
+    `  'write.format.default'='parquet',\n` +
+    `  'write.metadata.fingerprint'='${schema.fingerprint}'\n` +
+    `);`
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Step 4: write schema fingerprints to registry
+// ---------------------------------------------------------------------------
+function writeFingerprint(schema: TableSchema): void {
+  const registryDir = path.join(repoRoot, ".rad", "registry");
+  fs.mkdirSync(registryDir, { recursive: true });
+  const registryPath = path.join(registryDir, `${schema.tableName}.json`);
+
+  let existing: Record<string, unknown> = {};
+  if (fs.existsSync(registryPath)) {
+    existing = JSON.parse(fs.readFileSync(registryPath, "utf-8"));
+  }
+
+  const prev = (existing as { fingerprint?: string }).fingerprint;
+  const changed = prev && prev !== schema.fingerprint;
+
+  const record = {
+    tableName: schema.tableName,
+    fingerprint: schema.fingerprint,
+    columns: schema.columns,
+    rowCount: schema.rowCount,
+    seededAt: new Date().toISOString(),
+    env,
+    previousFingerprint: prev ?? null,
+  };
+
+  fs.writeFileSync(registryPath, JSON.stringify(record, null, 2));
+
+  if (changed) {
+    console.warn(
+      `  ⚠  Schema change detected for "${schema.tableName}": ` +
+        `${prev} → ${schema.fingerprint}. A migration may be required.`,
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Step 5: build a Python seed script and run it via django-iceberg
+// ---------------------------------------------------------------------------
+function buildPythonSeedScript(
+  schemas: TableSchema[],
+  csvPaths: string[],
+): string {
+  const namespace = env;
+  const warehouse = process.env.ICEBERG_WAREHOUSE ?? "data/warehouse";
+  const catalogUri =
+    process.env.ICEBERG_CATALOG_URI ?? "sqlite:///data/catalog.db";
+
+  const tableBlocks = schemas
+    .map((schema, i) => {
+      const ddl = buildCreateTableSql(schema, namespace).replace(/`/g, "\\`");
+      const csvPathEscaped = csvPaths[i].replace(/\\/g, "/");
+      return `
+    # --- ${schema.tableName} ---
+    ddl = """${ddl}"""
+    cat.execute(ddl)
+    df = pl.read_csv("${csvPathEscaped}", infer_schema_length=1000)
+    tbl = cat.load_table("${namespace}.${schema.tableName}")
+    tbl.overwrite(df.to_arrow())
+    print(f"  Seeded ${schema.tableName}: ${schema.rowCount} rows")
+`;
+    })
+    .join("\n");
+
+  return `
+import polars as pl
+from pyiceberg.catalog import load_catalog
+
+cat = load_catalog(
+    "default",
+    **{
+        "uri": "${catalogUri}",
+        "warehouse": "${warehouse}",
+    }
+)
+
+try:
+    cat.create_namespace("${namespace}")
+except Exception:
+    pass  # namespace already exists
+
+${tableBlocks}
+
+print("Seed complete.")
+`.trim();
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+async function main() {
+  console.log(`\nrad seed  env=${env}  source=${source}\n`);
+
+  const files = resolveFiles();
+  if (files.length === 0) {
+    console.log("No CSV files found. Nothing to seed.");
+    process.exit(0);
+  }
+
+  console.log(`Found ${files.length} CSV file(s):\n`);
+
+  const schemas: TableSchema[] = [];
+  for (const csvPath of files) {
+    const rel = path.relative(repoRoot, csvPath);
+    process.stdout.write(`  Validating ${rel} ... `);
+    try {
+      const schema = validateSchema(csvPath);
+      schemas.push(schema);
+      console.log(
+        `OK  [${schema.columns.length} cols, ${schema.rowCount} rows]`,
+      );
+    } catch (err) {
+      console.error(`FAIL\n${(err as Error).message}`);
+      process.exit(1);
+    }
+  }
+
+  // Write fingerprints before seeding so CI can detect schema drift
+  for (const schema of schemas) {
+    writeFingerprint(schema);
+  }
+
+  // Emit DDL summary
+  const ddlDir = path.join(repoRoot, ".rad", "ddl");
+  fs.mkdirSync(ddlDir, { recursive: true });
+  for (const schema of schemas) {
+    const ddl = buildCreateTableSql(schema, env);
+    fs.writeFileSync(path.join(ddlDir, `${schema.tableName}.sql`), ddl + "\n");
+  }
+
+  console.log(`\nDDL written to .rad/ddl/`);
+
+  // Run Python seed via pyiceberg if available
+  const seedScript = buildPythonSeedScript(schemas, files);
+  const scriptPath = path.join(repoRoot, ".rad", "tmp", "seed.py");
+  fs.mkdirSync(path.dirname(scriptPath), { recursive: true });
+  fs.writeFileSync(scriptPath, seedScript);
+
+  const pythonBin = process.env.PYTHON ?? "python";
+  try {
+    console.log(`\nSeeding Iceberg namespace "${env}" ...\n`);
+    // Runs an internally-generated seed script with the operator-configured
+    // python binary; no untrusted input, so no command-injection surface.
+    // nosemgrep: javascript.lang.security.detect-child-process.detect-child-process
+    execSync(`${pythonBin} "${scriptPath}"`, { stdio: "inherit" });
+  } catch {
+    console.error(
+      "\nPython seed step failed. Ensure pyiceberg and polars are installed:\n" +
+        "  pip install pyiceberg polars pyarrow\n\n" +
+        `Generated seed script: ${scriptPath}`,
+    );
+    process.exit(1);
+  }
+
+  console.log("\nDone.");
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

package/sync/engine.ts

@@ -0,0 +1,392 @@
+import { AxiosInstance } from "axios";
+import { createPersistence, SyncPersistence } from "./persistence";
+import { getEntity, getRegisteredEntities } from "./registry";
+import { SyncStore } from "./store";
+import { TransactionQueue } from "./transactions";
+import {
+  BootstrapResponse,
+  ChangesResponse,
+  EngineStatus,
+  EntityDescriptor,
+  EntityKey,
+  entityKey,
+  SyncActionPacket,
+  SyncEngineConfig,
+} from "./types";
+
+const CURSOR_META_KEY = "lastSyncId";
+const SCHEMA_META_KEY = "schemaHash";
+const READY_META_KEY = "readyEntities";
+
+/**
+ * The sync engine. Read path: local hydrate -> delta catch-up -> full
+ * bootstrap (only when there is no usable local state), then a long-poll
+ * `/changes` loop for the app's lifetime. Lazy entities are excluded from
+ * the initial bootstrap and partial-bootstrapped on first use. Write path:
+ * the TransactionQueue replays optimistic writes through the generated REST
+ * endpoints; deltas arriving here rebase and resolve those transactions.
+ * A server schema-hash change or a cursor past the server's change-log
+ * retention (HTTP 410) drops local state and re-bootstraps.
+ */
+export class SyncEngine {
+  readonly store: SyncStore;
+  readonly queue: TransactionQueue;
+  readonly persistence: SyncPersistence;
+
+  readonly api: AxiosInstance;
+  private syncBasePath: string;
+  private wait: number;
+  private onError?: (error: unknown) => void;
+
+  private status: EngineStatus = "idle";
+  private statusListeners = new Set<() => void>();
+  private running = false;
+  private lastSyncId: number | null = null;
+  private schemaHash: string | undefined;
+  private readyKeys = new Set<EntityKey>();
+  private inflightBootstraps = new Map<EntityKey, Promise<void>>();
+  private pendingEnsures = new Set<EntityKey>();
+  private applyLock: Promise<void> = Promise.resolve();
+  private pollController: AbortController | null = null;
+  private onOnline = () => this.wake();
+
+  constructor(api: AxiosInstance, config: SyncEngineConfig = {}) {
+    this.api = api;
+    this.syncBasePath = config.syncBasePath || "/api/sync/v1";
+    this.wait = config.wait ?? 25;
+    this.onError = config.onError;
+    this.store = new SyncStore();
+    this.persistence = createPersistence(
+      config.persist !== false,
+      config.databaseName || "rad-sync",
+    );
+    this.queue = new TransactionQueue({
+      api,
+      persistence: this.persistence,
+      store: this.store,
+      getEntity: (key) => getEntity(key),
+      onError: config.onError
+        ? (error) => config.onError!(error)
+        : undefined,
+    });
+  }
+
+  getStatus(): EngineStatus {
+    return this.status;
+  }
+
+  subscribeStatus(listener: () => void): () => void {
+    this.statusListeners.add(listener);
+    return () => {
+      this.statusListeners.delete(listener);
+    };
+  }
+
+  async start(): Promise<void> {
+    if (this.running) {
+      return;
+    }
+    this.running = true;
+    if (typeof window !== "undefined") {
+      window.addEventListener("online", this.onOnline);
+    }
+    try {
+      this.setStatus("hydrating");
+      const cursor = await this.persistence.getMeta(CURSOR_META_KEY);
+      this.schemaHash = await this.persistence.getMeta(SCHEMA_META_KEY);
+      const readyList: EntityKey[] =
+        (await this.persistence.getMeta(READY_META_KEY)) || [];
+      this.readyKeys = new Set(readyList);
+      if (typeof cursor === "number") {
+        this.lastSyncId = cursor;
+        await this.hydrateFromLocal();
+      }
+      await this.queue.resume();
+
+      const instant = this.entities().filter((entity) => !entity.lazy);
+      const missing = instant
+        .map(entityKey)
+        .filter((key) => !this.readyKeys.has(key));
+      if (this.lastSyncId === null || missing.length > 0) {
+        this.setStatus("bootstrapping");
+        await this.bootstrapEntities(
+          this.lastSyncId === null ? instant.map(entityKey) : missing,
+        );
+      }
+      if (this.pendingEnsures.size > 0) {
+        const ensures = Array.from(this.pendingEnsures);
+        this.pendingEnsures.clear();
+        await this.bootstrapEntities(
+          ensures.filter((key) => !this.readyKeys.has(key)),
+        );
+      }
+      void this.changesLoop();
+    } catch (error) {
+      this.setStatus("error");
+      this.onError?.(error);
+      throw error;
+    }
+  }
+
+  stop(): void {
+    this.running = false;
+    this.pollController?.abort();
+    if (typeof window !== "undefined") {
+      window.removeEventListener("online", this.onOnline);
+    }
+    this.setStatus("idle");
+  }
+
+  /**
+   * Kick the engine: cancels the in-flight long poll so a fresh `/changes`
+   * request fires immediately, and retries any queued writes. This is the
+   * hook a push transport (Pusher, API Gateway WebSocket) calls — push only
+   * wakes the client, the HTTP protocol stays the source of truth.
+   */
+  wake(): void {
+    this.queue.wake();
+    this.pollController?.abort();
+  }
+
+  /**
+   * Make sure an entity's data is (being) loaded. No-op once it's ready;
+   * for lazy entities this performs the on-demand partial bootstrap. Read
+   * hooks call this on mount, so lazy data loads the first time a component
+   * asks for it (and late-registered entities still get synced).
+   */
+  ensureEntity(key: EntityKey): Promise<void> {
+    if (this.readyKeys.has(key) || !getEntity(key)) {
+      return Promise.resolve();
+    }
+    if (!this.running) {
+      // Requested before start() — remembered and loaded right after.
+      this.pendingEnsures.add(key);
+      return Promise.resolve();
+    }
+    return this.bootstrapEntities([key]);
+  }
+
+  isEntityReady(key: EntityKey): boolean {
+    return this.readyKeys.has(key);
+  }
+
+  /** Drop local state and re-sync from scratch. */
+  async reset(): Promise<void> {
+    this.lastSyncId = null;
+    this.schemaHash = undefined;
+    this.readyKeys = new Set();
+    this.inflightBootstraps.clear();
+    await this.persistence.clear();
+    this.store.clear();
+    this.setStatus("bootstrapping");
+    // Instant entities reload now; lazy ones re-fetch on next use.
+    await this.bootstrapEntities(
+      this.entities()
+        .filter((entity) => !entity.lazy)
+        .map(entityKey),
+    );
+  }
+
+  private setStatus(status: EngineStatus): void {
+    if (this.status !== status) {
+      this.status = status;
+      this.statusListeners.forEach((listener) => listener());
+    }
+  }
+
+  private entities(): EntityDescriptor[] {
+    return getRegisteredEntities();
+  }
+
+  private async hydrateFromLocal(): Promise<void> {
+    for (const entity of this.entities()) {
+      const key = entityKey(entity);
+      if (!this.readyKeys.has(key)) {
+        continue; // never bootstrapped (e.g. lazy and untouched) — no local truth
+      }
+      const rows = await this.persistence.getRecords(key);
+      this.store.hydrate(key, rows, entity.idField || "id");
+    }
+  }
+
+  /** Bootstrap (or partial-bootstrap) the given entities from the server. */
+  private bootstrapEntities(keys: EntityKey[]): Promise<void> {
+    const wanted = keys.filter(
+      (key) => !this.readyKeys.has(key) && !this.inflightBootstraps.has(key),
+    );
+    if (wanted.length === 0) {
+      const inflight = keys
+        .map((key) => this.inflightBootstraps.get(key))
+        .filter(Boolean) as Promise<void>[];
+      return Promise.all(inflight).then(() => undefined);
+    }
+    const run = this.fetchBootstrap(wanted).finally(() => {
+      wanted.forEach((key) => this.inflightBootstraps.delete(key));
+    });
+    wanted.forEach((key) => this.inflightBootstraps.set(key, run));
+    return run;
+  }
+
+  private async fetchBootstrap(keys: EntityKey[]): Promise<void> {
+    if (keys.length === 0) {
+      return;
+    }
+    const response = await this.api.get<BootstrapResponse>(
+      `${this.syncBasePath}/bootstrap/`,
+      { params: { models: keys.join(",") } },
+    );
+    const payload = response.data;
+    for (const key of keys) {
+      const entity = getEntity(key);
+      if (!entity) {
+        continue;
+      }
+      const idField = entity.idField || "id";
+      const rows = payload.models[key] || [];
+      await this.persistence.replaceRecords(
+        key,
+        rows.map((row) => ({ id: String(row[idField]), data: row })),
+      );
+      this.store.hydrate(key, rows, idField);
+      this.readyKeys.add(key);
+    }
+    await this.persistence.setMeta(READY_META_KEY, Array.from(this.readyKeys));
+
+    // The cursor only ever advances: a partial bootstrap snapshot is at
+    // least as new as any cursor we already hold.
+    if (this.lastSyncId === null || payload.last_sync_id > this.lastSyncId) {
+      this.lastSyncId = payload.last_sync_id;
+      await this.persistence.setMeta(CURSOR_META_KEY, this.lastSyncId);
+    }
+    if (payload.schema_hash) {
+      this.schemaHash = payload.schema_hash;
+      await this.persistence.setMeta(SCHEMA_META_KEY, this.schemaHash);
+    }
+  }
+
+  private async changesLoop(): Promise<void> {
+    let backoff = 1000;
+    while (this.running) {
+      this.pollController = new AbortController();
+      try {
+        const response = await this.api.get<ChangesResponse>(
+          `${this.syncBasePath}/changes/`,
+          {
+            params: { since: this.lastSyncId ?? 0, wait: this.wait },
+            signal: this.pollController.signal,
+          },
+        );
+        const changed = await this.checkSchemaHash(
+          response.data.schema_hash,
+        );
+        if (!changed) {
+          await this.applyChanges(response.data);
+        }
+        this.setStatus("live");
+        backoff = 1000;
+      } catch (error: any) {
+        if (!this.running) {
+          break;
+        }
+        if (this.pollController.signal.aborted) {
+          // wake() cancelled the poll on purpose — go straight around.
+          continue;
+        }
+        const status = error?.response?.status;
+        if (status === 410) {
+          // Cursor fell out of the server's change-log retention.
+          try {
+            await this.reset();
+            continue;
+          } catch (resetError) {
+            this.onError?.(resetError);
+          }
+        }
+        this.setStatus("offline");
+        this.onError?.(error);
+        await new Promise((resolve) => setTimeout(resolve, backoff));
+        backoff = Math.min(backoff * 2, 30000);
+      }
+    }
+  }
+
+  /**
+   * Returns true (after resetting) when the server's schema hash no longer
+   * matches the one local state was built against.
+   */
+  private async checkSchemaHash(hash: string | undefined): Promise<boolean> {
+    if (!hash) {
+      return false;
+    }
+    if (this.schemaHash === undefined) {
+      this.schemaHash = hash;
+      await this.persistence.setMeta(SCHEMA_META_KEY, hash);
+      return false;
+    }
+    if (this.schemaHash === hash) {
+      return false;
+    }
+    await this.reset();
+    return true;
+  }
+
+  private applyChanges(payload: ChangesResponse): Promise<void> {
+    // Serialize applications so a wake-triggered poll can never interleave
+    // with one still being written.
+    this.applyLock = this.applyLock.then(() =>
+      this.applyChangesExclusive(payload),
+    );
+    return this.applyLock;
+  }
+
+  private async applyChangesExclusive(payload: ChangesResponse): Promise<void> {
+    for (const action of payload.actions) {
+      await this.applyAction(action);
+    }
+    if (payload.last_sync_id > (this.lastSyncId ?? 0)) {
+      this.lastSyncId = payload.last_sync_id;
+      await this.persistence.setMeta(CURSOR_META_KEY, this.lastSyncId);
+    }
+  }
+
+  private async applyAction(action: SyncActionPacket): Promise<void> {
+    const entity = getEntity(action.model);
+    if (!entity) {
+      return; // an entity this bundle doesn't use
+    }
+    const key = action.model;
+    if (entity.lazy && !this.readyKeys.has(key)) {
+      return; // not loaded yet — the on-demand bootstrap will include it
+    }
+    const idField = entity.idField || "id";
+    const id = String(action.id);
+    const deleted =
+      action.op === "D" || (action.data && action.data.deleted_at != null);
+
+    if (deleted) {
+      await this.persistence.deleteRecord(key, id);
+      this.store.remove(key, id);
+    } else if (action.data) {
+      await this.persistence.putRecords(key, [{ id, data: action.data }]);
+      const pending = this.queue.pendingFor(key, id);
+      if (pending.some((txn) => txn.op === "delete")) {
+        // Locally deleted, not yet confirmed: keep it gone from the UI.
+      } else if (pending.length > 0) {
+        // Rebase: server row underneath, unconfirmed local edits on top
+        // (last-writer-wins — the local write supersedes on its next send).
+        const overlay = pending.reduce(
+          (acc, txn) => ({ ...acc, ...(txn.payload || {}) }),
+          {} as Record<string, any>,
+        );
+        this.store.upsert(key, id, {
+          ...action.data,
+          ...overlay,
+          [idField]: action.data[idField],
+        });
+      } else {
+        this.store.upsert(key, id, action.data);
+      }
+    }
+    await this.queue.resolveForRow(key, id);
+  }
+}