rastack 0.0.16

package/dist/seed.js

@@ -0,0 +1,252 @@
+#!/usr/bin/env node
+"use strict";
+/**
+ * rad seed
+ *
+ * Loads CSV files from a local directory or GCS bucket into the target
+ * Iceberg namespace. Performs schema inference, PII check, and idempotent
+ * table creation before writing rows.
+ *
+ * Usage:
+ *   rad seed [--env dev|staging] [--source gs://bucket/ | --source ./myapp-dev/]
+ *
+ * Environment variables (set via CI secrets or .env):
+ *   GCP_PROJECT             GCP project ID
+ *   ICEBERG_WAREHOUSE       gs://... path to Iceberg warehouse root
+ *   ICEBERG_CATALOG_URI     sqlite:///... or http://... catalog URI
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var _a, _b;
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const child_process_1 = require("child_process");
+const csv_schema_1 = require("./csv-schema");
+// ---------------------------------------------------------------------------
+// Args
+// ---------------------------------------------------------------------------
+const args = process.argv.slice(2);
+function getArg(flag) {
+    const i = args.indexOf(flag);
+    return i !== -1 && args[i + 1] ? args[i + 1] : null;
+}
+const env = (_a = getArg("--env")) !== null && _a !== void 0 ? _a : "dev";
+const source = (_b = getArg("--source")) !== null && _b !== void 0 ? _b : path_1.default.join(process.cwd(), "myapp-dev");
+const repoRoot = process.cwd();
+// ---------------------------------------------------------------------------
+// Step 1: resolve CSV files
+// ---------------------------------------------------------------------------
+function resolveFiles() {
+    if (source.startsWith("gs://")) {
+        return downloadFromGcs(source);
+    }
+    return (0, csv_schema_1.discoverCsvFiles)(source);
+}
+function downloadFromGcs(gcsUri) {
+    const tmpDir = path_1.default.join(repoRoot, ".rad", "tmp", "seed");
+    fs_1.default.mkdirSync(tmpDir, { recursive: true });
+    console.log(`Downloading CSVs from ${gcsUri} ...`);
+    try {
+        (0, child_process_1.execSync)(`gcloud storage cp "${gcsUri}*.csv" "${tmpDir}/" --quiet`, {
+            stdio: "inherit",
+        });
+    }
+    catch (_a) {
+        // gcloud storage cp uses gsutil-style globs — fall back
+        (0, child_process_1.execSync)(`gsutil -m cp "${gcsUri}*.csv" "${tmpDir}/"`, { stdio: "inherit" });
+    }
+    return (0, csv_schema_1.discoverCsvFiles)(tmpDir);
+}
+// ---------------------------------------------------------------------------
+// Step 2: validate schemas (PII check — hard fail in CI)
+// ---------------------------------------------------------------------------
+function validateSchema(csvPath) {
+    const { schema, piiViolations } = (0, csv_schema_1.inferSchema)(csvPath, repoRoot);
+    if (piiViolations.length > 0) {
+        const list = piiViolations.map((v) => `  ${v.column}: ${v.reason}`).join("\n");
+        throw new Error(`PII detected in ${path_1.default.basename(csvPath)}:\n${list}\n` +
+            `Remove PII data before seeding. Run "rad scan" to review.`);
+    }
+    return schema;
+}
+// ---------------------------------------------------------------------------
+// Step 3: emit Iceberg DDL for a table
+// ---------------------------------------------------------------------------
+function icebergType(col) {
+    switch (col) {
+        case "integer":
+            return "long";
+        case "float":
+            return "double";
+        case "boolean":
+            return "boolean";
+        case "date":
+            return "date";
+        case "datetime":
+            return "timestamp";
+        default:
+            return "string";
+    }
+}
+function buildCreateTableSql(schema, namespace) {
+    const cols = schema.columns
+        .map((c) => {
+        const type = icebergType(c.type);
+        const nullable = c.nullable ? "" : " NOT NULL";
+        return `  ${c.name} ${type}${nullable}`;
+    })
+        .join(",\n");
+    return (`CREATE TABLE IF NOT EXISTS ${namespace}.${schema.tableName} (\n` +
+        `${cols}\n` +
+        `) USING iceberg\n` +
+        `TBLPROPERTIES (\n` +
+        `  'write.format.default'='parquet',\n` +
+        `  'write.metadata.fingerprint'='${schema.fingerprint}'\n` +
+        `);`);
+}
+// ---------------------------------------------------------------------------
+// Step 4: write schema fingerprints to registry
+// ---------------------------------------------------------------------------
+function writeFingerprint(schema) {
+    const registryDir = path_1.default.join(repoRoot, ".rad", "registry");
+    fs_1.default.mkdirSync(registryDir, { recursive: true });
+    const registryPath = path_1.default.join(registryDir, `${schema.tableName}.json`);
+    let existing = {};
+    if (fs_1.default.existsSync(registryPath)) {
+        existing = JSON.parse(fs_1.default.readFileSync(registryPath, "utf-8"));
+    }
+    const prev = existing.fingerprint;
+    const changed = prev && prev !== schema.fingerprint;
+    const record = {
+        tableName: schema.tableName,
+        fingerprint: schema.fingerprint,
+        columns: schema.columns,
+        rowCount: schema.rowCount,
+        seededAt: new Date().toISOString(),
+        env,
+        previousFingerprint: prev !== null && prev !== void 0 ? prev : null,
+    };
+    fs_1.default.writeFileSync(registryPath, JSON.stringify(record, null, 2));
+    if (changed) {
+        console.warn(`  ⚠  Schema change detected for "${schema.tableName}": ` +
+            `${prev} → ${schema.fingerprint}. A migration may be required.`);
+    }
+}
+// ---------------------------------------------------------------------------
+// Step 5: build a Python seed script and run it via django-iceberg
+// ---------------------------------------------------------------------------
+function buildPythonSeedScript(schemas, csvPaths) {
+    var _a, _b;
+    const namespace = env;
+    const warehouse = (_a = process.env.ICEBERG_WAREHOUSE) !== null && _a !== void 0 ? _a : "data/warehouse";
+    const catalogUri = (_b = process.env.ICEBERG_CATALOG_URI) !== null && _b !== void 0 ? _b : "sqlite:///data/catalog.db";
+    const tableBlocks = schemas
+        .map((schema, i) => {
+        const ddl = buildCreateTableSql(schema, namespace).replace(/`/g, "\\`");
+        const csvPathEscaped = csvPaths[i].replace(/\\/g, "/");
+        return `
+    # --- ${schema.tableName} ---
+    ddl = """${ddl}"""
+    cat.execute(ddl)
+    df = pl.read_csv("${csvPathEscaped}", infer_schema_length=1000)
+    tbl = cat.load_table("${namespace}.${schema.tableName}")
+    tbl.overwrite(df.to_arrow())
+    print(f"  Seeded ${schema.tableName}: ${schema.rowCount} rows")
+`;
+    })
+        .join("\n");
+    return `
+import polars as pl
+from pyiceberg.catalog import load_catalog
+
+cat = load_catalog(
+    "default",
+    **{
+        "uri": "${catalogUri}",
+        "warehouse": "${warehouse}",
+    }
+)
+
+try:
+    cat.create_namespace("${namespace}")
+except Exception:
+    pass  # namespace already exists
+
+${tableBlocks}
+
+print("Seed complete.")
+`.trim();
+}
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+function main() {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a;
+        console.log(`\nrad seed  env=${env}  source=${source}\n`);
+        const files = resolveFiles();
+        if (files.length === 0) {
+            console.log("No CSV files found. Nothing to seed.");
+            process.exit(0);
+        }
+        console.log(`Found ${files.length} CSV file(s):\n`);
+        const schemas = [];
+        for (const csvPath of files) {
+            const rel = path_1.default.relative(repoRoot, csvPath);
+            process.stdout.write(`  Validating ${rel} ... `);
+            try {
+                const schema = validateSchema(csvPath);
+                schemas.push(schema);
+                console.log(`OK  [${schema.columns.length} cols, ${schema.rowCount} rows]`);
+            }
+            catch (err) {
+                console.error(`FAIL\n${err.message}`);
+                process.exit(1);
+            }
+        }
+        // Write fingerprints before seeding so CI can detect schema drift
+        for (const schema of schemas) {
+            writeFingerprint(schema);
+        }
+        // Emit DDL summary
+        const ddlDir = path_1.default.join(repoRoot, ".rad", "ddl");
+        fs_1.default.mkdirSync(ddlDir, { recursive: true });
+        for (const schema of schemas) {
+            const ddl = buildCreateTableSql(schema, env);
+            fs_1.default.writeFileSync(path_1.default.join(ddlDir, `${schema.tableName}.sql`), ddl + "\n");
+        }
+        console.log(`\nDDL written to .rad/ddl/`);
+        // Run Python seed via pyiceberg if available
+        const seedScript = buildPythonSeedScript(schemas, files);
+        const scriptPath = path_1.default.join(repoRoot, ".rad", "tmp", "seed.py");
+        fs_1.default.mkdirSync(path_1.default.dirname(scriptPath), { recursive: true });
+        fs_1.default.writeFileSync(scriptPath, seedScript);
+        const pythonBin = (_a = process.env.PYTHON) !== null && _a !== void 0 ? _a : "python";
+        try {
+            console.log(`\nSeeding Iceberg namespace "${env}" ...\n`);
+            (0, child_process_1.execSync)(`${pythonBin} "${scriptPath}"`, { stdio: "inherit" });
+        }
+        catch (_b) {
+            console.error("\nPython seed step failed. Ensure pyiceberg and polars are installed:\n" +
+                "  pip install pyiceberg polars pyarrow\n\n" +
+                `Generated seed script: ${scriptPath}`);
+            process.exit(1);
+        }
+        console.log("\nDone.");
+    });
+}
+main().catch((err) => {
+    console.error(err);
+    process.exit(1);
+});

package/hooks/form/form.ts

@@ -0,0 +1,207 @@
+import { useFormik, FormikErrors } from "formik";
+import {
+  FormFieldStructure,
+  FormProps,
+  FormStructure,
+  FormUpdate,
+  FormValuesType,
+} from "./interfaces";
+import { FlattenedRecord } from "../../types";
+import { formikJSONSchemaValidation } from "./validate-schema";
+import { getFieldStructure } from "./structure";
+
+export function useForm<
+  TFormValues extends object,
+  TPostParams,
+  TPutParams,
+  TPatchParams,
+  TPostPayload,
+  TPutPayload,
+  TPatchPayload,
+  TPostResult,
+  TPutResult,
+  TPatchResult,
+  TFormUpdate extends FormUpdate,
+  TSourceId extends string | undefined,
+>({
+  id,
+  sourceData,
+  formValues,
+  formatResponse,
+  transformRequest,
+  transformResponse,
+  mutations,
+  updateType,
+  onSuccess,
+}: FormProps<
+  TFormValues,
+  TPostParams,
+  TPutParams,
+  TPatchParams,
+  TPostPayload,
+  TPutPayload,
+  TPatchPayload,
+  TPostResult,
+  TPutResult,
+  TPatchResult,
+  TFormUpdate,
+  TSourceId
+>): FormStructure<
+  TFormValues,
+  FormValuesType<
+    TFormUpdate,
+    TSourceId,
+    TPutParams,
+    TPutPayload,
+    TPostParams,
+    TPostPayload,
+    TPatchParams,
+    TPatchPayload
+  >
+> {
+  const updateMethod = id ? (updateType === "put" ? "put" : "patch") : "post";
+
+  const formatRequestValues = (values: TFormValues) => {
+    if (formatResponse) {
+      return (Object.keys(values) as Array<keyof TFormValues>).reduce(
+        (acc, key) => {
+          const formattedValue = formatResponse[key](values[key]);
+          return {
+            ...acc,
+            ...formattedValue,
+          };
+        },
+        {} as FormValuesType<
+          TFormUpdate,
+          TSourceId,
+          TPutParams,
+          TPutPayload,
+          TPostParams,
+          TPostPayload,
+          TPatchParams,
+          TPatchPayload
+        >,
+      );
+    } else if (transformRequest) {
+      return transformRequest(values);
+    } else {
+      return values;
+    }
+  };
+
+  const isSourceDataLoading = sourceData
+    ? Object.values(sourceData).some((source) =>
+        // @ts-ignore create a base type for list and fetch data extending react query. This will ensure that the isLoading property is available
+        source ? source.isLoading : false,
+      )
+    : false;
+
+  const formik = useFormik<TFormValues>({
+    initialValues: formValues,
+    validate: (values) => {
+      if (mutations) {
+        const payload = formatRequestValues(values);
+        return formikJSONSchemaValidation(
+          payload,
+          mutations[updateMethod]?.validation,
+        );
+      } else {
+        return {} as FormikErrors<TFormValues>;
+      }
+    },
+    onSubmit: (values, formikHelpers) => {
+      const mutation = mutations ? mutations[updateMethod].mutation : undefined;
+      const formattedResponse = formatRequestValues(values);
+      if (mutation) {
+        mutation.mutate(
+          {
+            //@ts-ignore
+            params: {
+              id,
+              ...formattedResponse,
+            },
+            //@ts-ignore
+            payload: formattedResponse,
+          },
+          {
+            onSuccess: (result) => {
+              if (onSuccess) {
+                // @ts-ignore
+                onSuccess(result);
+              }
+              formikHelpers.setSubmitting(false);
+            },
+            onError: () => {
+              formikHelpers.setSubmitting(false);
+            },
+          },
+        );
+      }
+    },
+    enableReinitialize: true,
+  });
+
+  const isSubmitDisabled =
+    formik.isValid === undefined ? false : !formik.isValid;
+  const isSubmitting = formik.isSubmitting;
+  const submitForm = () => {
+    formik.handleSubmit();
+  };
+
+  const getFormField = <
+    TFieldName extends Extract<keyof FlattenedRecord<TFormValues>, string>,
+  >(
+    fieldName: TFieldName,
+  ): FormFieldStructure<FlattenedRecord<TFormValues>[TFieldName]> => {
+    const { value: value, checked: isCheckedValue } =
+      formik.getFieldProps(fieldName);
+    const { touched: isTouched, error: errorMessage } =
+      formik.getFieldMeta(fieldName);
+    const { setValue: setFormValue, setTouched } =
+      formik.getFieldHelpers(fieldName);
+
+    let { type, isRequired } = getFieldStructure(
+      mutations ? mutations[updateMethod]?.validation : undefined,
+      fieldName,
+    );
+
+    const error = errorMessage ?? "";
+    const isError = errorMessage !== undefined;
+    const isDisabled = false;
+    const isChecked = isCheckedValue ?? false;
+    const setValue = (value: FlattenedRecord<TFormValues>[TFieldName]) => {
+      if (type === "number" && value) {
+        const numericValue = Number(value);
+        if (!isNaN(numericValue)) {
+          setFormValue(numericValue);
+        }
+      } else {
+        setFormValue(value);
+      }
+    };
+    return {
+      type,
+      value,
+      error,
+      isChecked,
+      isError,
+      isDisabled,
+      isRequired,
+      setValue,
+      setTouched,
+    };
+  };
+
+  return {
+    formValues: formik.values,
+    // @ts-ignore
+    responseValues: formatRequestValues(formik.values),
+    errors: formik.errors,
+    getFormField,
+    onSubmit: submitForm,
+    resetForm: formik.resetForm,
+    isSubmitDisabled,
+    isSourceDataLoading,
+    isSubmitting,
+  };
+}

package/hooks/form/index.ts

@@ -0,0 +1,8 @@
+export { useForm } from "./form";
+export type {
+  FormStructure,
+  FormUpdateMethod,
+  SetupUpdateEndpointReturn,
+  FormProps,
+  FormUpdate,
+} from "./interfaces";

package/hooks/form/interfaces.ts

@@ -0,0 +1,217 @@
+import { FlattenedRecord } from "../../types";
+import { SchemaObject } from "ajv";
+import { IUpdateQueryResult } from "../../hooks";
+import { FormikValues } from "formik";
+
+export interface FormFieldStructure<TValue> {
+  value: TValue;
+  error: string;
+  type: FormType;
+  isChecked: boolean;
+  isError: boolean;
+  isDisabled: boolean;
+  isRequired: boolean;
+  setValue: (value: TValue) => void;
+  setTouched: (isTouched: boolean) => void;
+}
+
+interface FormMutation<
+  TPostParams,
+  TPutParams,
+  TPatchParams,
+  TPostPayload,
+  TPutPayload,
+  TPatchPayload,
+  TPostResult,
+  TPutResult,
+  TPatchResult,
+> {
+  post: {
+    mutation?: IUpdateQueryResult<TPostParams, TPostPayload, TPostResult>;
+    validation?: SchemaObject | undefined;
+  };
+  put: {
+    mutation?: IUpdateQueryResult<TPutParams, TPutPayload, TPutResult>;
+    validation?: SchemaObject | undefined;
+  };
+  patch: {
+    mutation?: IUpdateQueryResult<TPatchParams, TPatchPayload, TPatchResult>;
+    validation?: SchemaObject | undefined;
+  };
+}
+
+export type FormUpdate = "post_or_patch" | "put";
+
+export interface FormProps<
+  TFormValues extends object,
+  TPostParams,
+  TPutParams,
+  TPatchParams,
+  TPostPayload,
+  TPutPayload,
+  TPatchPayload,
+  TPostResult,
+  TPutResult,
+  TPatchResult,
+  TFormUpdate extends FormUpdate,
+  TSourceId extends string | undefined,
+> {
+  id?: TSourceId & string;
+  sourceData?: SourceData;
+  formValues: TFormValues & object;
+  formatResponse?: ForwardMapping<
+    TFormValues,
+    TFormUpdate extends "put"
+      ? TPutParams & TPutResult
+      : TSourceId extends undefined
+        ? TPostParams & TPostResult
+        : TPatchParams & TPatchResult
+  >;
+  transformRequest?: ForwardMap<
+    TFormValues,
+    TFormUpdate extends "put"
+      ? TPutParams & TPutResult
+      : TSourceId extends undefined
+        ? TPostParams & TPostResult
+        : TPatchParams & TPatchResult
+  >;
+  transformResponse?: ReverseMap<
+    TFormValues,
+    TFormUpdate extends "put"
+      ? TPutParams & TPutResult
+      : TSourceId extends undefined
+        ? TPostParams & TPostResult
+        : TPatchParams & TPatchResult
+  >;
+  updateType?: TFormUpdate;
+  mutations?: FormMutation<
+    TPostParams,
+    TPutParams,
+    TPatchParams,
+    TPostPayload,
+    TPutPayload,
+    TPatchPayload,
+    TPostResult,
+    TPutResult,
+    TPatchResult
+  >;
+  onSuccess?: (
+    values: TFormUpdate extends "put"
+      ? TPutResult
+      : TSourceId extends undefined
+        ? TPostResult
+        : TPatchResult,
+  ) => void;
+}
+
+export interface FormStructure<TFormValues, TResponse> {
+  formValues: TFormValues | undefined;
+  responseValues: TResponse | undefined;
+  errors: any;
+  onSubmit: () => void;
+  resetForm: () => void;
+  isSubmitDisabled: boolean;
+  isSourceDataLoading: boolean;
+  isSubmitting: boolean;
+  getFormField: <TFieldName extends keyof FlattenedRecord<TFormValues>>(
+    fieldName: TFieldName,
+  ) => FormFieldStructure<FlattenedRecord<TFormValues>[TFieldName]>;
+}
+
+export type FormType = "number" | "text" | "array";
+
+export interface SetupUpdateEndpointReturn<TParams, TPayload, TResult> {
+  mutation: IUpdateQueryResult<TParams, TPayload, TResult>;
+  validation: SchemaObject;
+}
+
+export type FormUpdateMethod = "post_or_patch" | "put";
+
+export interface SourceData {
+  [key: string]: unknown;
+}
+
+export type ForwardMapping<TFormValues, TResponse> = {
+  [key in keyof TFormValues]: (value: TFormValues[key]) => {
+    [key in keyof TResponse]: TResponse[keyof TResponse];
+  };
+};
+
+export type ReverseMapping<TFormValues, TResponse> = {
+  [key in keyof TResponse]: (value: TResponse[key]) => {
+    [key in keyof TFormValues]: TFormValues[keyof TFormValues];
+  };
+};
+
+export type ForwardMap<TFormValues, TResponse> = (value: TFormValues) => {
+  [key in keyof TResponse]: TResponse[keyof TResponse];
+};
+
+export type ReverseMap<TFormValues, TResponse> = (value: TResponse) => {
+  [key in keyof TFormValues]: TFormValues[keyof TFormValues];
+};
+
+type EnsureObject<T> = T extends object ? T : Record<string, never>;
+
+export type FormValuesType<
+  TFormUpdate extends "put" | "post_or_patch",
+  TSourceId extends string | undefined,
+  TPutParams,
+  TPutPayload,
+  TPostParams,
+  TPostPayload,
+  TPatchParams,
+  TPatchPayload,
+> = FormikValues &
+  EnsureObject<
+    TFormUpdate extends "put"
+      ? TPutParams & TPutPayload
+      : TSourceId extends undefined
+        ? TPostParams & TPostPayload
+        : TPatchParams & TPatchPayload
+  >;
+
+export type FormParamsType<
+  TFormUpdate extends "put" | "post_or_patch",
+  TSourceId extends string | undefined,
+  TPutParams,
+  TPostParams,
+  TPatchParams,
+> = FormikValues &
+  EnsureObject<
+    TFormUpdate extends "put"
+      ? TPutParams
+      : TSourceId extends undefined
+        ? TPostParams
+        : TPatchParams
+  >;
+
+export type FormPayloadType<
+  TFormUpdate extends "put" | "post_or_patch",
+  TSourceId extends string | undefined,
+  TPutPayload,
+  TPostPayload,
+  TPatchPayload,
+> = FormikValues &
+  EnsureObject<
+    TFormUpdate extends "put"
+      ? TPutPayload
+      : TSourceId extends undefined
+        ? TPostPayload
+        : TPatchPayload
+  >;
+
+export type FormResultType<
+  TFormUpdate extends "put" | "post_or_patch",
+  TSourceId extends string | undefined,
+  TPutResult,
+  TPostResult,
+  TPatchResult,
+> = FormikValues &
+  EnsureObject<
+    TFormUpdate extends "put"
+      ? TPutResult
+      : TSourceId extends undefined
+        ? TPostResult
+        : TPatchResult
+  >;