rastack 0.0.60 → 0.0.62

package/CHANGELOG.md

@@ -2,6 +2,30 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.0.62](https://github.com/theserverkid/reactapistack/compare/v0.0.61...v0.0.62) (2026-07-19)
+
+
+### Features
+
+* **admin:** Iceberg table discovery — surface warehouse tables the manifest doesn't declare ([9a3b687](https://github.com/theserverkid/reactapistack/commit/9a3b68768878c545e3d1c4c7a95bc3ba75c94856))
+* **catalog:** Iceberg REST Catalog so DuckDB/PyIceberg can write the warehouse (phase 2) ([bf402eb](https://github.com/theserverkid/reactapistack/commit/bf402eb9d2b5ff99ac8074d55fc132a9f26466ab))
+* **store:** make the native engine Iceberg-native — read rows by walking the manifests ([b15fe8b](https://github.com/theserverkid/reactapistack/commit/b15fe8bdbb7053ed4fcadd7dc750c2a4ce313878))
+* **store:** publish real Apache Iceberg tables so DuckDB/PyIceberg can read the warehouse (phase 1) ([13835cd](https://github.com/theserverkid/reactapistack/commit/13835cdeab8f2a48360947ac767808e43d5bd2f8))
+
+
+### Bug Fixes
+
+* **ci:** avoid raw f-string SQL in verify_iceberg.py (Semgrep SAST) ([cffd674](https://github.com/theserverkid/reactapistack/commit/cffd6742ddb7ec04ed8f7b65c293c8de5eca1060))
+
+### [0.0.61](https://github.com/theserverkid/reactapistack/compare/v0.0.60...v0.0.61) (2026-07-19)
+
+
+### Features
+
+* **engine:** activate typed Parquet columns from the manifest ([45711c3](https://github.com/theserverkid/reactapistack/commit/45711c3bfc9c69cc2e841e1d8a31bc38728e58de))
+* **engine:** implement local-first sync server endpoints ([10d7e86](https://github.com/theserverkid/reactapistack/commit/10d7e861e8d574e0bb059ef3e3ce3538069d818f))
+* **store:** real typed per-field Parquet columns (Iceberg encoding) ([12d6f1f](https://github.com/theserverkid/reactapistack/commit/12d6f1fbe3d6326f54c79c33a3c9ea576d389f8f))
+
 ### [0.0.60](https://github.com/theserverkid/reactapistack/compare/v0.0.59...v0.0.60) (2026-07-18)
 
 

package/README.md

@@ -15,7 +15,7 @@ S3 at the bottom, with a stateless Rust API in between.
             └───────────────────────▲─────────────────────┘
                                     │  your types, resolved via the manifest
             ┌───────────────────────┴─────────────────────┐
-            │   TypeScript resources → `rastack compile`       │   API EXPOSURE
+            │   TypeScript resources → compile step        │   API EXPOSURE
             │   FK inferred from types → manifest+OpenAPI  │   SURFACE
             └───────────────────────▲─────────────────────┘
                                     │  schema.rastack.json (manifest)
@@ -41,7 +41,7 @@ S3 at the bottom, with a stateless Rust API in between.
 
 | Surface | What it is | Package(s) |
 | --- | --- | --- |
-| **API exposure** | TypeScript resources → `rastack compile` (FK inferred from types) → manifest + OpenAPI → pass your type to `useData` / `useForm` (typed TanStack Query hooks, no codegen) | the repo root, published as `rastack` (npm) — `components/`, `hooks/`, `theme/`, … |
+| **API exposure** | TypeScript resources → in-process compile (FK inferred from types) → manifest + OpenAPI → pass your type to `useData` / `useForm` (typed TanStack Query hooks, no codegen) | the repo root, published as `rastack` (npm) — `components/`, `hooks/`, `theme/`, … |
 | **Database storage** | A stateless Rust API that stores every model as an Apache Iceberg table on object storage — snapshots, ACID on S3, columnar Parquet | `engine/` (`serve/rastack-server`) |
 | **DevOps** | Serverless Rust on AWS Lambda + S3: stateless compute, state in object storage, infrastructure-as-code, scale-to-zero — driven by `rastack init` / `deploy` / `ci` | `deploy/cloudformation/`, `examples/serverless/` |
 
@@ -81,9 +81,9 @@ export const Terminal = resource("airports", "terminal", {
 ```
 
 ```bash
-rastack compile resources .rastack # TS → schema.rastack.json + openapi.json (FK detected)
-rastack dev                        # the app + admin, in-browser over the WASM engine
-rastack admin                      # Django-admin-style DB browser
+# No compile step to run — it happens in-process. Author your resources, then:
+rastack dev                        # compiles + serves your app in-browser over the WASM engine
+rastack admin                      # Django-admin-style DB browser over the same engine
 ```
 
 ```tsx
@@ -106,22 +106,26 @@ the importer, the server, and the in-browser WASM engine alike.
 
 ## Ship it — one command per stage
 
-The DevOps surface is a CLI too. A new repo is scaffolded, configured, and
-deployed with three commands, and GitHub Actions stays a one-liner because the
-pipeline lives in the CLI, not the YAML:
+The DevOps surface is a CLI too. A new repo goes from nothing to deployed in
+three commands, and GitHub Actions stays a one-liner because the pipeline lives
+in the CLI, not the YAML:
 
 ```bash
-rastack init                       # template .github/workflows + .rastack/deploy.json
-rastack deploy                     # configure the deployment; print the stack launch command
+rastack bootstrap                  # once per AWS account+region: buckets + GitHub OIDC
+rastack init                       # per app: scaffold .github/workflows + .rastack/deploy.json AND stand the backend up
 rastack ci deploy                  # what CI runs: build the arm64 Lambda, bake the JWKS, ship the code
 ```
 
-`init` scaffolds the base GitHub Actions and a deploy config (defaulting the
-GitHub coordinates from your `origin` remote); `deploy` fills that config in and
-prints the one-time `aws cloudformation deploy` plus the repo Variables to wire;
-`ci` runs the build/deploy pipeline so a workflow is just
-`run: rastack ci deploy` (`rastack ci --plan deploy` prints every step). Full
-guide: **[docs/serverless-deploy.md](docs/serverless-deploy.md)**.
+`bootstrap` stands the tier-0 account stack up once (the template-hosting
+buckets + the GitHub OIDC provider), decoupled from any app. `init` is per-app:
+it scaffolds the base GitHub Actions and a deploy config (defaulting the GitHub
+coordinates from your `origin` remote), then stands the serverless backend up —
+deploying `app.yaml`, reading the stack Outputs back into the config, and wiring
+the GitHub repo Variables so the first push deploys. From then on infrastructure
+is fixed and CI only ships code: `rastack ci deploy` builds and runs
+`update-function-code` (`rastack deploy` is an alias for it), so a workflow step
+is just `run: rastack ci deploy` (`rastack ci --plan deploy` prints every step).
+Full guide: **[docs/serverless-deploy.md](docs/serverless-deploy.md)**.
 
 ## Run the whole API in the browser
 

package/admin/AdminApp.tsx

@@ -23,7 +23,9 @@ import type { RastackEngine } from "../provider/types";
 import {
   AdminResource,
   changelistHeaders,
+  DISCOVER_PATH,
   findResource,
+  mergeDiscovered,
 } from "./adapters";
 import {
   useDashboardStats,
@@ -653,12 +655,45 @@ function useSidebarCollapsed(): [boolean, () => void] {
   return [collapsed, toggle];
 }
 
+/**
+ * Fold warehouse-discovered tables (Iceberg discovery, `GET /api/discover/`)
+ * into the manifest-declared resources. Runs once against the ready engine —
+ * the same seam in both modes (WASM locally, HTTP to `rastack-server`
+ * remotely). Discovery is best-effort: an older server without the endpoint, or
+ * a non-admin identity, just leaves the declared set untouched.
+ */
+function useDiscoveredResources(
+  engine: RastackEngine,
+  declared: AdminResource[],
+): AdminResource[] {
+  const [resources, setResources] = useState(declared);
+  useEffect(() => {
+    let live = true;
+    setResources(declared);
+    (async () => {
+      try {
+        const res = await engine.handle("GET", DISCOVER_PATH, {}, undefined);
+        if (live && res && res.status === 200) {
+          setResources(mergeDiscovered(declared, res.body));
+        }
+      } catch {
+        /* discovery is optional — keep the manifest resources */
+      }
+    })();
+    return () => {
+      live = false;
+    };
+  }, [engine, declared]);
+  return resources;
+}
+
 export function AdminApp(props: {
   engine: RastackEngine;
   resources: AdminResource[];
 }): React.JSX.Element {
   const route = useRoute();
-  const { engine, resources } = props;
+  const { engine } = props;
+  const resources = useDiscoveredResources(engine, props.resources);
   const [collapsed, toggleCollapsed] = useSidebarCollapsed();
 
   const activeKey =

package/admin/adapters.ts

@@ -47,6 +47,12 @@ export interface AdminResource {
   listDisplay: string[] | null;
   /** Whether this resource is shown in the admin (admin !== false). */
   adminEnabled: boolean;
+  /**
+   * True when this table was found in the warehouse (Iceberg discovery) but is
+   * *not* declared in the compiled manifest. Its fields are inferred from a
+   * sample of its rows rather than authored, so the UI badges it as discovered.
+   */
+  discovered?: boolean;
 }
 
 interface RawField {
@@ -69,6 +75,19 @@ interface RawManifest {
   resources?: RawResource[];
 }
 
+/** Map one raw manifest/discovery field to an admin field descriptor. */
+function toAdminField(f: RawField): AdminField {
+  return {
+    name: f.name,
+    type: f.type,
+    primaryKey: f.primaryKey === true,
+    maxLength: f.maxLength,
+    nullable: f.null === true,
+    unique: f.unique === true,
+    relation: f.relation,
+  };
+}
+
 /** Parse a manifest (object or JSON string) into admin resource descriptors. */
 export function parseManifest(manifest: object | string): AdminResource[] {
   const doc: RawManifest =
@@ -78,15 +97,7 @@ export function parseManifest(manifest: object | string): AdminResource[] {
     model: r.model,
     key: `${r.app}.${r.model}`,
     label: humanizeKey(r.model),
-    fields: (r.fields ?? []).map((f) => ({
-      name: f.name,
-      type: f.type,
-      primaryKey: f.primaryKey === true,
-      maxLength: f.maxLength,
-      nullable: f.null === true,
-      unique: f.unique === true,
-      relation: f.relation,
-    })),
+    fields: (r.fields ?? []).map(toAdminField),
     search: r.search ?? [],
     listDisplay:
       r.admin && Array.isArray(r.admin.listDisplay) ? r.admin.listDisplay : null,
@@ -94,6 +105,66 @@ export function parseManifest(manifest: object | string): AdminResource[] {
   }));
 }
 
+// -- Iceberg table discovery (`GET /api/discover/`) -------------------------
+
+/** The discovery endpoint the WASM/HTTP engine answers, over the `handle` seam. */
+export const DISCOVER_PATH = "/api/discover/";
+
+/** One table in the discovery payload (`{ tables: [...] }`). */
+interface DiscoveredTable {
+  app: string;
+  model: string;
+  declared: boolean;
+  adminEnabled?: boolean;
+  rowCount?: number;
+  fields?: RawField[];
+}
+
+/** Read the `{ tables: [...] }` discovery body defensively. */
+export function parseDiscovered(body: unknown): DiscoveredTable[] {
+  const tables = (body as { tables?: unknown } | null)?.tables;
+  if (!Array.isArray(tables)) return [];
+  return tables.filter(
+    (t): t is DiscoveredTable =>
+      !!t && typeof t === "object" && typeof (t as any).app === "string" &&
+      typeof (t as any).model === "string",
+  );
+}
+
+/**
+ * Merge warehouse-discovered tables over the manifest-declared resources. The
+ * manifest is authoritative — a declared table keeps its authored schema. A
+ * table present in the warehouse but absent from the manifest is appended as a
+ * `discovered` resource, its fields inferred from the sampled rows, so it shows
+ * up in the console even though the compiled schema never mentioned it.
+ */
+export function mergeDiscovered(
+  declared: AdminResource[],
+  body: unknown,
+): AdminResource[] {
+  const have = new Set(declared.map((r) => r.key));
+  const extra: AdminResource[] = [];
+  for (const t of parseDiscovered(body)) {
+    const key = `${t.app}.${t.model}`;
+    if (t.declared || have.has(key)) continue;
+    have.add(key);
+    extra.push({
+      app: t.app,
+      model: t.model,
+      key,
+      label: humanizeKey(t.model),
+      fields: (t.fields ?? []).map(toAdminField),
+      search: [],
+      listDisplay: null,
+      adminEnabled: t.adminEnabled !== false,
+      discovered: true,
+    });
+  }
+  // Declared first (stable), then discovered tables in app/model order.
+  extra.sort((a, b) => a.key.localeCompare(b.key));
+  return [...declared, ...extra];
+}
+
 /** The resources shown in the admin index (admin not disabled). */
 export function adminResources(resources: AdminResource[]): AdminResource[] {
   return resources.filter((r) => r.adminEnabled);

package/admin/ui/Sidebar.tsx

@@ -80,6 +80,14 @@ export function Sidebar(props: {
                 >
                   <IconTable size={16} className="ras-ico" />
                   <span className="ras-nav-text">{r.label}</span>
+                  {r.discovered ? (
+                    <span
+                      className="ras-nav-tag"
+                      title="Discovered from the warehouse — not in the compiled schema"
+                    >
+                      discovered
+                    </span>
+                  ) : null}
                 </a>
               );
             })}

package/admin/ui/styles.ts

@@ -145,6 +145,19 @@ a { color: inherit; text-decoration: none; }
   font-weight: 600;
 }
 .ras-nav-link.is-active .ras-ico { color: var(--color-accent); }
+.ras-nav-tag {
+  margin-left: auto;
+  font-size: 9px;
+  font-weight: 700;
+  letter-spacing: 0.04em;
+  text-transform: uppercase;
+  padding: 2px 6px;
+  border-radius: 999px;
+  color: var(--color-warning);
+  background: color-mix(in srgb, var(--color-warning) 16%, transparent);
+  white-space: nowrap;
+}
+.is-collapsed .ras-nav-tag { display: none; }
 .ras-nav-link.is-active::before {
   content: "";
   position: absolute;