quiverdb 0.9.15 → 0.10.0

package/README.md

@@ -99,6 +99,7 @@ bun run example.ts
 ### Read (IDs)
 
 - `readElementIds(collection)` -- Read all element IDs in a collection
+- `numberOfElements(collection)` -- Current number of elements in a collection
 
 ### Metadata
 
@@ -130,6 +131,10 @@ Parameters are passed as an array of `number | string | null`.
 - `commit()` -- Commit current transaction
 - `rollback()` -- Rollback current transaction
 - `inTransaction()` -- Check if transaction is active
+- `beginDryRun()` -- Begin a dry run: a transaction `endDryRun()` always rolls back. While it is
+  active, `beginTransaction`/`commit`/`rollback` are absorbed (no-ops)
+- `endDryRun()` -- End the active dry run, rolling back everything it covered
+- `inDryRun()` -- Check if a dry run is active
 
 ### CSV
 
@@ -152,7 +157,8 @@ Parameters are passed as an array of `number | string | null`.
 ### Lua
 
 - `LuaRunner(db)` -- Create Lua script runner with database access
-- `run(script)` -- Execute a Lua script
+- `run(script)` -- Execute a Lua script; returns its return value as a JSON string, or `""` if it
+  returned nothing
 - `close()` -- Close the Lua runner
 
 ## Types

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "quiverdb",
-  "version": "0.9.15",
+  "version": "0.10.0",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -31,4 +31,4 @@
     "@biomejs/biome": "^2.4.6",
     "@types/bun": "latest"
   }
-}
+}

package/src/create.ts

@@ -8,6 +8,7 @@ import {
   readPtrOut,
   toCString,
 } from "./ffi-helpers.ts";
+import { type GroupColumns, updateGroupColumns } from "./group-columns.ts";
 import { getSymbols, type NativePointer } from "./loader.ts";
 import type { ElementData, Value } from "./types.ts";
 
@@ -38,7 +39,9 @@ function setElementArray(
     const allIntegers = (values as number[]).every((v) => Number.isInteger(v));
     if (allIntegers) {
       const arr = allocNativeInt64(values as number[]);
-      check(lib.quiver_element_set_array_integer(elemPtr, nameBuf.buf, arr.buf, values.length, null));
+      check(
+        lib.quiver_element_set_array_integer(elemPtr, nameBuf.buf, arr.buf, values.length, null),
+      );
     } else {
       const arr = allocNativeFloat64(values as number[]);
       check(lib.quiver_element_set_array_float(elemPtr, nameBuf.buf, arr.buf, values.length, null));
@@ -48,7 +51,9 @@ function setElementArray(
 
   if (typeof first === "string") {
     const { table, keepalive: _keepalive } = allocNativeStringArray(values as string[]);
-    check(lib.quiver_element_set_array_string(elemPtr, nameBuf.buf, table.buf, values.length, null));
+    check(
+      lib.quiver_element_set_array_string(elemPtr, nameBuf.buf, table.buf, values.length, null),
+    );
     return;
   }
 
@@ -148,3 +153,48 @@ Database.prototype.deleteElement = function (this: Database, collection: string,
   const collBuf = toCString(collection);
   check(lib.quiver_database_delete_element(this._handle, collBuf.buf, BigInt(id)));
 };
+
+/**
+ * Replace all of an element's rows in one *named* vector group, from column arrays keyed by name.
+ *
+ * Pass `{}` to clear the group. Prefer this over routing the group's columns through
+ * updateElement when a column name is shared by two groups of the collection (legal for foreign
+ * keys): (collection, group) names exactly one table, a column name alone does not, and
+ * updateElement writes an ambiguous column to every match.
+ */
+Database.prototype.updateVectorGroup = function (
+  this: Database,
+  collection: string,
+  group: string,
+  id: number,
+  data: GroupColumns,
+): void {
+  updateGroupColumns(
+    this._handle,
+    "updateVectorGroup",
+    getSymbols().quiver_database_update_vector_group,
+    collection,
+    group,
+    id,
+    data,
+  );
+};
+
+/** Set-group counterpart of updateVectorGroup. */
+Database.prototype.updateSetGroup = function (
+  this: Database,
+  collection: string,
+  group: string,
+  id: number,
+  data: GroupColumns,
+): void {
+  updateGroupColumns(
+    this._handle,
+    "updateSetGroup",
+    getSymbols().quiver_database_update_set_group,
+    collection,
+    group,
+    id,
+    data,
+  );
+};

package/src/database.ts

@@ -1,6 +1,7 @@
 import type { CsvOptions } from "./csv.ts";
 import { check, QuiverError } from "./errors.ts";
 import { allocPtrOut, makeDefaultOptions, readPtrOut, toCString } from "./ffi-helpers.ts";
+import type { GroupColumns } from "./group-columns.ts";
 import type { NativePointer } from "./loader.ts";
 import { getSymbols } from "./loader.ts";
 import type { GroupMetadata, ScalarMetadata } from "./metadata.ts";
@@ -103,6 +104,7 @@ export class Database {
     id: number,
   ) => string | null;
   declare readElementIds: (collection: string) => number[];
+  declare numberOfElements: (collection: string) => number;
   declare readVectorIntegers: (collection: string, attribute: string) => number[][];
   declare readVectorFloats: (collection: string, attribute: string) => number[][];
   declare readVectorStrings: (collection: string, attribute: string) => string[][];
@@ -126,6 +128,9 @@ export class Database {
   declare commit: () => void;
   declare rollback: () => void;
   declare inTransaction: () => boolean;
+  declare beginDryRun: () => void;
+  declare endDryRun: () => void;
+  declare inDryRun: () => boolean;
 
   // --- Metadata (implemented in metadata.ts) ---
   declare getScalarMetadata: (collection: string, attribute: string) => ScalarMetadata;
@@ -151,6 +156,18 @@ export class Database {
     id: number,
     data: TimeSeriesData,
   ) => void;
+  declare updateVectorGroup: (
+    collection: string,
+    group: string,
+    id: number,
+    data: GroupColumns,
+  ) => void;
+  declare updateSetGroup: (
+    collection: string,
+    group: string,
+    id: number,
+    data: GroupColumns,
+  ) => void;
   declare upsertTimeSeriesRow: (
     collection: string,
     group: string,

package/src/group-columns.ts

@@ -0,0 +1,161 @@
+import { type Pointer, ptr } from "bun:ffi";
+import { check, QuiverError } from "./errors.ts";
+import {
+  allocNativeFloat64,
+  allocNativeInt64,
+  allocNativePtrTable,
+  allocNativeStringArray,
+  toCString,
+} from "./ffi-helpers.ts";
+import type { NativePointer } from "./loader.ts";
+import { DATA_TYPE_FLOAT, DATA_TYPE_INTEGER, DATA_TYPE_STRING, type Allocation } from "./types.ts";
+
+/** Column-oriented group payload: one array of cells per column name, `null` for SQL NULL. */
+export type GroupColumns = Record<string, (number | string | null)[]>;
+
+/**
+ * The parallel-array signature every columnar group update C function shares
+ * (quiver_database_update_{time_series,vector,set}_group).
+ */
+type ColumnUpdateFn = (
+  db: NativePointer,
+  collection: Uint8Array,
+  group: Uint8Array,
+  id: bigint,
+  names: Uint8Array | null,
+  types: Uint8Array | null,
+  data: Uint8Array | null,
+  masks: Uint8Array | null,
+  columnCount: bigint,
+  rowCount: bigint,
+) => number;
+
+/**
+ * Marshal a column-oriented payload and forward it to one of the columnar group update C
+ * functions. Shared by updateTimeSeriesGroup / updateVectorGroup / updateSetGroup: the three
+ * differ only in which C entry point they call.
+ *
+ * Pass `{}` (no columns) to clear the group.
+ */
+export function updateGroupColumns(
+  handle: NativePointer,
+  caller: string,
+  update: ColumnUpdateFn,
+  collection: string,
+  group: string,
+  id: number,
+  data: GroupColumns,
+): void {
+  const collBuf = toCString(collection);
+  const grpBuf = toCString(group);
+  const entries = Object.entries(data);
+
+  if (entries.length === 0) {
+    check(update(handle, collBuf.buf, grpBuf.buf, BigInt(id), null, null, null, null, 0n, 0n));
+    return;
+  }
+
+  const columnCount = entries.length;
+  const rowCount = entries[0][1].length;
+
+  // Validate before marshalling: a jagged column would desync the parallel arrays the C API
+  // reads against row_count, and a zero-length column (with columns present) would otherwise
+  // marshal a null data pointer. Named-but-empty columns are a caller mistake -- pass {} to
+  // clear the group instead. The C API rejects both too; failing here names the column.
+  for (const [name, values] of entries) {
+    if (values.length !== rowCount) {
+      throw new QuiverError(
+        `Cannot ${caller}: column '${name}' has length ${values.length} but expected ${rowCount}`,
+      );
+    }
+  }
+  if (rowCount === 0) {
+    const names = entries.map(([name]) => name).join(", ");
+    throw new QuiverError(
+      `Cannot ${caller}: columns [${names}] contain no rows; pass {} to clear the group`,
+    );
+  }
+
+  const keepalive: Allocation[] = [];
+
+  // Build column names as native string array
+  const colNames = entries.map(([name]) => name);
+  const { table: namesTable, keepalive: namesPtrs } = allocNativeStringArray(colNames);
+  keepalive.push(namesTable, ...namesPtrs);
+
+  // Build column types, data, and per-cell NULL masks. A null cell becomes mask 0 + a
+  // placeholder in the data array (the C API never reads it). An all-null column is tagged
+  // FLOAT with zeroed data — the type tag is ignored for masked-out cells.
+  const typesBuf = new Uint8Array(columnCount * 4);
+  const typesDv = new DataView(typesBuf.buffer);
+  const dataPtrs: (Pointer | null)[] = [];
+  const maskPtrs: (Pointer | null)[] = [];
+
+  for (let c = 0; c < columnCount; c++) {
+    const [colName, values] = entries[c];
+    const first = values.find((v) => v !== null);
+
+    // Mask via direct indexing — never a DataView, to avoid the documented
+    // .buffer-materialization pitfall between ptr() and the FFI call.
+    const maskBuf = new Uint8Array(rowCount);
+    for (let r = 0; r < rowCount; r++) maskBuf[r] = values[r] === null ? 0 : 1;
+    const maskAlloc: Allocation = { ptr: ptr(maskBuf), buf: maskBuf };
+    keepalive.push(maskAlloc);
+    maskPtrs.push(maskAlloc.ptr);
+
+    if (first === undefined) {
+      // All-null column
+      typesDv.setInt32(c * 4, DATA_TYPE_FLOAT, true);
+      const p = allocNativeFloat64(new Array(rowCount).fill(0));
+      keepalive.push(p);
+      dataPtrs.push(p.ptr);
+    } else if (typeof first === "string") {
+      typesDv.setInt32(c * 4, DATA_TYPE_STRING, true);
+      const { table, keepalive: strPtrs } = allocNativeStringArray(
+        values.map((v) => (v === null ? null : (v as string))),
+      );
+      keepalive.push(table, ...strPtrs);
+      dataPtrs.push(table.ptr);
+    } else if (typeof first === "number") {
+      const nonNull = values.filter((v) => v !== null) as number[];
+      const sanitized = values.map((v) => (v === null ? 0 : (v as number)));
+      if (nonNull.every((v) => Number.isInteger(v))) {
+        typesDv.setInt32(c * 4, DATA_TYPE_INTEGER, true);
+        const p = allocNativeInt64(sanitized);
+        keepalive.push(p);
+        dataPtrs.push(p.ptr);
+      } else {
+        typesDv.setInt32(c * 4, DATA_TYPE_FLOAT, true);
+        const p = allocNativeFloat64(sanitized);
+        keepalive.push(p);
+        dataPtrs.push(p.ptr);
+      }
+    } else {
+      throw new QuiverError(
+        `Cannot ${caller}: column '${colName}' has unsupported value type ${typeof first}`,
+      );
+    }
+  }
+
+  const typesAlloc: Allocation = { ptr: ptr(typesBuf), buf: typesBuf };
+  keepalive.push(typesAlloc);
+  const dataTable = allocNativePtrTable(dataPtrs);
+  keepalive.push(dataTable);
+  const maskTable = allocNativePtrTable(maskPtrs);
+  keepalive.push(maskTable);
+
+  check(
+    update(
+      handle,
+      collBuf.buf,
+      grpBuf.buf,
+      BigInt(id),
+      namesTable.buf,
+      typesAlloc.buf,
+      dataTable.buf,
+      maskTable.buf,
+      BigInt(columnCount),
+      BigInt(rowCount),
+    ),
+  );
+}

package/src/index.ts

@@ -11,6 +11,7 @@ import "./composites.ts";
 export type { CsvOptions } from "./csv.ts";
 export { Database } from "./database.ts";
 export { QuiverError } from "./errors.ts";
+export type { GroupColumns } from "./group-columns.ts";
 export { LUA_DB_API_REFERENCE } from "./lua-api.ts";
 export { LuaRunner } from "./lua-runner.ts";
 export type { GroupMetadata, ScalarMetadata } from "./metadata.ts";

package/src/loader.ts

@@ -81,6 +81,7 @@ const readSymbols = {
   quiver_database_read_set_floats_by_id: { args: [P, BUF, BUF, I64, P, P], returns: I32 },
   quiver_database_read_set_strings_by_id: { args: [P, BUF, BUF, I64, P, P], returns: I32 },
   quiver_database_read_element_ids: { args: [P, BUF, P, P], returns: I32 },
+  quiver_database_number_of_elements: { args: [P, BUF, P], returns: I32 },
 } as const;
 
 const querySymbols = {
@@ -97,6 +98,9 @@ const transactionSymbols = {
   quiver_database_commit: { args: [P], returns: I32 },
   quiver_database_rollback: { args: [P], returns: I32 },
   quiver_database_in_transaction: { args: [P, P], returns: I32 },
+  quiver_database_begin_dry_run: { args: [P], returns: I32 },
+  quiver_database_end_dry_run: { args: [P], returns: I32 },
+  quiver_database_in_dry_run: { args: [P, P], returns: I32 },
 } as const;
 
 const metadataSymbols = {
@@ -131,6 +135,14 @@ const timeSeriesSymbols = {
     args: [P, BUF, BUF, I64, P, P, P, P, USIZE, USIZE],
     returns: I32,
   },
+  quiver_database_update_vector_group: {
+    args: [P, BUF, BUF, I64, P, P, P, P, USIZE, USIZE],
+    returns: I32,
+  },
+  quiver_database_update_set_group: {
+    args: [P, BUF, BUF, I64, P, P, P, P, USIZE, USIZE],
+    returns: I32,
+  },
   quiver_database_free_time_series_data: { args: [P, P, P, P, USIZE, USIZE], returns: I32 },
   quiver_database_has_time_series_files: { args: [P, BUF, P], returns: I32 },
   quiver_database_list_time_series_files_columns: { args: [P, BUF, P, P], returns: I32 },
@@ -160,7 +172,8 @@ const freeSymbols = {
 const luaSymbols = {
   quiver_lua_runner_new: { args: [P, P], returns: I32 },
   quiver_lua_runner_free: { args: [P], returns: I32 },
-  quiver_lua_runner_run: { args: [P, BUF], returns: I32 },
+  quiver_lua_runner_run: { args: [P, BUF, BUF], returns: I32 },
+  quiver_lua_runner_free_string: { args: [P], returns: I32 },
 } as const;
 
 // Combined symbol map for dlopen.

package/src/lua-api.ts

@@ -1,16 +1,21 @@
 // Agent-facing reference for the Lua `db` API available inside run_lua scripts.
 //
 // Authority: `src/lua_runner.cpp` `bind_database()` (this repo) — extracted by hand, NOT imported.
-// The shipped quiverdb native binding is the runtime truth; this is docs. Whenever the C++ Lua
-// binding changes, re-diff every signature below against `bind_database()` (names, arg order, arg
-// types, return shapes).
+// The shipped quiverdb native binding is the runtime truth; this is docs.
+//
+// SYNC: `test/lua-api-sync.test.ts` derives the bound surface from `src/lua_runner.cpp` and checks
+// it AUTOMATICALLY — every `db:`/`quiver.*` name is documented, no documented name has been
+// removed, and the stdlib sentence matches `open_libraries` exactly. What it CANNOT check, and you
+// must still re-diff by hand when the binding changes: arg order, arity, arg types, return shapes,
+// and whether the prose is semantically true.
 //
 // NOTE: the binary/expression subsystems are bound in the native binding and documented below.
 // File-touching operations (db:open_file, db:bin_to_csv, db:csv_to_bin, expr:save) are sandboxed
 // to the database file's directory; the pure-metadata builders stay under the quiver.* global.
 //
-// FORMAT CONVENTION: every db: method should appear at least once as the literal token
-// `db:<snake_case_name>` so coverage is greppable.
+// FORMAT CONVENTION: every db: method appears at least once as the literal token
+// `db:<snake_case_name>`, and every quiver.* function as `quiver.<name>`, so coverage is greppable
+// (the sync test relies on this).
 export const LUA_DB_API_REFERENCE = `
 # Quiver Lua API Reference
 
@@ -66,19 +71,51 @@ datetime surface — there are no DateTime wrapper helpers, unlike Julia/Dart/Py
 - **Errors abort the script.** Any error thrown by a \`db:\` call stops the script and surfaces as
   \`Failed to run Lua script: <message>\`. Validation failures roll back whatever the current
   transaction covered.
-- **Standard library.** Only the \`base\`, \`string\`, and \`table\` libraries are loaded — there is NO
-  \`math\` (use \`//\` for integer division), and no \`os\`, \`io\`, or \`require\`; \`dofile\` and
-  \`loadfile\` are removed (string-form \`load\` is available).
+- **Standard library.** Loaded standard libraries: base, string, table, math, coroutine, utf8.
+  That is the pure-computation set — there is no \`os\`, \`io\`, \`debug\`, or \`package\`/\`require\`,
+  and \`dofile\`/\`loadfile\` are removed (string-form \`load\` stays available). Integer division is
+  the Lua 5.4 \`//\` operator — a language operator, unrelated to \`math\`.
 - **Filesystem sandbox.** Every file-touching operation (\`db:export_csv\`, \`db:import_csv\`,
   \`db:open_file\`, \`db:bin_to_csv\`, \`db:csv_to_bin\`, \`expr:save\`) resolves relative paths against
   the directory containing the database file and rejects anything outside it (subdirectories are
   fine; \`..\` escapes and outside absolute paths throw \`Cannot <op>: path '...' escapes the
   database directory ...\`). On an in-memory database these operations throw
   \`Cannot <op>: database is in-memory, file operations are unavailable\`.
-- **Output.** Scripts cannot return values to the tool — use \`print()\` for anything you need to
-  see (it is captured). Arrays are 1-indexed (iterate with \`ipairs\`); reading a NULL yields \`nil\`,
-  writing \`nil\` stores NULL where NULL is accepted (query params, ts rows, file columns — but NOT
-  element scalar attributes; see CRUD).
+- **Output.** A script can \`return\` one value and the host receives it as JSON — prefer this over
+  \`print()\` when you need structured data back (\`print()\` still works and is captured). Only the
+  **first** returned value is encoded. Arrays are 1-indexed (iterate with \`ipairs\`); reading a NULL
+  yields \`nil\`, writing \`nil\` stores NULL where NULL is accepted (query params, ts rows, file
+  columns — but NOT element scalar attributes; see CRUD).
+
+  \`\`\`lua
+  return { ids = db:read_element_ids("Collection"), total = 3 }
+  -- host receives: {"ids":[1,2,3],"total":3}
+  \`\`\`
+
+  | Returned                   | JSON                                                    |
+  | -------------------------- | ------------------------------------------------------- |
+  | nothing                    | \`""\` (empty — distinct from \`nil\`)                      |
+  | \`nil\`                      | \`null\`                                                  |
+  | integer / float / string   | the value; a float keeps its shortest round-trip form   |
+  | boolean                    | \`true\` / \`false\`                                        |
+  | NaN, \`math.huge\`           | \`null\` (JSON has no NaN/Infinity)                       |
+  | table keyed \`1..n\`         | array — an empty table \`{}\` encodes as \`[]\`             |
+  | any other table            | object, keys sorted; integer keys stringify              |
+
+  **A table with holes is an object, not an array.** A bulk read of a nullable column returns
+  \`nil\` holes (see Reading), so \`return db:read_scalar_integers(c, a)\` encodes as
+  \`{"1":10,"3":30}\` — not \`[10,null,30]\` — and the keys sort as text (\`"1","11","2"\`). When the host
+  needs positional data, return the ids alongside and fill the holes yourself:
+  \`local v = db:read_scalar_integers(c, a); local out = {}; for i in ipairs(ids) do out[i] = v[i] or false end\`.
+
+  Returning a function, a coroutine, or a userdata (including \`db\` itself) raises
+  \`Cannot run: script returned an unsupported Lua type\`; nesting deeper than 32 levels raises
+  \`Cannot run: script return value nests deeper than 32 levels\` (this is also what stops a
+  self-referencing table); a result over 64 MB raises
+  \`Cannot run: script return value exceeds ... bytes of JSON\`; a string holding bytes that are not
+  valid UTF-8 raises \`Cannot run: script return value contains a string that is not valid UTF-8\`;
+  and a table where an integer key and a string key spell the same thing (\`{[1] = 'a', ['1'] = 'b'}\`)
+  raises \`Cannot run: script returned a table with duplicate key '1'\`.
 - **Embedding harness may restrict further.** The library itself allows transactions and the
   (sandboxed) CSV/file operations below. A host that runs your script (e.g. a hosted \`run_lua\`
   tool) may disable some of them and report \`disabled in the run_lua sandbox\` — that limit comes
@@ -92,6 +129,7 @@ datetime surface — there are no DateTime wrapper helpers, unlike Julia/Dart/Py
 db:is_healthy()                    -- boolean
 db:current_version()               -- integer (current migration version)
 db:path()                          -- string (database file path)
+db:number_of_elements(collection)  -- integer: how many elements the collection holds right now
 db:describe()                      -- string: whole-DB text report (returns it, does NOT print)
 db:describe_collection(collection) -- string: one collection's structure (text report)
 db:summarize_collection(collection)-- string: per-scalar null/non-null counts, low-cardinality
@@ -121,14 +159,54 @@ db:transaction(function(db)
 end)   -- both writes commit together; if either throws, both roll back
 \`\`\`
 
-**Caveat:** if the host already runs your script inside a transaction, an explicit
-\`db:begin_transaction()\` will error (\`cannot start a transaction within a transaction\`) and a
-mid-script \`db:commit()\` would prematurely end the host's transaction. When unsure whether a
-transaction is already open, check \`db:in_transaction()\` first, or just issue writes directly —
+**Caveat:** if the host already runs your script inside a plain transaction (not a dry run), an
+explicit \`db:begin_transaction()\` will error (\`cannot start a transaction within a transaction\`)
+and a mid-script \`db:commit()\` would prematurely end the host's transaction. When unsure whether
+a transaction is already open, check \`db:in_transaction()\` first, or just issue writes directly —
 each \`db:\` write is durable on its own.
 
 ---
 
+## Dry runs
+
+A dry run executes writes and then throws them away. Use it to check that a sequence works — that
+the collections exist, the types match, the foreign keys resolve — before committing to it.
+
+\`\`\`lua
+db:dry_run(fn)           -- run fn(db), roll everything back, return fn's result
+db:begin_dry_run()       -- start one explicitly
+db:end_dry_run()         -- end it, rolling back everything it covered
+db:in_dry_run()          -- boolean: is a dry run currently active?
+\`\`\`
+
+\`\`\`lua
+local preview = db:dry_run(function(db)
+    db:create_element("Collection", { label = "Item 1", some_integer = 42 })
+    return db:read_element_ids("Collection")   -- reads see the uncommitted writes
+end)
+-- nothing was kept; preview holds what the reads saw
+\`\`\`
+
+Rules worth knowing:
+
+- **Nested transaction control is absorbed.** Inside a dry run, \`db:begin_transaction\`,
+  \`db:commit\` and \`db:rollback\` become no-ops, so the \`db:transaction\` pattern above composes
+  instead of erroring — \`db:transaction(fn)\` still runs \`fn\` and still performs its writes, only
+  its BEGIN/COMMIT are absorbed. The flip side: a nested \`db:rollback\` does **not** partially
+  undo — everything is undone when the dry run ends, whatever the nested calls asked for.
+- \`db:in_transaction()\` still reports \`true\` during a dry run: a real transaction is open.
+- \`db:import_csv\` cannot run inside a dry run (it toggles a pragma that is a no-op mid-transaction)
+  and throws \`Cannot import_csv: transaction already active\`.
+- **Dry runs do not nest.** The host may have already opened one around your whole script, in which
+  case both \`db:begin_dry_run()\` and \`db:dry_run(fn)\` (which calls it internally) error with
+  \`Cannot begin_dry_run: dry run already active\` — check \`db:in_dry_run()\` first and skip the
+  wrapper when one is already active. Do **not** call \`db:end_dry_run()\` to get around it: that
+  ends the host's dry run, and everything you write afterwards is committed for real.
+- For a rough sense of how much a run touched, \`db:query_integer("SELECT total_changes()")\` gives
+  the number of rows inserted, updated or deleted on this connection.
+
+---
+
 ## CRUD
 
 \`\`\`lua
@@ -164,8 +242,9 @@ Notes:
   type can't be inferred from an empty array), so it is silently dropped.
 - **No \`nil\` scalar attributes.** In Lua a key set to \`nil\` is dropped from the table, so
   \`{ x = nil }\` is identical to \`{}\`; an update/create table that ends up with no attributes
-  **throws** (\`element must have at least one attribute\`). To leave a column unchanged, omit the
-  key — you cannot set a scalar to NULL via the element table. (\`nil\` → NULL is only accepted by
+  **throws** (\`...must have at least one scalar attribute\` on create, \`...at least one attribute
+  to update\` on update). To leave a column unchanged, omit the key — you cannot set a scalar to
+  NULL via the element table. (\`nil\` → NULL is only accepted by
   \`upsert_time_series_row\` and \`update_time_series_files\`.)
 
 ---
@@ -206,6 +285,36 @@ db:read_set_strings(collection, attribute)
 
 ---
 
+## Replace a whole vector or set group (column-oriented)
+
+\`update_vector_group\` / \`update_set_group\` replace **all** of one element's rows in one *named*
+group, taking the same column-oriented shape as the time-series writer:
+
+\`\`\`lua
+db:update_vector_group("Child", "refs", id, { parent_ref = { 1, 2, 3 } })
+db:update_set_group("Child", "parents", id, { parent_ref = { 1, 2 } })
+
+db:update_vector_group("Child", "refs", id, {})   -- clears the group
+\`\`\`
+
+Use these instead of routing a group's columns through \`update_element\` whenever a column name is
+shared by two groups of the collection (legal for foreign-key columns): \`update_element\` routes an
+array **by column name**, so it writes to *every* group table that has that column — silently
+rewriting groups you never named. \`(collection, group)\` names exactly one table.
+
+Rules:
+- **Row count is the largest index any column reaches.** Shorter or sparse columns write NULL in
+  the gaps, so \`nil\` holes from a read round-trip.
+- **\`{}\` (no columns) clears the group.** Naming a column whose array is empty is an error, not a
+  clear — a typo'd column name must not destroy data.
+- **\`id\` and \`vector_index\` are managed by the group** (the element and the row's position) and
+  are rejected if passed.
+- **Foreign-key columns accept a label string** and resolve it to the referenced id, exactly as in
+  \`create_element\` / \`update_element\`.
+- The element id must exist, same as \`update_element\` / \`delete_element\`.
+
+---
+
 ## Composite by-id reads (Lua convenience helpers)
 
 \`\`\`lua
@@ -489,7 +598,8 @@ db:csv_to_bin(path)
 
 local e = (quiver.expression(r) + 10.0) * 2.0        -- files auto-wrap; scalars either side
 e = quiver.abs(e); e = quiver.sqrt(e)                -- also quiver.log / quiver.exp
-local cond_e = quiver.gt(e, 3.0)                     -- gt/lt/gte/lte/eq/neq -> 1.0/0.0 (NaN -> NaN)
+local cond_e = quiver.gt(e, 3.0)                     -- also quiver.lt/quiver.gte/quiver.lte/quiver.eq/quiver.neq
+                                                     -- all -> 1.0/0.0 per element (NaN operand -> NaN)
 cond_e = cond_e & ~quiver.lt(e, 1.0)                 -- boolean logic via & | ~ operators (and/or/not are keywords)
 e = quiver.ifelse(cond_e, then_e, else_e)            -- build cond_e with comparison + logical operators
 e = e:aggregate("stage", "sum")                      -- sum/mean/min/max/percentile

package/src/lua-runner.ts

@@ -1,6 +1,6 @@
 import type { Database } from "./database.ts";
 import { check, QuiverError } from "./errors.ts";
-import { allocPtrOut, readPtrOut, toCString } from "./ffi-helpers.ts";
+import { allocPtrOut, decodeStringFromBuf, readPtrOut, toCString } from "./ffi-helpers.ts";
 import type { NativePointer } from "./loader.ts";
 import { getSymbols } from "./loader.ts";
 
@@ -15,11 +15,21 @@ export class LuaRunner {
     this._ptr = readPtrOut(outRunner);
   }
 
-  run(script: string): void {
+  /**
+   * Runs a Lua script and returns its return value encoded as JSON, or "" if it returned nothing.
+   *
+   * To execute a script without keeping its writes, wrap the call in
+   * `db.beginDryRun()` / `db.endDryRun()`.
+   */
+  run(script: string): string {
     this.ensureOpen();
     const lib = getSymbols();
     const scriptBuf = toCString(script);
-    check(lib.quiver_lua_runner_run(this._ptr, scriptBuf.buf));
+    const outResult = allocPtrOut();
+    check(lib.quiver_lua_runner_run(this._ptr, scriptBuf.buf, outResult.buf));
+    const result = decodeStringFromBuf(outResult);
+    lib.quiver_lua_runner_free_string(readPtrOut(outResult));
+    return result;
   }
 
   close(): void {

package/src/read.ts

@@ -211,6 +211,14 @@ Database.prototype.readElementIds = function (this: Database, collection: string
   return result;
 };
 
+Database.prototype.numberOfElements = function (this: Database, collection: string): number {
+  const lib = getSymbols();
+  const collBuf = toCString(collection);
+  const outBuf = new Uint8Array(8);
+  check(lib.quiver_database_number_of_elements(this._handle, collBuf.buf, outBuf));
+  return Number(new DataView(outBuf.buffer).getBigInt64(0, true));
+};
+
 // --- Vector bulk reads ---
 
 function readBulkIntegers(

package/src/time-series.ts

@@ -17,6 +17,7 @@ import {
   readUint64Out,
   toCString,
 } from "./ffi-helpers.ts";
+import { updateGroupColumns } from "./group-columns.ts";
 import { getSymbols } from "./loader.ts";
 import {
   type Allocation,
@@ -188,133 +189,14 @@ Database.prototype.updateTimeSeriesGroup = function (
   id: number,
   data: TimeSeriesData,
 ): void {
-  const lib = getSymbols();
-  const collBuf = toCString(collection);
-  const grpBuf = toCString(group);
-  const entries = Object.entries(data);
-
-  if (entries.length === 0) {
-    check(
-      lib.quiver_database_update_time_series_group(
-        this._handle,
-        collBuf.buf,
-        grpBuf.buf,
-        BigInt(id),
-        null,
-        null,
-        null,
-        null,
-        0n,
-        0n,
-      ),
-    );
-    return;
-  }
-
-  const columnCount = entries.length;
-  const rowCount = entries[0][1].length;
-
-  // Validate before marshalling: a jagged column would desync the parallel
-  // arrays the C API reads against row_count, and a zero-length column (with
-  // columns present) would otherwise marshal a null data pointer that the C API
-  // dereferences. Named-but-empty columns are a caller mistake -- pass {} to
-  // clear the group instead.
-  for (const [name, values] of entries) {
-    if (values.length !== rowCount) {
-      throw new QuiverError(
-        `Cannot updateTimeSeriesGroup: column '${name}' has length ${values.length} but expected ${rowCount}`,
-      );
-    }
-  }
-  if (rowCount === 0) {
-    const names = entries.map(([name]) => name).join(", ");
-    throw new QuiverError(
-      `Cannot updateTimeSeriesGroup: columns [${names}] contain no rows; pass {} to clear the group`,
-    );
-  }
-
-  const keepalive: Allocation[] = [];
-
-  // Build column names as native string array
-  const colNames = entries.map(([name]) => name);
-  const { table: namesTable, keepalive: namesPtrs } = allocNativeStringArray(colNames);
-  keepalive.push(namesTable, ...namesPtrs);
-
-  // Build column types, data, and per-cell NULL masks. A null cell becomes
-  // mask 0 + a placeholder in the data array (the C API never reads it). An
-  // all-null column is tagged FLOAT with zeroed data — the type tag is ignored
-  // for masked-out cells.
-  const typesBuf = new Uint8Array(columnCount * 4);
-  const typesDv = new DataView(typesBuf.buffer);
-  const dataPtrs: (Pointer | null)[] = [];
-  const maskPtrs: (Pointer | null)[] = [];
-
-  for (let c = 0; c < columnCount; c++) {
-    const [colName, values] = entries[c];
-    const first = values.find((v) => v !== null);
-
-    // Mask via direct indexing — never a DataView, to avoid the documented
-    // .buffer-materialization pitfall between ptr() and the FFI call.
-    const maskBuf = new Uint8Array(rowCount);
-    for (let r = 0; r < rowCount; r++) maskBuf[r] = values[r] === null ? 0 : 1;
-    const maskAlloc: Allocation = { ptr: ptr(maskBuf), buf: maskBuf };
-    keepalive.push(maskAlloc);
-    maskPtrs.push(maskAlloc.ptr);
-
-    if (first === undefined) {
-      // All-null column
-      typesDv.setInt32(c * 4, DATA_TYPE_FLOAT, true);
-      const p = allocNativeFloat64(new Array(rowCount).fill(0));
-      keepalive.push(p);
-      dataPtrs.push(p.ptr);
-    } else if (typeof first === "string") {
-      typesDv.setInt32(c * 4, DATA_TYPE_STRING, true);
-      const { table, keepalive: strPtrs } = allocNativeStringArray(
-        values.map((v) => (v === null ? null : (v as string))),
-      );
-      keepalive.push(table, ...strPtrs);
-      dataPtrs.push(table.ptr);
-    } else if (typeof first === "number") {
-      const nonNull = values.filter((v) => v !== null) as number[];
-      const sanitized = values.map((v) => (v === null ? 0 : (v as number)));
-      if (nonNull.every((v) => Number.isInteger(v))) {
-        typesDv.setInt32(c * 4, DATA_TYPE_INTEGER, true);
-        const p = allocNativeInt64(sanitized);
-        keepalive.push(p);
-        dataPtrs.push(p.ptr);
-      } else {
-        typesDv.setInt32(c * 4, DATA_TYPE_FLOAT, true);
-        const p = allocNativeFloat64(sanitized);
-        keepalive.push(p);
-        dataPtrs.push(p.ptr);
-      }
-    } else {
-      throw new QuiverError(
-        `Cannot updateTimeSeriesGroup: column '${colName}' has unsupported value type ${typeof first}`,
-      );
-    }
-  }
-
-  const typesAlloc: Allocation = { ptr: ptr(typesBuf), buf: typesBuf };
-  keepalive.push(typesAlloc);
-  const dataTable = allocNativePtrTable(dataPtrs);
-  keepalive.push(dataTable);
-  const maskTable = allocNativePtrTable(maskPtrs);
-  keepalive.push(maskTable);
-
-  check(
-    lib.quiver_database_update_time_series_group(
-      this._handle,
-      collBuf.buf,
-      grpBuf.buf,
-      BigInt(id),
-      namesTable.buf,
-      typesAlloc.buf,
-      dataTable.buf,
-      maskTable.buf,
-      BigInt(columnCount),
-      BigInt(rowCount),
-    ),
+  updateGroupColumns(
+    this._handle,
+    "updateTimeSeriesGroup",
+    getSymbols().quiver_database_update_time_series_group,
+    collection,
+    group,
+    id,
+    data,
   );
 };
 

package/src/transaction.ts

@@ -23,3 +23,27 @@ Database.prototype.inTransaction = function (this: Database): boolean {
   check(lib.quiver_database_in_transaction(this._handle, outBuf));
   return new DataView(outBuf.buffer).getInt32(0, true) !== 0;
 };
+
+/**
+ * Begin a dry run: a transaction that endDryRun always rolls back.
+ *
+ * While it is active, beginTransaction/commit/rollback are absorbed (no-ops), so code that
+ * manages its own transactions composes instead of erroring on a nested BEGIN. A nested rollback
+ * is therefore not partial -- everything is undone when the dry run ends.
+ */
+Database.prototype.beginDryRun = function (this: Database): void {
+  const lib = getSymbols();
+  check(lib.quiver_database_begin_dry_run(this._handle));
+};
+
+Database.prototype.endDryRun = function (this: Database): void {
+  const lib = getSymbols();
+  check(lib.quiver_database_end_dry_run(this._handle));
+};
+
+Database.prototype.inDryRun = function (this: Database): boolean {
+  const lib = getSymbols();
+  const outBuf = new Uint8Array(4);
+  check(lib.quiver_database_in_dry_run(this._handle, outBuf));
+  return new DataView(outBuf.buffer).getInt32(0, true) !== 0;
+};