quiverdb 0.9.14 → 0.9.16

package/README.md

@@ -130,6 +130,10 @@ Parameters are passed as an array of `number | string | null`.
 - `commit()` -- Commit current transaction
 - `rollback()` -- Rollback current transaction
 - `inTransaction()` -- Check if transaction is active
+- `beginDryRun()` -- Begin a dry run: a transaction `endDryRun()` always rolls back. While it is
+  active, `beginTransaction`/`commit`/`rollback` are absorbed (no-ops)
+- `endDryRun()` -- End the active dry run, rolling back everything it covered
+- `inDryRun()` -- Check if a dry run is active
 
 ### CSV
 
@@ -152,7 +156,8 @@ Parameters are passed as an array of `number | string | null`.
 ### Lua
 
 - `LuaRunner(db)` -- Create Lua script runner with database access
-- `run(script)` -- Execute a Lua script
+- `run(script)` -- Execute a Lua script; returns its return value as a JSON string, or `""` if it
+  returned nothing
 - `close()` -- Close the Lua runner
 
 ## Types

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "quiverdb",
-  "version": "0.9.14",
+  "version": "0.9.16",
   "license": "MIT",
   "repository": {
     "type": "git",

package/src/create.ts

@@ -22,7 +22,7 @@ function setElementArray(
   const nameBuf = toCString(name);
 
   if (values.length === 0) {
-    check(lib.quiver_element_set_array_integer(elemPtr, nameBuf.buf, null, 0));
+    check(lib.quiver_element_set_array_integer(elemPtr, nameBuf.buf, null, 0, null));
     return;
   }
 
@@ -30,7 +30,7 @@ function setElementArray(
 
   if (typeof first === "bigint") {
     const arr = allocNativeInt64(values as bigint[]);
-    check(lib.quiver_element_set_array_integer(elemPtr, nameBuf.buf, arr.buf, values.length));
+    check(lib.quiver_element_set_array_integer(elemPtr, nameBuf.buf, arr.buf, values.length, null));
     return;
   }
 
@@ -38,17 +38,17 @@ function setElementArray(
     const allIntegers = (values as number[]).every((v) => Number.isInteger(v));
     if (allIntegers) {
       const arr = allocNativeInt64(values as number[]);
-      check(lib.quiver_element_set_array_integer(elemPtr, nameBuf.buf, arr.buf, values.length));
+      check(lib.quiver_element_set_array_integer(elemPtr, nameBuf.buf, arr.buf, values.length, null));
     } else {
       const arr = allocNativeFloat64(values as number[]);
-      check(lib.quiver_element_set_array_float(elemPtr, nameBuf.buf, arr.buf, values.length));
+      check(lib.quiver_element_set_array_float(elemPtr, nameBuf.buf, arr.buf, values.length, null));
     }
     return;
   }
 
   if (typeof first === "string") {
     const { table, keepalive: _keepalive } = allocNativeStringArray(values as string[]);
-    check(lib.quiver_element_set_array_string(elemPtr, nameBuf.buf, table.buf, values.length));
+    check(lib.quiver_element_set_array_string(elemPtr, nameBuf.buf, table.buf, values.length, null));
     return;
   }
 

package/src/database.ts

@@ -126,6 +126,9 @@ export class Database {
   declare commit: () => void;
   declare rollback: () => void;
   declare inTransaction: () => boolean;
+  declare beginDryRun: () => void;
+  declare endDryRun: () => void;
+  declare inDryRun: () => boolean;
 
   // --- Metadata (implemented in metadata.ts) ---
   declare getScalarMetadata: (collection: string, attribute: string) => ScalarMetadata;

package/src/loader.ts

@@ -50,9 +50,9 @@ const elementSymbols = {
   quiver_element_set_float: { args: [P, BUF, F64], returns: I32 },
   quiver_element_set_string: { args: [P, BUF, BUF], returns: I32 },
   quiver_element_set_null: { args: [P, BUF], returns: I32 },
-  quiver_element_set_array_integer: { args: [P, BUF, P, I32], returns: I32 },
-  quiver_element_set_array_float: { args: [P, BUF, P, I32], returns: I32 },
-  quiver_element_set_array_string: { args: [P, BUF, P, I32], returns: I32 },
+  quiver_element_set_array_integer: { args: [P, BUF, P, I32, P], returns: I32 },
+  quiver_element_set_array_float: { args: [P, BUF, P, I32, P], returns: I32 },
+  quiver_element_set_array_string: { args: [P, BUF, P, I32, P], returns: I32 },
 } as const;
 
 const crudSymbols = {
@@ -97,6 +97,9 @@ const transactionSymbols = {
   quiver_database_commit: { args: [P], returns: I32 },
   quiver_database_rollback: { args: [P], returns: I32 },
   quiver_database_in_transaction: { args: [P, P], returns: I32 },
+  quiver_database_begin_dry_run: { args: [P], returns: I32 },
+  quiver_database_end_dry_run: { args: [P], returns: I32 },
+  quiver_database_in_dry_run: { args: [P, P], returns: I32 },
 } as const;
 
 const metadataSymbols = {
@@ -160,7 +163,8 @@ const freeSymbols = {
 const luaSymbols = {
   quiver_lua_runner_new: { args: [P, P], returns: I32 },
   quiver_lua_runner_free: { args: [P], returns: I32 },
-  quiver_lua_runner_run: { args: [P, BUF], returns: I32 },
+  quiver_lua_runner_run: { args: [P, BUF, BUF], returns: I32 },
+  quiver_lua_runner_free_string: { args: [P], returns: I32 },
 } as const;
 
 // Combined symbol map for dlopen.

package/src/lua-api.ts

@@ -1,16 +1,21 @@
 // Agent-facing reference for the Lua `db` API available inside run_lua scripts.
 //
 // Authority: `src/lua_runner.cpp` `bind_database()` (this repo) — extracted by hand, NOT imported.
-// The shipped quiverdb native binding is the runtime truth; this is docs. Whenever the C++ Lua
-// binding changes, re-diff every signature below against `bind_database()` (names, arg order, arg
-// types, return shapes).
+// The shipped quiverdb native binding is the runtime truth; this is docs.
+//
+// SYNC: `test/lua-api-sync.test.ts` derives the bound surface from `src/lua_runner.cpp` and checks
+// it AUTOMATICALLY — every `db:`/`quiver.*` name is documented, no documented name has been
+// removed, and the stdlib sentence matches `open_libraries` exactly. What it CANNOT check, and you
+// must still re-diff by hand when the binding changes: arg order, arity, arg types, return shapes,
+// and whether the prose is semantically true.
 //
 // NOTE: the binary/expression subsystems are bound in the native binding and documented below.
 // File-touching operations (db:open_file, db:bin_to_csv, db:csv_to_bin, expr:save) are sandboxed
 // to the database file's directory; the pure-metadata builders stay under the quiver.* global.
 //
-// FORMAT CONVENTION: every db: method should appear at least once as the literal token
-// `db:<snake_case_name>` so coverage is greppable.
+// FORMAT CONVENTION: every db: method appears at least once as the literal token
+// `db:<snake_case_name>`, and every quiver.* function as `quiver.<name>`, so coverage is greppable
+// (the sync test relies on this).
 export const LUA_DB_API_REFERENCE = `
 # Quiver Lua API Reference
 
@@ -66,19 +71,51 @@ datetime surface — there are no DateTime wrapper helpers, unlike Julia/Dart/Py
 - **Errors abort the script.** Any error thrown by a \`db:\` call stops the script and surfaces as
   \`Failed to run Lua script: <message>\`. Validation failures roll back whatever the current
   transaction covered.
-- **Standard library.** Only the \`base\`, \`string\`, and \`table\` libraries are loaded — there is NO
-  \`math\` (use \`//\` for integer division), and no \`os\`, \`io\`, or \`require\`; \`dofile\` and
-  \`loadfile\` are removed (string-form \`load\` is available).
+- **Standard library.** Loaded standard libraries: base, string, table, math, coroutine, utf8.
+  That is the pure-computation set — there is no \`os\`, \`io\`, \`debug\`, or \`package\`/\`require\`,
+  and \`dofile\`/\`loadfile\` are removed (string-form \`load\` stays available). Integer division is
+  the Lua 5.4 \`//\` operator — a language operator, unrelated to \`math\`.
 - **Filesystem sandbox.** Every file-touching operation (\`db:export_csv\`, \`db:import_csv\`,
   \`db:open_file\`, \`db:bin_to_csv\`, \`db:csv_to_bin\`, \`expr:save\`) resolves relative paths against
   the directory containing the database file and rejects anything outside it (subdirectories are
   fine; \`..\` escapes and outside absolute paths throw \`Cannot <op>: path '...' escapes the
   database directory ...\`). On an in-memory database these operations throw
   \`Cannot <op>: database is in-memory, file operations are unavailable\`.
-- **Output.** Scripts cannot return values to the tool — use \`print()\` for anything you need to
-  see (it is captured). Arrays are 1-indexed (iterate with \`ipairs\`); reading a NULL yields \`nil\`,
-  writing \`nil\` stores NULL where NULL is accepted (query params, ts rows, file columns — but NOT
-  element scalar attributes; see CRUD).
+- **Output.** A script can \`return\` one value and the host receives it as JSON — prefer this over
+  \`print()\` when you need structured data back (\`print()\` still works and is captured). Only the
+  **first** returned value is encoded. Arrays are 1-indexed (iterate with \`ipairs\`); reading a NULL
+  yields \`nil\`, writing \`nil\` stores NULL where NULL is accepted (query params, ts rows, file
+  columns — but NOT element scalar attributes; see CRUD).
+
+  \`\`\`lua
+  return { ids = db:read_element_ids("Collection"), total = 3 }
+  -- host receives: {"ids":[1,2,3],"total":3}
+  \`\`\`
+
+  | Returned                   | JSON                                                    |
+  | -------------------------- | ------------------------------------------------------- |
+  | nothing                    | \`""\` (empty — distinct from \`nil\`)                      |
+  | \`nil\`                      | \`null\`                                                  |
+  | integer / float / string   | the value; a float keeps its shortest round-trip form   |
+  | boolean                    | \`true\` / \`false\`                                        |
+  | NaN, \`math.huge\`           | \`null\` (JSON has no NaN/Infinity)                       |
+  | table keyed \`1..n\`         | array — an empty table \`{}\` encodes as \`[]\`             |
+  | any other table            | object, keys sorted; integer keys stringify              |
+
+  **A table with holes is an object, not an array.** A bulk read of a nullable column returns
+  \`nil\` holes (see Reading), so \`return db:read_scalar_integers(c, a)\` encodes as
+  \`{"1":10,"3":30}\` — not \`[10,null,30]\` — and the keys sort as text (\`"1","11","2"\`). When the host
+  needs positional data, return the ids alongside and fill the holes yourself:
+  \`local v = db:read_scalar_integers(c, a); local out = {}; for i in ipairs(ids) do out[i] = v[i] or false end\`.
+
+  Returning a function, a coroutine, or a userdata (including \`db\` itself) raises
+  \`Cannot run: script returned an unsupported Lua type\`; nesting deeper than 32 levels raises
+  \`Cannot run: script return value nests deeper than 32 levels\` (this is also what stops a
+  self-referencing table); a result over 64 MB raises
+  \`Cannot run: script return value exceeds ... bytes of JSON\`; a string holding bytes that are not
+  valid UTF-8 raises \`Cannot run: script return value contains a string that is not valid UTF-8\`;
+  and a table where an integer key and a string key spell the same thing (\`{[1] = 'a', ['1'] = 'b'}\`)
+  raises \`Cannot run: script returned a table with duplicate key '1'\`.
 - **Embedding harness may restrict further.** The library itself allows transactions and the
   (sandboxed) CSV/file operations below. A host that runs your script (e.g. a hosted \`run_lua\`
   tool) may disable some of them and report \`disabled in the run_lua sandbox\` — that limit comes
@@ -121,14 +158,54 @@ db:transaction(function(db)
 end)   -- both writes commit together; if either throws, both roll back
 \`\`\`
 
-**Caveat:** if the host already runs your script inside a transaction, an explicit
-\`db:begin_transaction()\` will error (\`cannot start a transaction within a transaction\`) and a
-mid-script \`db:commit()\` would prematurely end the host's transaction. When unsure whether a
-transaction is already open, check \`db:in_transaction()\` first, or just issue writes directly —
+**Caveat:** if the host already runs your script inside a plain transaction (not a dry run), an
+explicit \`db:begin_transaction()\` will error (\`cannot start a transaction within a transaction\`)
+and a mid-script \`db:commit()\` would prematurely end the host's transaction. When unsure whether
+a transaction is already open, check \`db:in_transaction()\` first, or just issue writes directly —
 each \`db:\` write is durable on its own.
 
 ---
 
+## Dry runs
+
+A dry run executes writes and then throws them away. Use it to check that a sequence works — that
+the collections exist, the types match, the foreign keys resolve — before committing to it.
+
+\`\`\`lua
+db:dry_run(fn)           -- run fn(db), roll everything back, return fn's result
+db:begin_dry_run()       -- start one explicitly
+db:end_dry_run()         -- end it, rolling back everything it covered
+db:in_dry_run()          -- boolean: is a dry run currently active?
+\`\`\`
+
+\`\`\`lua
+local preview = db:dry_run(function(db)
+    db:create_element("Collection", { label = "Item 1", some_integer = 42 })
+    return db:read_element_ids("Collection")   -- reads see the uncommitted writes
+end)
+-- nothing was kept; preview holds what the reads saw
+\`\`\`
+
+Rules worth knowing:
+
+- **Nested transaction control is absorbed.** Inside a dry run, \`db:begin_transaction\`,
+  \`db:commit\` and \`db:rollback\` become no-ops, so the \`db:transaction\` pattern above composes
+  instead of erroring — \`db:transaction(fn)\` still runs \`fn\` and still performs its writes, only
+  its BEGIN/COMMIT are absorbed. The flip side: a nested \`db:rollback\` does **not** partially
+  undo — everything is undone when the dry run ends, whatever the nested calls asked for.
+- \`db:in_transaction()\` still reports \`true\` during a dry run: a real transaction is open.
+- \`db:import_csv\` cannot run inside a dry run (it toggles a pragma that is a no-op mid-transaction)
+  and throws \`Cannot import_csv: transaction already active\`.
+- **Dry runs do not nest.** The host may have already opened one around your whole script, in which
+  case both \`db:begin_dry_run()\` and \`db:dry_run(fn)\` (which calls it internally) error with
+  \`Cannot begin_dry_run: dry run already active\` — check \`db:in_dry_run()\` first and skip the
+  wrapper when one is already active. Do **not** call \`db:end_dry_run()\` to get around it: that
+  ends the host's dry run, and everything you write afterwards is committed for real.
+- For a rough sense of how much a run touched, \`db:query_integer("SELECT total_changes()")\` gives
+  the number of rows inserted, updated or deleted on this connection.
+
+---
+
 ## CRUD
 
 \`\`\`lua
@@ -164,8 +241,9 @@ Notes:
   type can't be inferred from an empty array), so it is silently dropped.
 - **No \`nil\` scalar attributes.** In Lua a key set to \`nil\` is dropped from the table, so
   \`{ x = nil }\` is identical to \`{}\`; an update/create table that ends up with no attributes
-  **throws** (\`element must have at least one attribute\`). To leave a column unchanged, omit the
-  key — you cannot set a scalar to NULL via the element table. (\`nil\` → NULL is only accepted by
+  **throws** (\`...must have at least one scalar attribute\` on create, \`...at least one attribute
+  to update\` on update). To leave a column unchanged, omit the key — you cannot set a scalar to
+  NULL via the element table. (\`nil\` → NULL is only accepted by
   \`upsert_time_series_row\` and \`update_time_series_files\`.)
 
 ---
@@ -489,7 +567,8 @@ db:csv_to_bin(path)
 
 local e = (quiver.expression(r) + 10.0) * 2.0        -- files auto-wrap; scalars either side
 e = quiver.abs(e); e = quiver.sqrt(e)                -- also quiver.log / quiver.exp
-local cond_e = quiver.gt(e, 3.0)                     -- gt/lt/gte/lte/eq/neq -> 1.0/0.0 (NaN -> NaN)
+local cond_e = quiver.gt(e, 3.0)                     -- also quiver.lt/quiver.gte/quiver.lte/quiver.eq/quiver.neq
+                                                     -- all -> 1.0/0.0 per element (NaN operand -> NaN)
 cond_e = cond_e & ~quiver.lt(e, 1.0)                 -- boolean logic via & | ~ operators (and/or/not are keywords)
 e = quiver.ifelse(cond_e, then_e, else_e)            -- build cond_e with comparison + logical operators
 e = e:aggregate("stage", "sum")                      -- sum/mean/min/max/percentile

package/src/lua-runner.ts

@@ -1,6 +1,6 @@
 import type { Database } from "./database.ts";
 import { check, QuiverError } from "./errors.ts";
-import { allocPtrOut, readPtrOut, toCString } from "./ffi-helpers.ts";
+import { allocPtrOut, decodeStringFromBuf, readPtrOut, toCString } from "./ffi-helpers.ts";
 import type { NativePointer } from "./loader.ts";
 import { getSymbols } from "./loader.ts";
 
@@ -15,11 +15,21 @@ export class LuaRunner {
     this._ptr = readPtrOut(outRunner);
   }
 
-  run(script: string): void {
+  /**
+   * Runs a Lua script and returns its return value encoded as JSON, or "" if it returned nothing.
+   *
+   * To execute a script without keeping its writes, wrap the call in
+   * `db.beginDryRun()` / `db.endDryRun()`.
+   */
+  run(script: string): string {
     this.ensureOpen();
     const lib = getSymbols();
     const scriptBuf = toCString(script);
-    check(lib.quiver_lua_runner_run(this._ptr, scriptBuf.buf));
+    const outResult = allocPtrOut();
+    check(lib.quiver_lua_runner_run(this._ptr, scriptBuf.buf, outResult.buf));
+    const result = decodeStringFromBuf(outResult);
+    lib.quiver_lua_runner_free_string(readPtrOut(outResult));
+    return result;
   }
 
   close(): void {

package/src/transaction.ts

@@ -23,3 +23,27 @@ Database.prototype.inTransaction = function (this: Database): boolean {
   check(lib.quiver_database_in_transaction(this._handle, outBuf));
   return new DataView(outBuf.buffer).getInt32(0, true) !== 0;
 };
+
+/**
+ * Begin a dry run: a transaction that endDryRun always rolls back.
+ *
+ * While it is active, beginTransaction/commit/rollback are absorbed (no-ops), so code that
+ * manages its own transactions composes instead of erroring on a nested BEGIN. A nested rollback
+ * is therefore not partial -- everything is undone when the dry run ends.
+ */
+Database.prototype.beginDryRun = function (this: Database): void {
+  const lib = getSymbols();
+  check(lib.quiver_database_begin_dry_run(this._handle));
+};
+
+Database.prototype.endDryRun = function (this: Database): void {
+  const lib = getSymbols();
+  check(lib.quiver_database_end_dry_run(this._handle));
+};
+
+Database.prototype.inDryRun = function (this: Database): boolean {
+  const lib = getSymbols();
+  const outBuf = new Uint8Array(4);
+  check(lib.quiver_database_in_dry_run(this._handle, outBuf));
+  return new DataView(outBuf.buffer).getInt32(0, true) !== 0;
+};