quiverdb 0.12.9 → 0.13.1

package/README.md

@@ -194,10 +194,10 @@ as an exact INTEGER, a `boolean` as the INTEGER 1 or 0).
 
 ### Lua
 
-- `LuaRunner(db)` -- Create Lua script runner with database access
+- `Sandbox(db)` -- Create Lua script runner with database access
 - `run(script)` -- Execute a Lua script; returns its return value as a JSON string, or `""` if it
   returned nothing
-- `close()` -- Close the Lua runner
+- `close()` -- Close the Sandbox
 
 ## Types
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "quiverdb",
-  "version": "0.12.9",
+  "version": "0.13.1",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -20,6 +20,7 @@
   ],
   "scripts": {
     "test": "bun test test",
+    "typecheck": "tsc --noEmit",
     "lint": "biome check",
     "format": "biome format --write"
   },
@@ -29,6 +30,7 @@
   },
   "devDependencies": {
     "@biomejs/biome": "^2.4.6",
-    "@types/bun": "latest"
+    "@types/bun": "latest",
+    "typescript": "7.0.2"
   }
 }

package/src/group-columns.ts

@@ -42,11 +42,11 @@ export type TimeSeriesData = Record<string, (number | string | null)[]>;
  * (quiver_database_update_{time_series,vector,set}_group and their _by_label forms). The 4th
  * argument addresses the element: an id for the by-id forms, a NUL-terminated label otherwise.
  */
-type ColumnUpdateFn = (
+type ColumnUpdateFn<Key extends number | string> = (
   db: NativePointer,
   collection: Uint8Array,
   group: Uint8Array,
-  key: bigint | Uint8Array,
+  key: Key extends string ? Uint8Array : bigint,
   names: Uint8Array | null,
   types: Uint8Array | null,
   data: Uint8Array | null,
@@ -90,18 +90,20 @@ export function numericCells(
  *
  * Pass `{}` (no columns) to clear the group.
  */
-export function updateGroupColumns(
+export function updateGroupColumns<Key extends number | string>(
   handle: NativePointer,
   caller: string,
-  update: ColumnUpdateFn,
+  update: ColumnUpdateFn<Key>,
   collection: string,
   group: string,
-  key: number | string,
+  key: Key,
   data: GroupColumns,
 ): void {
   const collBuf = toCString(collection);
   const grpBuf = toCString(group);
-  const keyArg = typeof key === "string" ? toCString(key).buf : BigInt(key);
+  const keyArg = (typeof key === "string" ? toCString(key).buf : BigInt(key)) as Key extends string
+    ? Uint8Array
+    : bigint;
   const entries = Object.entries(data);
 
   if (entries.length === 0) {

package/src/index.ts

@@ -12,8 +12,8 @@ export type { CsvOptions } from "./csv.ts";
 export { Database } from "./database.ts";
 export { QuiverError } from "./errors.ts";
 export type { GroupColumns, TimeSeriesData } from "./group-columns.ts";
-export { LUA_DB_API_REFERENCE } from "./lua-api.ts";
-export { LuaRunner } from "./lua-runner.ts";
+export { SANDBOX_API_REFERENCE } from "./sandbox-api.ts";
+export { Sandbox } from "./sandbox.ts";
 export type { GroupMetadata, ScalarMetadata } from "./metadata.ts";
 export type {
   ArrayValue,

package/src/loader.ts

@@ -196,11 +196,11 @@ const freeSymbols = {
   quiver_database_free_group_metadata_array: { args: [P, USIZE], returns: I32 },
 } as const;
 
-const luaSymbols = {
-  quiver_lua_runner_new: { args: [P, P], returns: I32 },
-  quiver_lua_runner_free: { args: [P], returns: I32 },
-  quiver_lua_runner_run: { args: [P, BUF, BUF], returns: I32 },
-  quiver_lua_runner_free_string: { args: [P], returns: I32 },
+const sandboxSymbols = {
+  quiver_sandbox_new: { args: [P, P], returns: I32 },
+  quiver_sandbox_free: { args: [P], returns: I32 },
+  quiver_sandbox_run: { args: [P, BUF, BUF], returns: I32 },
+  quiver_sandbox_free_string: { args: [P], returns: I32 },
 } as const;
 
 // Combined symbol map for dlopen.
@@ -216,7 +216,7 @@ const allSymbols = {
   ...timeSeriesSymbols,
   ...csvSymbols,
   ...freeSymbols,
-  ...luaSymbols,
+  ...sandboxSymbols,
 } as const;
 
 // ---------------------------------------------------------------------------
@@ -288,7 +288,17 @@ function initLibrary(): QuiverLib {
     }
   }
 
-  // Tier 2: Dev mode -- walk up directories looking for build/bin/.
+  // Tier 2: Standalone Bun executables ship native siblings beside the executable.
+  const executableDir = dirname(process.execPath);
+  if (existsSync(join(executableDir, C_API_LIB))) {
+    try {
+      return openLibrary(executableDir);
+    } catch (e) {
+      lastError = e;
+    }
+  }
+
+  // Tier 3: Dev mode -- walk up directories looking for build/bin/.
   for (const dir of getSearchPaths()) {
     try {
       return openLibrary(dir);
@@ -297,7 +307,7 @@ function initLibrary(): QuiverLib {
     }
   }
 
-  // Tier 3: System PATH fallback -- the core lib is expected to be discoverable
+  // Tier 4: System PATH fallback -- the core lib is expected to be discoverable
   // on PATH alongside the C API lib.
   try {
     return dlopen(C_API_LIB, allSymbols);
@@ -307,6 +317,7 @@ function initLibrary(): QuiverLib {
 
   const searched = [
     join(__dirname, "..", "libs", platformKey),
+    executableDir,
     ...getSearchPaths(),
     "system PATH",
   ].join(", ");

package/src/{lua-api.ts → sandbox-api.ts}

@@ -1,33 +1,33 @@
 // Agent-facing reference for the Lua `db` API available inside run_lua scripts.
 //
-// Authority: `src/lua_runner.cpp` `bind_database()` (this repo) — extracted by hand, NOT imported.
+// Authority: the binders under `src/sandbox/` (`bind_database` through `bind_expression`,
+// this repo) — extracted by hand, NOT imported.
 // The shipped quiverdb native binding is the runtime truth; this is docs.
 //
-// SYNC: `test/lua-api-sync.test.ts` derives the bound surface from `src/lua_runner.cpp` and checks
+// SYNC: `test/sandbox-api-sync.test.ts` derives the bound surface from every `.cpp`/`.h` under
+// `src/sandbox/` and checks
 // it AUTOMATICALLY — every `db:`/`quiver.*` name is documented, no documented name has been
 // removed, and the stdlib sentence matches `open_libraries` exactly. What it CANNOT check, and you
 // must still re-diff by hand when the binding changes: arg order, arity, arg types, return shapes,
 // and whether the prose is semantically true.
 //
 // NOTE: the binary/expression subsystems are bound in the native binding and documented below.
-// File-touching operations (db:open_file, db:bin_to_csv, db:csv_to_bin, db:validate_migrations,
-// expr:save) are sandboxed to the database file's directory; the pure-metadata builders stay under
-// the quiver.* global.
+// File-touching operations are sandboxed to the database file's directory (the "Filesystem
+// sandbox" bullet lists all of them); the pure-metadata builders stay under the quiver.* global.
 //
 // FORMAT CONVENTION: every db: method appears at least once as the literal token
 // `db:<snake_case_name>`, and every quiver.* function as `quiver.<name>`, so coverage is greppable
 // (the sync test relies on this).
-export const LUA_DB_API_REFERENCE = `
-# Quiver Lua API Reference
+export const SANDBOX_API_REFERENCE = `
+# Quiver Sandbox API Reference
 
-Quiver embeds a Lua scripting layer (via \`LuaRunner\`) that exposes the same database API as the
-other language bindings. This document is a complete reference of every method available to a Lua
-script.
+Quiver embeds a Lua sandbox scripting layer that exposes the same database API as the other language
+bindings. This document is a complete reference of every method available to a Lua script.
 
 ## Running scripts
 
 The database is provided to your script as a global userdata named \`db\`. There are **no
-open/close lifecycle methods in Lua** — the \`LuaRunner\` opens the database and hands you \`db\`
+open/close lifecycle methods in Lua** — the \`Sandbox\` opens the database and hands you \`db\`
 already connected. All methods are called with the colon syntax:
 
 \`\`\`lua
@@ -100,18 +100,27 @@ midnight.
   and do not \`pcall\` inside it: a caught error lets the block commit whatever ran.
 - **Standard library.** Loaded standard libraries: base, string, table, math, coroutine, utf8.
   That is the pure-computation set — there is no \`os\`, \`io\`, \`debug\`, or \`package\`/\`require\`,
-  and \`dofile\`/\`loadfile\` are removed (string-form \`load\` stays available). Integer division is
-  the Lua 5.4 \`//\` operator — a language operator, unrelated to \`math\`. No \`io\` does **not** mean
-  a data file on disk is out of reach: read it with \`db:read_csv\` / \`db:read_csv_stream\` (see
-  the CSV file reading section below). Never copy, paste, or re-type a data file's contents into
+  and \`dofile\`/\`loadfile\` are removed (string-form \`load\` stays for source text; a precompiled
+  binary chunk is refused). Integer division is the Lua 5.4 \`//\` operator — a language operator,
+  unrelated to \`math\`. No \`io\` does **not** mean
+  a data file on disk is out of reach: use \`db:read_csv\` / \`db:read_csv_stream\` for CSV or
+  \`db:read_xlsx\` / \`db:read_xlsx_stream\` for XLSX (see the file reading sections below).
+  Never copy, paste, or re-type a data file's contents into
   the script as literals — read the file.
 - **Filesystem sandbox.** Every file-touching operation (\`db:export_csv\`, \`db:import_csv\`,
   \`db:open_file\`, \`db:bin_to_csv\`, \`db:csv_to_bin\`, \`db:validate_migrations\`, \`db:read_csv\`,
-  \`db:read_csv_stream\`, \`db:write_csv\`, \`expr:save\`) resolves
+  \`db:read_csv_stream\`, \`db:write_csv\`, \`db:read_xlsx\`, \`db:read_xlsx_stream\`,
+  \`save\` on a file or an expression) resolves
   relative paths against the directory containing the database file and rejects anything outside it
   (subdirectories are fine; \`..\` escapes and outside absolute paths throw \`Cannot <op>: path '...' escapes the
   database directory ...\`). On an in-memory database these operations throw
   \`Cannot <op>: database is in-memory, file operations are unavailable\`.
+- **What the sandbox does not limit.** The sandbox controls which files a script can touch and
+  which standard libraries exist. It does not bound how much work a script does: there is no
+  instruction-count limit, no memory cap and no wall-clock timeout (\`while true do end\` runs
+  until the host stops it). Globals persist across \`run()\` calls on the same runner, so a global
+  one script sets is visible to the next; use \`local\`. A host that runs untrusted scripts has to
+  impose those limits outside the library.
 - **Output.** A script can \`return\` one value and the host receives it as JSON — prefer this over
   \`print()\` when you need structured data back (\`print()\` still works and is captured). Only the
   **first** returned value is encoded. Arrays are 1-indexed. Iterate with \`ipairs\` only where no
@@ -193,7 +202,7 @@ db:begin_transaction()   -- start an explicit transaction
 db:commit()              -- commit it
 db:rollback()            -- roll it back
 db:in_transaction()      -- boolean: is a transaction currently open?
-db:transaction(fn)       -- run fn(db) inside begin/commit; rollback + rethrow if fn errors
+db:transaction(fn)       -- run fn(db) inside begin/commit; rollback + rethrow if fn errors or the commit fails
 \`\`\`
 
 To make a group of writes atomic, prefer the \`db:transaction\` wrapper:
@@ -303,8 +312,13 @@ Notes:
   which only the new label resolves. Because the label form delegates to the id form, failures
   that validate the *element* (an empty table, a type mismatch) report
   \`Cannot update_element: ...\`.
-- **Empty arrays are skipped.** An attribute whose value is \`{}\` writes no vector/set (the element
-  type can't be inferred from an empty array), so it is silently dropped.
+- **An empty array clears on update.** On \`update_element\` / \`update_element_by_label\`,
+  \`{ col = {} }\` clears the whole group holding \`col\` (all its columns, and every group that
+  shares the column name). An empty column beside a non-empty column of the same group throws a
+  length error, and a misspelled empty column throws
+  \`array '<name>' does not match any vector, set, or time series table ...\`. \`create_element\`
+  skips an empty array. To leave a group alone, omit its column: writing back a
+  \`read_vectors_by_id\` result clears a group whose read column came back empty or all-NULL.
 - **Arrays must be dense.** A vector/set read returns a NULL cell as a \`nil\` hole, but an element
   array cannot carry one: \`create_element\` / \`update_element\` throw \`array '<name>' has a nil
   hole ...\` rather than cut the array short at the hole. Write NULL cells with
@@ -691,7 +705,7 @@ own) — it throws \`Cannot import_csv: transaction already active\`. Call it ou
 
 ## CSV file reading
 
-Read a CSV file from disk directly into Lua — the only way to get file data into a script, since
+Read a CSV file from disk directly into Lua, since
 \`io\` is deliberately absent from the sandbox. \`path\` is sandboxed the same way as every other
 file-touching operation (see Critical rules).
 
@@ -712,8 +726,8 @@ no header at all, so \`csv.header\` is absent (\`nil\`, not an empty table) and
 file's first line — useful for a file with a junk title row and/or a units row around the real
 header (skip them by naming the header row and slicing \`csv.rows\` in the script). A \`header_row\`
 past the end of the file throws. Passing the separator positionally (\`db:read_csv(path, ";")\`)
-throws \`Cannot read_csv: options must be a table\` instead of silently parsing with a comma; an
-unknown key, a separator that isn't a single character (or is a quote, CR, LF or NUL — none of
+throws \`Cannot read_csv: options must be a table, got string\` instead of silently parsing with
+a comma; an unknown key, a separator that isn't a single character (or is a quote, CR, LF or NUL — none of
 those can be a delimiter), a non-string option key, or a \`header_row\` that isn't a
 non-negative integer also throws.
 
@@ -777,6 +791,47 @@ end
 
 ---
 
+## XLSX file reading
+
+Read one worksheet of an XLSX workbook through the same filesystem sandbox as CSV. XLSX only:
+legacy XLS and encrypted workbooks are unsupported. These methods read cells; they do not write
+the workbook or import rows into the database.
+
+\`\`\`lua
+local data = db:read_xlsx("input.xlsx", { sheet = "Data", header_row = 1 })
+-- data.header: array of strings; data.rows: array of arrays of strings
+local count = db:read_xlsx_stream("input.xlsx", function(row, index, header)
+    -- header is available even on the first callback; index counts emitted data rows.
+    -- Only an explicit false stops. No return, nil, 0 and strings all continue.
+end, { sheet = 2 })
+\`\`\`
+
+The optional options table accepts exactly \`sheet\` and \`header_row\`. \`sheet\` is an exact
+worksheet name or a positive 1-based integer, default \`1\`. Chart sheets do not count; hidden
+worksheets do. Missing worksheets throw. \`header_row\` is the physical worksheet row number,
+default \`1\`; rows above it are skipped. A missing or completely blank header throws.
+\`header_row = 0\` reads all data rows with no header (\`data.header\` and the callback header are
+\`nil\`). Omit options, pass \`nil\`, or pass \`{}\` to use defaults; unknown keys and wrong
+types throw \`Cannot <operation>: ...\`.
+
+Every cell is a string. Text, including leading zeros, Unicode, whitespace and rich-text content,
+is preserved. Numbers keep their stored locale-independent text, booleans are \`"1"\`/\`"0"\`,
+blank cells are \`""\`, and Excel errors such as \`#DIV/0!\` stay text. Formatting is ignored:
+an Excel date stored as \`45292.5\` remains \`"45292.5"\`, and an explicit date stored as text
+stays text. Formulas return their saved cached value, never the expression or a recalculated value.
+A formula with no cache throws, naming the sheet and cell; recalculate and save it in Excel first.
+
+Completely blank data rows are skipped. Rows and headers are padded with \`""\` to the worksheet's
+last column containing a value or formula, so column positions stay aligned; formatting-only
+cells do not expand that width. Merged cells contribute only their stored values (normally the
+top-left cell); values are not copied across the merged area. Header-only sheets return no data
+rows; an empty sheet is readable with \`header_row = 0\`.
+
+Both forms use the same reader. The callback form returns the number of rows delivered, including
+the row that returned false; callback errors propagate unchanged and handles close on every exit.
+Unlike CSV streaming, XLSX callbacks avoid the full Lua rows table but still load worksheet XML
+and shared strings into memory. For very large files, use CSV streaming instead.
+
 ## CSV file writing
 
 Write a CSV file to disk — the only way to get data out of a script onto disk, since \`io\` is
@@ -870,9 +925,16 @@ File I/O is db-scoped (\`db:open_file\` / \`db:bin_to_csv\` / \`db:csv_to_bin\`)
 base paths, sandboxed to the database directory (see Critical rules), and \`get_file_path()\` returns
 the resolved absolute path. The pure-metadata builders and expression constructors live under the
 global \`quiver\` table. Mirrors the Julia surface; aggregation ops are strings (Lua has no enums);
-operators are \`+ - * /\` and unary \`-\`, with scalars allowed on either side. A file handle does not
-outlive the \`run()\` that opened it: any handle still open when the script returns is closed (and a
-writer flushed), so reopen the file in each script.
+operators are \`+ - * /\` and unary \`-\`, with scalars allowed on either side. A binary file **is** an
+expression: every operator, every \`quiver.*\` expression function and every expression method
+(\`aggregate\`, \`aggregate_agents\`, \`select_agents\`, \`rename_agents\`, \`save\`, \`get_metadata\`) takes
+a file handle directly; \`quiver.expression(f)\` stays as an explicit conversion, and saving from a
+file reads it by path and leaves the handle open. A wrong operand raises \`Cannot <op>: operand must
+be an expression or a binary file, got <type>\`, and extra arguments to a \`quiver.*\` expression
+function (or to an operator metamethod called directly) raise
+\`Cannot <op>: too many arguments (expected N, got M)\`; expression methods ignore extra arguments,
+like every other method. A file handle does not outlive the \`run()\` that opened it: any handle still open when the script returns
+is closed (and a writer flushed), so reopen the file in each script.
 
 \`\`\`lua
 local md = quiver.metadata{
@@ -892,7 +954,9 @@ quiver.metadata_from_toml(text); quiver.metadata_from_element(tbl)
 db:bin_to_csv(path)                                  -- aggregate=true by default; pass false to keep time dims as columns
 db:csv_to_bin(path)
 
-local e = (quiver.expression(r) + 10.0) * 2.0        -- files auto-wrap; scalars either side
+local e = (r + 10.0) * 2.0                           -- a file is an expression; scalars either side
+local per_stage = r:aggregate("stage", "sum")        -- every expression method works on a file too
+r:save(copy_path)                                    -- reads the file by path; r stays open
 e = quiver.abs(e); e = quiver.sqrt(e)                -- also quiver.log / quiver.exp
 local cond_e = quiver.gt(e, 3.0)                     -- also quiver.lt/quiver.gte/quiver.lte/quiver.eq/quiver.neq
                                                      -- all -> 1.0/0.0 per element (NaN operand -> NaN)
@@ -902,7 +966,7 @@ e = e:aggregate("stage", "sum")                      -- sum/mean/min/max/percent
 e = e:aggregate("stage", "percentile", 0.9)          -- percentile needs the fraction
 e = e:aggregate_agents("mean")                       -- collapse the label axis
 e = e:select_agents({"v2"}); e = e:rename_agents({v1 = "alpha"})
-e:save(out_path); e:metadata()                       -- save path is sandboxed like db:open_file
+e:save(out_path); e:get_metadata()                   -- save path is sandboxed like db:open_file
 \`\`\`
 
 **\`quiver.metadata{...}\` kwargs and defaults:** \`version\` defaults to \`"1"\`; \`initial_datetime\`

package/src/{lua-runner.ts → sandbox.ts}

@@ -4,19 +4,19 @@ import { allocPtrOut, decodeStringFromBuf, readPtrOut, toCString } from "./ffi-h
 import type { NativePointer } from "./loader.ts";
 import { getSymbols } from "./loader.ts";
 
-export class LuaRunner {
+export class Sandbox {
   private _ptr: NativePointer;
   private _closed = false;
 
   constructor(db: Database) {
     const lib = getSymbols();
     const outRunner = allocPtrOut();
-    check(lib.quiver_lua_runner_new(db._handle, outRunner.buf));
+    check(lib.quiver_sandbox_new(db._handle, outRunner.buf));
     this._ptr = readPtrOut(outRunner);
   }
 
   /**
-   * Runs a Lua script and returns its return value encoded as JSON, or "" if it returned nothing.
+   * Runs a script and returns its return value encoded as JSON, or "" if it returned nothing.
    *
    * To execute a script without keeping its writes, wrap the call in
    * `db.beginDryRun()` / `db.endDryRun()`.
@@ -26,22 +26,22 @@ export class LuaRunner {
     const lib = getSymbols();
     const scriptBuf = toCString(script);
     const outResult = allocPtrOut();
-    check(lib.quiver_lua_runner_run(this._ptr, scriptBuf.buf, outResult.buf));
+    check(lib.quiver_sandbox_run(this._ptr, scriptBuf.buf, outResult.buf));
     const result = decodeStringFromBuf(outResult);
-    lib.quiver_lua_runner_free_string(readPtrOut(outResult));
+    lib.quiver_sandbox_free_string(readPtrOut(outResult));
     return result;
   }
 
   close(): void {
     if (this._closed) return;
     const lib = getSymbols();
-    check(lib.quiver_lua_runner_free(this._ptr));
+    check(lib.quiver_sandbox_free(this._ptr));
     this._closed = true;
   }
 
   private ensureOpen(): void {
     if (this._closed) {
-      throw new QuiverError("LuaRunner is closed");
+      throw new QuiverError("Sandbox is closed");
     }
   }
 }

package/src/time-series.ts

@@ -149,11 +149,11 @@ Database.prototype.updateTimeSeriesGroupByLabel = function (
  * (quiver_database_upsert_time_series_row and its _by_label form). `key` is an id for the by-id
  * form, a NUL-terminated label otherwise.
  */
-type UpsertRowFn = (
+type UpsertRowFn<Key extends number | string> = (
   db: NativePointer,
   collection: Uint8Array,
   group: Uint8Array,
-  key: bigint | Uint8Array,
+  key: Key extends string ? Uint8Array : bigint,
   names: Uint8Array | null,
   types: Uint8Array | null,
   data: Uint8Array | null,
@@ -165,18 +165,20 @@ type UpsertRowFn = (
  * C functions. Shared by upsertTimeSeriesRow / upsertTimeSeriesRowByLabel: they differ only in
  * which C entry point they call and whether `key` is an id or a label.
  */
-function upsertRowColumns(
+function upsertRowColumns<Key extends number | string>(
   handle: NativePointer,
   caller: string,
-  upsert: UpsertRowFn,
+  upsert: UpsertRowFn<Key>,
   collection: string,
   group: string,
-  key: number | string,
+  key: Key,
   row: Record<string, number | bigint | string | boolean>,
 ): void {
   const collBuf = toCString(collection);
   const grpBuf = toCString(group);
-  const keyArg = typeof key === "string" ? toCString(key).buf : BigInt(key);
+  const keyArg = (typeof key === "string" ? toCString(key).buf : BigInt(key)) as Key extends string
+    ? Uint8Array
+    : bigint;
   const entries = Object.entries(row);
   const columnCount = entries.length;
   const keepalive: Allocation[] = [];