quiverdb 0.12.8 → 0.13.0

package/README.md

@@ -62,8 +62,9 @@ bun run example.ts
 
 ### Lifecycle
 
-- `Database.fromSchema(dbPath, schemaPath)` -- Create database from SQL schema file
-- `Database.fromMigrations(dbPath, migrationsPath)` -- Create database from migrations directory
+- `Database.fromSchema(dbPath, schemaPath, options?)` -- Create database from SQL schema file
+- `Database.fromMigrations(dbPath, migrationsPath, options?)` -- Create database from migrations directory
+- `Database.open(dbPath, options?)` -- Open an existing database file; the schema loads on first use
 - `Database.validateMigrations(migrationsPath)` -- Validate a migrations directory (every `up.sql`, then every `down.sql`, ending with no table left behind) in-memory; throws on failure
 - `close()` -- Close the database connection
 
@@ -74,6 +75,16 @@ bun run example.ts
 - `deleteElement(collection, id)` -- Delete element by ID
 - `updateElementByLabel(collection, label, data)` -- Update element by label
 - `deleteElementByLabel(collection, label)` -- Delete element by label
+- `updateRelation(collectionFrom, collectionTo, relationType, id, targetLabel)` -- Point a scalar
+  relation at the element labelled `targetLabel` (`null` clears it)
+- `updateRelationByLabel(collectionFrom, collectionTo, relationType, label, targetLabel)` -- Same,
+  addressing the element by label
+- `updateVectorGroup(collection, group, id, data)` -- Replace an element's rows in one named vector
+  group (`{}` clears it)
+- `updateVectorGroupByLabel(collection, group, label, data)` -- Same, addressing the element by label
+- `updateSetGroup(collection, group, id, data)` -- Replace an element's rows in one named set group
+  (`{}` clears it)
+- `updateSetGroupByLabel(collection, group, label, data)` -- Same, addressing the element by label
 
 ### Read (bulk)
 
@@ -127,6 +138,18 @@ bun run example.ts
 
 - `readTimeSeriesGroup(collection, group, id)` -- Read time series data for an element
 - `updateTimeSeriesGroup(collection, group, id, data)` -- Write time series data for an element
+  (`{}` clears it)
+- `updateTimeSeriesGroupByLabel(collection, group, label, data)` -- Same, addressing the element by
+  label
+- `readTimeSeriesRow(collection, group, attribute, dateTime)` -- One value per element: the last
+  non-null value at or before `dateTime`, or `null` if there is none
+- `upsertTimeSeriesRow(collection, group, id, row)` -- Insert or replace one row, keyed by its
+  dimension value
+- `upsertTimeSeriesRowByLabel(collection, group, label, row)` -- Same, addressing the element by label
+- `hasTimeSeriesFiles(collection)` -- Check whether the collection has a time series files table
+- `listTimeSeriesFilesColumns(collection)` -- List the time series files columns
+- `readTimeSeriesFiles(collection)` -- Read the file paths (`null` for an unset column)
+- `updateTimeSeriesFiles(collection, data)` -- Write the file paths
 
 ### Query
 
@@ -162,27 +185,31 @@ as an exact INTEGER, a `boolean` as the INTEGER 1 or 0).
 
 ### Introspection
 
-- `describe()` -- Print schema info to stdout
+- `describe()` -- Whole-database text report (returns a string)
+- `describeCollection(collection)` -- One collection's structure (returns a string)
+- `summarizeCollection(collection)` -- Per-attribute null/value summary (returns a string)
 - `isHealthy()` -- Check database health
 - `path()` -- Get database file path
 - `currentVersion()` -- Get current schema version
 
 ### Lua
 
-- `LuaRunner(db)` -- Create Lua script runner with database access
+- `Sandbox(db)` -- Create Lua script runner with database access
 - `run(script)` -- Execute a Lua script; returns its return value as a JSON string, or `""` if it
   returned nothing
-- `close()` -- Close the Lua runner
+- `close()` -- Close the Sandbox
 
 ## Types
 
 Exported types available for TypeScript consumers:
 
-- `ScalarValue` -- `number | bigint | string | null`
-- `ArrayValue` -- `number[] | bigint[] | string[]`
+- `ScalarValue` -- `number | bigint | boolean | string | null` (a `boolean` is stored as INTEGER 1/0)
+- `ArrayValue` -- `number[] | bigint[] | boolean[] | string[]`
 - `Value` -- `ScalarValue | ArrayValue`
 - `ElementData` -- `Record<string, Value | undefined>`
 - `QueryParam` -- `number | bigint | boolean | string | null`
+- `GroupColumns` -- `Record<string, (number | bigint | string | boolean | null)[]>`, the column-oriented
+  payload of `updateTimeSeriesGroup` / `updateVectorGroup` / `updateSetGroup` (and their `ByLabel` forms)
 - `QuiverError` -- Error class for all Quiver operations
 - `ScalarMetadata` -- Scalar attribute metadata
 - `GroupMetadata` -- Vector/set/time series group metadata

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "quiverdb",
-  "version": "0.12.8",
+  "version": "0.13.0",
   "license": "MIT",
   "repository": {
     "type": "git",

package/src/index.ts

@@ -12,8 +12,8 @@ export type { CsvOptions } from "./csv.ts";
 export { Database } from "./database.ts";
 export { QuiverError } from "./errors.ts";
 export type { GroupColumns, TimeSeriesData } from "./group-columns.ts";
-export { LUA_DB_API_REFERENCE } from "./lua-api.ts";
-export { LuaRunner } from "./lua-runner.ts";
+export { SANDBOX_API_REFERENCE } from "./sandbox-api.ts";
+export { Sandbox } from "./sandbox.ts";
 export type { GroupMetadata, ScalarMetadata } from "./metadata.ts";
 export type {
   ArrayValue,

package/src/loader.ts

@@ -196,11 +196,11 @@ const freeSymbols = {
   quiver_database_free_group_metadata_array: { args: [P, USIZE], returns: I32 },
 } as const;
 
-const luaSymbols = {
-  quiver_lua_runner_new: { args: [P, P], returns: I32 },
-  quiver_lua_runner_free: { args: [P], returns: I32 },
-  quiver_lua_runner_run: { args: [P, BUF, BUF], returns: I32 },
-  quiver_lua_runner_free_string: { args: [P], returns: I32 },
+const sandboxSymbols = {
+  quiver_sandbox_new: { args: [P, P], returns: I32 },
+  quiver_sandbox_free: { args: [P], returns: I32 },
+  quiver_sandbox_run: { args: [P, BUF, BUF], returns: I32 },
+  quiver_sandbox_free_string: { args: [P], returns: I32 },
 } as const;
 
 // Combined symbol map for dlopen.
@@ -216,7 +216,7 @@ const allSymbols = {
   ...timeSeriesSymbols,
   ...csvSymbols,
   ...freeSymbols,
-  ...luaSymbols,
+  ...sandboxSymbols,
 } as const;
 
 // ---------------------------------------------------------------------------

package/src/{lua-api.ts → sandbox-api.ts}

@@ -1,33 +1,33 @@
 // Agent-facing reference for the Lua `db` API available inside run_lua scripts.
 //
-// Authority: `src/lua_runner.cpp` `bind_database()` (this repo) — extracted by hand, NOT imported.
+// Authority: the binders under `src/sandbox/` (`bind_database` through `bind_expression`,
+// this repo) — extracted by hand, NOT imported.
 // The shipped quiverdb native binding is the runtime truth; this is docs.
 //
-// SYNC: `test/lua-api-sync.test.ts` derives the bound surface from `src/lua_runner.cpp` and checks
+// SYNC: `test/sandbox-api-sync.test.ts` derives the bound surface from every `.cpp`/`.h` under
+// `src/sandbox/` and checks
 // it AUTOMATICALLY — every `db:`/`quiver.*` name is documented, no documented name has been
 // removed, and the stdlib sentence matches `open_libraries` exactly. What it CANNOT check, and you
 // must still re-diff by hand when the binding changes: arg order, arity, arg types, return shapes,
 // and whether the prose is semantically true.
 //
 // NOTE: the binary/expression subsystems are bound in the native binding and documented below.
-// File-touching operations (db:open_file, db:bin_to_csv, db:csv_to_bin, db:validate_migrations,
-// expr:save) are sandboxed to the database file's directory; the pure-metadata builders stay under
-// the quiver.* global.
+// File-touching operations are sandboxed to the database file's directory (the "Filesystem
+// sandbox" bullet lists all of them); the pure-metadata builders stay under the quiver.* global.
 //
 // FORMAT CONVENTION: every db: method appears at least once as the literal token
 // `db:<snake_case_name>`, and every quiver.* function as `quiver.<name>`, so coverage is greppable
 // (the sync test relies on this).
-export const LUA_DB_API_REFERENCE = `
-# Quiver Lua API Reference
+export const SANDBOX_API_REFERENCE = `
+# Quiver Sandbox API Reference
 
-Quiver embeds a Lua scripting layer (via \`LuaRunner\`) that exposes the same database API as the
-other language bindings. This document is a complete reference of every method available to a Lua
-script.
+Quiver embeds a Lua sandbox scripting layer that exposes the same database API as the other language
+bindings. This document is a complete reference of every method available to a Lua script.
 
 ## Running scripts
 
 The database is provided to your script as a global userdata named \`db\`. There are **no
-open/close lifecycle methods in Lua** — the \`LuaRunner\` opens the database and hands you \`db\`
+open/close lifecycle methods in Lua** — the \`Sandbox\` opens the database and hands you \`db\`
 already connected. All methods are called with the colon syntax:
 
 \`\`\`lua
@@ -100,18 +100,25 @@ midnight.
   and do not \`pcall\` inside it: a caught error lets the block commit whatever ran.
 - **Standard library.** Loaded standard libraries: base, string, table, math, coroutine, utf8.
   That is the pure-computation set — there is no \`os\`, \`io\`, \`debug\`, or \`package\`/\`require\`,
-  and \`dofile\`/\`loadfile\` are removed (string-form \`load\` stays available). Integer division is
-  the Lua 5.4 \`//\` operator — a language operator, unrelated to \`math\`. No \`io\` does **not** mean
+  and \`dofile\`/\`loadfile\` are removed (string-form \`load\` stays for source text; a precompiled
+  binary chunk is refused). Integer division is the Lua 5.4 \`//\` operator — a language operator,
+  unrelated to \`math\`. No \`io\` does **not** mean
   a data file on disk is out of reach: read it with \`db:read_csv\` / \`db:read_csv_stream\` (see
   the CSV file reading section below). Never copy, paste, or re-type a data file's contents into
   the script as literals — read the file.
 - **Filesystem sandbox.** Every file-touching operation (\`db:export_csv\`, \`db:import_csv\`,
   \`db:open_file\`, \`db:bin_to_csv\`, \`db:csv_to_bin\`, \`db:validate_migrations\`, \`db:read_csv\`,
-  \`db:read_csv_stream\`, \`db:write_csv\`, \`expr:save\`) resolves
+  \`db:read_csv_stream\`, \`db:write_csv\`, \`save\` on a file or an expression) resolves
   relative paths against the directory containing the database file and rejects anything outside it
   (subdirectories are fine; \`..\` escapes and outside absolute paths throw \`Cannot <op>: path '...' escapes the
   database directory ...\`). On an in-memory database these operations throw
   \`Cannot <op>: database is in-memory, file operations are unavailable\`.
+- **What the sandbox does not limit.** The sandbox controls which files a script can touch and
+  which standard libraries exist. It does not bound how much work a script does: there is no
+  instruction-count limit, no memory cap and no wall-clock timeout (\`while true do end\` runs
+  until the host stops it). Globals persist across \`run()\` calls on the same runner, so a global
+  one script sets is visible to the next; use \`local\`. A host that runs untrusted scripts has to
+  impose those limits outside the library.
 - **Output.** A script can \`return\` one value and the host receives it as JSON — prefer this over
   \`print()\` when you need structured data back (\`print()\` still works and is captured). Only the
   **first** returned value is encoded. Arrays are 1-indexed. Iterate with \`ipairs\` only where no
@@ -193,7 +200,7 @@ db:begin_transaction()   -- start an explicit transaction
 db:commit()              -- commit it
 db:rollback()            -- roll it back
 db:in_transaction()      -- boolean: is a transaction currently open?
-db:transaction(fn)       -- run fn(db) inside begin/commit; rollback + rethrow if fn errors
+db:transaction(fn)       -- run fn(db) inside begin/commit; rollback + rethrow if fn errors or the commit fails
 \`\`\`
 
 To make a group of writes atomic, prefer the \`db:transaction\` wrapper:
@@ -303,8 +310,13 @@ Notes:
   which only the new label resolves. Because the label form delegates to the id form, failures
   that validate the *element* (an empty table, a type mismatch) report
   \`Cannot update_element: ...\`.
-- **Empty arrays are skipped.** An attribute whose value is \`{}\` writes no vector/set (the element
-  type can't be inferred from an empty array), so it is silently dropped.
+- **An empty array clears on update.** On \`update_element\` / \`update_element_by_label\`,
+  \`{ col = {} }\` clears the whole group holding \`col\` (all its columns, and every group that
+  shares the column name). An empty column beside a non-empty column of the same group throws a
+  length error, and a misspelled empty column throws
+  \`array '<name>' does not match any vector, set, or time series table ...\`. \`create_element\`
+  skips an empty array. To leave a group alone, omit its column: writing back a
+  \`read_vectors_by_id\` result clears a group whose read column came back empty or all-NULL.
 - **Arrays must be dense.** A vector/set read returns a NULL cell as a \`nil\` hole, but an element
   array cannot carry one: \`create_element\` / \`update_element\` throw \`array '<name>' has a nil
   hole ...\` rather than cut the array short at the hole. Write NULL cells with
@@ -712,8 +724,8 @@ no header at all, so \`csv.header\` is absent (\`nil\`, not an empty table) and
 file's first line — useful for a file with a junk title row and/or a units row around the real
 header (skip them by naming the header row and slicing \`csv.rows\` in the script). A \`header_row\`
 past the end of the file throws. Passing the separator positionally (\`db:read_csv(path, ";")\`)
-throws \`Cannot read_csv: options must be a table\` instead of silently parsing with a comma; an
-unknown key, a separator that isn't a single character (or is a quote, CR, LF or NUL — none of
+throws \`Cannot read_csv: options must be a table, got string\` instead of silently parsing with
+a comma; an unknown key, a separator that isn't a single character (or is a quote, CR, LF or NUL — none of
 those can be a delimiter), a non-string option key, or a \`header_row\` that isn't a
 non-negative integer also throws.
 
@@ -870,9 +882,16 @@ File I/O is db-scoped (\`db:open_file\` / \`db:bin_to_csv\` / \`db:csv_to_bin\`)
 base paths, sandboxed to the database directory (see Critical rules), and \`get_file_path()\` returns
 the resolved absolute path. The pure-metadata builders and expression constructors live under the
 global \`quiver\` table. Mirrors the Julia surface; aggregation ops are strings (Lua has no enums);
-operators are \`+ - * /\` and unary \`-\`, with scalars allowed on either side. A file handle does not
-outlive the \`run()\` that opened it: any handle still open when the script returns is closed (and a
-writer flushed), so reopen the file in each script.
+operators are \`+ - * /\` and unary \`-\`, with scalars allowed on either side. A binary file **is** an
+expression: every operator, every \`quiver.*\` expression function and every expression method
+(\`aggregate\`, \`aggregate_agents\`, \`select_agents\`, \`rename_agents\`, \`save\`, \`get_metadata\`) takes
+a file handle directly; \`quiver.expression(f)\` stays as an explicit conversion, and saving from a
+file reads it by path and leaves the handle open. A wrong operand raises \`Cannot <op>: operand must
+be an expression or a binary file, got <type>\`, and extra arguments to a \`quiver.*\` expression
+function (or to an operator metamethod called directly) raise
+\`Cannot <op>: too many arguments (expected N, got M)\`; expression methods ignore extra arguments,
+like every other method. A file handle does not outlive the \`run()\` that opened it: any handle still open when the script returns
+is closed (and a writer flushed), so reopen the file in each script.
 
 \`\`\`lua
 local md = quiver.metadata{
@@ -892,7 +911,9 @@ quiver.metadata_from_toml(text); quiver.metadata_from_element(tbl)
 db:bin_to_csv(path)                                  -- aggregate=true by default; pass false to keep time dims as columns
 db:csv_to_bin(path)
 
-local e = (quiver.expression(r) + 10.0) * 2.0        -- files auto-wrap; scalars either side
+local e = (r + 10.0) * 2.0                           -- a file is an expression; scalars either side
+local per_stage = r:aggregate("stage", "sum")        -- every expression method works on a file too
+r:save(copy_path)                                    -- reads the file by path; r stays open
 e = quiver.abs(e); e = quiver.sqrt(e)                -- also quiver.log / quiver.exp
 local cond_e = quiver.gt(e, 3.0)                     -- also quiver.lt/quiver.gte/quiver.lte/quiver.eq/quiver.neq
                                                      -- all -> 1.0/0.0 per element (NaN operand -> NaN)
@@ -902,7 +923,7 @@ e = e:aggregate("stage", "sum")                      -- sum/mean/min/max/percent
 e = e:aggregate("stage", "percentile", 0.9)          -- percentile needs the fraction
 e = e:aggregate_agents("mean")                       -- collapse the label axis
 e = e:select_agents({"v2"}); e = e:rename_agents({v1 = "alpha"})
-e:save(out_path); e:metadata()                       -- save path is sandboxed like db:open_file
+e:save(out_path); e:get_metadata()                   -- save path is sandboxed like db:open_file
 \`\`\`
 
 **\`quiver.metadata{...}\` kwargs and defaults:** \`version\` defaults to \`"1"\`; \`initial_datetime\`

package/src/{lua-runner.ts → sandbox.ts}

@@ -4,19 +4,19 @@ import { allocPtrOut, decodeStringFromBuf, readPtrOut, toCString } from "./ffi-h
 import type { NativePointer } from "./loader.ts";
 import { getSymbols } from "./loader.ts";
 
-export class LuaRunner {
+export class Sandbox {
   private _ptr: NativePointer;
   private _closed = false;
 
   constructor(db: Database) {
     const lib = getSymbols();
     const outRunner = allocPtrOut();
-    check(lib.quiver_lua_runner_new(db._handle, outRunner.buf));
+    check(lib.quiver_sandbox_new(db._handle, outRunner.buf));
     this._ptr = readPtrOut(outRunner);
   }
 
   /**
-   * Runs a Lua script and returns its return value encoded as JSON, or "" if it returned nothing.
+   * Runs a script and returns its return value encoded as JSON, or "" if it returned nothing.
    *
    * To execute a script without keeping its writes, wrap the call in
    * `db.beginDryRun()` / `db.endDryRun()`.
@@ -26,22 +26,22 @@ export class LuaRunner {
     const lib = getSymbols();
     const scriptBuf = toCString(script);
     const outResult = allocPtrOut();
-    check(lib.quiver_lua_runner_run(this._ptr, scriptBuf.buf, outResult.buf));
+    check(lib.quiver_sandbox_run(this._ptr, scriptBuf.buf, outResult.buf));
     const result = decodeStringFromBuf(outResult);
-    lib.quiver_lua_runner_free_string(readPtrOut(outResult));
+    lib.quiver_sandbox_free_string(readPtrOut(outResult));
     return result;
   }
 
   close(): void {
     if (this._closed) return;
     const lib = getSymbols();
-    check(lib.quiver_lua_runner_free(this._ptr));
+    check(lib.quiver_sandbox_free(this._ptr));
     this._closed = true;
   }
 
   private ensureOpen(): void {
     if (this._closed) {
-      throw new QuiverError("LuaRunner is closed");
+      throw new QuiverError("Sandbox is closed");
     }
   }
 }