quiverdb 0.9.6 → 0.9.8

package/mod.ts

@@ -22,4 +22,4 @@ export type {
   TimeSeriesData,
   Value,
 } from "./src/index.ts";
-export { Database, LuaRunner, QuiverError } from "./src/index.ts";
+export { Database, LUA_DB_API_REFERENCE, LuaRunner, QuiverError } from "./src/index.ts";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "quiverdb",
-  "version": "0.9.6",
+  "version": "0.9.8",
   "license": "MIT",
   "repository": {
     "type": "git",

package/src/database.ts

@@ -180,7 +180,9 @@ export class Database {
   declare isHealthy: () => boolean;
   declare currentVersion: () => number;
   declare path: () => string;
-  declare describe: () => void;
+  declare describe: () => string;
+  declare describeCollection: (collection: string) => string;
+  declare summarizeCollection: (collection: string) => string;
 
   // --- Composite helpers (implemented in composites.ts) ---
   declare readScalarsById: (

package/src/ffi-helpers.ts

@@ -141,14 +141,17 @@ export function allocNativeString(str: string): Allocation {
   return { ptr: ptr(buf), buf };
 }
 
-/** Build a native string pointer table (const char* const*) from an array of strings. */
-export function allocNativeStringArray(strings: string[]): {
+/**
+ * Build a native string pointer table (const char* const*) from an array of strings.
+ * A `null` entry becomes a NULL pointer in the table (used for SQL-NULL string cells).
+ */
+export function allocNativeStringArray(strings: (string | null)[]): {
   table: Allocation;
   keepalive: Allocation[];
 } {
-  const strAllocs = strings.map((s) => allocNativeString(s));
-  const table = allocNativePtrTable(strAllocs.map((a) => a.ptr));
-  return { table, keepalive: strAllocs };
+  const strAllocs = strings.map((s) => (s === null ? null : allocNativeString(s)));
+  const table = allocNativePtrTable(strAllocs.map((a) => (a ? a.ptr : null)));
+  return { table, keepalive: strAllocs.filter((a): a is Allocation => a !== null) };
 }
 
 /** Get the native address of a pointer as BigInt. */

package/src/index.ts

@@ -11,16 +11,10 @@ import "./composites.ts";
 export type { CsvOptions } from "./csv.ts";
 export { Database } from "./database.ts";
 export { QuiverError } from "./errors.ts";
+export { LUA_DB_API_REFERENCE } from "./lua-api.ts";
 export { LuaRunner } from "./lua-runner.ts";
 export type { GroupMetadata, ScalarMetadata } from "./metadata.ts";
 export type { TimeSeriesData } from "./time-series.ts";
-export {
-  LOG_LEVEL_DEBUG,
-  LOG_LEVEL_ERROR,
-  LOG_LEVEL_INFO,
-  LOG_LEVEL_OFF,
-  LOG_LEVEL_WARN,
-} from "./types.ts";
 export type {
   ArrayValue,
   DatabaseOptions,
@@ -29,3 +23,10 @@ export type {
   ScalarValue,
   Value,
 } from "./types.ts";
+export {
+  LOG_LEVEL_DEBUG,
+  LOG_LEVEL_ERROR,
+  LOG_LEVEL_INFO,
+  LOG_LEVEL_OFF,
+  LOG_LEVEL_WARN,
+} from "./types.ts";

package/src/introspection.ts

@@ -1,6 +1,6 @@
 import { Database } from "./database.ts";
 import { check } from "./errors.ts";
-import { allocPtrOut, decodeStringFromBuf } from "./ffi-helpers.ts";
+import { allocPtrOut, decodeStringFromBuf, readPtrOut, toCString } from "./ffi-helpers.ts";
 import { getSymbols } from "./loader.ts";
 
 Database.prototype.isHealthy = function (this: Database): boolean {
@@ -24,7 +24,31 @@ Database.prototype.path = function (this: Database): string {
   return decodeStringFromBuf(outPath);
 };
 
-Database.prototype.describe = function (this: Database): void {
+Database.prototype.describe = function (this: Database): string {
   const lib = getSymbols();
-  check(lib.quiver_database_describe(this._handle));
+  const out = allocPtrOut();
+  check(lib.quiver_database_describe(this._handle, out.buf));
+  const result = decodeStringFromBuf(out);
+  lib.quiver_database_free_string(readPtrOut(out));
+  return result;
+};
+
+Database.prototype.describeCollection = function (this: Database, collection: string): string {
+  const lib = getSymbols();
+  const collBuf = toCString(collection);
+  const out = allocPtrOut();
+  check(lib.quiver_database_describe_collection(this._handle, collBuf.buf, out.buf));
+  const result = decodeStringFromBuf(out);
+  lib.quiver_database_free_string(readPtrOut(out));
+  return result;
+};
+
+Database.prototype.summarizeCollection = function (this: Database, collection: string): string {
+  const lib = getSymbols();
+  const collBuf = toCString(collection);
+  const out = allocPtrOut();
+  check(lib.quiver_database_summarize_collection(this._handle, collBuf.buf, out.buf));
+  const result = decodeStringFromBuf(out);
+  lib.quiver_database_free_string(readPtrOut(out));
+  return result;
 };

package/src/loader.ts

@@ -40,7 +40,7 @@ const lifecycleSymbols = {
   quiver_database_is_healthy: { args: [P, P], returns: I32 },
   quiver_database_path: { args: [P, P], returns: I32 },
   quiver_database_current_version: { args: [P, P], returns: I32 },
-  quiver_database_describe: { args: [P], returns: I32 },
+  quiver_database_describe: { args: [P, P], returns: I32 },
 } as const;
 
 const elementSymbols = {
@@ -112,15 +112,23 @@ const metadataSymbols = {
   quiver_database_list_time_series_groups: { args: [P, BUF, P, P], returns: I32 },
 } as const;
 
+const describeSymbols = {
+  quiver_database_describe_collection: { args: [P, BUF, P], returns: I32 },
+  quiver_database_summarize_collection: { args: [P, BUF, P], returns: I32 },
+} as const;
+
 const timeSeriesSymbols = {
-  quiver_database_read_time_series_group: { args: [P, BUF, BUF, I64, P, P, P, P, P], returns: I32 },
+  quiver_database_read_time_series_group: {
+    args: [P, BUF, BUF, I64, P, P, P, P, P, P],
+    returns: I32,
+  },
   quiver_database_read_time_series_row: { args: [P, BUF, BUF, BUF, BUF, P, P, P], returns: I32 },
   quiver_database_add_time_series_row: { args: [P, BUF, BUF, I64, P, P, P, USIZE], returns: I32 },
   quiver_database_update_time_series_group: {
-    args: [P, BUF, BUF, I64, P, P, P, USIZE, USIZE],
+    args: [P, BUF, BUF, I64, P, P, P, P, USIZE, USIZE],
     returns: I32,
   },
-  quiver_database_free_time_series_data: { args: [P, P, P, USIZE, USIZE], returns: I32 },
+  quiver_database_free_time_series_data: { args: [P, P, P, P, USIZE, USIZE], returns: I32 },
   quiver_database_has_time_series_files: { args: [P, BUF, P], returns: I32 },
   quiver_database_list_time_series_files_columns: { args: [P, BUF, P, P], returns: I32 },
   quiver_database_read_time_series_files: { args: [P, BUF, P, P, P], returns: I32 },
@@ -160,6 +168,7 @@ const allSymbols = {
   ...querySymbols,
   ...transactionSymbols,
   ...metadataSymbols,
+  ...describeSymbols,
   ...timeSeriesSymbols,
   ...csvSymbols,
   ...freeSymbols,

package/src/lua-api.ts

@@ -0,0 +1,512 @@
+// Agent-facing reference for the Lua `db` API available inside run_lua scripts.
+//
+// Authority: `src/lua_runner.cpp` `bind_database()` (this repo) — extracted by hand, NOT imported.
+// The shipped quiverdb native binding is the runtime truth; this is docs. Whenever the C++ Lua
+// binding changes, re-diff every signature below against `bind_database()` (names, arg order, arg
+// types, return shapes).
+//
+// NOTE: the binary/expression subsystems are bound in the native binding and documented below.
+// File-touching operations (db:open_file, db:bin_to_csv, db:csv_to_bin, expr:save) are sandboxed
+// to the database file's directory; the pure-metadata builders stay under the quiver.* global.
+//
+// FORMAT CONVENTION: every db: method should appear at least once as the literal token
+// `db:<snake_case_name>` so coverage is greppable.
+export const LUA_DB_API_REFERENCE = `
+# Quiver Lua API Reference
+
+Quiver embeds a Lua scripting layer (via \`LuaRunner\`) that exposes the same database API as the
+other language bindings. This document is a complete reference of every method available to a Lua
+script.
+
+## Running scripts
+
+The database is provided to your script as a global userdata named \`db\`. There are **no
+open/close lifecycle methods in Lua** — the \`LuaRunner\` opens the database and hands you \`db\`
+already connected. All methods are called with the colon syntax:
+
+\`\`\`lua
+db:create_element("Collection", { label = "Item 1", value = 42 })
+local ids = db:read_element_ids("Collection")
+\`\`\`
+
+If a script raises an error (including any error thrown by a \`db:\` call), it surfaces to the host
+as:
+
+\`\`\`
+Failed to run Lua script: <message>
+\`\`\`
+
+## Value type mapping
+
+Lua values map to Quiver column values as follows:
+
+| Lua value          | Quiver value | Notes                                          |
+| ------------------ | ------------ | ---------------------------------------------- |
+| integer            | INTEGER      | Lua integers map to int64.                     |
+| number (float)     | REAL         | REAL columns reject bare integers: write 75.0, not 75. |
+| string             | TEXT         | Also used for \`date_time\` columns (ISO 8601).  |
+| \`nil\`              | NULL         | In query params, file paths, and ts rows.      |
+| table (1-indexed)  | array        | Used for vectors/sets and column-oriented data.|
+
+**Unsupported types throw.** Passing a boolean, a function, or a nested table where a scalar is
+expected raises an error rather than silently dropping the value. This applies to element
+attributes, time-series rows, and query parameters. A skipped positional query parameter would
+shift every later parameter and bind NULL to the trailing placeholder, so this is rejected loudly.
+
+Dates are plain strings in ISO 8601 format: \`YYYY-MM-DDTHH:MM:SS\`. (Lua keeps a string-based
+datetime surface — there are no DateTime wrapper helpers, unlike Julia/Dart/Python.)
+
+---
+
+## Critical rules
+
+- **STRICT typing.** A REAL column rejects integer literals — write \`75.0\`, not \`75\`; an INTEGER
+  column rejects floats. A mismatch raises a validation error and rolls the whole script back.
+- **Errors abort the script.** Any error thrown by a \`db:\` call stops the script and surfaces as
+  \`Failed to run Lua script: <message>\`. Validation failures roll back whatever the current
+  transaction covered.
+- **Standard library.** Only the \`base\`, \`string\`, and \`table\` libraries are loaded — there is NO
+  \`math\` (use \`//\` for integer division), and no \`os\`, \`io\`, or \`require\`; \`dofile\` and
+  \`loadfile\` are removed (string-form \`load\` is available).
+- **Filesystem sandbox.** Every file-touching operation (\`db:export_csv\`, \`db:import_csv\`,
+  \`db:open_file\`, \`db:bin_to_csv\`, \`db:csv_to_bin\`, \`expr:save\`) resolves relative paths against
+  the directory containing the database file and rejects anything outside it (subdirectories are
+  fine; \`..\` escapes and outside absolute paths throw \`Cannot <op>: path '...' escapes the
+  database directory ...\`). On an in-memory database these operations throw
+  \`Cannot <op>: database is in-memory, file operations are unavailable\`.
+- **Output.** Scripts cannot return values to the tool — use \`print()\` for anything you need to
+  see (it is captured). Arrays are 1-indexed (iterate with \`ipairs\`); reading a NULL yields \`nil\`,
+  writing \`nil\` stores NULL where NULL is accepted (query params, ts rows, file columns — but NOT
+  element scalar attributes; see CRUD).
+
+---
+
+## Database info
+
+\`\`\`lua
+db:is_healthy()                    -- boolean
+db:current_version()               -- integer (current migration version)
+db:path()                          -- string (database file path)
+db:describe()                      -- string: whole-DB text report (returns it, does NOT print)
+db:describe_collection(collection) -- string: one collection's structure (text report)
+db:summarize_collection(collection)-- string: per-scalar null/non-null counts, low-cardinality
+                                   --         integer value distributions, per-group sizes
+\`\`\`
+
+All three \`describe*\`/\`summarize*\` methods **return** a string — \`print()\` it to see it.
+
+---
+
+## Transactions
+
+\`\`\`lua
+db:begin_transaction()   -- start an explicit transaction
+db:commit()              -- commit it
+db:rollback()            -- roll it back
+db:in_transaction()      -- boolean: is a transaction currently open?
+db:transaction(fn)       -- run fn(db) inside begin/commit; rollback + rethrow if fn errors
+\`\`\`
+
+To make a group of writes atomic, prefer the \`db:transaction\` wrapper:
+
+\`\`\`lua
+db:transaction(function(db)
+    local id = db:create_element("Collection", { label = "Item 1", some_integer = 42 })
+    db:update_element("Collection", id, { some_integer = 99 })
+end)   -- both writes commit together; if either throws, both roll back
+\`\`\`
+
+**Caveat:** if the host already runs your script inside a transaction, an explicit
+\`db:begin_transaction()\` will error (\`cannot start a transaction within a transaction\`) and a
+mid-script \`db:commit()\` would prematurely end the host's transaction. When unsure whether a
+transaction is already open, check \`db:in_transaction()\` first, or just issue writes directly —
+each \`db:\` write is durable on its own.
+
+---
+
+## CRUD
+
+\`\`\`lua
+local id = db:create_element(collection, element_table)   -- returns new integer id
+db:update_element(collection, id, element_table)
+db:delete_element(collection, id)
+\`\`\`
+
+The element table holds scalar attributes as \`key = value\`, and vector/set attributes as
+1-indexed arrays. The element type of an array is inferred from its **first** entry:
+
+\`\`\`lua
+local id = db:create_element("Collection", {
+    label        = "Item 1",   -- scalar string
+    some_integer = 42,         -- scalar integer
+    some_float   = 3.14,       -- scalar real
+    value_int    = { 1, 2, 3 },          -- vector/set of integers
+    tags         = { "a", "b", "c" },    -- vector/set of strings
+})
+\`\`\`
+
+\`update_element\` only touches the attributes you pass:
+
+\`\`\`lua
+db:update_element("Collection", id, { some_integer = 999 })
+\`\`\`
+
+Notes:
+- **Empty arrays are skipped.** An attribute whose value is \`{}\` writes no vector/set (the element
+  type can't be inferred from an empty array), so it is silently dropped.
+- **No \`nil\` scalar attributes.** Passing \`nil\` for a scalar attribute **throws** (unsupported
+  type). To leave a column unset, omit the key — you cannot set a scalar to NULL via the element
+  table. (\`nil\` → NULL is only accepted by \`add_time_series_row\` and \`update_time_series_files\`.)
+
+---
+
+## Scalar reads (bulk, across all elements)
+
+Each returns a flat array (1-indexed table), one value per element, in id order.
+
+\`\`\`lua
+db:read_scalar_integers(collection, attribute)   -- { 42, 37, ... }
+db:read_scalar_floats(collection, attribute)      -- { 3.14, 2.71, ... }
+db:read_scalar_strings(collection, attribute)     -- { "Item 1", "Item 2", ... }
+\`\`\`
+
+---
+
+## Vector reads (bulk)
+
+Each returns an array of arrays — one inner array per element.
+
+\`\`\`lua
+db:read_vector_integers(collection, attribute)   -- { {1,2,3}, {2,3,4}, ... }
+db:read_vector_floats(collection, attribute)
+db:read_vector_strings(collection, attribute)
+\`\`\`
+
+---
+
+## Set reads (bulk)
+
+Same shape as vector reads — an array of arrays.
+
+\`\`\`lua
+db:read_set_integers(collection, attribute)
+db:read_set_floats(collection, attribute)
+db:read_set_strings(collection, attribute)
+\`\`\`
+
+---
+
+## Composite by-id reads (Lua convenience helpers)
+
+\`\`\`lua
+db:read_element_ids(collection)                  -- { 1, 2, 3, ... }
+
+db:read_scalars_by_id(collection, id)            -- { attr = value, ... } (missing -> nil)
+db:read_vectors_by_id(collection, id)            -- { column = { v1, v2, ... }, ... }
+db:read_sets_by_id(collection, id)               -- { column = { v1, v2, ... }, ... }
+db:read_element_by_id(collection, id)            -- scalars + vectors + sets merged into one table
+\`\`\`
+
+\`read_element_by_id\` merges every scalar, vector, and set for the element into a single table.
+Scalar attributes with no value come back as \`nil\`.
+
+---
+
+## Time series
+
+Time-series group data is **column-oriented** in Lua: \`{ column = { v1, v2, ... }, ... }\`. The
+dimension (ordering) column is a \`date_*\` text column holding ISO 8601 timestamps.
+
+### Read a whole group (column-oriented)
+
+\`\`\`lua
+local ts = db:read_time_series_group(collection, group, id)
+-- ts = { date_time = { "2024-01-01T00:00:00", ... }, value = { 10.5, 20.0, ... } }
+-- returns an empty table {} if the element has no rows
+\`\`\`
+
+A \`NULL\` value cell comes back as a \`nil\` hole (the index is simply absent), and an all-\`NULL\`
+value column is an empty table \`{}\` (its key is still present). Because \`nil\` cannot occupy an
+array slot, **take the row count from the dimension column** (\`#ts.date_time\`), never from a value
+column.
+
+### Read one value per element at a date (\`read_time_series_row\`)
+
+\`\`\`lua
+local values = db:read_time_series_row(collection, group, attribute, date_time)
+-- { v_elem1, v_elem2, ... } in element-id order
+\`\`\`
+
+One value per element using **last non-null value at or before \`date_time\`** semantics. Elements
+with no matching data yield \`nil\` in the array. \`date_time\` is an ISO 8601 string.
+
+### Replace a whole group (column-oriented — SAME shape as the read)
+
+\`update_time_series_group\` takes the **exact column-oriented shape \`read_time_series_group\`
+returns**: a table mapping each column name to a 1-indexed array of its values. Read → modify →
+write round-trips. Passing an empty table \`{}\` clears the group.
+
+\`\`\`lua
+db:update_time_series_group("Items", "data", id, {
+    date_time = { "2024-01-01T00:00:00", "2024-01-02T00:00:00", "2024-01-03T00:00:00" },
+    value     = { 10.5, 20.0, 30.0 },
+})
+
+db:update_time_series_group("Items", "data", id, {})   -- clears the group
+\`\`\`
+
+A read-modify-write looks like this:
+
+\`\`\`lua
+local ts = db:read_time_series_group("Items", "data", id)
+ts.value[2] = 125.0                                    -- edit the 2nd row's value
+db:update_time_series_group("Items", "data", id, ts)   -- write the whole group back
+\`\`\`
+
+**DO NOT pass an array of row tables** (\`{ { date_time = ..., value = ... }, ... }\`) — that is the
+\`add_time_series_row\` shape, not this one. Doing so raises
+\`stack index -1, expected string, received number\` (the integer array indices 1, 2, 3 are not
+column names). Each value of the top-level table must be an **array**, not a scalar.
+
+**Rules** (validation throws, rolling the script back):
+- Every column value must be an array — a bare scalar throws \`column '...' must be an array of values\`.
+- The **dimension column(s) set the row count** and must be present and fully populated: the
+  \`date_*\` ordering column, plus any extra primary-key columns in a multi-dimensional group (e.g.
+  \`block\`). A missing one throws \`missing dimension column '...'\`; a \`nil\` inside one throws
+  \`dimension column '...' has nil at index N\` (they are primary-key columns and cannot be NULL).
+- **Value columns may be shorter, sparser, or absent** relative to the dimension column — every
+  missing cell is written as \`NULL\`. So \`value = { 10.0, nil, 30.0 }\` or a too-short \`value = { 10.0 }\`
+  both write NULLs for the gaps; this is how you round-trip the \`nil\` holes a read produces. A value
+  column **longer** than the dimension column throws \`column '...' has length N but expected M\`.
+- Named columns whose dimension transposes to zero rows throw (\`contain no rows; pass an empty
+  table {} to clear the group\`) — only a bare \`{}\` clears.
+- Integer values are accepted for REAL columns (converted on insert). Booleans, functions, and
+  other unsupported Lua types throw \`column '...' has unsupported Lua type\`.
+
+### Append/upsert a single row (\`add_time_series_row\` — ROW-oriented, the one exception)
+
+Unlike the column-oriented group update above, this takes **one row table of scalars** (dimension
+column + value column(s)), and upserts that single row:
+
+\`\`\`lua
+db:add_time_series_row("Items", "data", id, {
+    date_time = "2024-01-04T00:00:00",
+    value     = 40.0,
+})
+\`\`\`
+
+---
+
+## Time series files
+
+For schemas that reference external time-series files (the \`{Collection}_time_series_files\`
+singleton table):
+
+\`\`\`lua
+db:has_time_series_files(collection)              -- boolean
+db:list_time_series_files_columns(collection)     -- { "data_file", "metadata_file", ... }
+db:read_time_series_files(collection)             -- { data_file = "path", metadata_file = nil, ... }
+db:update_time_series_files(collection, { data_file = "path/to/data.bin", metadata_file = nil })
+\`\`\`
+
+In \`update_time_series_files\`, a \`nil\` value clears that column.
+
+---
+
+## Metadata
+
+### Single attribute / group
+
+\`\`\`lua
+db:get_scalar_metadata(collection, attribute)        -- scalar metadata table (below)
+db:get_vector_metadata(collection, group_name)       -- group metadata table (below)
+db:get_set_metadata(collection, group_name)          -- group metadata table
+db:get_time_series_metadata(collection, group_name)  -- group metadata table (+ dimension_column)
+\`\`\`
+
+### Lists (one entry per attribute/group)
+
+\`\`\`lua
+db:list_scalar_attributes(collection)    -- array of scalar metadata tables
+db:list_vector_groups(collection)        -- array of group metadata tables
+db:list_set_groups(collection)           -- array of group metadata tables
+db:list_time_series_groups(collection)   -- array of group metadata tables (+ dimension_column)
+\`\`\`
+
+### Scalar metadata table shape
+
+\`\`\`lua
+{
+    name                   = "value",
+    data_type              = "integer",   -- "integer" | "real" | "text" | "date_time"
+    not_null               = true,
+    primary_key            = false,
+    default_value          = nil,         -- string, or nil
+    is_foreign_key         = false,
+    references_collection  = nil,         -- string, or nil
+    references_column      = nil,         -- string, or nil
+}
+\`\`\`
+
+### Group metadata table shape
+
+\`\`\`lua
+{
+    group_name      = "data",
+    value_columns   = { <scalar metadata table>, ... },
+    dimension_column = "date_time",   -- present ONLY for time-series groups
+}
+\`\`\`
+
+The \`dimension_column\` key is present only in **time-series** group metadata
+(\`get_time_series_metadata\` / \`list_time_series_groups\`). Vector and set group metadata omit it
+entirely.
+
+---
+
+## Query (parameterized SQL)
+
+Positional \`?\` placeholders; \`params\` is an optional 1-indexed array. Each returns the first
+column of the first row as the requested type, or \`nil\` if there is no result.
+
+\`\`\`lua
+db:query_string(sql, params)    -- string or nil
+db:query_integer(sql, params)   -- integer or nil
+db:query_float(sql, params)     -- number or nil
+\`\`\`
+
+Example:
+
+\`\`\`lua
+local label = db:query_string(
+    "SELECT label FROM Collection WHERE some_integer = ?",
+    { 42 }
+)
+local count = db:query_integer("SELECT COUNT(*) FROM Collection")
+\`\`\`
+
+---
+
+## CSV import / export
+
+Export a time-series group to a CSV file, or import one from a CSV file. \`path\` is sandboxed:
+relative paths resolve against the database file's directory and must stay inside it (see
+Critical rules); \`options\` is optional.
+
+\`\`\`lua
+db:export_csv(collection, group, path, options)
+db:import_csv(collection, group, path, options)
+\`\`\`
+
+The optional \`options\` table has two keys:
+
+\`\`\`lua
+{
+    date_time_format = "%Y-%m-%d",   -- strftime-style format for the dimension column
+
+    -- enum_labels: write/read integer codes as human labels. Three nested levels:
+    --   attribute name -> locale -> { label = integer_id }
+    enum_labels = {
+        status = {
+            en = { active = 1, inactive = 0 },
+            pt = { ativo  = 1, inativo  = 0 },
+        },
+    },
+}
+\`\`\`
+
+**Precondition:** \`db:import_csv\` cannot run inside an open transaction (it toggles
+\`PRAGMA foreign_keys\`, a no-op mid-transaction) — it throws \`Cannot import_csv: transaction already
+active\`. Call it outside any \`db:transaction\` / \`db:begin_transaction\` block.
+
+---
+
+## Complete example
+
+\`\`\`lua
+-- create
+db:create_element("Configuration", { label = "Configuration" })
+
+local item1 = db:create_element("Collection",
+    { label = "Item 1", some_integer = 42, some_float = 3.14, value_int = { 1, 2, 3 } })
+local item2 = db:create_element("Collection",
+    { label = "Item 2", some_integer = 37, some_float = 2.71, value_int = { 2, 3, 4 } })
+
+-- read every element
+local ids = db:read_element_ids("Collection")
+for _, id in ipairs(ids) do
+    local element = db:read_element_by_id("Collection", id)
+    print("Element " .. id .. ", label = " .. tostring(element.label))
+end
+
+-- update
+db:update_element("Collection", item1, { some_integer = 999 })
+
+-- delete
+db:delete_element("Collection", item2)
+\`\`\`
+
+---
+
+## Binary & expression subsystems
+
+Dense N-dimensional \`float64\` arrays (\`.qvr\` + \`.toml\` sidecar) plus lazy arithmetic over them.
+File I/O is db-scoped (\`db:open_file\` / \`db:bin_to_csv\` / \`db:csv_to_bin\`); paths are extensionless
+base paths, sandboxed to the database directory (see Critical rules), and \`get_file_path()\` returns
+the resolved absolute path. The pure-metadata builders and expression constructors live under the
+global \`quiver\` table. Mirrors the Julia surface; aggregation ops are strings (Lua has no enums);
+operators are \`+ - * /\` and unary \`-\`, with scalars allowed on either side.
+
+\`\`\`lua
+local md = quiver.metadata{
+    initial_datetime = "2025-01-01T00:00:00", unit = "MW",
+    labels = {"v1", "v2"}, dimensions = {"stage", "block"}, dimension_sizes = {4, 31},
+    time_dimensions = {"stage", "block"}, frequencies = {"monthly", "daily"},
+}
+local f = db:open_file(path, "w", md)               -- mode "r"/"w"; md required for "w"
+f:write({1.0, 2.0}, {stage = 1, block = 1})         -- data table, dims table
+f:close()
+local r = db:open_file(path, "r")
+local cell = r:read({stage = 1, block = 1})         -- { v1, v2 }; pass true as 2nd arg to allow NaN
+r:get_metadata(); r:get_file_path(); r:is_open()
+md:get_unit(); md:get_version(); md:get_initial_datetime()
+md:get_labels(); md:get_dimensions(); md:get_number_of_time_dimensions(); md:to_toml()
+quiver.metadata_from_toml(text); quiver.metadata_from_element(tbl)
+db:bin_to_csv(path)                                  -- aggregate=true by default; pass false to keep time dims as columns
+db:csv_to_bin(path)
+
+local e = (quiver.expression(r) + 10.0) * 2.0        -- files auto-wrap; scalars either side
+e = quiver.abs(e); e = quiver.sqrt(e)                -- also quiver.log / quiver.exp
+e = quiver.ifelse(cond_e, then_e, else_e)
+e = e:aggregate("stage", "sum")                      -- sum/mean/min/max/percentile
+e = e:aggregate("stage", "percentile", 0.9)          -- percentile needs the fraction
+e = e:aggregate_agents("mean")                       -- collapse the label axis
+e = e:select_agents({"v2"}); e = e:rename_agents({v1 = "alpha"})
+e:save(out_path); e:metadata()                       -- save path is sandboxed like db:open_file
+\`\`\`
+
+**\`quiver.metadata{...}\` kwargs and defaults:** \`version\` defaults to \`"1"\`; \`initial_datetime\`
+and \`unit\` default to \`""\`; \`labels\`, \`dimensions\`, \`dimension_sizes\`, \`time_dimensions\`, and
+\`frequencies\` default to empty arrays.
+
+**\`get_dimensions()\` / \`get_metadata()\` dimension shape** — returns an array of dimension tables:
+
+\`\`\`lua
+{
+    name                   = "stage",
+    size                   = 4,
+    is_time_dimension      = true,
+    frequency              = "monthly",   -- nil for non-time dimensions
+    initial_value          = 1,           -- nil for non-time dimensions
+    parent_dimension_index = -1,          -- nil for non-time dimensions
+}
+\`\`\`
+
+---
+
+## What Lua does *not* expose
+
+DateTime wrapper helpers (Lua uses ISO 8601 strings) and \`_by_id\` single-scalar variants (use the
+composite by-id readers or the bulk readers instead). Everything else the native binding exposes —
+CRUD, reads, time series, metadata, query, CSV, and the binary/expression subsystems — is
+documented above and callable.`;

package/src/time-series.ts

@@ -26,7 +26,7 @@ import {
   DATA_TYPE_STRING,
 } from "./types.ts";
 
-export type TimeSeriesData = Record<string, (number | string)[]>;
+export type TimeSeriesData = Record<string, (number | string | null)[]>;
 
 Database.prototype.readTimeSeriesGroup = function (
   this: Database,
@@ -40,6 +40,7 @@ Database.prototype.readTimeSeriesGroup = function (
   const outNames = allocPtrOut();
   const outTypes = allocPtrOut();
   const outData = allocPtrOut();
+  const outHasValue = allocPtrOut();
   const outColCount = allocUint64Out();
   const outRowCount = allocUint64Out();
 
@@ -52,6 +53,7 @@ Database.prototype.readTimeSeriesGroup = function (
       outNames.buf,
       outTypes.buf,
       outData.buf,
+      outHasValue.buf,
       outColCount.buf,
       outRowCount.buf,
     ),
@@ -64,26 +66,49 @@ Database.prototype.readTimeSeriesGroup = function (
   const namesPtr = readPtrOut(outNames);
   const typesPtr = readPtrOut(outTypes);
   const dataPtr = readPtrOut(outData);
+  const hasValuePtr = readPtrOut(outHasValue);
 
   const colNames = decodeStringArray(namesPtr, colCount);
   const typesAb = toArrayBuffer(typesPtr as Pointer, 0, colCount * 4);
   const types = Array.from(new Int32Array(typesAb));
   const dataPtrs = decodePtrArray(dataPtr, colCount);
+  const maskPtrs = decodePtrArray(hasValuePtr, colCount);
 
+  // Per-cell NULL mask: mask[r] === 0 means SQL NULL, surfaced as JS null. The
+  // dimension column's mask is always all 1, so it stays dense.
   const result: TimeSeriesData = {};
   for (let c = 0; c < colCount; c++) {
     const colName = colNames[c];
+    const maskPtr = maskPtrs[c];
+    const mask = maskPtr ? new Uint8Array(toArrayBuffer(maskPtr as Pointer, 0, rowCount)) : null;
     switch (types[c]) {
-      case DATA_TYPE_INTEGER:
-        result[colName] = decodeInt64Array(dataPtrs[c], rowCount);
+      case DATA_TYPE_INTEGER: {
+        const vals = decodeInt64Array(dataPtrs[c], rowCount);
+        result[colName] = mask ? vals.map((v, r) => (mask[r] ? v : null)) : vals;
         break;
-      case DATA_TYPE_FLOAT:
-        result[colName] = decodeFloat64Array(dataPtrs[c], rowCount);
+      }
+      case DATA_TYPE_FLOAT: {
+        const vals = decodeFloat64Array(dataPtrs[c], rowCount);
+        result[colName] = mask ? vals.map((v, r) => (mask[r] ? v : null)) : vals;
         break;
+      }
       case DATA_TYPE_STRING:
-      case DATA_TYPE_DATE_TIME:
-        result[colName] = decodeStringArray(dataPtrs[c], rowCount);
+      case DATA_TYPE_DATE_TIME: {
+        // Read pointer-by-pointer (not decodeStringArray): a masked-out cell is a
+        // NULL char* that CString cannot construct from.
+        const base = dataPtrs[c];
+        const col: (string | null)[] = new Array(rowCount);
+        for (let r = 0; r < rowCount; r++) {
+          if (mask && !mask[r]) {
+            col[r] = null;
+            continue;
+          }
+          const strPtr = base ? read.ptr(base as Pointer, r * 8) : 0;
+          col[r] = strPtr === 0 ? null : new CString(strPtr as Pointer).toString();
+        }
+        result[colName] = col;
         break;
+      }
     }
   }
 
@@ -91,6 +116,7 @@ Database.prototype.readTimeSeriesGroup = function (
     namesPtr,
     typesPtr,
     dataPtr,
+    hasValuePtr,
     BigInt(colCount),
     BigInt(rowCount),
   );
@@ -177,6 +203,7 @@ Database.prototype.updateTimeSeriesGroup = function (
         null,
         null,
         null,
+        null,
         0n,
         0n,
       ),
@@ -213,32 +240,58 @@ Database.prototype.updateTimeSeriesGroup = function (
   const { table: namesTable, keepalive: namesPtrs } = allocNativeStringArray(colNames);
   keepalive.push(namesTable, ...namesPtrs);
 
-  // Build column types and data
+  // Build column types, data, and per-cell NULL masks. A null cell becomes
+  // mask 0 + a placeholder in the data array (the C API never reads it). An
+  // all-null column is tagged FLOAT with zeroed data — the type tag is ignored
+  // for masked-out cells.
   const typesBuf = new Uint8Array(columnCount * 4);
   const typesDv = new DataView(typesBuf.buffer);
   const dataPtrs: (Pointer | null)[] = [];
+  const maskPtrs: (Pointer | null)[] = [];
 
   for (let c = 0; c < columnCount; c++) {
-    const values = entries[c][1];
-
-    if (typeof values[0] === "string") {
+    const [colName, values] = entries[c];
+    const first = values.find((v) => v !== null);
+
+    // Mask via direct indexing — never a DataView, to avoid the documented
+    // .buffer-materialization pitfall between ptr() and the FFI call.
+    const maskBuf = new Uint8Array(rowCount);
+    for (let r = 0; r < rowCount; r++) maskBuf[r] = values[r] === null ? 0 : 1;
+    const maskAlloc: Allocation = { ptr: ptr(maskBuf), buf: maskBuf };
+    keepalive.push(maskAlloc);
+    maskPtrs.push(maskAlloc.ptr);
+
+    if (first === undefined) {
+      // All-null column
+      typesDv.setInt32(c * 4, DATA_TYPE_FLOAT, true);
+      const p = allocNativeFloat64(new Array(rowCount).fill(0));
+      keepalive.push(p);
+      dataPtrs.push(p.ptr);
+    } else if (typeof first === "string") {
       typesDv.setInt32(c * 4, DATA_TYPE_STRING, true);
-      const { table, keepalive: strPtrs } = allocNativeStringArray(values as string[]);
+      const { table, keepalive: strPtrs } = allocNativeStringArray(
+        values.map((v) => (v === null ? null : (v as string))),
+      );
       keepalive.push(table, ...strPtrs);
       dataPtrs.push(table.ptr);
-    } else if (typeof values[0] === "number") {
-      const allIntegers = (values as number[]).every((v) => Number.isInteger(v));
-      if (allIntegers) {
+    } else if (typeof first === "number") {
+      const nonNull = values.filter((v) => v !== null) as number[];
+      const sanitized = values.map((v) => (v === null ? 0 : (v as number)));
+      if (nonNull.every((v) => Number.isInteger(v))) {
         typesDv.setInt32(c * 4, DATA_TYPE_INTEGER, true);
-        const p = allocNativeInt64(values as number[]);
+        const p = allocNativeInt64(sanitized);
         keepalive.push(p);
         dataPtrs.push(p.ptr);
       } else {
         typesDv.setInt32(c * 4, DATA_TYPE_FLOAT, true);
-        const p = allocNativeFloat64(values as number[]);
+        const p = allocNativeFloat64(sanitized);
         keepalive.push(p);
         dataPtrs.push(p.ptr);
       }
+    } else {
+      throw new QuiverError(
+        `Cannot updateTimeSeriesGroup: column '${colName}' has unsupported value type ${typeof first}`,
+      );
     }
   }
 
@@ -246,6 +299,8 @@ Database.prototype.updateTimeSeriesGroup = function (
   keepalive.push(typesAlloc);
   const dataTable = allocNativePtrTable(dataPtrs);
   keepalive.push(dataTable);
+  const maskTable = allocNativePtrTable(maskPtrs);
+  keepalive.push(maskTable);
 
   check(
     lib.quiver_database_update_time_series_group(
@@ -256,6 +311,7 @@ Database.prototype.updateTimeSeriesGroup = function (
       namesTable.buf,
       typesAlloc.buf,
       dataTable.buf,
+      maskTable.buf,
       BigInt(columnCount),
       BigInt(rowCount),
     ),