quiverdb 0.9.7 → 0.9.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "quiverdb",
-  "version": "0.9.7",
+  "version": "0.9.8",
   "license": "MIT",
   "repository": {
     "type": "git",

package/src/ffi-helpers.ts

@@ -141,14 +141,17 @@ export function allocNativeString(str: string): Allocation {
   return { ptr: ptr(buf), buf };
 }
 
-/** Build a native string pointer table (const char* const*) from an array of strings. */
-export function allocNativeStringArray(strings: string[]): {
+/**
+ * Build a native string pointer table (const char* const*) from an array of strings.
+ * A `null` entry becomes a NULL pointer in the table (used for SQL-NULL string cells).
+ */
+export function allocNativeStringArray(strings: (string | null)[]): {
   table: Allocation;
   keepalive: Allocation[];
 } {
-  const strAllocs = strings.map((s) => allocNativeString(s));
-  const table = allocNativePtrTable(strAllocs.map((a) => a.ptr));
-  return { table, keepalive: strAllocs };
+  const strAllocs = strings.map((s) => (s === null ? null : allocNativeString(s)));
+  const table = allocNativePtrTable(strAllocs.map((a) => (a ? a.ptr : null)));
+  return { table, keepalive: strAllocs.filter((a): a is Allocation => a !== null) };
 }
 
 /** Get the native address of a pointer as BigInt. */

package/src/loader.ts

@@ -118,14 +118,17 @@ const describeSymbols = {
 } as const;
 
 const timeSeriesSymbols = {
-  quiver_database_read_time_series_group: { args: [P, BUF, BUF, I64, P, P, P, P, P], returns: I32 },
+  quiver_database_read_time_series_group: {
+    args: [P, BUF, BUF, I64, P, P, P, P, P, P],
+    returns: I32,
+  },
   quiver_database_read_time_series_row: { args: [P, BUF, BUF, BUF, BUF, P, P, P], returns: I32 },
   quiver_database_add_time_series_row: { args: [P, BUF, BUF, I64, P, P, P, USIZE], returns: I32 },
   quiver_database_update_time_series_group: {
-    args: [P, BUF, BUF, I64, P, P, P, USIZE, USIZE],
+    args: [P, BUF, BUF, I64, P, P, P, P, USIZE, USIZE],
     returns: I32,
   },
-  quiver_database_free_time_series_data: { args: [P, P, P, USIZE, USIZE], returns: I32 },
+  quiver_database_free_time_series_data: { args: [P, P, P, P, USIZE, USIZE], returns: I32 },
   quiver_database_has_time_series_files: { args: [P, BUF, P], returns: I32 },
   quiver_database_list_time_series_files_columns: { args: [P, BUF, P, P], returns: I32 },
   quiver_database_read_time_series_files: { args: [P, BUF, P, P, P], returns: I32 },

package/src/lua-api.ts

@@ -5,9 +5,9 @@
 // binding changes, re-diff every signature below against `bind_database()` (names, arg order, arg
 // types, return shapes).
 //
-// NOTE: the binary/expression subsystems (quiver.* globals + file:/expr: methods) are bound in the
-// native binding and documented below. They are quiver.*/file:/expr: calls, not db:<name> tokens,
-// and they read/write files on the host filesystem.
+// NOTE: the binary/expression subsystems are bound in the native binding and documented below.
+// File-touching operations (db:open_file, db:bin_to_csv, db:csv_to_bin, expr:save) are sandboxed
+// to the database file's directory; the pure-metadata builders stay under the quiver.* global.
 //
 // FORMAT CONVENTION: every db: method should appear at least once as the literal token
 // `db:<snake_case_name>` so coverage is greppable.
@@ -66,7 +66,14 @@ datetime surface — there are no DateTime wrapper helpers, unlike Julia/Dart/Py
   \`Failed to run Lua script: <message>\`. Validation failures roll back whatever the current
   transaction covered.
 - **Standard library.** Only the \`base\`, \`string\`, and \`table\` libraries are loaded — there is NO
-  \`math\` (use \`//\` for integer division), and no \`os\`, \`io\`, \`require\`, \`load\`, or \`dofile\`.
+  \`math\` (use \`//\` for integer division), and no \`os\`, \`io\`, or \`require\`; \`dofile\` and
+  \`loadfile\` are removed (string-form \`load\` is available).
+- **Filesystem sandbox.** Every file-touching operation (\`db:export_csv\`, \`db:import_csv\`,
+  \`db:open_file\`, \`db:bin_to_csv\`, \`db:csv_to_bin\`, \`expr:save\`) resolves relative paths against
+  the directory containing the database file and rejects anything outside it (subdirectories are
+  fine; \`..\` escapes and outside absolute paths throw \`Cannot <op>: path '...' escapes the
+  database directory ...\`). On an in-memory database these operations throw
+  \`Cannot <op>: database is in-memory, file operations are unavailable\`.
 - **Output.** Scripts cannot return values to the tool — use \`print()\` for anything you need to
   see (it is captured). Arrays are 1-indexed (iterate with \`ipairs\`); reading a NULL yields \`nil\`,
   writing \`nil\` stores NULL where NULL is accepted (query params, ts rows, file columns — but NOT
@@ -218,6 +225,11 @@ local ts = db:read_time_series_group(collection, group, id)
 -- returns an empty table {} if the element has no rows
 \`\`\`
 
+A \`NULL\` value cell comes back as a \`nil\` hole (the index is simply absent), and an all-\`NULL\`
+value column is an empty table \`{}\` (its key is still present). Because \`nil\` cannot occupy an
+array slot, **take the row count from the dimension column** (\`#ts.date_time\`), never from a value
+column.
+
 ### Read one value per element at a date (\`read_time_series_row\`)
 
 \`\`\`lua
@@ -258,14 +270,18 @@ column names). Each value of the top-level table must be an **array**, not a sca
 
 **Rules** (validation throws, rolling the script back):
 - Every column value must be an array — a bare scalar throws \`column '...' must be an array of values\`.
-- All columns must have the **same length** — a mismatch throws \`column '...' has length N but expected M\`.
-- Named-but-empty columns throw (\`contain no rows; pass an empty table {} to clear\`) rather than
-  silently clearing — only a bare \`{}\` clears.
-- Write REAL-column values as float literals (\`30.0\`, not \`30\`); STRICT validation rejects an
-  integer for a REAL column. Other Lua types throw \`column '...' has unsupported Lua type\`.
-- \`nil\` is **not** accepted inside a column here (it throws \`unsupported Lua type\`); columns must
-  be fully populated. To write a NULL into a single row, use \`add_time_series_row\` instead, which
-  does accept \`nil\`.
+- The **dimension column(s) set the row count** and must be present and fully populated: the
+  \`date_*\` ordering column, plus any extra primary-key columns in a multi-dimensional group (e.g.
+  \`block\`). A missing one throws \`missing dimension column '...'\`; a \`nil\` inside one throws
+  \`dimension column '...' has nil at index N\` (they are primary-key columns and cannot be NULL).
+- **Value columns may be shorter, sparser, or absent** relative to the dimension column — every
+  missing cell is written as \`NULL\`. So \`value = { 10.0, nil, 30.0 }\` or a too-short \`value = { 10.0 }\`
+  both write NULLs for the gaps; this is how you round-trip the \`nil\` holes a read produces. A value
+  column **longer** than the dimension column throws \`column '...' has length N but expected M\`.
+- Named columns whose dimension transposes to zero rows throw (\`contain no rows; pass an empty
+  table {} to clear the group\`) — only a bare \`{}\` clears.
+- Integer values are accepted for REAL columns (converted on insert). Booleans, functions, and
+  other unsupported Lua types throw \`column '...' has unsupported Lua type\`.
 
 ### Append/upsert a single row (\`add_time_series_row\` — ROW-oriented, the one exception)
 
@@ -373,8 +389,9 @@ local count = db:query_integer("SELECT COUNT(*) FROM Collection")
 
 ## CSV import / export
 
-Export a time-series group to a CSV file, or import one from a CSV file. \`path\` is resolved by the
-host filesystem; \`options\` is optional.
+Export a time-series group to a CSV file, or import one from a CSV file. \`path\` is sandboxed:
+relative paths resolve against the database file's directory and must stay inside it (see
+Critical rules); \`options\` is optional.
 
 \`\`\`lua
 db:export_csv(collection, group, path, options)
@@ -433,10 +450,12 @@ db:delete_element("Collection", item2)
 
 ## Binary & expression subsystems
 
-Dense N-dimensional \`float64\` arrays (\`.qvr\` + \`.toml\` sidecar) plus lazy arithmetic over them,
-under a global \`quiver\` table (not \`db\`). Mirrors the Julia surface; aggregation ops are strings
-(Lua has no enums); operators are \`+ - * /\` and unary \`-\`, with scalars allowed on either side.
-These read/write files on the host filesystem (\`path\` is host-resolved).
+Dense N-dimensional \`float64\` arrays (\`.qvr\` + \`.toml\` sidecar) plus lazy arithmetic over them.
+File I/O is db-scoped (\`db:open_file\` / \`db:bin_to_csv\` / \`db:csv_to_bin\`); paths are extensionless
+base paths, sandboxed to the database directory (see Critical rules), and \`get_file_path()\` returns
+the resolved absolute path. The pure-metadata builders and expression constructors live under the
+global \`quiver\` table. Mirrors the Julia surface; aggregation ops are strings (Lua has no enums);
+operators are \`+ - * /\` and unary \`-\`, with scalars allowed on either side.
 
 \`\`\`lua
 local md = quiver.metadata{
@@ -444,17 +463,17 @@ local md = quiver.metadata{
     labels = {"v1", "v2"}, dimensions = {"stage", "block"}, dimension_sizes = {4, 31},
     time_dimensions = {"stage", "block"}, frequencies = {"monthly", "daily"},
 }
-local f = quiver.open_file(path, "w", md)          -- mode "r"/"w"; md required for "w"
+local f = db:open_file(path, "w", md)               -- mode "r"/"w"; md required for "w"
 f:write({1.0, 2.0}, {stage = 1, block = 1})         -- data table, dims table
 f:close()
-local r = quiver.open_file(path, "r")
+local r = db:open_file(path, "r")
 local cell = r:read({stage = 1, block = 1})         -- { v1, v2 }; pass true as 2nd arg to allow NaN
 r:get_metadata(); r:get_file_path(); r:is_open()
 md:get_unit(); md:get_version(); md:get_initial_datetime()
 md:get_labels(); md:get_dimensions(); md:get_number_of_time_dimensions(); md:to_toml()
 quiver.metadata_from_toml(text); quiver.metadata_from_element(tbl)
-quiver.bin_to_csv(path)                              -- aggregate=true by default; pass false to keep time dims as columns
-quiver.csv_to_bin(path)
+db:bin_to_csv(path)                                  -- aggregate=true by default; pass false to keep time dims as columns
+db:csv_to_bin(path)
 
 local e = (quiver.expression(r) + 10.0) * 2.0        -- files auto-wrap; scalars either side
 e = quiver.abs(e); e = quiver.sqrt(e)                -- also quiver.log / quiver.exp
@@ -463,7 +482,7 @@ e = e:aggregate("stage", "sum")                      -- sum/mean/min/max/percent
 e = e:aggregate("stage", "percentile", 0.9)          -- percentile needs the fraction
 e = e:aggregate_agents("mean")                       -- collapse the label axis
 e = e:select_agents({"v2"}); e = e:rename_agents({v1 = "alpha"})
-e:save(out_path); e:metadata()
+e:save(out_path); e:metadata()                       -- save path is sandboxed like db:open_file
 \`\`\`
 
 **\`quiver.metadata{...}\` kwargs and defaults:** \`version\` defaults to \`"1"\`; \`initial_datetime\`

package/src/time-series.ts

@@ -26,7 +26,7 @@ import {
   DATA_TYPE_STRING,
 } from "./types.ts";
 
-export type TimeSeriesData = Record<string, (number | string)[]>;
+export type TimeSeriesData = Record<string, (number | string | null)[]>;
 
 Database.prototype.readTimeSeriesGroup = function (
   this: Database,
@@ -40,6 +40,7 @@ Database.prototype.readTimeSeriesGroup = function (
   const outNames = allocPtrOut();
   const outTypes = allocPtrOut();
   const outData = allocPtrOut();
+  const outHasValue = allocPtrOut();
   const outColCount = allocUint64Out();
   const outRowCount = allocUint64Out();
 
@@ -52,6 +53,7 @@ Database.prototype.readTimeSeriesGroup = function (
       outNames.buf,
       outTypes.buf,
       outData.buf,
+      outHasValue.buf,
       outColCount.buf,
       outRowCount.buf,
     ),
@@ -64,26 +66,49 @@ Database.prototype.readTimeSeriesGroup = function (
   const namesPtr = readPtrOut(outNames);
   const typesPtr = readPtrOut(outTypes);
   const dataPtr = readPtrOut(outData);
+  const hasValuePtr = readPtrOut(outHasValue);
 
   const colNames = decodeStringArray(namesPtr, colCount);
   const typesAb = toArrayBuffer(typesPtr as Pointer, 0, colCount * 4);
   const types = Array.from(new Int32Array(typesAb));
   const dataPtrs = decodePtrArray(dataPtr, colCount);
+  const maskPtrs = decodePtrArray(hasValuePtr, colCount);
 
+  // Per-cell NULL mask: mask[r] === 0 means SQL NULL, surfaced as JS null. The
+  // dimension column's mask is always all 1, so it stays dense.
   const result: TimeSeriesData = {};
   for (let c = 0; c < colCount; c++) {
     const colName = colNames[c];
+    const maskPtr = maskPtrs[c];
+    const mask = maskPtr ? new Uint8Array(toArrayBuffer(maskPtr as Pointer, 0, rowCount)) : null;
     switch (types[c]) {
-      case DATA_TYPE_INTEGER:
-        result[colName] = decodeInt64Array(dataPtrs[c], rowCount);
+      case DATA_TYPE_INTEGER: {
+        const vals = decodeInt64Array(dataPtrs[c], rowCount);
+        result[colName] = mask ? vals.map((v, r) => (mask[r] ? v : null)) : vals;
         break;
-      case DATA_TYPE_FLOAT:
-        result[colName] = decodeFloat64Array(dataPtrs[c], rowCount);
+      }
+      case DATA_TYPE_FLOAT: {
+        const vals = decodeFloat64Array(dataPtrs[c], rowCount);
+        result[colName] = mask ? vals.map((v, r) => (mask[r] ? v : null)) : vals;
         break;
+      }
       case DATA_TYPE_STRING:
-      case DATA_TYPE_DATE_TIME:
-        result[colName] = decodeStringArray(dataPtrs[c], rowCount);
+      case DATA_TYPE_DATE_TIME: {
+        // Read pointer-by-pointer (not decodeStringArray): a masked-out cell is a
+        // NULL char* that CString cannot construct from.
+        const base = dataPtrs[c];
+        const col: (string | null)[] = new Array(rowCount);
+        for (let r = 0; r < rowCount; r++) {
+          if (mask && !mask[r]) {
+            col[r] = null;
+            continue;
+          }
+          const strPtr = base ? read.ptr(base as Pointer, r * 8) : 0;
+          col[r] = strPtr === 0 ? null : new CString(strPtr as Pointer).toString();
+        }
+        result[colName] = col;
         break;
+      }
     }
   }
 
@@ -91,6 +116,7 @@ Database.prototype.readTimeSeriesGroup = function (
     namesPtr,
     typesPtr,
     dataPtr,
+    hasValuePtr,
     BigInt(colCount),
     BigInt(rowCount),
   );
@@ -177,6 +203,7 @@ Database.prototype.updateTimeSeriesGroup = function (
         null,
         null,
         null,
+        null,
         0n,
         0n,
       ),
@@ -213,32 +240,58 @@ Database.prototype.updateTimeSeriesGroup = function (
   const { table: namesTable, keepalive: namesPtrs } = allocNativeStringArray(colNames);
   keepalive.push(namesTable, ...namesPtrs);
 
-  // Build column types and data
+  // Build column types, data, and per-cell NULL masks. A null cell becomes
+  // mask 0 + a placeholder in the data array (the C API never reads it). An
+  // all-null column is tagged FLOAT with zeroed data — the type tag is ignored
+  // for masked-out cells.
   const typesBuf = new Uint8Array(columnCount * 4);
   const typesDv = new DataView(typesBuf.buffer);
   const dataPtrs: (Pointer | null)[] = [];
+  const maskPtrs: (Pointer | null)[] = [];
 
   for (let c = 0; c < columnCount; c++) {
-    const values = entries[c][1];
-
-    if (typeof values[0] === "string") {
+    const [colName, values] = entries[c];
+    const first = values.find((v) => v !== null);
+
+    // Mask via direct indexing — never a DataView, to avoid the documented
+    // .buffer-materialization pitfall between ptr() and the FFI call.
+    const maskBuf = new Uint8Array(rowCount);
+    for (let r = 0; r < rowCount; r++) maskBuf[r] = values[r] === null ? 0 : 1;
+    const maskAlloc: Allocation = { ptr: ptr(maskBuf), buf: maskBuf };
+    keepalive.push(maskAlloc);
+    maskPtrs.push(maskAlloc.ptr);
+
+    if (first === undefined) {
+      // All-null column
+      typesDv.setInt32(c * 4, DATA_TYPE_FLOAT, true);
+      const p = allocNativeFloat64(new Array(rowCount).fill(0));
+      keepalive.push(p);
+      dataPtrs.push(p.ptr);
+    } else if (typeof first === "string") {
       typesDv.setInt32(c * 4, DATA_TYPE_STRING, true);
-      const { table, keepalive: strPtrs } = allocNativeStringArray(values as string[]);
+      const { table, keepalive: strPtrs } = allocNativeStringArray(
+        values.map((v) => (v === null ? null : (v as string))),
+      );
       keepalive.push(table, ...strPtrs);
       dataPtrs.push(table.ptr);
-    } else if (typeof values[0] === "number") {
-      const allIntegers = (values as number[]).every((v) => Number.isInteger(v));
-      if (allIntegers) {
+    } else if (typeof first === "number") {
+      const nonNull = values.filter((v) => v !== null) as number[];
+      const sanitized = values.map((v) => (v === null ? 0 : (v as number)));
+      if (nonNull.every((v) => Number.isInteger(v))) {
         typesDv.setInt32(c * 4, DATA_TYPE_INTEGER, true);
-        const p = allocNativeInt64(values as number[]);
+        const p = allocNativeInt64(sanitized);
         keepalive.push(p);
         dataPtrs.push(p.ptr);
       } else {
         typesDv.setInt32(c * 4, DATA_TYPE_FLOAT, true);
-        const p = allocNativeFloat64(values as number[]);
+        const p = allocNativeFloat64(sanitized);
         keepalive.push(p);
         dataPtrs.push(p.ptr);
       }
+    } else {
+      throw new QuiverError(
+        `Cannot updateTimeSeriesGroup: column '${colName}' has unsupported value type ${typeof first}`,
+      );
     }
   }
 
@@ -246,6 +299,8 @@ Database.prototype.updateTimeSeriesGroup = function (
   keepalive.push(typesAlloc);
   const dataTable = allocNativePtrTable(dataPtrs);
   keepalive.push(dataTable);
+  const maskTable = allocNativePtrTable(maskPtrs);
+  keepalive.push(maskTable);
 
   check(
     lib.quiver_database_update_time_series_group(
@@ -256,6 +311,7 @@ Database.prototype.updateTimeSeriesGroup = function (
       namesTable.buf,
       typesAlloc.buf,
       dataTable.buf,
+      maskTable.buf,
       BigInt(columnCount),
       BigInt(rowCount),
     ),