quiverdb 0.13.0 → 0.13.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "quiverdb",
-  "version": "0.13.0",
+  "version": "0.13.2",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -20,6 +20,7 @@
   ],
   "scripts": {
     "test": "bun test test",
+    "typecheck": "tsc --noEmit",
     "lint": "biome check",
     "format": "biome format --write"
   },
@@ -29,6 +30,7 @@
   },
   "devDependencies": {
     "@biomejs/biome": "^2.4.6",
-    "@types/bun": "latest"
+    "@types/bun": "latest",
+    "typescript": "7.0.2"
   }
 }

package/src/group-columns.ts

@@ -42,11 +42,11 @@ export type TimeSeriesData = Record<string, (number | string | null)[]>;
  * (quiver_database_update_{time_series,vector,set}_group and their _by_label forms). The 4th
  * argument addresses the element: an id for the by-id forms, a NUL-terminated label otherwise.
  */
-type ColumnUpdateFn = (
+type ColumnUpdateFn<Key extends number | string> = (
   db: NativePointer,
   collection: Uint8Array,
   group: Uint8Array,
-  key: bigint | Uint8Array,
+  key: Key extends string ? Uint8Array : bigint,
   names: Uint8Array | null,
   types: Uint8Array | null,
   data: Uint8Array | null,
@@ -90,18 +90,20 @@ export function numericCells(
  *
  * Pass `{}` (no columns) to clear the group.
  */
-export function updateGroupColumns(
+export function updateGroupColumns<Key extends number | string>(
   handle: NativePointer,
   caller: string,
-  update: ColumnUpdateFn,
+  update: ColumnUpdateFn<Key>,
   collection: string,
   group: string,
-  key: number | string,
+  key: Key,
   data: GroupColumns,
 ): void {
   const collBuf = toCString(collection);
   const grpBuf = toCString(group);
-  const keyArg = typeof key === "string" ? toCString(key).buf : BigInt(key);
+  const keyArg = (typeof key === "string" ? toCString(key).buf : BigInt(key)) as Key extends string
+    ? Uint8Array
+    : bigint;
   const entries = Object.entries(data);
 
   if (entries.length === 0) {

package/src/loader.ts

@@ -288,7 +288,17 @@ function initLibrary(): QuiverLib {
     }
   }
 
-  // Tier 2: Dev mode -- walk up directories looking for build/bin/.
+  // Tier 2: Standalone Bun executables ship native siblings beside the executable.
+  const executableDir = dirname(process.execPath);
+  if (existsSync(join(executableDir, C_API_LIB))) {
+    try {
+      return openLibrary(executableDir);
+    } catch (e) {
+      lastError = e;
+    }
+  }
+
+  // Tier 3: Dev mode -- walk up directories looking for build/bin/.
   for (const dir of getSearchPaths()) {
     try {
       return openLibrary(dir);
@@ -297,7 +307,7 @@ function initLibrary(): QuiverLib {
     }
   }
 
-  // Tier 3: System PATH fallback -- the core lib is expected to be discoverable
+  // Tier 4: System PATH fallback -- the core lib is expected to be discoverable
   // on PATH alongside the C API lib.
   try {
     return dlopen(C_API_LIB, allSymbols);
@@ -307,6 +317,7 @@ function initLibrary(): QuiverLib {
 
   const searched = [
     join(__dirname, "..", "libs", platformKey),
+    executableDir,
     ...getSearchPaths(),
     "system PATH",
   ].join(", ");

package/src/sandbox-api.ts

@@ -103,12 +103,14 @@ midnight.
   and \`dofile\`/\`loadfile\` are removed (string-form \`load\` stays for source text; a precompiled
   binary chunk is refused). Integer division is the Lua 5.4 \`//\` operator — a language operator,
   unrelated to \`math\`. No \`io\` does **not** mean
-  a data file on disk is out of reach: read it with \`db:read_csv\` / \`db:read_csv_stream\` (see
-  the CSV file reading section below). Never copy, paste, or re-type a data file's contents into
+  a data file on disk is out of reach: use \`db:read_csv\` / \`db:read_csv_stream\` for CSV or
+  \`db:read_xlsx\` / \`db:read_xlsx_stream\` for XLSX (see the file reading sections below).
+  Never copy, paste, or re-type a data file's contents into
   the script as literals — read the file.
 - **Filesystem sandbox.** Every file-touching operation (\`db:export_csv\`, \`db:import_csv\`,
   \`db:open_file\`, \`db:bin_to_csv\`, \`db:csv_to_bin\`, \`db:validate_migrations\`, \`db:read_csv\`,
-  \`db:read_csv_stream\`, \`db:write_csv\`, \`save\` on a file or an expression) resolves
+  \`db:read_csv_stream\`, \`db:write_csv\`, \`db:read_xlsx\`, \`db:read_xlsx_stream\`,
+  \`save\` on a file or an expression) resolves
   relative paths against the directory containing the database file and rejects anything outside it
   (subdirectories are fine; \`..\` escapes and outside absolute paths throw \`Cannot <op>: path '...' escapes the
   database directory ...\`). On an in-memory database these operations throw
@@ -325,10 +327,11 @@ Notes:
   \`{ x = nil }\` is identical to \`{}\`; an update/create table that ends up with no attributes
   **throws** (\`...must have at least one scalar attribute\` on create, \`...at least one attribute
   to update\` on update). To leave a column unchanged, omit the key — you cannot set a scalar to
-  NULL via the element table. (\`nil\` → NULL is accepted by \`update_relation\` and in query params
-  (not a trailing one), and as a cell in the group writers; \`upsert_time_series_row\` and
-  \`update_time_series_files\` replace the whole row, so a column you leave out (or set to \`nil\`,
-  the same thing) is cleared — see Time series and Time series files.)
+  NULL via the element table — nor via \`update_time_series_files\`, keyed by column name too,
+  where an omitted column keeps its current value. (\`nil\` → NULL is accepted by
+  \`update_relation\` and in query params (not a trailing one), and as a cell in the group writers;
+  \`upsert_time_series_row\` replaces the whole row, so a column you leave out (or set to \`nil\`,
+  the same thing) is cleared — see Time series.)
 - **\`update_relation\` points one scalar foreign-key relation at another element**, named by the
   target's label. The column is derived from the naming convention —
   \`lowercase(collection_to) .. "_" .. relation_type\`, so
@@ -557,15 +560,12 @@ singleton table):
 db:has_time_series_files(collection)              -- boolean
 db:list_time_series_files_columns(collection)     -- { "data_file", "metadata_file", ... }
 db:read_time_series_files(collection)             -- { data_file = "path", metadata_file = nil, ... }
-db:update_time_series_files(collection, { data_file = "path/to/data.bin" })  -- metadata_file is cleared
+db:update_time_series_files(collection, { data_file = "path/to/data.bin" })
 \`\`\`
 
-\`update_time_series_files\` **replaces the whole row**: every column you do not give a string is
-set to NULL (or to its DEFAULT, if the schema declares one), and in Lua a \`nil\` value and a
-missing key are the same thing. A table with no string values (\`{}\`, or only \`nil\` values)
-changes nothing and clears nothing. A value that is not a string throws. To change one path, read
-the row with \`db:read_time_series_files(collection)\`, change that one entry, and pass the whole
-table back.
+Only the columns you name are written, so the call above leaves \`metadata_file\` alone. A column
+cannot be cleared from Lua here: \`{ metadata_file = nil }\` is \`{}\`, which names nothing — so
+feeding a \`read_time_series_files\` result straight back in is a no-op.
 
 ---
 
@@ -703,7 +703,7 @@ own) — it throws \`Cannot import_csv: transaction already active\`. Call it ou
 
 ## CSV file reading
 
-Read a CSV file from disk directly into Lua — the only way to get file data into a script, since
+Read a CSV file from disk directly into Lua, since
 \`io\` is deliberately absent from the sandbox. \`path\` is sandboxed the same way as every other
 file-touching operation (see Critical rules).
 
@@ -789,6 +789,47 @@ end
 
 ---
 
+## XLSX file reading
+
+Read one worksheet of an XLSX workbook through the same filesystem sandbox as CSV. XLSX only:
+legacy XLS and encrypted workbooks are unsupported. These methods read cells; they do not write
+the workbook or import rows into the database.
+
+\`\`\`lua
+local data = db:read_xlsx("input.xlsx", { sheet = "Data", header_row = 1 })
+-- data.header: array of strings; data.rows: array of arrays of strings
+local count = db:read_xlsx_stream("input.xlsx", function(row, index, header)
+    -- header is available even on the first callback; index counts emitted data rows.
+    -- Only an explicit false stops. No return, nil, 0 and strings all continue.
+end, { sheet = 2 })
+\`\`\`
+
+The optional options table accepts exactly \`sheet\` and \`header_row\`. \`sheet\` is an exact
+worksheet name or a positive 1-based integer, default \`1\`. Chart sheets do not count; hidden
+worksheets do. Missing worksheets throw. \`header_row\` is the physical worksheet row number,
+default \`1\`; rows above it are skipped. A missing or completely blank header throws.
+\`header_row = 0\` reads all data rows with no header (\`data.header\` and the callback header are
+\`nil\`). Omit options, pass \`nil\`, or pass \`{}\` to use defaults; unknown keys and wrong
+types throw \`Cannot <operation>: ...\`.
+
+Every cell is a string. Text, including leading zeros, Unicode, whitespace and rich-text content,
+is preserved. Numbers keep their stored locale-independent text, booleans are \`"1"\`/\`"0"\`,
+blank cells are \`""\`, and Excel errors such as \`#DIV/0!\` stay text. Formatting is ignored:
+an Excel date stored as \`45292.5\` remains \`"45292.5"\`, and an explicit date stored as text
+stays text. Formulas return their saved cached value, never the expression or a recalculated value.
+A formula with no cache throws, naming the sheet and cell; recalculate and save it in Excel first.
+
+Completely blank data rows are skipped. Rows and headers are padded with \`""\` to the worksheet's
+last column containing a value or formula, so column positions stay aligned; formatting-only
+cells do not expand that width. Merged cells contribute only their stored values (normally the
+top-left cell); values are not copied across the merged area. Header-only sheets return no data
+rows; an empty sheet is readable with \`header_row = 0\`.
+
+Both forms use the same reader. The callback form returns the number of rows delivered, including
+the row that returned false; callback errors propagate unchanged and handles close on every exit.
+Unlike CSV streaming, XLSX callbacks avoid the full Lua rows table but still load worksheet XML
+and shared strings into memory. For very large files, use CSV streaming instead.
+
 ## CSV file writing
 
 Write a CSV file to disk — the only way to get data out of a script onto disk, since \`io\` is

package/src/time-series.ts

@@ -149,11 +149,11 @@ Database.prototype.updateTimeSeriesGroupByLabel = function (
  * (quiver_database_upsert_time_series_row and its _by_label form). `key` is an id for the by-id
  * form, a NUL-terminated label otherwise.
  */
-type UpsertRowFn = (
+type UpsertRowFn<Key extends number | string> = (
   db: NativePointer,
   collection: Uint8Array,
   group: Uint8Array,
-  key: bigint | Uint8Array,
+  key: Key extends string ? Uint8Array : bigint,
   names: Uint8Array | null,
   types: Uint8Array | null,
   data: Uint8Array | null,
@@ -165,18 +165,20 @@ type UpsertRowFn = (
  * C functions. Shared by upsertTimeSeriesRow / upsertTimeSeriesRowByLabel: they differ only in
  * which C entry point they call and whether `key` is an id or a label.
  */
-function upsertRowColumns(
+function upsertRowColumns<Key extends number | string>(
   handle: NativePointer,
   caller: string,
-  upsert: UpsertRowFn,
+  upsert: UpsertRowFn<Key>,
   collection: string,
   group: string,
-  key: number | string,
+  key: Key,
   row: Record<string, number | bigint | string | boolean>,
 ): void {
   const collBuf = toCString(collection);
   const grpBuf = toCString(group);
-  const keyArg = typeof key === "string" ? toCString(key).buf : BigInt(key);
+  const keyArg = (typeof key === "string" ? toCString(key).buf : BigInt(key)) as Key extends string
+    ? Uint8Array
+    : bigint;
   const entries = Object.entries(row);
   const columnCount = entries.length;
   const keepalive: Allocation[] = [];