quiverdb 0.9.16 → 0.10.0

package/README.md

@@ -99,6 +99,7 @@ bun run example.ts
 ### Read (IDs)
 
 - `readElementIds(collection)` -- Read all element IDs in a collection
+- `numberOfElements(collection)` -- Current number of elements in a collection
 
 ### Metadata
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "quiverdb",
-  "version": "0.9.16",
+  "version": "0.10.0",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -31,4 +31,4 @@
     "@biomejs/biome": "^2.4.6",
     "@types/bun": "latest"
   }
-}
+}

package/src/create.ts

@@ -8,6 +8,7 @@ import {
   readPtrOut,
   toCString,
 } from "./ffi-helpers.ts";
+import { type GroupColumns, updateGroupColumns } from "./group-columns.ts";
 import { getSymbols, type NativePointer } from "./loader.ts";
 import type { ElementData, Value } from "./types.ts";
 
@@ -38,7 +39,9 @@ function setElementArray(
     const allIntegers = (values as number[]).every((v) => Number.isInteger(v));
     if (allIntegers) {
       const arr = allocNativeInt64(values as number[]);
-      check(lib.quiver_element_set_array_integer(elemPtr, nameBuf.buf, arr.buf, values.length, null));
+      check(
+        lib.quiver_element_set_array_integer(elemPtr, nameBuf.buf, arr.buf, values.length, null),
+      );
     } else {
       const arr = allocNativeFloat64(values as number[]);
       check(lib.quiver_element_set_array_float(elemPtr, nameBuf.buf, arr.buf, values.length, null));
@@ -48,7 +51,9 @@ function setElementArray(
 
   if (typeof first === "string") {
     const { table, keepalive: _keepalive } = allocNativeStringArray(values as string[]);
-    check(lib.quiver_element_set_array_string(elemPtr, nameBuf.buf, table.buf, values.length, null));
+    check(
+      lib.quiver_element_set_array_string(elemPtr, nameBuf.buf, table.buf, values.length, null),
+    );
     return;
   }
 
@@ -148,3 +153,48 @@ Database.prototype.deleteElement = function (this: Database, collection: string,
   const collBuf = toCString(collection);
   check(lib.quiver_database_delete_element(this._handle, collBuf.buf, BigInt(id)));
 };
+
+/**
+ * Replace all of an element's rows in one *named* vector group, from column arrays keyed by name.
+ *
+ * Pass `{}` to clear the group. Prefer this over routing the group's columns through
+ * updateElement when a column name is shared by two groups of the collection (legal for foreign
+ * keys): (collection, group) names exactly one table, a column name alone does not, and
+ * updateElement writes an ambiguous column to every match.
+ */
+Database.prototype.updateVectorGroup = function (
+  this: Database,
+  collection: string,
+  group: string,
+  id: number,
+  data: GroupColumns,
+): void {
+  updateGroupColumns(
+    this._handle,
+    "updateVectorGroup",
+    getSymbols().quiver_database_update_vector_group,
+    collection,
+    group,
+    id,
+    data,
+  );
+};
+
+/** Set-group counterpart of updateVectorGroup. */
+Database.prototype.updateSetGroup = function (
+  this: Database,
+  collection: string,
+  group: string,
+  id: number,
+  data: GroupColumns,
+): void {
+  updateGroupColumns(
+    this._handle,
+    "updateSetGroup",
+    getSymbols().quiver_database_update_set_group,
+    collection,
+    group,
+    id,
+    data,
+  );
+};

package/src/database.ts

@@ -1,6 +1,7 @@
 import type { CsvOptions } from "./csv.ts";
 import { check, QuiverError } from "./errors.ts";
 import { allocPtrOut, makeDefaultOptions, readPtrOut, toCString } from "./ffi-helpers.ts";
+import type { GroupColumns } from "./group-columns.ts";
 import type { NativePointer } from "./loader.ts";
 import { getSymbols } from "./loader.ts";
 import type { GroupMetadata, ScalarMetadata } from "./metadata.ts";
@@ -103,6 +104,7 @@ export class Database {
     id: number,
   ) => string | null;
   declare readElementIds: (collection: string) => number[];
+  declare numberOfElements: (collection: string) => number;
   declare readVectorIntegers: (collection: string, attribute: string) => number[][];
   declare readVectorFloats: (collection: string, attribute: string) => number[][];
   declare readVectorStrings: (collection: string, attribute: string) => string[][];
@@ -154,6 +156,18 @@ export class Database {
     id: number,
     data: TimeSeriesData,
   ) => void;
+  declare updateVectorGroup: (
+    collection: string,
+    group: string,
+    id: number,
+    data: GroupColumns,
+  ) => void;
+  declare updateSetGroup: (
+    collection: string,
+    group: string,
+    id: number,
+    data: GroupColumns,
+  ) => void;
   declare upsertTimeSeriesRow: (
     collection: string,
     group: string,

package/src/group-columns.ts

@@ -0,0 +1,161 @@
+import { type Pointer, ptr } from "bun:ffi";
+import { check, QuiverError } from "./errors.ts";
+import {
+  allocNativeFloat64,
+  allocNativeInt64,
+  allocNativePtrTable,
+  allocNativeStringArray,
+  toCString,
+} from "./ffi-helpers.ts";
+import type { NativePointer } from "./loader.ts";
+import { DATA_TYPE_FLOAT, DATA_TYPE_INTEGER, DATA_TYPE_STRING, type Allocation } from "./types.ts";
+
+/** Column-oriented group payload: one array of cells per column name, `null` for SQL NULL. */
+export type GroupColumns = Record<string, (number | string | null)[]>;
+
+/**
+ * The parallel-array signature every columnar group update C function shares
+ * (quiver_database_update_{time_series,vector,set}_group).
+ */
+type ColumnUpdateFn = (
+  db: NativePointer,
+  collection: Uint8Array,
+  group: Uint8Array,
+  id: bigint,
+  names: Uint8Array | null,
+  types: Uint8Array | null,
+  data: Uint8Array | null,
+  masks: Uint8Array | null,
+  columnCount: bigint,
+  rowCount: bigint,
+) => number;
+
+/**
+ * Marshal a column-oriented payload and forward it to one of the columnar group update C
+ * functions. Shared by updateTimeSeriesGroup / updateVectorGroup / updateSetGroup: the three
+ * differ only in which C entry point they call.
+ *
+ * Pass `{}` (no columns) to clear the group.
+ */
+export function updateGroupColumns(
+  handle: NativePointer,
+  caller: string,
+  update: ColumnUpdateFn,
+  collection: string,
+  group: string,
+  id: number,
+  data: GroupColumns,
+): void {
+  const collBuf = toCString(collection);
+  const grpBuf = toCString(group);
+  const entries = Object.entries(data);
+
+  if (entries.length === 0) {
+    check(update(handle, collBuf.buf, grpBuf.buf, BigInt(id), null, null, null, null, 0n, 0n));
+    return;
+  }
+
+  const columnCount = entries.length;
+  const rowCount = entries[0][1].length;
+
+  // Validate before marshalling: a jagged column would desync the parallel arrays the C API
+  // reads against row_count, and a zero-length column (with columns present) would otherwise
+  // marshal a null data pointer. Named-but-empty columns are a caller mistake -- pass {} to
+  // clear the group instead. The C API rejects both too; failing here names the column.
+  for (const [name, values] of entries) {
+    if (values.length !== rowCount) {
+      throw new QuiverError(
+        `Cannot ${caller}: column '${name}' has length ${values.length} but expected ${rowCount}`,
+      );
+    }
+  }
+  if (rowCount === 0) {
+    const names = entries.map(([name]) => name).join(", ");
+    throw new QuiverError(
+      `Cannot ${caller}: columns [${names}] contain no rows; pass {} to clear the group`,
+    );
+  }
+
+  const keepalive: Allocation[] = [];
+
+  // Build column names as native string array
+  const colNames = entries.map(([name]) => name);
+  const { table: namesTable, keepalive: namesPtrs } = allocNativeStringArray(colNames);
+  keepalive.push(namesTable, ...namesPtrs);
+
+  // Build column types, data, and per-cell NULL masks. A null cell becomes mask 0 + a
+  // placeholder in the data array (the C API never reads it). An all-null column is tagged
+  // FLOAT with zeroed data — the type tag is ignored for masked-out cells.
+  const typesBuf = new Uint8Array(columnCount * 4);
+  const typesDv = new DataView(typesBuf.buffer);
+  const dataPtrs: (Pointer | null)[] = [];
+  const maskPtrs: (Pointer | null)[] = [];
+
+  for (let c = 0; c < columnCount; c++) {
+    const [colName, values] = entries[c];
+    const first = values.find((v) => v !== null);
+
+    // Mask via direct indexing — never a DataView, to avoid the documented
+    // .buffer-materialization pitfall between ptr() and the FFI call.
+    const maskBuf = new Uint8Array(rowCount);
+    for (let r = 0; r < rowCount; r++) maskBuf[r] = values[r] === null ? 0 : 1;
+    const maskAlloc: Allocation = { ptr: ptr(maskBuf), buf: maskBuf };
+    keepalive.push(maskAlloc);
+    maskPtrs.push(maskAlloc.ptr);
+
+    if (first === undefined) {
+      // All-null column
+      typesDv.setInt32(c * 4, DATA_TYPE_FLOAT, true);
+      const p = allocNativeFloat64(new Array(rowCount).fill(0));
+      keepalive.push(p);
+      dataPtrs.push(p.ptr);
+    } else if (typeof first === "string") {
+      typesDv.setInt32(c * 4, DATA_TYPE_STRING, true);
+      const { table, keepalive: strPtrs } = allocNativeStringArray(
+        values.map((v) => (v === null ? null : (v as string))),
+      );
+      keepalive.push(table, ...strPtrs);
+      dataPtrs.push(table.ptr);
+    } else if (typeof first === "number") {
+      const nonNull = values.filter((v) => v !== null) as number[];
+      const sanitized = values.map((v) => (v === null ? 0 : (v as number)));
+      if (nonNull.every((v) => Number.isInteger(v))) {
+        typesDv.setInt32(c * 4, DATA_TYPE_INTEGER, true);
+        const p = allocNativeInt64(sanitized);
+        keepalive.push(p);
+        dataPtrs.push(p.ptr);
+      } else {
+        typesDv.setInt32(c * 4, DATA_TYPE_FLOAT, true);
+        const p = allocNativeFloat64(sanitized);
+        keepalive.push(p);
+        dataPtrs.push(p.ptr);
+      }
+    } else {
+      throw new QuiverError(
+        `Cannot ${caller}: column '${colName}' has unsupported value type ${typeof first}`,
+      );
+    }
+  }
+
+  const typesAlloc: Allocation = { ptr: ptr(typesBuf), buf: typesBuf };
+  keepalive.push(typesAlloc);
+  const dataTable = allocNativePtrTable(dataPtrs);
+  keepalive.push(dataTable);
+  const maskTable = allocNativePtrTable(maskPtrs);
+  keepalive.push(maskTable);
+
+  check(
+    update(
+      handle,
+      collBuf.buf,
+      grpBuf.buf,
+      BigInt(id),
+      namesTable.buf,
+      typesAlloc.buf,
+      dataTable.buf,
+      maskTable.buf,
+      BigInt(columnCount),
+      BigInt(rowCount),
+    ),
+  );
+}

package/src/index.ts

@@ -11,6 +11,7 @@ import "./composites.ts";
 export type { CsvOptions } from "./csv.ts";
 export { Database } from "./database.ts";
 export { QuiverError } from "./errors.ts";
+export type { GroupColumns } from "./group-columns.ts";
 export { LUA_DB_API_REFERENCE } from "./lua-api.ts";
 export { LuaRunner } from "./lua-runner.ts";
 export type { GroupMetadata, ScalarMetadata } from "./metadata.ts";

package/src/loader.ts

@@ -81,6 +81,7 @@ const readSymbols = {
   quiver_database_read_set_floats_by_id: { args: [P, BUF, BUF, I64, P, P], returns: I32 },
   quiver_database_read_set_strings_by_id: { args: [P, BUF, BUF, I64, P, P], returns: I32 },
   quiver_database_read_element_ids: { args: [P, BUF, P, P], returns: I32 },
+  quiver_database_number_of_elements: { args: [P, BUF, P], returns: I32 },
 } as const;
 
 const querySymbols = {
@@ -134,6 +135,14 @@ const timeSeriesSymbols = {
     args: [P, BUF, BUF, I64, P, P, P, P, USIZE, USIZE],
     returns: I32,
   },
+  quiver_database_update_vector_group: {
+    args: [P, BUF, BUF, I64, P, P, P, P, USIZE, USIZE],
+    returns: I32,
+  },
+  quiver_database_update_set_group: {
+    args: [P, BUF, BUF, I64, P, P, P, P, USIZE, USIZE],
+    returns: I32,
+  },
   quiver_database_free_time_series_data: { args: [P, P, P, P, USIZE, USIZE], returns: I32 },
   quiver_database_has_time_series_files: { args: [P, BUF, P], returns: I32 },
   quiver_database_list_time_series_files_columns: { args: [P, BUF, P, P], returns: I32 },

package/src/lua-api.ts

@@ -129,6 +129,7 @@ datetime surface — there are no DateTime wrapper helpers, unlike Julia/Dart/Py
 db:is_healthy()                    -- boolean
 db:current_version()               -- integer (current migration version)
 db:path()                          -- string (database file path)
+db:number_of_elements(collection)  -- integer: how many elements the collection holds right now
 db:describe()                      -- string: whole-DB text report (returns it, does NOT print)
 db:describe_collection(collection) -- string: one collection's structure (text report)
 db:summarize_collection(collection)-- string: per-scalar null/non-null counts, low-cardinality
@@ -284,6 +285,36 @@ db:read_set_strings(collection, attribute)
 
 ---
 
+## Replace a whole vector or set group (column-oriented)
+
+\`update_vector_group\` / \`update_set_group\` replace **all** of one element's rows in one *named*
+group, taking the same column-oriented shape as the time-series writer:
+
+\`\`\`lua
+db:update_vector_group("Child", "refs", id, { parent_ref = { 1, 2, 3 } })
+db:update_set_group("Child", "parents", id, { parent_ref = { 1, 2 } })
+
+db:update_vector_group("Child", "refs", id, {})   -- clears the group
+\`\`\`
+
+Use these instead of routing a group's columns through \`update_element\` whenever a column name is
+shared by two groups of the collection (legal for foreign-key columns): \`update_element\` routes an
+array **by column name**, so it writes to *every* group table that has that column — silently
+rewriting groups you never named. \`(collection, group)\` names exactly one table.
+
+Rules:
+- **Row count is the largest index any column reaches.** Shorter or sparse columns write NULL in
+  the gaps, so \`nil\` holes from a read round-trip.
+- **\`{}\` (no columns) clears the group.** Naming a column whose array is empty is an error, not a
+  clear — a typo'd column name must not destroy data.
+- **\`id\` and \`vector_index\` are managed by the group** (the element and the row's position) and
+  are rejected if passed.
+- **Foreign-key columns accept a label string** and resolve it to the referenced id, exactly as in
+  \`create_element\` / \`update_element\`.
+- The element id must exist, same as \`update_element\` / \`delete_element\`.
+
+---
+
 ## Composite by-id reads (Lua convenience helpers)
 
 \`\`\`lua

package/src/read.ts

@@ -211,6 +211,14 @@ Database.prototype.readElementIds = function (this: Database, collection: string
   return result;
 };
 
+Database.prototype.numberOfElements = function (this: Database, collection: string): number {
+  const lib = getSymbols();
+  const collBuf = toCString(collection);
+  const outBuf = new Uint8Array(8);
+  check(lib.quiver_database_number_of_elements(this._handle, collBuf.buf, outBuf));
+  return Number(new DataView(outBuf.buffer).getBigInt64(0, true));
+};
+
 // --- Vector bulk reads ---
 
 function readBulkIntegers(

package/src/time-series.ts

@@ -17,6 +17,7 @@ import {
   readUint64Out,
   toCString,
 } from "./ffi-helpers.ts";
+import { updateGroupColumns } from "./group-columns.ts";
 import { getSymbols } from "./loader.ts";
 import {
   type Allocation,
@@ -188,133 +189,14 @@ Database.prototype.updateTimeSeriesGroup = function (
   id: number,
   data: TimeSeriesData,
 ): void {
-  const lib = getSymbols();
-  const collBuf = toCString(collection);
-  const grpBuf = toCString(group);
-  const entries = Object.entries(data);
-
-  if (entries.length === 0) {
-    check(
-      lib.quiver_database_update_time_series_group(
-        this._handle,
-        collBuf.buf,
-        grpBuf.buf,
-        BigInt(id),
-        null,
-        null,
-        null,
-        null,
-        0n,
-        0n,
-      ),
-    );
-    return;
-  }
-
-  const columnCount = entries.length;
-  const rowCount = entries[0][1].length;
-
-  // Validate before marshalling: a jagged column would desync the parallel
-  // arrays the C API reads against row_count, and a zero-length column (with
-  // columns present) would otherwise marshal a null data pointer that the C API
-  // dereferences. Named-but-empty columns are a caller mistake -- pass {} to
-  // clear the group instead.
-  for (const [name, values] of entries) {
-    if (values.length !== rowCount) {
-      throw new QuiverError(
-        `Cannot updateTimeSeriesGroup: column '${name}' has length ${values.length} but expected ${rowCount}`,
-      );
-    }
-  }
-  if (rowCount === 0) {
-    const names = entries.map(([name]) => name).join(", ");
-    throw new QuiverError(
-      `Cannot updateTimeSeriesGroup: columns [${names}] contain no rows; pass {} to clear the group`,
-    );
-  }
-
-  const keepalive: Allocation[] = [];
-
-  // Build column names as native string array
-  const colNames = entries.map(([name]) => name);
-  const { table: namesTable, keepalive: namesPtrs } = allocNativeStringArray(colNames);
-  keepalive.push(namesTable, ...namesPtrs);
-
-  // Build column types, data, and per-cell NULL masks. A null cell becomes
-  // mask 0 + a placeholder in the data array (the C API never reads it). An
-  // all-null column is tagged FLOAT with zeroed data — the type tag is ignored
-  // for masked-out cells.
-  const typesBuf = new Uint8Array(columnCount * 4);
-  const typesDv = new DataView(typesBuf.buffer);
-  const dataPtrs: (Pointer | null)[] = [];
-  const maskPtrs: (Pointer | null)[] = [];
-
-  for (let c = 0; c < columnCount; c++) {
-    const [colName, values] = entries[c];
-    const first = values.find((v) => v !== null);
-
-    // Mask via direct indexing — never a DataView, to avoid the documented
-    // .buffer-materialization pitfall between ptr() and the FFI call.
-    const maskBuf = new Uint8Array(rowCount);
-    for (let r = 0; r < rowCount; r++) maskBuf[r] = values[r] === null ? 0 : 1;
-    const maskAlloc: Allocation = { ptr: ptr(maskBuf), buf: maskBuf };
-    keepalive.push(maskAlloc);
-    maskPtrs.push(maskAlloc.ptr);
-
-    if (first === undefined) {
-      // All-null column
-      typesDv.setInt32(c * 4, DATA_TYPE_FLOAT, true);
-      const p = allocNativeFloat64(new Array(rowCount).fill(0));
-      keepalive.push(p);
-      dataPtrs.push(p.ptr);
-    } else if (typeof first === "string") {
-      typesDv.setInt32(c * 4, DATA_TYPE_STRING, true);
-      const { table, keepalive: strPtrs } = allocNativeStringArray(
-        values.map((v) => (v === null ? null : (v as string))),
-      );
-      keepalive.push(table, ...strPtrs);
-      dataPtrs.push(table.ptr);
-    } else if (typeof first === "number") {
-      const nonNull = values.filter((v) => v !== null) as number[];
-      const sanitized = values.map((v) => (v === null ? 0 : (v as number)));
-      if (nonNull.every((v) => Number.isInteger(v))) {
-        typesDv.setInt32(c * 4, DATA_TYPE_INTEGER, true);
-        const p = allocNativeInt64(sanitized);
-        keepalive.push(p);
-        dataPtrs.push(p.ptr);
-      } else {
-        typesDv.setInt32(c * 4, DATA_TYPE_FLOAT, true);
-        const p = allocNativeFloat64(sanitized);
-        keepalive.push(p);
-        dataPtrs.push(p.ptr);
-      }
-    } else {
-      throw new QuiverError(
-        `Cannot updateTimeSeriesGroup: column '${colName}' has unsupported value type ${typeof first}`,
-      );
-    }
-  }
-
-  const typesAlloc: Allocation = { ptr: ptr(typesBuf), buf: typesBuf };
-  keepalive.push(typesAlloc);
-  const dataTable = allocNativePtrTable(dataPtrs);
-  keepalive.push(dataTable);
-  const maskTable = allocNativePtrTable(maskPtrs);
-  keepalive.push(maskTable);
-
-  check(
-    lib.quiver_database_update_time_series_group(
-      this._handle,
-      collBuf.buf,
-      grpBuf.buf,
-      BigInt(id),
-      namesTable.buf,
-      typesAlloc.buf,
-      dataTable.buf,
-      maskTable.buf,
-      BigInt(columnCount),
-      BigInt(rowCount),
-    ),
+  updateGroupColumns(
+    this._handle,
+    "updateTimeSeriesGroup",
+    getSymbols().quiver_database_update_time_series_group,
+    collection,
+    group,
+    id,
+    data,
   );
 };