vectrojs 0.0.1 → 0.1.0

package/src/sort.mjs

@@ -0,0 +1,102 @@
+// Sorting helpers for tables (see Table.sort / Table.top).
+//
+// Sort keys are given as text: a column name or any expression, sorted from
+// small to large; a leading "-" reverses it ('-price', '-(price * qty)').
+// A category column sorts by its labels (A to Z, or Z to A with "-").
+// Several keys: ['region', '-revenue']. Ties keep table order (stable).
+// Missing values (NaN) always come last.
+
+const IDENT = /^\s*(-?)\s*([A-Za-z_][A-Za-z0-9_]*)\s*$/;
+
+// [{ text: expression to compute | null, column: category column | null, desc }]
+export function parseKeys(by, cols) {
+  const list = Array.isArray(by) ? by : [by];
+  if (!list.length) throw new Error('vectro: sort needs at least one key, e.g. "-price"');
+  return list.map((k) => {
+    if (typeof k !== 'string' || !k.trim()) throw new Error(`vectro: a sort key must be a column or expression, got ${JSON.stringify(k)}`);
+    const m = IDENT.exec(k);
+    if (m && cols.get(m[2])?.dict) return { column: m[2], desc: m[1] === '-', text: null };
+    if (m && cols.get(m[2])?.strings) throw new Error(`vectro: can't sort by free-text column "${m[2]}" (load it as a category)`);
+    return { column: null, desc: false, text: k };
+  });
+}
+
+// Ranks for a category column's codes: code -> position of its label A..Z.
+export function labelRanks(dict, desc) {
+  const order = dict.map((_, i) => i).sort((a, b) => (dict[a] < dict[b] ? -1 : dict[a] > dict[b] ? 1 : 0));
+  const rank = new Float64Array(dict.length);
+  order.forEach((code, r) => { rank[code] = desc ? -r : r; });
+  return rank;
+}
+
+// Stable sort of row indices 0..n-1 by several Float64Array keys (first key
+// most important): LSD radix sort, last key first. Each key becomes two
+// uint32 words that sort as unsigned numbers; the words travel with the
+// indices (so every pass reads in order instead of gathering at random), in
+// 11-bit digits (2,048 buckets fit in cache). All digit counts are taken in
+// one pass per key, and digits that are the same in every row are skipped,
+// so small integer keys take one or two passes instead of six. Keys that are
+// whole numbers within 2^53 of their minimum (codes, IDs, counts) sort as
+// that offset, so only the digits that vary cost a pass.
+const DIGITS = [[0, 0], [0, 11], [0, 22], [1, 0], [1, 11], [1, 22]]; // [word (0 = low), shift]
+export function sortIndex(keys, n) {
+  let idx = new Uint32Array(n);
+  for (let i = 0; i < n; i++) idx[i] = i;
+  if (n <= 4096) return smallSort(keys, idx); // a comparison sort is quicker than bucket passes
+  let lo = new Uint32Array(n), hi = new Uint32Array(n);
+  let idx2 = new Uint32Array(n), lo2 = new Uint32Array(n), hi2 = new Uint32Array(n);
+  const hist = new Uint32Array(6 * 2048);
+  for (let k = keys.length - 1; k >= 0; k--) {
+    // float64 -> two uint32 halves that sort as unsigned numbers, in the current order
+    const f = keys[k];
+    let min = Infinity, max = -Infinity, whole = true;
+    for (let i = 0; i < n; i++) { const v = f[i]; if (v < min) min = v; if (v > max) max = v; if (v !== Math.floor(v)) { whole = false; break; } }
+    whole = whole && max - min < 2 ** 53; // (NaN and infinities fail `whole`)
+    const bits = new Uint32Array(f.buffer, f.byteOffset, n * 2);
+    hist.fill(0);
+    for (let i = 0; i < n; i++) {
+      const r = idx[i];
+      let h, l;
+      if (whole) { const o = f[r] - min; h = Math.floor(o / 4294967296); l = o - h * 4294967296; }
+      else {
+        h = bits[2 * r + 1]; l = bits[2 * r];
+        if (f[r] !== f[r]) { h = 0xffffffff; l = 0xffffffff; } // NaN: after everything
+        else if (h & 0x80000000) { h = ~h >>> 0; l = ~l >>> 0; } // negative: flip all
+        else h = (h | 0x80000000) >>> 0;                         // positive: flip the sign
+      }
+      hi[i] = h; lo[i] = l;
+      hist[l & 2047]++; hist[2048 + ((l >>> 11) & 2047)]++; hist[4096 + (l >>> 22)]++;
+      hist[6144 + (h & 2047)]++; hist[8192 + ((h >>> 11) & 2047)]++; hist[10240 + (h >>> 22)]++;
+    }
+    for (let d = 0; d < 6; d++) {
+      const [word, shift] = DIGITS[d], base = d * 2048;
+      const src = word ? hi : lo;
+      if (hist[base + ((src[0] >>> shift) & 2047)] === n) continue; // every row has this digit
+      let sum = 0;
+      for (let b = 0; b < 2048; b++) { const c = hist[base + b]; hist[base + b] = sum; sum += c; }
+      for (let i = 0; i < n; i++) {
+        const p = hist[base + ((src[i] >>> shift) & 2047)]++;
+        idx2[p] = idx[i]; lo2[p] = lo[i]; hi2[p] = hi[i];
+      }
+      [idx, idx2] = [idx2, idx]; [lo, lo2] = [lo2, lo]; [hi, hi2] = [hi2, hi];
+    }
+  }
+  return idx;
+}
+
+// Stable comparison sort of row indices for small inputs (same order as the
+// radix sort: by each key in turn, NaN last, ties in table order).
+function smallSort(keys, idx) {
+  const order = Array.from(idx);
+  order.sort((a, b) => {
+    for (const f of keys) {
+      const x = f[a], y = f[b];
+      if (x === y) continue;
+      if (x !== x) { if (y !== y) continue; return 1; }
+      if (y !== y) return -1;
+      return x < y ? -1 : 1;
+    }
+    return a - b;
+  });
+  return Uint32Array.from(order);
+}

package/src/split-worker-core.mjs

@@ -0,0 +1,252 @@
+// Worker side of split tables. Each worker has its own single-thread engine
+// and owns slices of tables (no shared memory). The page sends requests
+// { rid, op, args } and gets back { rid, ok, value | error }.
+import { Engine } from './engine.mjs';
+import { Table } from './table.mjs';
+import { parse } from './fuse.mjs';
+import { createStreams } from './stream-worker.mjs';
+import { parseKeys } from './sort.mjs';
+import { searchTable } from './vectors.mjs';
+import { decodeOne } from './pack.mjs';
+import { SKETCH_SIZE, sketchAdd, hashNumber, hashText } from './sketch.mjs';
+
+export function serveSplitWorker(post, listen) {
+  let engine;
+  const tables = new Map(); // table id -> Table (this worker's slice)
+  const streams = createStreams(() => engine); // live streams (see stream-worker.mjs)
+
+  const table = (id) => {
+    const t = tables.get(id) ?? streams.view(id);
+    if (!t) throw new Error(`vectro: no table ${id} in this worker`);
+    return t;
+  };
+  // A filtered view: the page keeps filters as text and sends them with each query.
+  const view = (id, preds = []) => preds.reduce((t, p) => t.filter(p.expr, p.params), table(id));
+  const dictsOf = (t) => Object.fromEntries([...t.cols].filter(([, c]) => c.dict).map(([name, c]) => [name, c.dict]));
+
+  // A worker's rows come in segments (blocks of the original table, in this
+  // worker's order); the page needs rows per segment to put rows from all
+  // workers back in table order. With filters, the compiled count(filter)
+  // runs on each segment's rows. With `limit`, counting stops once that many
+  // rows matched: later segments can't be among the first `limit` rows.
+  function segmentCounts(id, preds, segs, limit = Infinity) {
+    if (!preds?.length) return Float64Array.from(segs);
+    const v = view(id, preds), t = table(id);
+    const used = new Set();
+    (function walk(node) {
+      if (node.t === 'var') used.add(node.name);
+      for (const k of ['x', 'y', 'c', 'l', 'r']) if (node[k]) walk(node[k]);
+      if (node.args) node.args.forEach(walk);
+    })(parse(v.pred));
+    const columns = [...used].filter((name) => t.cols.has(name)).map((name) => [name, t.cols.get(name)]);
+    const scalars = Object.fromEntries([...used].filter((name) => !t.cols.has(name)).map((name) => [name, v.params[name]]));
+    const src = `count(${v.pred})`;
+    const out = [];
+    let r = 0, total = 0;
+    for (let k = 0; k < segs.length && total < limit; k++) {
+      const vars = { ...scalars };
+      for (const [name, c] of columns) { // (memory may have grown: live views)
+        const part = engine.live(c.data).subarray(r, r + segs[k]);
+        vars[name] = c.enc ? { data: part, enc: c.enc, n: segs[k], type: t.type === Float32Array ? 'f32' : 'f64' } : part;
+      }
+      const c = segs[k] ? engine.eval(src, vars) : 0;
+      out.push(c);
+      total += c;
+      r += segs[k];
+    }
+    return Float64Array.from(out);
+  }
+
+  const handlers = {
+    init({ engineOptions }) {
+      engine = new Engine(engineOptions);
+      return { jit: engine.jit };
+    },
+    // A slice of columns. Text columns get this worker's own dictionary; the
+    // page merges them and sends `remap` so every worker uses the same codes.
+    load({ id, columns }) {
+      const t = engine.table(columns);
+      tables.set(id, t);
+      return { n: t.length, dicts: dictsOf(t), names: t.columnNames };
+    },
+    // Columns already encoded by the page: numbers, plus text as codes into a
+    // shared dictionary (so nothing needs renumbering afterwards).
+    loadEncoded({ id, order, numbers, text, float32 = false, dates = [] }) {
+      const T = float32 ? Float32Array : Float64Array; // one number type per table
+      const base = Object.keys(numbers).length ? engine.table(numbers, { type: T }) : null;
+      const cols = new Map();
+      for (const name of order) {
+        if (text[name]) {
+          const { codes, dict } = text[name];
+          const data = engine.alloc(T, codes.length);
+          data.set(codes);
+          cols.set(name, { data, dict, codes: new Map(dict.map((v, i) => [v, i])), strings: null });
+        } else {
+          cols.set(name, base.cols.get(name));
+        }
+      }
+      for (const c of cols.values()) c.data = engine.live(c.data); // later allocations may have moved memory
+      for (const name of dates) cols.get(name).date = true;
+      const n = base ? base.length : (Object.values(text)[0]?.codes.length ?? 0);
+      const t = new Table(engine, cols, n, { owned: [...order] });
+      tables.set(id, t);
+      return { n: t.length, names: t.columnNames };
+    },
+    // CSV: a header plus this worker's pieces of the file, back to back.
+    // `pieces` are their byte lengths; returns the rows in each piece.
+    loadCSV({ id, bytes, opts, pieces }) {
+      let segs = null;
+      const t = engine.parseCSV(bytes, { ...opts, pieces, onPieceRows: (rows) => { segs = rows; } });
+      tables.set(id, t);
+      return { n: t.length, segs, dicts: dictsOf(t), names: t.columnNames, strings: [...t.cols].filter(([, c]) => c.strings).map(([n]) => n), dates: [...t.cols].filter(([, c]) => c.date).map(([n]) => n) };
+    },
+    // Switch text columns to the shared dictionary: code -> lut[code].
+    remap({ id, maps }) {
+      const t = table(id);
+      let repack = false;
+      for (const [name, { lut, dict }] of Object.entries(maps)) {
+        const c = t.cols.get(name);
+        if (c.enc) { // packed codes: unpack, renumber, pack again below
+          const plain = engine.alloc(c.plainType, t.length), codes = engine.live(c.data);
+          for (let i = 0; i < t.length; i++) plain[i] = decodeOne(c.enc, codes[i], c.plainType === Float32Array);
+          engine.free(c.data);
+          c.data = plain;
+          delete c.enc; delete c.plainType; delete c.rows;
+          repack = true;
+        }
+        const data = engine.live(c.data);
+        for (let i = 0; i < t.length; i++) data[i] = lut[data[i]];
+        c.dict = dict;
+        c.codes = new Map(dict.map((s, i) => [s, i]));
+      }
+      if (repack) t.compress();
+      return true;
+    },
+    agg({ id, preds, spec, params }) {
+      return view(id, preds).agg(spec, params);
+    },
+    // Partial group results, as columns: one per key (labels for text keys)
+    // + one array per aggregate.
+    groupBy({ id, preds, keys, spec, params }) {
+      const g = view(id, preds).groupBy(keys).agg(spec, params);
+      try {
+        const keyCols = keys.map((key) => {
+          const k = g.cols.get(key);
+          const codes = engine.live(k.data);
+          return k.dict ? Array.from(codes, (c) => k.dict[c]) : Float64Array.from(codes);
+        });
+        const cols = {};
+        for (const out of Object.keys(spec)) cols[out] = Float64Array.from(engine.live(g.cols.get(out).data));
+        return { keyCols, cols };
+      } finally {
+        g.dispose();
+      }
+    },
+    collect({ id, preds, newId, segs }) {
+      const counts = segmentCounts(id, preds, segs);
+      const t = view(id, preds).collect();
+      tables.set(newId, t);
+      return { n: t.length, segs: counts };
+    },
+    derive({ id, newId, name, expr, params }) {
+      const t = table(id).derive(name, expr, params);
+      tables.set(newId, t);
+      return t.length;
+    },
+    // This worker's first k rows by `by`, with each expression key's values
+    // (text keys are compared by label on the page).
+    top({ id, preds, k, by, columns = null }) {
+      const v = view(id, preds);
+      const t = (columns ? v.select(columns) : v).top(k, by);
+      try {
+        const keys = parseKeys(by, t.cols);
+        const values = keys.map((key) => {
+          if (key.column) return null;
+          const d = t.derive('__np_k', key.text);
+          try { return Float64Array.from(engine.live(d.cols.get('__np_k').data)); } finally { d.dispose(); }
+        });
+        return { rows: t.toRows(), keys: keys.map((key) => (key.column ? { label: key.column, desc: key.desc } : null)), values };
+      } finally {
+        t.dispose();
+      }
+    },
+    // Vector search over this worker's rows (see vectors.mjs).
+    vectorTop({ id, preds, query, opts }) {
+      return searchTable(view(id, preds), query, opts);
+    },
+    // Distinct (group..., value) pairs of this worker's rows, or with sketch
+    // one distinct-count sketch per group (the page merges them).
+    distinct({ id, preds, keys, arg, params, sketch }) {
+      const v = view(id, preds);
+      const plain = /^[A-Za-z_]\w*$/.test(arg.trim()) && v.cols.has(arg.trim()) && !keys.includes(arg.trim()); // (a key: count a copy)
+      const t = plain ? v : v.derive('__np_dv', arg, params);
+      const name = plain ? arg.trim() : '__np_dv';
+      const g = t.groupBy([...keys, name]).agg({ __np_n: 'count()' });
+      try {
+        const colOf = (k) => { const c = g.cols.get(k), d = engine.live(c.data); return c.dict ? Array.from(d, (x) => c.dict[x]) : Float64Array.from(d); };
+        let keyCols = keys.map(colOf), values = colOf(name);
+        if (Array.isArray(values) && values.includes('')) { // (missing text: not a value to count)
+          const keep = values.map((x) => x !== '');
+          keyCols = keyCols.map((c) => c.filter((_, r) => keep[r]));
+          values = values.filter((_, r) => keep[r]);
+        }
+        if (!sketch) return { keyCols, values };
+        const index = new Map(), groupCols = keys.map(() => []), regs = [];
+        for (let r = 0; r < values.length; r++) {
+          const tuple = keyCols.map((c) => c[r]);
+          const id = tuple.map((x) => (typeof x === 'string' ? JSON.stringify(x) : String(x))).join(',');
+          let at = index.get(id);
+          if (at === undefined) { at = index.size; index.set(id, at); tuple.forEach((x, k) => groupCols[k].push(x)); regs.push(new Uint8Array(SKETCH_SIZE)); }
+          const x = values[r];
+          if (typeof x === 'string') sketchAdd(regs[at], 0, hashText(x));
+          else if (x === x) sketchAdd(regs[at], 0, hashNumber(x));
+        }
+        const sketches = new Uint8Array(regs.length * SKETCH_SIZE);
+        regs.forEach((r, i) => sketches.set(r, i * SKETCH_SIZE));
+        return { keyCols: groupCols, sketches };
+      } finally {
+        g.dispose();
+        if (!plain) t.dispose();
+      }
+    },
+    columns({ id, preds, segs }) {
+      return { columns: view(id, preds).toColumns(), segs: segmentCounts(id, preds, segs) };
+    },
+    toRows({ id, preds, limit, segs }) {
+      return { rows: view(id, preds).toRows(limit), segs: segmentCounts(id, preds, segs, limit ?? Infinity) };
+    },
+    ...streams.handlers,
+    streamSnapshot: (args) => streams.handlers.streamSnapshot({ ...args, tables }),
+    dispose({ id }) {
+      if (streams.close(id)) return true;
+      tables.get(id)?.dispose();
+      tables.delete(id);
+      return true;
+    },
+  };
+
+  listen(async ({ rid, op, args }) => {
+    try {
+      const value = await handlers[op](args);
+      post({ rid, ok: true, value }, transferList(value));
+    } catch (err) {
+      post({ rid, ok: false, error: (globalThis.process?.env?.VECTRO_DEBUG ? err?.stack : err?.message) ?? String(err) });
+    }
+  });
+}
+
+// Typed arrays in a group result move to the page without copying.
+function transferList(value) {
+  const out = [];
+  if (value && typeof value === 'object') {
+    if (ArrayBuffer.isView(value.keys)) out.push(value.keys.buffer);
+    if (value.keyCols) for (const k of value.keyCols) if (ArrayBuffer.isView(k)) out.push(k.buffer);
+    if (ArrayBuffer.isView(value.sketches)) out.push(value.sketches.buffer);
+    if (ArrayBuffer.isView(value.values)) out.push(value.values.buffer);
+    if (value.cols) for (const c of Object.values(value.cols)) out.push(c.buffer);
+    if (ArrayBuffer.isView(value.segs)) out.push(value.segs.buffer);
+    if (value.columns) for (const c of Object.values(value.columns)) if (ArrayBuffer.isView(c) || ArrayBuffer.isView(c.codes)) out.push((c.codes ?? c).buffer);
+    if (Array.isArray(value.values)) for (const v of value.values) if (ArrayBuffer.isView(v)) out.push(v.buffer);
+  }
+  return out;
+}

package/src/split-worker-node.mjs

@@ -0,0 +1,8 @@
+// Node worker thread for split tables (see split-worker-core.mjs).
+import { parentPort } from 'node:worker_threads';
+import { serveSplitWorker } from './split-worker-core.mjs';
+
+serveSplitWorker(
+  (msg, transfer) => parentPort.postMessage(msg, transfer),
+  (handle) => parentPort.on('message', handle),
+);

package/src/split-worker.mjs

@@ -0,0 +1,7 @@
+// Browser Web Worker for split tables (see split-worker-core.mjs).
+import { serveSplitWorker } from './split-worker-core.mjs';
+
+serveSplitWorker(
+  (msg, transfer) => self.postMessage(msg, transfer),
+  (handle) => { self.onmessage = (ev) => handle(ev.data); },
+);