vectrojs 0.0.1 → 0.1.1

package/src/window.mjs

@@ -0,0 +1,160 @@
+// Window functions over rows already sorted by (partition keys, order keys):
+// each function gives one value per row, from the rows of its partition in
+// its frame. Plain loops, one pass each (running and moving sums from prefix
+// sums, moving min / max with a monotonic queue). Missing values (NaN) follow
+// SQL: sum / avg / min / max skip them, count(x) counts values present.
+//
+// spec items: { fn, arg (Float64Array | null), frame, offset, def, k, p }
+//   fn: row_number rank dense_rank percent_rank cume_dist ntile
+//       lag lead first_value last_value nth_value
+//       sum avg min max count (count with arg null: rows)
+//       percentile_cont percentile_disc (p; arg sorted within the partition)
+//   frame: { unit: 'rows' | 'range', start, end } with start / end each
+//       { kind: 'unbounded' | 'preceding' | 'following' | 'current', n }
+//       (null: SQL's default: with ORDER BY, RANGE from the partition start to
+//       the current row and its ties; without, the whole partition)
+
+// Partition starts: part[i] = first row of row i's partition; peers likewise
+// for rows with equal order keys inside a partition (peers: false skips them,
+// for functions that don't look at ties).
+export function bounds(n, partKeys, orderKeys, peers = true) {
+  const newPart = new Uint8Array(n), partStart = new Int32Array(n), partEnd = new Int32Array(n);
+  if (n) newPart[0] = 1;
+  for (const k of partKeys) markChanges(k, newPart, n);
+  spans(newPart, partStart, partEnd, n);
+  if (!peers) return { partStart, partEnd, peerStart: partStart, peerEnd: partEnd };
+  const newPeer = newPart.slice(), peerStart = new Int32Array(n), peerEnd = new Int32Array(n);
+  for (const k of orderKeys) markChanges(k, newPeer, n);
+  spans(newPeer, peerStart, peerEnd, n);
+  return { partStart, partEnd, peerStart, peerEnd };
+}
+
+// mark[i] = 1 where k[i] differs from k[i - 1] (NaN equals NaN here)
+function markChanges(k, mark, n) {
+  for (let i = 1; i < n; i++) { const a = k[i], b = k[i - 1]; if (a !== b && (a === a || b === b)) mark[i] = 1; }
+}
+
+// Each row's run [start, end) from the marks of where runs begin.
+function spans(mark, start, end, n) {
+  let s = 0;
+  for (let i = 1; i <= n; i++) {
+    if (i === n || mark[i]) {
+      for (let j = s; j < i; j++) { start[j] = s; end[j] = i; }
+      s = i;
+    }
+  }
+}
+
+// Whether some item reads peers (rows with equal order keys): ranks, and
+// frames by RANGE (the default with ORDER BY counts the current row's ties).
+const NO_PEERS = new Set(['row_number', 'ntile', 'lag', 'lead', 'percentile_cont', 'percentile_disc']);
+export const readsPeers = (items) => items.some((it) => !NO_PEERS.has(it.fn) && it.frame?.unit !== 'rows');
+
+// Each row's frame [lo[i], hi[i]) (row indexes, within its partition).
+// orderKey: the single ORDER BY key, for RANGE with distances (desc: it falls).
+function frames(n, b, frame, hasOrder, orderKey, desc) {
+  const lo = new Int32Array(n), hi = new Int32Array(n);
+  const f = frame ?? (hasOrder ? { unit: 'range', start: { kind: 'unbounded' }, end: { kind: 'current' } } : { unit: 'rows', start: { kind: 'unbounded' }, end: { kind: 'unbounded' } });
+  const { partStart: ps, partEnd: pe, peerStart: qs, peerEnd: qe } = b;
+  const byValue = f.unit === 'range' && (f.start.n !== undefined || f.end.n !== undefined);
+  if (byValue && !orderKey) throw new Error('vectro: RANGE with a distance needs exactly one ORDER BY key, a number or date');
+  for (let i = 0; i < n; i++) {
+    for (const [e, isEnd] of [[f.start, false], [f.end, true]]) {
+      let at;
+      if (e.kind === 'unbounded') at = isEnd ? pe[i] : ps[i];
+      else if (f.unit === 'rows') {
+        const j = e.kind === 'current' ? i : e.kind === 'preceding' ? i - e.n : i + e.n;
+        at = Math.min(pe[i], Math.max(ps[i], isEnd ? j + 1 : j));
+      } else if (e.kind === 'current') at = isEnd ? qe[i] : qs[i]; // (RANGE: the current row and its ties)
+      else continue; // (RANGE n PRECEDING / FOLLOWING: by value, below)
+      if (isEnd) hi[i] = at; else lo[i] = at;
+    }
+  }
+  if (byValue) { // two pointers per partition over the keys, in their order
+    const sign = desc ? -1 : 1, off = (e) => (e.kind === 'preceding' ? -e.n : e.kind === 'following' ? e.n : 0);
+    for (let s = 0; s < n; s = pe[s]) {
+      const end = pe[s];
+      let a = s, z = s;
+      for (let i = s; i < end; i++) {
+        const v = sign * orderKey[i];
+        if (v !== v) { lo[i] = qs[i]; hi[i] = qe[i]; continue; } // (missing keys: their ties)
+        if (f.start.n !== undefined) { const t = v + off(f.start); while (a < end && sign * orderKey[a] < t) a++; lo[i] = a; }
+        if (f.end.n !== undefined) { const t = v + off(f.end); while (z < end && sign * orderKey[z] <= t) z++; hi[i] = z; }
+      }
+    }
+  }
+  for (let i = 0; i < n; i++) if (hi[i] < lo[i]) hi[i] = lo[i]; // (an empty frame)
+  return { lo, hi };
+}
+
+// The values of one window function, for every row.
+export function windowValues(n, b, item, hasOrder, orderKey, desc) {
+  const out = new Float64Array(n);
+  const { partStart: ps, partEnd: pe, peerStart: qs, peerEnd: qe } = b;
+  const x = item.arg;
+  switch (item.fn) {
+    case 'row_number': for (let i = 0; i < n; i++) out[i] = i - ps[i] + 1; return out;
+    case 'rank': for (let i = 0; i < n; i++) out[i] = qs[i] - ps[i] + 1; return out;
+    case 'dense_rank': for (let i = 0, r = 0; i < n; i++) { if (i === ps[i]) r = 1; else if (qs[i] === i) r++; out[i] = r; } return out;
+    case 'percent_rank': for (let i = 0; i < n; i++) { const m = pe[i] - ps[i]; out[i] = m > 1 ? (qs[i] - ps[i]) / (m - 1) : 0; } return out;
+    case 'cume_dist': for (let i = 0; i < n; i++) out[i] = (qe[i] - ps[i]) / (pe[i] - ps[i]); return out;
+    case 'ntile': for (let i = 0; i < n; i++) { // the first (m mod k) tiles get one row more
+      const m = pe[i] - ps[i], k = item.k, j = i - ps[i], small = Math.floor(m / k), big = m % k;
+      out[i] = j < big * (small + 1) ? Math.floor(j / (small + 1)) + 1 : big + Math.floor((j - big * (small + 1)) / Math.max(1, small)) + 1;
+    } return out;
+    case 'lag': case 'lead': {
+      const k = (item.fn === 'lag' ? -1 : 1) * (item.offset ?? 1);
+      for (let i = 0; i < n; i++) { const j = i + k; out[i] = j >= ps[i] && j < pe[i] ? x[j] : (item.def ?? NaN); }
+      return out;
+    }
+    case 'percentile_cont': case 'percentile_disc': { // over the partition's values present (sorted: the window orders by the value)
+      for (let s = 0; s < n; ) {
+        const e = pe[s];
+        let a = s, z = e; // present values: NaN sort last
+        while (z > a && x[z - 1] !== x[z - 1]) z--;
+        const m = z - a;
+        let v = NaN;
+        if (m) {
+          if (item.fn === 'percentile_disc') v = x[a + Math.max(0, Math.ceil(item.p * m) - 1)];
+          else { const r = item.p * (m - 1), lo = Math.floor(r); v = x[a + lo] + (r - lo) * ((x[a + Math.min(m - 1, lo + 1)]) - x[a + lo]); }
+        }
+        out.fill(v, s, e);
+        s = e;
+      }
+      return out;
+    }
+  }
+  // functions of the frame
+  const { lo, hi } = frames(n, b, item.frame, hasOrder, orderKey, desc);
+  switch (item.fn) {
+    case 'first_value': for (let i = 0; i < n; i++) out[i] = hi[i] > lo[i] ? x[lo[i]] : NaN; return out;
+    case 'last_value': for (let i = 0; i < n; i++) out[i] = hi[i] > lo[i] ? x[hi[i] - 1] : NaN; return out;
+    case 'nth_value': for (let i = 0; i < n; i++) { const j = lo[i] + item.k - 1; out[i] = j < hi[i] ? x[j] : NaN; } return out;
+    case 'sum': case 'avg': case 'count': {
+      // prefix sums of the values present, and of how many are present
+      const s = new Float64Array(n + 1), c = new Float64Array(n + 1);
+      for (let i = 0; i < n; i++) { const v = x ? x[i] : 1; const ok = v === v; s[i + 1] = s[i] + (ok ? v : 0); c[i + 1] = c[i] + (ok ? 1 : 0); }
+      for (let i = 0; i < n; i++) {
+        const cnt = c[hi[i]] - c[lo[i]], sum = s[hi[i]] - s[lo[i]];
+        out[i] = item.fn === 'count' ? cnt : item.fn === 'avg' ? (cnt ? sum / cnt : NaN) : cnt ? sum : NaN; // (sum of nothing: NULL)
+      }
+      if (item.fn === 'sum' && item.exactSmall !== false) { // short frames: add directly (prefix differences lose digits)
+        for (let i = 0; i < n; i++) if (hi[i] - lo[i] <= 64) { let t = 0, m = 0; for (let j = lo[i]; j < hi[i]; j++) { const v = x[j]; if (v === v) { t += v; m++; } } out[i] = m ? t : NaN; }
+      }
+      return out;
+    }
+    case 'min': case 'max': { // frames only move forward: a monotonic queue of candidates
+      const better = item.fn === 'min' ? (a, c) => a <= c : (a, c) => a >= c;
+      const q = new Int32Array(n);
+      let head = 0, tail = 0, added = 0, part = -1;
+      for (let i = 0; i < n; i++) {
+        if (ps[i] !== part) { part = ps[i]; head = tail = 0; added = ps[i]; }
+        while (added < hi[i]) { const v = x[added]; if (v === v) { while (tail > head && better(v, x[q[tail - 1]])) tail--; q[tail++] = added; } added++; }
+        while (tail > head && q[head] < lo[i]) head++;
+        out[i] = tail > head ? x[q[head]] : NaN;
+      }
+      return out;
+    }
+  }
+  throw new Error(`vectro: unknown window function ${item.fn}`);
+}

package/src/worker-browser.mjs

@@ -0,0 +1,21 @@
+// Browser Web Worker: runs the shared worker loop. Messages arrive only
+// between event-loop turns, so they are queued here and the loop awaits them
+// when a task needs one (pullAsync).
+import { workerLoop } from './worker-loop.mjs';
+
+self.onmessage = (first) => {
+  self.onmessage = null;
+  const data = first.data;
+  const { port } = data;
+  const inbox = [];
+  let wake = null;
+  port.onmessage = (ev) => {
+    inbox.push(ev.data);
+    if (wake) { const w = wake; wake = null; w(); }
+  };
+  workerLoop(data, {
+    post: (msg) => port.postMessage(msg),
+    pullSync(handle) { while (inbox.length) handle(inbox.shift()); },
+    pullAsync: () => new Promise((resolve) => { wake = resolve; }),
+  }).catch((err) => port.postMessage({ kind: 'error', message: err?.stack ?? String(err) }));
+};

package/src/worker-loop.mjs

@@ -0,0 +1,106 @@
+// The worker side of the engine, shared by Node (worker.mjs) and browsers
+// (worker-browser.mjs). A worker blocks on the control block between tasks
+// (Atomics.wait is allowed in workers everywhere) and only needs messages for
+// two things: newly compiled kernels and arrays outside engine memory.
+//
+// host = {
+//   post(msg)          send a message to the main thread
+//   pullSync(handle)   hand every message that has already arrived to handle()
+//   pullAsync()        browsers: a promise for the next message (a browser
+//                      worker only receives messages between event-loop turns);
+//                      Node: undefined (messages can be read synchronously)
+// }
+import { C, OP, WASM_MEMORY, SEQ_WRITING, MAX_ARGS, TAG_SHIFT, decodeArgs, runUnits, tagOf } from './kernels.mjs';
+
+export async function workerLoop({ ctrlBuf, scalarBuf, resultBuf, spin, module, memory, index }, host) {
+  const ctrl = new Int32Array(ctrlBuf);
+  const sc = new Float64Array(scalarBuf);
+  const res = new Float64Array(resultBuf);
+  const buffers = new Map(); // bufferId -> SharedArrayBuffer (arrays outside WASM memory)
+  const fusedModules = new Map(); // kernel id -> compiled WebAssembly.Module
+  const fusedKernels = new Map(); // kernel id -> this thread's instance of it
+
+  // Same compiled module, same shared memory as every other thread.
+  const W = new WebAssembly.Instance(module, { env: { memory } }).exports;
+
+  // memory.buffer is re-read each time so views see memory that grew since.
+  // The engine tells other threads about a grow asynchronously; grow(0) catches up.
+  function bufferFor(id, bytesNeeded) {
+    if (id !== WASM_MEMORY) return buffers.get(id);
+    if (memory.buffer.byteLength < bytesNeeded) memory.grow(0);
+    return memory.buffer;
+  }
+
+  const handle = (msg) => {
+    if (msg.kind === 'kernel') fusedModules.set(msg.id, msg.module);
+    else buffers.set(msg.id, msg.buffer);
+  };
+
+  const usesKernel = (op) => op === OP.FUSED || op === OP.GROUP || op === OP.GROUP_RANGE || op === OP.UNITS || op === OP.TASKS || op === OP.COMPACT;
+
+  // Does the current task need a kernel or array we haven't received yet?
+  function missing(op) {
+    if (usesKernel(op) && !fusedModules.has(ctrl[C.KERNEL])) return true;
+    for (let k = 0; k < MAX_ARGS; k++) {
+      const id = ctrl[C.ARGS + k * 4];
+      if (id > WASM_MEMORY && !buffers.has(id)) return true;
+    }
+    return false;
+  }
+
+  function fused(id) {
+    let k = fusedKernels.get(id);
+    if (!k) {
+      k = new WebAssembly.Instance(fusedModules.get(id), { env: { memory } }).exports.k;
+      fusedKernels.set(id, k);
+    }
+    return k;
+  }
+
+  let seen = 0;
+  for (;;) {
+    // Spin briefly (cheap wake-up for back-to-back calls), then sleep.
+    let s = Atomics.load(ctrl, C.SEQ);
+    for (let k = 0; (s === seen || s === SEQ_WRITING) && k < spin; k++) s = Atomics.load(ctrl, C.SEQ);
+    while (s === seen || s === SEQ_WRITING) {
+      if (s === seen) Atomics.wait(ctrl, C.SEQ, seen);
+      s = Atomics.load(ctrl, C.SEQ);
+    }
+    seen = s;
+
+    const op = ctrl[C.OP];
+    if (op === OP.EXIT) break;
+
+    // If we woke up late, the main thread may already be on a newer task and
+    // rewriting these fields. Reading them can then fail or mix two tasks, so
+    // re-check SEQ: a replaced task is skipped (claiming would fail anyway).
+    let A, F;
+    try {
+      host.pullSync(handle);
+      // A browser worker gets messages only between event-loop turns: if the
+      // task needs something that was sent but not yet received, step out of
+      // the loop until it arrives (the main thread sent it before the task).
+      while (host.pullAsync && missing(op) && Atomics.load(ctrl, C.SEQ) === s) {
+        await host.pullAsync();
+        host.pullSync(handle);
+      }
+      A = decodeArgs(ctrl, bufferFor);
+      F = usesKernel(op) ? fused(ctrl[C.KERNEL])
+        : op === OP.CSV_PARSE ? (msg) => host.post(msg) // results go back to the main thread
+        : op === OP.READ ? (jobs, u) => { // a file read straight into engine memory (Node: host.read)
+          const dst = jobs[u * 4 + 3], len = jobs[u * 4 + 2];
+          if (memory.buffer.byteLength < dst + len) memory.grow(0);
+          host.read(jobs[u * 4], new Uint8Array(memory.buffer, dst, len), jobs[u * 4 + 1]);
+        }
+        : null;
+    } catch (err) {
+      // a newer task is being written (SEQ changed), or ran without the
+      // workers (a fresh tag): these fields weren't our task's
+      if (Atomics.load(ctrl, C.SEQ) !== s || Atomics.load(ctrl, C.NEXT) >>> TAG_SHIFT !== tagOf(s)) continue;
+      throw err;
+    }
+    if (Atomics.load(ctrl, C.SEQ) !== s) continue;
+    runUnits(op, ctrl[C.THREADS], ctrl[C.N], A, sc, ctrl, ctrl[C.WASM] ? W : null, res, tagOf(s), F, index,
+      (err) => host.post({ kind: 'error', message: err?.stack ?? String(err) }));
+  }
+}

package/src/worker.mjs

@@ -0,0 +1,22 @@
+// Node worker thread: runs the shared worker loop. Messages are read
+// synchronously, so the loop never has to return to the event loop.
+import { workerData, receiveMessageOnPort } from 'node:worker_threads';
+import { readSync } from 'node:fs';
+import { workerLoop } from './worker-loop.mjs';
+
+const { port } = workerData;
+workerLoop(workerData, {
+  post: (msg) => port.postMessage(msg),
+  pullSync(handle) {
+    let m;
+    while ((m = receiveMessageOnPort(port))) handle(m.message);
+  },
+  pullAsync: undefined,
+  read(fd, view, pos) { // (datasets: chunks are read on every thread)
+    for (let done = 0; done < view.length; ) {
+      const got = readSync(fd, view, done, view.length - done, pos + done);
+      if (got === 0) throw new Error('vectro: dataset file ended early');
+      done += got;
+    }
+  },
+});

package/src/zones.mjs

@@ -0,0 +1,75 @@
+// Zone maps: the min and max of each block of rows, per column. A filter
+// such as `ts >= start && price > 900` can't match a block whose max ts is
+// below `start`, so a query only needs the rows from the first block that
+// can match to the last one. On time-ordered data (logs, orders, streams),
+// "the last hour" becomes a small slice of the table.
+//
+// Exact by construction: only `column <op> number` parts of a top-level
+// `&&` chain are used (number columns and text codes; never free text), and
+// the rows left out provably fail the filter.
+import { parse } from './fuse.mjs';
+
+export const ZONE_ROWS = 16_384;
+const FLIP = { '<': '>', '<=': '>=', '>': '<', '>=': '<=', '==': '==' };
+
+// Min and max per block, skipping NaN (a block of only NaN has min Infinity).
+export function computeZones(data, n, B = ZONE_ROWS) {
+  const blocks = Math.ceil(n / B);
+  const mins = new Float64Array(blocks).fill(Infinity), maxs = new Float64Array(blocks).fill(-Infinity);
+  for (let b = 0; b < blocks; b++) {
+    let lo = Infinity, hi = -Infinity;
+    for (let i = b * B, end = Math.min(n, i + B); i < end; i++) {
+      const v = data[i];
+      if (v < lo) lo = v;
+      if (v > hi) hi = v;
+    }
+    mins[b] = lo; maxs[b] = hi;
+  }
+  return { block: B, shift: 0, mins, maxs };
+}
+
+// `column <op> value` conditions every matching row must meet.
+export function conditions(pred, cols, params) {
+  const out = [];
+  const value = (m) => {
+    if (m.t === 'num') return m.v;
+    if (m.t === 'var' && !cols.has(m.name) && typeof params[m.name] === 'number') return params[m.name];
+    if (m.t === 'un' && m.op === '-') { const v = value(m.x); return v === undefined ? undefined : -v; }
+    return undefined;
+  };
+  const column = (m) => (m.t === 'var' && cols.has(m.name) && !cols.get(m.name).strings ? m.name : null);
+  (function visit(node) {
+    if (node.t === 'bin' && node.op === '&&') { visit(node.l); visit(node.r); return; }
+    if (node.t !== 'bin' || !FLIP[node.op]) return;
+    let col = column(node.l), v = value(node.r), op = node.op;
+    if (!col) { col = column(node.r); v = value(node.l); op = FLIP[op]; }
+    if (col && typeof v === 'number' && !Number.isNaN(v)) out.push({ col, op, v });
+  })(parse(pred));
+  return out;
+}
+
+export const mayMatch = (op, lo, hi, v) =>
+  op === '>' ? hi > v : op === '>=' ? hi >= v : op === '<' ? lo < v : op === '<=' ? lo <= v : lo <= v && v <= hi;
+
+// The rows [lo, hi) that can match `pred`, or null when zone maps can't
+// narrow it usefully. zonesOf(name) gives a column's zones (computed lazily).
+// round: how the compiled filter rounds numbers (Math.fround for float32
+// tables), so the zone check compares exactly like the filter will.
+export function zoneRange(pred, cols, params, n, zonesOf, round = (v) => v) {
+  const conds = conditions(pred, cols, params).map((c) => ({ ...c, v: round(c.v) }));
+  if (!conds.length) return null;
+  let first = Infinity, last = -Infinity, zones = null;
+  const maps = conds.map((c) => ({ ...c, z: zonesOf(c.col) }));
+  zones = maps[0].z;
+  const { block: B, shift } = zones;
+  const blocks = zones.mins.length;
+  for (let b = 0; b < blocks; b++) {
+    let ok = true;
+    for (const { op, v, z } of maps) if (!mayMatch(op, z.mins[b], z.maxs[b], v)) { ok = false; break; }
+    if (ok) { if (b < first) first = b; last = b; }
+  }
+  if (first === Infinity) return [0, 0];
+  const lo = Math.max(0, first * B - shift), hi = Math.min(n, (last + 1) * B - shift);
+  if (hi - lo > 0.9 * n) return null; // not worth a narrower run
+  return [lo, Math.max(lo, hi)];
+}