vectrojs 0.0.1 → 0.1.1

package/src/kernels.mjs

@@ -0,0 +1,312 @@
+// Kernels run identically on the main thread and on every worker. Work is
+// split into units that threads claim one at a time (see runUnits).
+
+import { csvScan, csvParse } from './csv-kernel.mjs';
+import { countPattern } from './bytes.mjs';
+
+export const OP = {
+  EXIT: 0, SUM: 1, MIN: 2, MAX: 3, ADD: 4, SCALE: 5, DOT: 6, HEAVY: 7,
+  COUNT_GT: 8, SORT_CHUNK: 9, MERGE: 10, COUNT_BYTES: 11, COPY: 12, FUSED: 13, GROUP: 14, COMPACT: 15, CSV_SCAN: 16, CSV_PARSE: 17, GROUP_RANGE: 18, UNITS: 19, READ: 20, TASKS: 21,
+};
+
+// Layout of the shared control block (Int32 slots).
+export const C = { SEQ: 0, DONE: 1, OP: 2, N: 3, THREADS: 4, NEXT: 5, WASM: 6, KERNEL: 7, NOUT: 8, NSCAL: 9, ERROR: 10, ARGS: 11 };
+export const MAX_ARGS = 256; // each array arg = [bufferId, byteOffset, length, typeCode]
+export const WASM_MEMORY = 0; // bufferId of the engine's WebAssembly memory
+// SEQ holds this while the main thread rewrites the task (a seqlock): a
+// worker that read the task during the rewrite sees SEQ change and skips it.
+export const SEQ_WRITING = -1;
+
+export const TYPES = [
+  Float64Array, Float32Array, Int32Array, Uint32Array,
+  Int16Array, Uint16Array, Int8Array, Uint8Array,
+];
+
+export const chunkStart = (i, T, n) => Math.floor((i * n) / T);
+
+// Work is cut into many small blocks that threads claim one at a time
+// (dynamic scheduling). Fast cores end up doing more blocks than slow ones,
+// and a thread that gets paused by the OS doesn't hold everyone up.
+const BLOCK = 16_384;
+const blockCount = (n, T) => Math.max(1, Math.min(Math.ceil(n / BLOCK), T * 16));
+
+// Offsets are read as unsigned so arrays can live anywhere in 4GB of memory.
+// bufferFor(id, bytesNeeded) returns the buffer an array lives in.
+export function decodeArgs(ctrl, bufferFor) {
+  const u = new Uint32Array(ctrl.buffer);
+  const A = [];
+  for (let k = 0; k < MAX_ARGS; k++) {
+    const base = C.ARGS + k * 4;
+    const id = ctrl[base];
+    if (id < 0) { A.push(null); continue; }
+    const Type = TYPES[u[base + 3]];
+    A.push(new Type(bufferFor(id, u[base + 1] + u[base + 2] * Type.BYTES_PER_ELEMENT), u[base + 1], u[base + 2]));
+  }
+  return A;
+}
+
+// Work is handed out as numbered units (blocks, sort chunks or merge pairs).
+// NEXT holds [12-bit task tag | 20-bit next unit], so a thread that wakes up
+// late and still holds an old task can never claim a unit of a newer one.
+export const TAG_SHIFT = 20;
+export const tagOf = (seq) => seq & 0xfff;
+
+export function unitCount(op, T, n, sc) {
+  if (op === OP.CSV_SCAN || op === OP.CSV_PARSE || op === OP.READ) return n; // n = number of chunks (READ: of file reads)
+  if (op === OP.SORT_CHUNK) return sc[1];
+  if (op === OP.GROUP_RANGE) return T; // a range of groups per thread
+  if (op === OP.TASKS) return n; // n tasks (e.g. one per partition)
+  if (op === OP.MERGE) return Math.ceil(sc[1] / (2 * sc[0]));
+  return blockCount(n, T);
+}
+export const maxUnits = (T) => T * 16;
+export const MAX_OUTPUTS = 8; // reductions per fused kernel
+
+function claim(ctrl, tag, units) {
+  for (;;) {
+    const v = Atomics.load(ctrl, C.NEXT);
+    if (v >>> TAG_SHIFT !== tag) return -1; // a newer task has started
+    const u = v & 0xfffff;
+    if (u >= units) return -1;              // nothing left
+    if (Atomics.compareExchange(ctrl, C.NEXT, v, v + 1) === v) return u;
+  }
+}
+
+// Claim and run units until none are left. Each unit's result goes in its own
+// slot of res, so reductions combine in the same order on every run.
+// Callers never wait for threads, only for units: a sleeping thread that wakes
+// up after the work is done simply finds nothing to claim.
+// F is this thread's instance of a generated kernel (FUSED, GROUP, COMPACT);
+// self is this thread's index (0 = calling thread). If a unit throws, the
+// error goes to report() and the ERROR flag is set; the unit still counts as
+// done, so the caller never waits forever.
+export function runUnits(op, T, n, A, sc, ctrl, W, res, tag, F, self = 0, report = null) {
+  const units = unitCount(op, T, n, sc);
+  for (let u; (u = claim(ctrl, tag, units)) >= 0; ) {
+    try {
+      runUnit(op, u, units, T, n, A, sc, ctrl, W, res, F, self);
+    } catch (err) {
+      Atomics.store(ctrl, C.ERROR, 1);
+      if (report) report(err); else throw err;
+    } finally {
+      if (Atomics.add(ctrl, C.DONE, 1) + 1 === units) Atomics.notify(ctrl, C.DONE);
+    }
+  }
+}
+
+function runUnit(op, u, units, T, n, A, sc, ctrl, W, res, F, self) {
+    if (op === OP.FUSED) {
+      runFused(F, u, units, n, A, sc, ctrl, res);
+    } else if (op === OP.GROUP) {
+      runGroup(F, u, units, n, A, sc, ctrl, self);
+    } else if (op === OP.GROUP_RANGE) {
+      runGroupRange(F, u, units, n, A, sc, ctrl);
+    } else if (op === OP.UNITS) {
+      runIndexed(F, u, units, n, sc, ctrl);
+    } else if (op === OP.TASKS) {
+      runTask(F, u, sc, ctrl, self);
+    } else if (op === OP.COMPACT) {
+      runCompact(F, u, units, n, A, sc, ctrl);
+    } else if (op === OP.CSV_SCAN) {
+      res[u] = csvScan(u, A, sc);
+    } else if (op === OP.CSV_PARSE) {
+      csvParse(u, A, sc, F); // F reports each chunk's text dictionary
+    } else if (op === OP.READ) {
+      F(A[0], u); // read job u: [fd, file position, bytes, memory offset] (datasets, Node)
+    } else if (op === OP.SORT_CHUNK) {
+      A[0].subarray(chunkStart(u, units, n), chunkStart(u + 1, units, n)).sort();
+    } else if (op === OP.MERGE) {
+      mergePair(A[0], A[1], u, sc[1], n, sc[0]);
+    } else {
+      res[u] = rangeKernel(op, chunkStart(u, units, n), chunkStart(u + 1, units, n), A, sc, W);
+    }
+}
+
+// A fused kernel takes (ptr per array..., length, scalars...) and returns its
+// reductions (none for an elementwise map).
+function runFused(F, u, units, n, A, sc, ctrl, res) {
+  const start = chunkStart(u, units, n);
+  const args = [];
+  for (const ta of A) if (ta) args.push(ta.byteOffset + start * ta.BYTES_PER_ELEMENT);
+  args.push(chunkStart(u + 1, units, n) - start);
+  for (let j = 0, ns = ctrl[C.NSCAL]; j < ns; j++) args.push(sc[j]);
+  const r = F(...args);
+  const R = ctrl[C.NOUT];
+  if (R === 1) res[u] = r;
+  else for (let j = 0; j < R; j++) res[u * R + j] = r[j];
+}
+
+// Group-by kernel: (column ptrs..., key ptr, length, accBase, K, scalars...).
+// Each thread adds into its own accumulators (sc holds base, stride and K
+// after the scalars), so no two threads ever write the same slot.
+function runGroup(F, u, units, n, A, sc, ctrl, self) {
+  const start = chunkStart(u, units, n);
+  const ns = ctrl[C.NSCAL];
+  const args = [];
+  for (const ta of A) if (ta) args.push(ta.byteOffset + start * ta.BYTES_PER_ELEMENT);
+  args.push(chunkStart(u + 1, units, n) - start, sc[ns] + self * sc[ns + 1], sc[ns + 2]);
+  for (let j = 0; j < ns; j++) args.push(sc[j]);
+  F(...args);
+}
+
+// Group-by over a range of groups: (column ptrs..., key ptr, length, accBase,
+// K, lo, hi, scalars...). Every unit reads all rows but only adds those whose
+// group is in its range [lo, hi), into ONE set of accumulators: no copies per
+// thread (they would be huge with millions of groups) and no two threads
+// writing the same slot.
+function runGroupRange(F, u, units, n, A, sc, ctrl) {
+  const ns = ctrl[C.NSCAL], K = sc[ns + 1];
+  const args = [];
+  for (const ta of A) if (ta) args.push(ta.byteOffset);
+  args.push(n, sc[ns], K, Math.floor((u * K) / units), Math.floor(((u + 1) * K) / units));
+  for (let j = 0; j < ns; j++) args.push(sc[j]);
+  F(...args);
+}
+
+// A kernel that is told its rows and its unit: (start, end, unit, scalars...).
+// Pointers travel as scalars; the unit number lets it use its own slots.
+function runIndexed(F, u, units, n, sc, ctrl) {
+  const args = [chunkStart(u, units, n), chunkStart(u + 1, units, n), u];
+  for (let j = 0, ns = ctrl[C.NSCAL]; j < ns; j++) args.push(sc[j]);
+  F(...args);
+}
+
+// A kernel run once per task: (task, thread, scalars...). The thread index
+// (0 = calling thread) lets a task use that thread's own scratch memory.
+function runTask(F, u, sc, ctrl, self) {
+  const args = [u, self];
+  for (let j = 0, ns = ctrl[C.NSCAL]; j < ns; j++) args.push(sc[j]);
+  F(...args);
+}
+
+// Compact kernel: (src ptrs..., length, dst ptrs..., scalars...). The last
+// array holds each unit's output offset (from a counting pass), so every unit
+// writes straight to its final position.
+function runCompact(F, u, units, n, A, sc, ctrl) {
+  const arrays = A.filter(Boolean);
+  const offsets = arrays[arrays.length - 1];
+  const S = ctrl[C.NOUT];
+  const start = chunkStart(u, units, n);
+  const args = [];
+  for (let k = 0; k < S; k++) args.push(arrays[k].byteOffset + start * arrays[k].BYTES_PER_ELEMENT);
+  args.push(chunkStart(u + 1, units, n) - start);
+  for (let k = S; k < arrays.length - 1; k++) args.push(arrays[k].byteOffset + offsets[u] * arrays[k].BYTES_PER_ELEMENT);
+  for (let j = 0, ns = ctrl[C.NSCAL]; j < ns; j++) args.push(sc[j]);
+  F(...args);
+}
+
+// Which ops have a WASM SIMD kernel, by element type.
+const WASM_NAMES = {
+  [OP.SUM]: { f64: 'sum_f64' },
+  [OP.MIN]: { f64: 'min_f64', f32: 'min_f32' },
+  [OP.MAX]: { f64: 'max_f64', f32: 'max_f32' },
+  [OP.DOT]: { f64: 'dot_f64' },
+  [OP.COUNT_GT]: { f64: 'countgt_f64', f32: 'countgt_f32' },
+  [OP.ADD]: { f64: 'add_f64', f32: 'add_f32' },
+  [OP.SCALE]: { f64: 'scale_f64', f32: 'scale_f32' },
+};
+const elemType = (ta) => (ta instanceof Float64Array ? 'f64' : ta instanceof Float32Array ? 'f32' : null);
+
+export function hasWasmKernel(op, A) {
+  const t = elemType(A[0]);
+  return Boolean(t && WASM_NAMES[op]?.[t] && A.every((x) => !x || elemType(x) === t));
+}
+
+// The actual work on [start, end). Also used directly for small inputs.
+// With W (WASM exports) it runs the SIMD kernel on raw memory addresses.
+export function rangeKernel(op, start, end, A, sc, W) {
+  if (W) {
+    const f = W[WASM_NAMES[op][elemType(A[0])]];
+    const len = end - start;
+    const at = (ta) => ta.byteOffset + start * ta.BYTES_PER_ELEMENT;
+    switch (op) {
+      case OP.SUM: case OP.MIN: case OP.MAX: return f(at(A[0]), len);
+      case OP.DOT: return f(at(A[0]), at(A[1]), len);
+      case OP.COUNT_GT: return f(at(A[0]), len, sc[0]);
+      case OP.ADD: f(at(A[0]), at(A[1]), at(A[2]), len); return 0;
+      case OP.SCALE: f(at(A[0]), at(A[1]), len, sc[0], sc[1]); return 0;
+    }
+  }
+  switch (op) {
+    case OP.SUM: return sum(A[0], start, end);
+    case OP.MIN: return min(A[0], start, end);
+    case OP.MAX: return max(A[0], start, end);
+    case OP.ADD: return add(A[0], A[1], A[2], start, end);
+    case OP.SCALE: return scale(A[0], A[1], sc[0], sc[1], start, end);
+    case OP.DOT: return dot(A[0], A[1], start, end);
+    case OP.HEAVY: return heavy(A[0], start, end);
+    case OP.COUNT_GT: return countGreater(A[0], sc[0], start, end);
+    case OP.COUNT_BYTES: return countBytes(A[0], A[1], start, end);
+    case OP.COPY: A[1].set(A[0].subarray(start, end), start); return 0;
+  }
+  throw new Error(`unknown op ${op}`);
+}
+
+export function sum(a, s, e) {
+  let r = 0;
+  for (let i = s; i < e; i++) r += a[i];
+  return r;
+}
+
+export function min(a, s, e) {
+  let r = Infinity;
+  for (let i = s; i < e; i++) if (a[i] < r) r = a[i];
+  return r;
+}
+
+export function max(a, s, e) {
+  let r = -Infinity;
+  for (let i = s; i < e; i++) if (a[i] > r) r = a[i];
+  return r;
+}
+
+export function add(a, b, out, s, e) {
+  for (let i = s; i < e; i++) out[i] = a[i] + b[i];
+  return 0;
+}
+
+// out = a * k + c
+export function scale(a, out, k, c, s, e) {
+  for (let i = s; i < e; i++) out[i] = a[i] * k + c;
+  return 0;
+}
+
+export function dot(a, b, s, e) {
+  let r = 0;
+  for (let i = s; i < e; i++) r += a[i] * b[i];
+  return r;
+}
+
+export function heavy(a, s, e) {
+  let r = 0;
+  for (let i = s; i < e; i++) {
+    const x = a[i];
+    r += Math.sqrt(x) * Math.sin(x) + Math.cos(x * 0.5);
+  }
+  return r;
+}
+
+export function countGreater(a, x, s, e) {
+  let r = 0;
+  for (let i = s; i < e; i++) if (a[i] > x) r++;
+  return r;
+}
+
+// One merge in a round of merge sort: pair p joins two neighbouring runs,
+// each `width` sorted chunks long, from src into dst.
+function mergePair(src, dst, p, chunks, n, width) {
+  const first = p * 2 * width;
+  const lo = chunkStart(first, chunks, n);
+  const mid = chunkStart(Math.min(first + width, chunks), chunks, n);
+  const hi = chunkStart(Math.min(first + 2 * width, chunks), chunks, n);
+  let l = lo, r = mid, o = lo;
+  while (l < mid && r < hi) dst[o++] = src[l] <= src[r] ? src[l++] : src[r++];
+  while (l < mid) dst[o++] = src[l++];
+  while (r < hi) dst[o++] = src[r++];
+}
+
+// Count occurrences of `pat` that start inside [s, e) (native indexOf finds
+// candidate first bytes, so each thread searches its slice at memchr speed).
+export function countBytes(text, pat, s, e) {
+  return countPattern(text, pat, s, e);
+}

package/src/ndjson.mjs

@@ -0,0 +1,136 @@
+// JSON lines (one flat object per line) straight into columns, without making
+// a JS object per row:
+//   {"ts":1700000000000,"price":12.5,"qty":3,"region":"EU"}
+// Only the wanted keys are read. Number columns take numbers (and numeric
+// strings); category columns take strings (and numbers, as their text).
+// null, missing keys and nested values give NaN (numbers) or "" (categories).
+// A line that isn't a JSON object is skipped and counted in `bad`.
+import { ByteDict, parseNumber } from './csv-kernel.mjs';
+import { encode } from './bytes.mjs';
+import { parseDateBytes } from './dates.mjs';
+
+const NL = 10, CR = 13, SPACE = 32, TAB = 9, QUOTE = 34, BACKSLASH = 92, COMMA = 44, COLON = 58;
+const LBRACE = 123, RBRACE = 125, LBRACKET = 91, RBRACKET = 93;
+const isSpace = (c) => c === SPACE || c === TAB || c === CR || c === NL;
+const unescapeJSON = (raw) => JSON.parse(`"${raw}"`);
+
+// names: column names; kinds: 'number' | 'category' | 'date' for each
+// (dates: ISO text or ms numbers, as ms).
+// Returns { rows, cols: Float64Array[], dicts: (string[] | null)[], bad }.
+export function parseNDJSON(bytes, start, end, names, kinds) {
+  const k = names.length;
+  const keyBytes = names.map((n) => encode(n));
+  let cap = 1;
+  for (let i = start; i < end; i++) if (bytes[i] === NL) cap++;
+  const cols = names.map(() => new Float64Array(cap));
+  const dicts = kinds.map((kind) => (kind === 'category' ? new ByteDict(bytes, unescapeJSON) : null));
+  const empty = dicts.map((d) => (d ? -1 : NaN)); // code of "" per category column, made on first use
+  const isDate = kinds.map((kind) => kind === 'date');
+  const seen = new Uint8Array(k);
+  let guess = new Int32Array(64).fill(-1); // column of the j-th key on the previous line
+  let rows = 0, bad = 0;
+
+  // Which column does the key in bytes[s, e) name? (-1: not wanted)
+  const columnOf = (s, e, j) => {
+    const g = j < guess.length ? guess[j] : -1;
+    if (g >= 0 && same(keyBytes[g], s, e)) return g;
+    for (let c = 0; c < k; c++) if (same(keyBytes[c], s, e)) { if (j < guess.length) guess[j] = c; return c; }
+    return -1;
+  };
+  const same = (kb, s, e) => {
+    if (kb.length !== e - s) return false;
+    for (let i = 0; i < kb.length; i++) if (kb[i] !== bytes[s + i]) return false;
+    return true;
+  };
+  const emptyCode = (c) => (empty[c] === -1 ? (empty[c] = dicts[c].code(0, 0, false)) : empty[c]);
+
+  let i = start;
+  while (i < end) {
+    // one line: [i, eol)
+    let eol = i;
+    while (eol < end && bytes[eol] !== NL) eol++;
+    let p = i;
+    while (p < eol && isSpace(bytes[p])) p++;
+    if (p === eol) { i = eol + 1; continue; } // blank line
+    const row = rows;
+    seen.fill(0);
+    let ok = bytes[p] === LBRACE;
+    p++;
+    let j = 0;
+    while (ok) {
+      while (p < eol && isSpace(bytes[p])) p++;
+      if (bytes[p] === RBRACE) { p++; break; }
+      if (j > 0) { if (bytes[p] !== COMMA) { ok = false; break; } p++; while (p < eol && isSpace(bytes[p])) p++; }
+      if (bytes[p] !== QUOTE) { ok = false; break; }
+      // key
+      const ks = ++p;
+      let kEsc = false;
+      while (p < eol && bytes[p] !== QUOTE) { if (bytes[p] === BACKSLASH) { kEsc = true; p++; } p++; }
+      if (p >= eol) { ok = false; break; }
+      const c = kEsc ? columnOfText(unescapeJSON(decodeLatin(bytes, ks, p)), names) : columnOf(ks, p, j);
+      p++;
+      while (p < eol && isSpace(bytes[p])) p++;
+      if (bytes[p] !== COLON) { ok = false; break; }
+      p++;
+      while (p < eol && isSpace(bytes[p])) p++;
+      // value
+      const b = bytes[p];
+      if (b === QUOTE) {
+        const vs = ++p;
+        let esc = false;
+        while (p < eol && bytes[p] !== QUOTE) { if (bytes[p] === BACKSLASH) { esc = true; p++; } p++; }
+        if (p >= eol) { ok = false; break; }
+        if (c >= 0) {
+          if (dicts[c]) cols[c][row] = dicts[c].code(vs, p, esc);
+          else if (isDate[c]) cols[c][row] = parseDateBytes(bytes, vs, p);
+          else cols[c][row] = esc ? Number(unescapeJSON(decodeLatin(bytes, vs, p))) || NaN : parseNumber(bytes, vs, p);
+          seen[c] = 1;
+        }
+        p++;
+      } else if (b === LBRACE || b === LBRACKET) { // nested: skip it
+        p = skipNested(bytes, p, eol);
+        if (p < 0) { ok = false; break; }
+      } else {
+        const vs = p;
+        while (p < eol && bytes[p] !== COMMA && bytes[p] !== RBRACE && !isSpace(bytes[p])) p++;
+        if (c >= 0) {
+          const first = bytes[vs];
+          if (first === 110) { /* null: leave empty */ }
+          else if (dicts[c]) { cols[c][row] = dicts[c].code(vs, p, false); seen[c] = 1; }
+          else { cols[c][row] = first === 116 ? 1 : first === 102 ? 0 : parseNumber(bytes, vs, p); seen[c] = 1; }
+        }
+      }
+      j++;
+    }
+    if (ok) {
+      while (p < eol && isSpace(bytes[p])) p++;
+      ok = p === eol;
+    }
+    if (ok) {
+      for (let c = 0; c < k; c++) if (!seen[c]) cols[c][row] = dicts[c] ? emptyCode(c) : NaN;
+      rows++;
+    } else {
+      bad++;
+    }
+    i = eol + 1;
+  }
+  return { rows, cols: cols.map((col) => col.subarray(0, rows)), dicts: dicts.map((d) => d?.strings ?? null), bad };
+}
+
+function columnOfText(text, names) { return names.indexOf(text); }
+
+function decodeLatin(bytes, s, e) {
+  return new TextDecoder().decode(bytes.subarray(s, e));
+}
+
+// Index just past the nested object or array starting at p (-1 if unbalanced).
+function skipNested(bytes, p, eol) {
+  let depth = 0;
+  for (; p < eol; p++) {
+    const c = bytes[p];
+    if (c === QUOTE) { for (p++; p < eol && bytes[p] !== QUOTE; p++) if (bytes[p] === BACKSLASH) p++; }
+    else if (c === LBRACE || c === LBRACKET) depth++;
+    else if (c === RBRACE || c === RBRACKET) { if (--depth === 0) return p + 1; }
+  }
+  return -1;
+}

package/src/node.mjs

@@ -0,0 +1,120 @@
+// Node.js entry: the engine with worker threads on all cores, plus file reading.
+//   import { Engine } from 'vectrojs';   ->   new Engine()  (all cores)
+import { Worker, MessageChannel, receiveMessageOnPort } from 'node:worker_threads';
+import { availableParallelism } from 'node:os';
+import { openSync, fstatSync, readSync, closeSync } from 'node:fs';
+import { open } from 'node:fs/promises';
+import { Engine as CoreEngine } from './engine.mjs';
+import { loadBytes, loadBytesAsync } from './csv.mjs';
+import { Dataset, DatasetWriter, writeDataset, csvToDataset, csvToDatasetAsync, openDatasetAsync, writeDatasetAsync, createDatasetAsync, setFiles } from './dataset.mjs';
+import { nodeFiles } from './files-node.mjs';
+setFiles(nodeFiles);
+
+export const nodePlatform = {
+  cpus: () => availableParallelism(),
+  spawn(workerData) {
+    const { port1, port2 } = new MessageChannel();
+    port1.unref();
+    const worker = new Worker(new URL('./worker.mjs', import.meta.url), {
+      workerData: { ...workerData, port: port2 },
+      transferList: [port2],
+    });
+    worker.unref();
+    return { worker, port: port1 };
+  },
+  receive: (port) => receiveMessageOnPort(port),
+};
+
+export class Engine extends CoreEngine {
+  constructor(opts = {}) {
+    super({ platform: nodePlatform, ...opts });
+  }
+
+  // Load a CSV file into a table, parsing on all threads.
+  readCSV(path, opts = {}) {
+    const fd = openSync(path, 'r');
+    try {
+      const size = fstatSync(fd).size;
+      const bytes = this.alloc(Uint8Array, size);
+      for (let off = 0; off < size; ) {
+        const got = readSync(fd, bytes, off, size - off, off);
+        if (got === 0) throw new Error(`vectro: ${path} ended early`);
+        off += got;
+      }
+      const t = loadBytes(this, bytes, opts);
+      return opts.compress ? t.compress() : t;
+    } finally {
+      closeSync(fd);
+    }
+  }
+
+  // Same, without blocking: the file is read with async I/O straight into
+  // engine memory, and parsing runs on the worker threads.
+  async readCSVAsync(path, opts = {}) {
+    const file = await open(path, 'r');
+    try {
+      const { size } = await file.stat();
+      const bytes = this.alloc(Uint8Array, size);
+      for (let off = 0; off < size; ) {
+        const { bytesRead } = await file.read(this.live(bytes), off, size - off, off);
+        if (bytesRead === 0) throw new Error(`vectro: ${path} ended early`);
+        off += bytesRead;
+      }
+      const t = await loadBytesAsync(this, bytes, opts);
+      return opts.compress ? await t.compressAsync() : t;
+    } finally {
+      await file.close();
+    }
+  }
+
+  // ---- datasets: tables bigger than memory, on disk (see dataset.mjs) ----------
+  // Open one for queries (filter, groupBy, join, sql... read it chunk by chunk).
+  openDataset(path) { return new Dataset(this, path); }
+  // The same from a path, a URL (HTTP range requests) or a Blob; async.
+  openDatasetAsync(source) { return openDatasetAsync(this, source); }
+  // A writer: append(table or columns) as often as needed, then close().
+  createDataset(path, opts) { return new DatasetWriter(this, path, opts); }
+  // A table, a query (over a dataset: written chunk by chunk) or columns.
+  writeDataset(path, source, opts) { return writeDataset(this, path, source, opts); }
+  // A CSV file of any size, read in pieces (async: also from a URL or Blob).
+  csvToDataset(csvPath, path, opts) { return csvToDataset(this, csvPath, path, opts); }
+  csvToDatasetAsync(csv, path, opts) { return csvToDatasetAsync(this, csv, path, opts); }
+  // writeDataset / createDataset without blocking (a writer with appendAsync / closeAsync).
+  writeDatasetAsync(path, source, opts) { return writeDatasetAsync(this, path, source, opts); }
+  createDatasetAsync(path, opts) { return createDatasetAsync(this, path, opts); }
+}
+
+export { canCompileWasm } from './engine.mjs';
+export { Dataset, DatasetWriter, blobFile, urlFile } from './dataset.mjs';
+export { memoryFile } from './files-browser.mjs';
+export { Table } from './table.mjs';
+
+// ---- split tables and streams: each worker thread owns its rows --------------
+//   import { SplitEngine } from 'vectrojs';
+//   const engine = new SplitEngine();               // one worker per core
+//   const live = await engine.streamAsync({ columns: { ts: 'number', region: 'category' } });
+import { SplitEngine as BaseSplitEngine } from './split.mjs';
+
+export function spawnSplitWorker() {
+  const w = new Worker(new URL('./split-worker-node.mjs', import.meta.url));
+  w.unref();
+  return {
+    post: (msg, transfer) => w.postMessage(msg, transfer ?? []),
+    onMessage: (fn) => w.on('message', fn),
+    onError: (fn) => { w.on('error', fn); w.on('exit', (code) => fn(new Error(`exited with code ${code}`))); },
+    terminate: () => w.terminate(),
+  };
+}
+
+export class SplitEngine extends BaseSplitEngine {
+  constructor(opts = {}) {
+    super({ workers: availableParallelism(), spawn: spawnSplitWorker, ...opts });
+  }
+}
+
+export { VectorIndex } from './vectors.mjs';
+export { Emitter } from './emitter.mjs';
+export { BatchEncoder, FrameReader, encodeSchema, encodeEnd } from './frames.mjs';
+export { readArrow, writeArrow, ArrowReader, ArrowEncoder } from './arrow.mjs';
+export { DistinctSketch } from './sketch.mjs';
+export { sql } from './sql.mjs';

package/src/pack.mjs

@@ -0,0 +1,114 @@
+// Compressed ("packed") number columns. A column whose values are all
+//   base + code * mul      (whole numbers: counts, codes, IDs, ms on whole seconds)
+//   code / div             (decimals: prices in cents, rates in basis points)
+// with small whole-number codes is stored as 1, 2 or 4 bytes per row instead
+// of 8 (or 4). Missing values (NaN) take the largest code of the type. Every
+// value is checked to decode back exactly, with the same arithmetic the
+// kernels use (float32 tables: float32), so packing never changes an answer.
+//
+// The fused kernels read packed columns directly (SIMD widen + convert, then
+// scale), so scans read 2-8x fewer bytes. Other operations decode the
+// columns they need first (see Table).
+
+export const KINDS = {
+  u8: { Type: Uint8Array, bytes: 1, nan: 255, max: 254, min: 0 },
+  u16: { Type: Uint16Array, bytes: 2, nan: 65535, max: 65534, min: 0 },
+  i32: { Type: Int32Array, bytes: 4, nan: -2147483648, max: 2147483647, min: -2147483647 },
+};
+const SLACK = 16; // bytes after the codes: SIMD loads read a little past the last row
+
+const MULS = [1, 1000, 60_000, 3_600_000, 86_400_000];
+const DIVS = [10, 100, 1000, 10_000];
+
+// Candidate encodings for a column with these statistics, smallest first.
+// (Decimals: the fewest decimals first; each candidate must still be checked
+// value by value.)
+export function encodingCandidates({ min, max, whole, nan, inf, negZero }, f32 = false) {
+  if (min === Infinity || inf || negZero) return [];
+  const fullBytes = f32 ? 4 : 8;
+  const kindFor = (lo, hi) => {
+    for (const [name, k] of Object.entries(KINDS)) if (lo >= k.min && hi <= k.max && k.bytes < fullBytes && !(f32 && name === 'i32' && (Math.abs(lo) > 2 ** 24 || Math.abs(hi) > 2 ** 24))) return name;
+    return null;
+  };
+  const out = [];
+  if (whole) {
+    for (const mul of MULS) { const kind = kindFor(0, Math.round((max - min) / mul)); if (kind) out.push({ kind, mul, div: 0, base: min, nan }); }
+    return out.sort((a, b) => KINDS[a.kind].bytes - KINDS[b.kind].bytes);
+  }
+  for (const div of DIVS) { const kind = kindFor(Math.round(min * div), Math.round(max * div)); if (kind) out.push({ kind, mul: 0, div, base: 0, nan }); }
+  return out;
+}
+
+// The encoding to use for values[0..n), or null if none saves memory.
+// enc: { kind, mul, div, base }
+export function chooseEncoding(values, n, f32 = false) {
+  if (!n) return null;
+  const R = f32 ? Math.fround : (x) => x;
+  let min = Infinity, max = -Infinity, nan = false, whole = true;
+  for (let i = 0; i < n; i++) {
+    const v = values[i];
+    if (v !== v) { nan = true; continue; }
+    if (v === Infinity || v === -Infinity) return null;
+    if (v < min) min = v;
+    if (v > max) max = v;
+    if (whole && v !== Math.floor(v)) whole = false;
+  }
+  if (min === Infinity) return null; // only missing values: leave it
+  const fullBytes = f32 ? 4 : 8;
+  const kindFor = (lo, hi) => {
+    for (const [name, k] of Object.entries(KINDS)) if (lo >= k.min && hi <= k.max && k.bytes < fullBytes && !(f32 && name === 'i32' && (Math.abs(lo) > 2 ** 24 || Math.abs(hi) > 2 ** 24))) return name;
+    return null;
+  };
+  const decode = (enc, code) => (enc.div ? R(R(code) / enc.div) : R(R(R(code) * enc.mul) + enc.base));
+  const encode = (enc, v) => (enc.div ? Math.round(v * enc.div) : Math.round((v - enc.base) / enc.mul));
+  const fits = (enc, sample) => { // every value (or a sample first) decodes back exactly
+    const step = sample ? Math.max(1, Math.floor(n / 512)) : 1;
+    for (let i = 0; i < n; i += step) {
+      const v = values[i];
+      if (v !== v) continue;
+      const code = encode(enc, v);
+      if (!(code >= KINDS[enc.kind].min && code <= KINDS[enc.kind].max) || decode(enc, code) !== v || Object.is(v, -0) !== Object.is(decode(enc, code), -0)) return false;
+    }
+    return true;
+  };
+  const candidates = [];
+  if (whole) {
+    for (const mul of MULS) {
+      const lo = 0, hi = Math.round((max - min) / mul);
+      const kind = kindFor(lo, hi);
+      if (kind) candidates.push({ kind, mul, div: 0, base: min });
+    }
+  } else {
+    for (const div of DIVS) { // the fewest decimals that give every value back
+      const kind = kindFor(Math.round(min * div), Math.round(max * div));
+      const enc = kind && { kind, mul: 0, div, base: 0 };
+      if (enc && fits(enc, true) && fits(enc, false)) return { ...enc, nan };
+    }
+    return null;
+  }
+  candidates.sort((a, b) => KINDS[a.kind].bytes - KINDS[b.kind].bytes);
+  for (const enc of candidates) if (fits(enc, true) && fits(enc, false)) return { ...enc, nan }; // nan: some codes mean missing
+  return null;
+}
+
+// Codes for values[0..n) in a new engine array (with room after the end).
+export function packInto(engine, values, n, enc) {
+  const k = KINDS[enc.kind];
+  const data = engine.alloc(k.Type, n + SLACK / k.bytes);
+  const v = engine.live(values);
+  for (let i = 0; i < n; i++) {
+    const x = v[i];
+    data[i] = x !== x ? k.nan : enc.div ? Math.round(x * enc.div) : Math.round((x - enc.base) / enc.mul);
+  }
+  return data;
+}
+
+// Plain JS decode of one code (kernels do this in SIMD).
+export function decodeOne(enc, code, f32 = false) {
+  if (code === KINDS[enc.kind].nan) return NaN;
+  const R = f32 ? Math.fround : (x) => x;
+  return enc.div ? R(R(code) / enc.div) : R(R(R(code) * enc.mul) + enc.base);
+}
+
+// Bytes per row stored, and what the column would take unpacked.
+export const packedBytes = (enc, n) => KINDS[enc.kind].bytes * n;