tinydecide 0.0.0-stage → 0.1.2

package/src/shadow.js

@@ -0,0 +1,415 @@
+// shadow(): wrap a decision API so it keeps working exactly as before, while a local model learns
+// from its answers and gradually takes over the calls it can answer within the agreement target.
+//
+// Modes
+//   learning  every call goes to your API; every answer is logged
+//   local     confident calls are answered locally, the rest go to your API
+//   guarded   a live check failed or a new label appeared; every call goes to your API until a
+//             compile on post-incident answers certifies again
+//
+// A uniform random sample of calls (rate `audit`) is also answered by your API whatever the route.
+// That sample is iid over traffic, so it drives both the live agreement check and later calibration;
+// calls logged only because they were deferred are used for training, never for certifying.
+//
+// Two live checks can switch to guarded:
+//   audit     a CUSUM test on the random sample finds overall agreement below target: about 2% false
+//             alarms over a model's life when agreement sits exactly at target, fewer above it
+//   deferred  on deferred calls (answered by your API anyway, so free to check), the model's guess
+//             matches your API far less often than it did on calibration, overall or for one label
+//             against the model's own calibrated confidence. It usually trips before the audit check.
+//
+// TinyDecide never makes a call fail that would have succeeded without it: if anything inside it
+// throws, the call goes to your API.
+
+import { compile } from './compile.js';
+import { logger } from './log.js';
+import { wilson } from './stats.js';
+import { memoryStore } from './store.js';
+
+const GUARD_Z = 3; // strict, because the guard is re-checked on every observation
+const CUSUM_ALTS = [2, 4, 8]; // error rates the audit check watches for, as multiples of 1 − target
+const CUSUM_H = 9;
+const DEFERRED_WINDOW = 400;
+const DEFERRED_MIN = 100;
+const DEFERRED_SLACK = 0.05;
+const LABEL_WINDOW = 200; // deferred calls per label the model leans toward
+const LABEL_MIN = 20;
+const LABEL_Z = 4; // stricter still: one check per label
+const LABEL_SLACK = 0.1; // room for the model's confidence being somewhat off on one label
+const LABEL_KEYS = ['choice', 'label', 'decision', 'answer'];
+const PROBS_KEYS = ['probs', 'probabilities'];
+const OPTIONS = ['target', 'delta', 'audit', 'labels', 'model', 'store', 'choice', 'respond', 'group', 'fallback', 'log', 'onEvent', 'autoCompile', 'compile', 'compiler', 'random'];
+
+/**
+ * @param {string} [name]  names the decision in logs; in Node it also names its folder in .tinydecide/
+ * @param {(input: any) => any} api  your existing decision call
+ * @param {object} [options]  see README
+ */
+export function shadow(name, api, options) {
+  if (typeof name === 'function') [name, api, options] = [undefined, name, api];
+  options ??= {};
+  validate(name, api, options);
+  const o = {
+    target: 0.98, delta: 0.05, audit: 0.02, labels: undefined, model: null, store: null,
+    choice: defaultChoice, respond: undefined, group: undefined, fallback: false, log: true,
+    onEvent: () => {}, compile: {}, compiler: compile, random: Math.random,
+    ...options,
+  };
+  o.labels ??= o.model?.labels; // a model you pass in fixes the label order
+  const auto = o.autoCompile === false ? null : { min: 2000, growth: 2, maxEvery: 20_000, window: 50_000, ...o.autoCompile };
+  const store = o.store ?? memoryStore();
+  const say = logger(name, o.log);
+
+  let model = null;
+  let mode = 'learning';
+  let shape = null; // how your API's responses look, so local answers come back the same way
+  let incident = null; // { t } while guarded
+  let epoch = 0; // bumps on every guard trip; a compile that started in an older epoch is discarded
+  let compiling = null;
+  let running = null; // AbortController of the compile in flight, so a guard trip can stop it
+  let closed = false;
+  let unreadable = false;
+  // The compile schedule: answers available, answers since the last compile, and rows it used.
+  let available = 0, fresh = 0, basis = 0, sinceIncident = 0;
+  const ring = new Uint8Array(500); // recent audits, for stats and the guard message
+  let ringN = 0, ringAgree = 0, ringAt = 0;
+  const alpha = 1 - o.target;
+  const steps = CUSUM_ALTS.map((m) => { const p = Math.min(0.5, m * alpha); return [Math.log((1 - p) / (1 - alpha)), Math.log(p / alpha)]; });
+  const cusum = CUSUM_ALTS.map(() => 0);
+  const dring = new Uint8Array(DEFERRED_WINDOW);
+  let dN = 0, dAgree = 0, dAt = 0;
+  const byLabel = new Map(); // label the model leans toward → its deferred calls: agreed, and confidence
+  const n = { calls: 0, local: 0, api: 0, fallback: 0, audits: 0, auditDisagree: 0, compiles: 0, errors: 0 };
+
+  const emit = (e) => {
+    e = { t: Date.now(), ...e };
+    say(e);
+    try { o.onEvent(e); } catch {}
+  };
+  const fail = (where, err) => { n.errors++; emit({ type: 'error', where, error: String(err?.message ?? err) }); };
+  const NOPE = Symbol('failed');
+  /** Runs fn; on a throw (or rejection) reports it and returns NOPE, so the caller can route around it. */
+  function safely(where, fn) {
+    try {
+      const v = fn();
+      if (v && typeof v.then === 'function') return v.then((x) => x, (err) => { fail(where, err); return NOPE; });
+      return v;
+    } catch (err) { fail(where, err); return NOPE; }
+  }
+
+  // ── Startup: resume from the store when it has a model or a guarded state ──────────────────────────
+
+  function adopt(saved, state) {
+    available = store.size ?? 0;
+    if (saved && saved !== NOPE) {
+      model = saved;
+      shape = saved.meta?.response ?? null;
+      basis = saved.meta?.rows?.total ?? 0;
+      mode = 'local';
+    }
+    if (state && state !== NOPE && state.guardedSince) {
+      mode = 'guarded';
+      incident = { t: state.guardedSince };
+      basis = 0;
+    }
+    if (mode === 'local') emit({ type: 'start', mode, share: model.meta?.expected?.share ?? 0, target: model.meta?.certificate?.target ?? o.target });
+    else if (mode === 'guarded') emit({ type: 'start', mode, since: incident.t });
+    else if (auto) emit({ type: 'start', mode, min: auto.min, logged: available });
+  }
+  const saved = o.model ?? safely('loading the saved model', () => store.loadModel?.() ?? null);
+  const state = safely('loading the saved state', () => store.loadState?.() ?? null);
+  const ready = [saved, state].some((v) => typeof v?.then === 'function')
+    ? Promise.all([saved, state]).then(([m, s]) => adopt(m, s))
+    : Promise.resolve(adopt(saved, state));
+  ready.then(() => maybeCompile());
+
+  // ── Live checks ────────────────────────────────────────────────────────────────────────────────────
+
+  function observe(agree) {
+    if (ringN === ring.length) ringAgree -= ring[ringAt]; else ringN++;
+    ring[ringAt] = agree ? 1 : 0;
+    ringAgree += ring[ringAt];
+    ringAt = (ringAt + 1) % ring.length;
+    if (mode !== 'local') return;
+    let alarm = false;
+    for (let j = 0; j < cusum.length; j++) {
+      cusum[j] = Math.max(0, cusum[j] + steps[j][agree ? 0 : 1]);
+      if (cusum[j] >= CUSUM_H) alarm = true;
+    }
+    if (alarm) trip('drift', { slice: 'audit', agreement: ringAgree / ringN, sampled: ringN, target: o.target });
+  }
+
+  function observeDeferred(guess, confidence, agree) {
+    if (dN === dring.length) dAgree -= dring[dAt]; else dN++;
+    dring[dAt] = agree ? 1 : 0;
+    dAgree += dring[dAt];
+    dAt = (dAt + 1) % dring.length;
+    const base = model?.meta?.expected?.deferred;
+    if (mode !== 'local' || !base) return;
+    if (dN >= DEFERRED_MIN && base.n >= 50) {
+      const upper = wilson(dAgree, dN, GUARD_Z)[1];
+      if (upper < base.agreement - DEFERRED_SLACK) return trip('drift', { slice: 'deferred', agreement: dAgree / dN, baseline: base.agreement, sampled: dN, upper });
+    }
+    // Per label: the model's calibrated confidence says how often your API should agree with its
+    // guess. Far fewer agreements than that, beyond chance and slack, means your API changed.
+    let r = byLabel.get(guess);
+    if (!r) byLabel.set(guess, (r = { hit: new Uint8Array(LABEL_WINDOW), p: new Float64Array(LABEL_WINDOW), n: 0, at: 0 }));
+    r.hit[r.at] = agree ? 1 : 0;
+    r.p[r.at] = confidence;
+    r.at = (r.at + 1) % LABEL_WINDOW;
+    if (r.n < LABEL_WINDOW) r.n++;
+    if (r.n < LABEL_MIN) return;
+    let seen = 0, expected = 0, variance = 0;
+    for (let k = 0; k < r.n; k++) { seen += r.hit[k]; expected += r.p[k]; variance += r.p[k] * (1 - r.p[k]); }
+    const ratio = Math.min(1, base.ratio?.[guess] ?? 1); // how well confidence predicted agreement on calibration
+    expected *= ratio;
+    variance *= ratio;
+    if (seen < expected - LABEL_SLACK * r.n - LABEL_Z * Math.sqrt(variance)) {
+      trip('drift', { slice: 'deferred', label: guess, agreement: seen / r.n, baseline: expected / r.n, sampled: r.n });
+    }
+  }
+
+  function resetChecks() { ringN = ringAgree = ringAt = 0; dN = dAgree = dAt = 0; byLabel.clear(); cusum.fill(0); }
+
+  function trip(reason, detail) {
+    if (mode === 'guarded') return;
+    mode = 'guarded';
+    epoch++;
+    running?.abort();
+    incident = { t: Date.now() };
+    basis = fresh = sinceIncident = 0;
+    resetChecks();
+    persist('saving the guarded state', () => store.saveState?.({ guardedSince: incident.t }));
+    emit({ type: 'guard', reason, ...detail });
+  }
+
+  // ── Calls ──────────────────────────────────────────────────────────────────────────────────────────
+
+  async function detailed(input) {
+    n.calls++;
+    const t0 = performance.now();
+    const sample = mode !== 'local' || o.random() < o.audit;
+    let d = null;
+    // Until your API has given one readable response, its shape is unknown, so it answers.
+    if (mode === 'local' && model && (shape || o.respond)) {
+      d = safely('the local model', () => model.decide(input, { probs: wantsProbs() }));
+      if (d === NOPE) d = null;
+      if (d?.local) {
+        const value = safely('respond', () => answer(d));
+        if (value !== NOPE) {
+          n.local++;
+          if (sample) audit(input, d.choice);
+          return { value, source: 'local', choice: d.choice, confidence: d.confidence, ms: performance.now() - t0 };
+        }
+      }
+    }
+    let resp;
+    try {
+      resp = await api(input);
+    } catch (err) {
+      if (!o.fallback || !model) throw err;
+      d ??= safely('the local model', () => model.decide(input, { probs: wantsProbs() }));
+      const value = d && d !== NOPE ? safely('respond', () => answer(d)) : NOPE;
+      if (value === NOPE) throw err;
+      n.fallback++;
+      return { value, source: 'fallback', choice: d.choice, confidence: d.confidence, ms: performance.now() - t0 };
+    }
+    n.api++;
+    const choice = record(input, resp, sample);
+    if (mode === 'local' && d && choice != null) observeDeferred(d.choice, d.confidence, d.choice === choice);
+    if (sample && mode === 'local') observe(true);
+    return { value: resp, source: 'api', choice, confidence: null, ms: performance.now() - t0 };
+  }
+
+  function audit(input, localChoice) {
+    n.audits++;
+    Promise.resolve().then(() => api(input)).then((resp) => {
+      const choice = record(input, resp, true);
+      if (choice == null) return;
+      const agree = choice === localChoice;
+      if (!agree) n.auditDisagree++;
+      emit({ type: 'audit', agree, local: localChoice, api: choice });
+      if (mode === 'local') observe(agree);
+    }, () => {}); // a failed audit is skipped
+  }
+
+  /** Logs one answer from your API. Returns the label, or null when none could be read. */
+  function record(input, resp, sample) {
+    // Index labels and probs arrays map only through labels you gave; a model's sorted labels are not your API's order.
+    const read = safely('choice', () => o.choice(resp, o.labels));
+    const choice = read === NOPE ? null : typeof read === 'string' ? read : read?.choice;
+    if (choice == null) {
+      if (!unreadable) { unreadable = true; emit({ type: 'unreadable', preview: preview(resp) }); }
+      return null;
+    }
+    shape ??= shapeOf(resp); // learned from a response whose label could be read
+    // A label nobody has seen yet joins the label set, so the next compile can learn it.
+    if (o.labels && !o.labels.includes(choice)) o.labels = [...o.labels, choice];
+    if (model && !model.labels.includes(choice)) trip('schema', { label: choice });
+    const rec = { t: Date.now(), input, choice };
+    if (read.probs) rec.probs = read.probs;
+    if (o.group) {
+      const g = safely('group', () => o.group(input));
+      if (g != null && g !== NOPE) rec.group = String(g);
+    }
+    if (!sample) rec.sample = false;
+    if (safely('saving an answer', () => store.append(rec)) !== NOPE) {
+      available++; fresh++;
+      if (mode === 'guarded') sinceIncident++;
+      maybeCompile();
+    }
+    return choice;
+  }
+
+  const wantsProbs = () => !o.respond && shape?.probsKey != null;
+
+  function answer(d) {
+    if (o.respond) return o.respond(d, shape);
+    if (!shape || shape.kind === 'string') return d.choice;
+    if (shape.probsArray && !o.labels) throw new Error('your API returns probs as an array, so pass labels: [...] in that order to answer locally');
+    const out = { [shape.key]: shape.numeric ? (o.labels ? d.index : Number(d.choice)) : d.choice };
+    if (shape.probsKey && d.probs) out[shape.probsKey] = shape.probsArray ? model.labels.map((l) => d.probs[l]) : d.probs;
+    if (shape.confidence) out.confidence = d.confidence;
+    return out;
+  }
+
+  // ── Compiles ───────────────────────────────────────────────────────────────────────────────────────
+
+  function maybeCompile() {
+    if (!auto || compiling || closed) return;
+    const pool = mode === 'guarded' ? sinceIncident : available;
+    if (pool < auto.min) return;
+    const gap = basis ? Math.min(auto.maxEvery, Math.max(auto.min, basis * (auto.growth - 1))) : 0;
+    if (fresh >= gap) recompile({ background: true }).catch((err) => fail('compile', err));
+  }
+
+  async function recompile({ background = false } = {}) {
+    if (compiling) return compiling;
+    if (closed) return null;
+    compiling = (async () => {
+      await ready;
+      const started = epoch;
+      const recovering = incident != null;
+      fresh = 0;
+      let rows = await store.load();
+      if (incident) rows = rows.filter((r) => r.t >= incident.t);
+      rows = rows.slice(-(auto?.window ?? 50_000));
+      emit({ type: 'compile:start', rows: rows.length });
+      n.compiles++;
+      // Keep the labels the model knows: one missing from this window is rare, not new.
+      const labels = o.labels ?? (model ? [...new Set([...model.labels, ...rows.map((r) => r.choice).filter((c) => typeof c === 'string')])] : undefined);
+      const ctrl = (running = new AbortController());
+      let result = null;
+      try {
+        result = await o.compiler(rows, {
+          ...o.compile, target: o.target, delta: o.delta, audit: o.audit, labels, signal: ctrl.signal, background,
+        });
+      } catch (err) {
+        if (err?.name !== 'AbortError') throw err;
+      } finally {
+        running = null;
+        // Set even when the compile fails, and to at least 1, so a failing compile waits for more answers.
+        if (epoch === started) basis = Math.max(rows.length, 1);
+      }
+      if (!result || epoch !== started || closed) {
+        // The guard tripped while this compile ran, so its rows straddle the change. Never install it.
+        emit({ type: 'discard', reason: closed ? 'closed' : 'guard tripped during compile' });
+        return null;
+      }
+      const { model: next, report } = result;
+      emit({ type: 'compile', report, recovering });
+      if (report.calib.share > 0) {
+        next.meta.response = shape;
+        model = next;
+        mode = 'local';
+        incident = null;
+        resetChecks();
+        emit({ type: 'install', share: report.calib.share, verdict: report.verdict, model: next, report });
+        await persist('saving the model', () => store.saveModel?.(next, report));
+        await persist('saving the state', () => store.saveState?.(null));
+      }
+      return report;
+    })();
+    try { return await compiling; } finally { compiling = null; maybeCompile(); }
+  }
+
+  function persist(where, fn) {
+    const v = safely(where, fn);
+    return v && typeof v.then === 'function' ? v : Promise.resolve(v);
+  }
+
+  // ── The decide function ────────────────────────────────────────────────────────────────────────────
+
+  const decide = async (input) => (await detailed(input)).value;
+  decide.detailed = detailed;
+  decide.local = (input) => (model ? model.decide(input) : null);
+  decide.compile = () => recompile();
+  decide.ready = ready;
+  decide.close = async () => {
+    closed = true;
+    running?.abort();
+    try { await compiling; } catch {}
+    await persist('closing the store', () => store.close?.());
+  };
+  decide.stats = () => ({
+    name: name ?? null, mode, ...n, logged: store.size ?? available,
+    share: n.calls ? n.local / n.calls : 0,
+    apiCallsSaved: n.local - n.audits,
+    live: { sampled: ringN, agreement: ringN ? ringAgree / ringN : null, ci: ringN ? wilson(ringAgree, ringN) : null },
+    deferred: { checked: dN, agreement: dN ? dAgree / dN : null, baseline: model?.meta?.expected?.deferred?.agreement ?? null },
+    model: model ? { labels: model.labels.length, cutoff: model.cutoff, expectedShare: model.meta?.expected?.share ?? null, created: model.meta?.created ?? null } : null,
+  });
+  Object.defineProperty(decide, 'name', { value: name ?? 'decide' });
+  Object.defineProperty(decide, 'model', { get: () => model });
+  Object.defineProperty(decide, 'mode', { get: () => mode });
+  if (Symbol.asyncDispose) decide[Symbol.asyncDispose] = decide.close;
+  return decide;
+}
+
+/** Reads { choice, probs } from common response shapes: a label string, or { choice | label | decision | answer, probs }. */
+export function defaultChoice(resp, labels) {
+  if (typeof resp === 'string') return { choice: resp };
+  if (!resp || typeof resp !== 'object') return { choice: null };
+  let choice = resp.choice ?? resp.label ?? resp.decision ?? resp.answer ?? null;
+  if (typeof choice === 'number' && labels) choice = labels[choice];
+  let probs = resp.probs ?? resp.probabilities ?? null;
+  if (Array.isArray(probs)) probs = labels ? Object.fromEntries(labels.map((l, k) => [l, probs[k]])) : null;
+  return { choice: choice == null ? null : String(choice), probs };
+}
+
+/** The parts of a response local answers mirror: the label's key and type, and probs and confidence if present. */
+function shapeOf(resp) {
+  if (typeof resp === 'string') return { kind: 'string' };
+  if (!resp || typeof resp !== 'object') return null;
+  const key = LABEL_KEYS.find((k) => resp[k] != null) ?? 'choice';
+  const probsKey = PROBS_KEYS.find((k) => resp[k] != null) ?? null;
+  return { kind: 'object', key, numeric: typeof resp[key] === 'number', probsKey, probsArray: Array.isArray(resp[probsKey]), confidence: 'confidence' in resp };
+}
+
+function preview(resp) {
+  let s;
+  try { s = JSON.stringify(resp); } catch { s = String(resp); }
+  s ??= String(resp);
+  return s.length > 120 ? `${s.slice(0, 117)}...` : s;
+}
+
+export const NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+
+export function validate(name, api, options) {
+  const fail = (msg) => { throw new TypeError(`tinydecide: ${msg}`); };
+  if (name !== undefined && (typeof name !== 'string' || !NAME.test(name))) {
+    fail(`the name must be 1–64 letters, digits, dots, dashes or underscores, like "support-intent"; got ${JSON.stringify(name)}`);
+  }
+  if (typeof api !== 'function') fail(`shadow(${name ? `"${name}", ` : ''}api) needs your API call as a function, like (text) => classify(text); got ${typeof api}`);
+  if (typeof options !== 'object') fail(`options must be an object; got ${typeof options}`);
+  for (const k of Object.keys(options)) {
+    if (k === 'dir') fail('"dir" needs a named decision in Node, like shadow("support", api, { dir })');
+    if (!OPTIONS.includes(k)) fail(`unknown option "${k}". Options: ${OPTIONS.join(', ')}`);
+  }
+  const unit = (k, lo, hi) => {
+    const v = options[k];
+    if (v !== undefined && !(typeof v === 'number' && v > lo && v < hi)) fail(`${k} must be a number between ${lo} and ${hi}; got ${v}`);
+  };
+  unit('target', 0, 1); unit('delta', 0, 1);
+  if (options.audit !== undefined && !(options.audit >= 0 && options.audit <= 1)) fail(`audit must be a rate from 0 to 1; got ${options.audit}`);
+}

package/src/stats.js

@@ -0,0 +1,78 @@
+// Binomial tails, Wilson intervals, and Learn-Then-Test cutoff certification.
+
+/** log P(X ≤ k) for X ~ Binomial(n, p), stable for large n. */
+export function logBinomCdf(k, n, p) {
+  if (k < 0) return -Infinity;
+  if (k >= n || p <= 0) return 0;
+  if (p >= 1) return -Infinity;
+  const lp = Math.log(p), lq = Math.log1p(-p);
+  let term = n * lq, acc = term;
+  for (let i = 0; i < k; i++) {
+    term += Math.log((n - i) / (i + 1)) + lp - lq;
+    acc = acc > term ? acc + Math.log1p(Math.exp(term - acc)) : term + Math.log1p(Math.exp(acc - term));
+  }
+  return Math.min(0, acc);
+}
+
+export function binomCdf(k, n, p) { return Math.exp(logBinomCdf(k, n, p)); }
+
+/** Wilson score interval for x successes in n trials. */
+export function wilson(x, n, z = 1.959963984540054) {
+  if (n === 0) return [0, 1];
+  const ph = x / n, z2 = z * z, den = 1 + z2 / n;
+  const mid = (ph + z2 / (2 * n)) / den;
+  const half = (z * Math.sqrt((ph * (1 - ph)) / n + z2 / (4 * n * n))) / den;
+  return [Math.max(0, mid - half), Math.min(1, mid + half)];
+}
+
+/**
+ * Learn-Then-Test with fixed-sequence testing (Angelopoulos et al., 2021).
+ * Walks the cutoffs from strictest to loosest and keeps the loosest one whose risk is certified ≤ alpha
+ * at level delta; it stops at the first cutoff that fails. The grid must not be chosen from the
+ * calibration rows themselves.
+ *
+ *   guarantee 'overall': risk = P(answered locally and wrong)   (n: all calibration rows)
+ *   guarantee 'local'  : risk = P(wrong | answered locally)     (n: rows answered locally)
+ *
+ * @param {Float64Array|number[]} score  per calibration row; answered locally when score ≥ cutoff
+ * @param {Uint8Array|boolean[]} wrong   per calibration row; the local answer differs from the API's
+ * @param {number[]} grid                candidate cutoffs, any order
+ * @returns {{ cutoff: number, local: number, errors: number, pValue: number, tested: number }}
+ */
+export function certify(score, wrong, grid, { alpha, delta = 0.05, guarantee = 'overall' }) {
+  const n = score.length;
+  const order = Array.from(score.keys()).sort((i, j) => score[j] - score[i]);
+  const cuts = [...new Set(grid)].sort((a, b) => b - a);
+  // For the local guarantee, testing starts at the first cutoff that answers localStart() rows:
+  // starting at the bare minimum fails the whole sequence on a single mislabelled row among the most
+  // confident few. Which cutoffs are skipped depends only on the scores, never on the errors, so the
+  // test sequence is still fixed given the scores.
+  const floor = guarantee === 'local' ? localStart(alpha, delta) : 1;
+  let best = { cutoff: Infinity, local: 0, errors: 0, pValue: 0, tested: 0 };
+  let i = 0, local = 0, errors = 0;
+  for (let c = 0; c < cuts.length; c++) {
+    const cut = cuts[c];
+    while (i < n && score[order[i]] >= cut) { local++; errors += wrong[order[i]] ? 1 : 0; i++; }
+    const trials = guarantee === 'local' ? local : n;
+    if (trials < floor) continue;
+    const pValue = binomCdf(errors, trials, alpha);
+    if (pValue > delta) return { ...best, tested: c + 1 };
+    best = { cutoff: cut, local, errors, pValue, tested: c + 1 };
+  }
+  return best;
+}
+
+/**
+ * Local answers needed before the local guarantee starts testing: the smallest sample on which an
+ * error rate of half the budget would still certify, so one stray label error cannot end the test.
+ */
+export function localStart(alpha, delta = 0.05) {
+  let m = minCalibration(alpha, delta);
+  while (binomCdf(Math.floor((m * alpha) / 2), m, alpha) > delta) m++;
+  return m;
+}
+
+/** Smallest calibration size at which an error-free sample can certify alpha at level delta. */
+export function minCalibration(alpha, delta = 0.05) {
+  return Math.ceil(Math.log(delta) / Math.log1p(-alpha));
+}

package/src/store.js

@@ -0,0 +1,21 @@
+// Where shadow() keeps your API's answers, the installed model, and whether it is guarded.
+//
+// A store needs append(record), load() → recent records (oldest first) and size. The model and
+// state methods are optional; without them a restart starts from scratch, which is always safe.
+// The file store lives in ./node.js so this module stays browser-safe.
+
+/** Keeps everything in memory: the default in browsers and workers. */
+export function memoryStore({ limit = 100_000, records = [] } = {}) {
+  const rows = records.slice(-limit);
+  let model = null, state = null;
+  return {
+    append(record) { rows.push(record); if (rows.length > limit * 1.25) rows.splice(0, rows.length - limit); },
+    load() { return rows.slice(-limit); },
+    get size() { return rows.length; },
+    get records() { return rows; },
+    loadModel: () => model,
+    saveModel(m) { model = m; },
+    loadState: () => state,
+    saveState(s) { state = s; },
+  };
+}

package/src/train.js

@@ -0,0 +1,129 @@
+// fastText-style trainer: mean-pooled hashed features → linear head, plain SGD with linear decay,
+// distilling your API's soft labels (one-hot when it gives only a label).
+
+import { U32Map, rng } from './hash.js';
+
+export const RECIPE = Object.freeze({ d: 32, lr: 0.5, minCount: 2, maxFeatures: 200_000, steps: 200_000, minEpochs: 5, maxEpochs: 25 });
+
+export const tick = makeTick();
+
+// Yield to the event loop between chunks of work. Browsers clamp nested setTimeout to 4 ms, so use a
+// MessageChannel there; Node has setImmediate.
+function makeTick() {
+  if (typeof setImmediate === 'function') return () => new Promise((r) => setImmediate(r));
+  if (typeof MessageChannel === 'function') {
+    const ch = new MessageChannel(), queue = [];
+    ch.port1.onmessage = () => queue.shift()?.();
+    return () => new Promise((r) => { queue.push(r); ch.port2.postMessage(0); });
+  }
+  return () => new Promise((r) => setTimeout(r, 0));
+}
+
+/**
+ * @param {{ w: Uint32Array, c: Uint32Array, t: Float32Array }[]} rows  features and target distribution; remapped in place
+ * @param {number} K
+ * @returns {Promise<{ d, keys: Uint32Array, E: Float32Array, W: Float32Array, b: Float32Array, epochs: number }>}
+ */
+export async function train(rows, K, opts = {}) {
+  const { d, lr, minCount, maxFeatures, steps, minEpochs, maxEpochs } = { ...RECIPE, ...opts };
+  const seed = opts.seed ?? 1;
+  const onProgress = opts.onProgress ?? (() => {});
+  const yieldMs = opts.yieldMs ?? 30;
+
+  // Vocabulary: features seen at least minCount times, the most frequent maxFeatures of them.
+  const counts = new U32Map(1 << 16);
+  for (const r of rows) { for (const f of r.w) counts.inc(f); for (const f of r.c) counts.inc(f); }
+  let kept = [];
+  counts.forEach((key, n) => { if (n >= minCount) kept.push([key, n]); });
+  if (kept.length > maxFeatures) kept = kept.sort((a, b) => b[1] - a[1] || a[0] - b[0]).slice(0, maxFeatures);
+  const keys = Uint32Array.from(kept.map((e) => e[0])).sort();
+  const index = new U32Map(keys.length);
+  for (let i = 0; i < keys.length; i++) index.set(keys[i], i);
+  const ids = (feats) => {
+    const out = [];
+    for (const f of feats) { const r = index.get(f); if (r >= 0) out.push(r); }
+    return Int32Array.from(out);
+  };
+  // In place, so each row's raw feature copy can be freed as soon as it is mapped.
+  // shortcut: rows still take about 22 KB each at 2,000-char inputs (0.7 GB at the default 50,000-row
+  // window); cap the window by total features if long-input decisions must fit in less memory.
+  for (let i = 0; i < rows.length; i++) rows[i] = { w: ids(rows[i].w), c: ids(rows[i].c), t: rows[i].t };
+  const data = rows;
+
+  const V = keys.length;
+  const E = new Float32Array(V * d);
+  const W = new Float32Array(K * d);
+  const b = new Float32Array(K);
+  const rand = rng(seed);
+  const a = Math.sqrt(6 / (d + K));
+  for (let i = 0; i < W.length; i++) W[i] = (rand() * 2 - 1) * a;
+  // Start the bias at the API's average distribution.
+  const prior = new Float64Array(K);
+  for (const r of data) for (let k = 0; k < K; k++) prior[k] += r.t[k];
+  for (let k = 0; k < K; k++) b[k] = Math.log((prior[k] + 1) / (data.length + K));
+
+  const N = data.length;
+  const epochs = Math.max(minEpochs, Math.min(maxEpochs, Math.round(steps / Math.max(1, N))));
+  const total = epochs * N;
+  const h = new Float32Array(d), hw = new Float32Array(d), hc = new Float32Array(d), gh = new Float32Array(d);
+  const p = new Float32Array(K);
+  const order = Int32Array.from({ length: N }, (_, i) => i);
+  let step = 0, last = performance.now();
+
+  for (let ep = 0; ep < epochs; ep++) {
+    for (let i = N - 1; i > 0; i--) { const j = Math.floor(rand() * (i + 1)); const x = order[i]; order[i] = order[j]; order[j] = x; }
+    for (let s = 0; s < N; s++, step++) {
+      const ex = data[order[s]];
+      const rate = lr * (1 - step / total);
+      const nw = ex.w.length, nc = ex.c.length;
+      if (nw + nc === 0) continue;
+      meanInto(hw, E, ex.w, d); meanInto(hc, E, ex.c, d);
+      for (let j = 0; j < d; j++) h[j] = hw[j] + hc[j];
+      let m = -Infinity;
+      for (let k = 0; k < K; k++) {
+        let z = b[k];
+        const o = k * d;
+        for (let j = 0; j < d; j++) z += W[o + j] * h[j];
+        p[k] = z; if (z > m) m = z;
+      }
+      let sum = 0;
+      for (let k = 0; k < K; k++) sum += p[k] = Math.exp(p[k] - m);
+      gh.fill(0);
+      for (let k = 0; k < K; k++) {
+        const g = p[k] / sum - ex.t[k];
+        if (g === 0) continue;
+        const o = k * d;
+        for (let j = 0; j < d; j++) { gh[j] += W[o + j] * g; W[o + j] -= rate * g * h[j]; }
+        b[k] -= rate * g;
+      }
+      if (nw) scatter(E, ex.w, gh, -rate / nw, d);
+      if (nc) scatter(E, ex.c, gh, -rate / nc, d);
+      if ((step & 255) === 0 && performance.now() - last > yieldMs) {
+        onProgress({ phase: 'train', done: step / total });
+        await tick();
+        if (opts.signal?.aborted) throw abortError();
+        last = performance.now();
+      }
+    }
+  }
+  onProgress({ phase: 'train', done: 1 });
+  return { d, keys, E, W, b, epochs };
+}
+
+function meanInto(out, E, ids, d) {
+  out.fill(0);
+  const n = ids.length;
+  if (!n) return;
+  for (let i = 0; i < n; i++) { const o = ids[i] * d; for (let j = 0; j < d; j++) out[j] += E[o + j]; }
+  for (let j = 0; j < d; j++) out[j] /= n;
+}
+
+function scatter(E, ids, g, scale, d) {
+  for (let i = 0; i < ids.length; i++) { const o = ids[i] * d; for (let j = 0; j < d; j++) E[o + j] += scale * g[j]; }
+}
+
+export function abortError() {
+  const err = new Error('compile aborted');
+  err.name = 'AbortError';
+  return err;
+}

package/src/worker.js

@@ -0,0 +1,12 @@
+// Runs one compile off the main thread and sends the model back as bytes.
+
+import { parentPort, workerData } from 'node:worker_threads';
+import { compile } from './compile.js';
+
+try {
+  const { model, report } = await compile(workerData.rows, workerData.options);
+  const bytes = model.toBytes();
+  parentPort.postMessage({ bytes, report }, [bytes.buffer]);
+} catch (err) {
+  parentPort.postMessage({ error: { name: err?.name ?? 'Error', message: String(err?.message ?? err) } });
+}