squeezr-ai 1.90.0 → 1.99.2

package/dist/dashboard.js

@@ -430,7 +430,7 @@ code{font-family:'Cascadia Code','SF Mono',Consolas,monospace;font-size:.9em}
         <div class="hero-card">
           <div class="hc-label">Requests</div>
           <div class="hc-val" id="h-reqs">—</div>
-          <div class="hc-sub"><span id="h-comp">—</span> AI-compressed · det. always on</div>
+          <div class="hc-sub"><span id="h-comp">—</span> AI-compressed · det. always on<span id="h-bypassed"></span></div>
         </div>
       </div>
 
@@ -498,16 +498,16 @@ code{font-family:'Cascadia Code','SF Mono',Consolas,monospace;font-size:.9em}
             <div style="margin-top:8px;font-size:11px;color:var(--text3);text-align:center">0 here is normal when AI compression is off</div>
           </div>
         </div>
-        <!-- AI Compression -->
+        <!-- Output shaping (output-side token reduction) — pinned in the old AI-Compression slot -->
         <div class="section" style="margin:0">
-          <div class="section-head"><span class="section-title">AI Compression</span><span style="font-size:11px;color:var(--text3)">today · persisted</span></div>
+          <div class="section-head"><span class="section-title">Output</span><span style="font-size:11px;color:var(--text3)" id="out-note">what the model writes back · needs [output] enabled</span></div>
           <div class="section-body">
             <div class="cache-row">
-              <div class="cache-card"><div class="cache-label">Calls</div><div class="cache-val" id="ai-calls">—</div></div>
-              <div class="cache-card"><div class="cache-label">Saved</div><div class="cache-val" id="ai-saved" style="color:var(--brand2)">—</div></div>
-              <div class="cache-card"><div class="cache-label">Spent</div><div class="cache-val" id="ai-spent">—</div></div>
+              <div class="cache-card"><div class="cache-label">Echo (restated)</div><div class="cache-val" id="out-echo">—</div></div>
+              <div class="cache-card"><div class="cache-label">Steered</div><div class="cache-val" id="out-steered">—</div></div>
+              <div class="cache-card"><div class="cache-label">Effort lowered</div><div class="cache-val" id="out-effort">—</div></div>
             </div>
-            <div id="ai-net" style="margin-top:8px;font-size:11px;color:var(--text3);text-align:center">—</div>
+            <div style="margin-top:8px;font-size:11px;color:var(--text3);text-align:center">Echo = % of the model's output that merely restated context it was already given (lower is better).</div>
           </div>
         </div>
       </div>
@@ -1022,6 +1022,9 @@ function render(d) {
   document.getElementById('h-cost').textContent  = fmtUsd(tCost);
   document.getElementById('h-reqs').textContent  = fmt(tReqs);
   document.getElementById('h-comp').textContent  = fmt(tComps);
+  var byp = (d.bypassed_requests != null) ? d.bypassed_requests : 0;
+  var bypEl = document.getElementById('h-bypassed');
+  if (bypEl) bypEl.textContent = byp > 0 ? ' · ' + fmt(byp) + ' bypassed (not counted)' : '';
 var perEl = document.getElementById('overview-period');
   if (perEl && today.date) perEl.textContent = 'today · ' + today.date;
   // (No quality banner in the Overview — only genuine quality issues, high expand
@@ -1118,6 +1121,22 @@ var eff = (today.efficiency_pct != null) ? today.efficiency_pct : null;
       }
     }
   }
+  // Output shaping card (all-time; only meaningful when [output] enabled)
+  var out = d.output || {};
+  var setOut = function(id, v){ var e = document.getElementById(id); if(e) e.textContent = v; };
+  var echoEl = document.getElementById('out-echo');
+  if (echoEl) {
+    if (out.echo_samples > 0) {
+      echoEl.textContent = out.avg_echo_pct + '%';
+      // lower echo is better: green under 15%, amber 15-35%, red above
+      echoEl.style.color = out.avg_echo_pct <= 15 ? 'var(--brand2)' : out.avg_echo_pct <= 35 ? '#fbbf24' : 'var(--red, #e5484d)';
+    } else {
+      echoEl.textContent = '—';
+    }
+  }
+  setOut('out-steered', out.steered > 0 ? fmt(out.steered) : '—');
+  setOut('out-effort', out.effort_lowered > 0 ? fmt(out.effort_lowered) : '—');
+
   // Cost Comparison is OVERVIEW = TODAY: use today's tokens (not all-time), so the
   // sub-lines match the today money. (modelCosts already comes from today's models.)
   var cmpIn     = today.original_tokens || 0;

package/dist/deterministic.d.ts

@@ -34,4 +34,4 @@ export declare function preprocessAssistant(text: string): string;
 export type CodeLang = 'ts' | 'py' | 'go' | 'rs' | 'java' | 'c' | 'cpp';
 export declare function detectCodeLanguage(text: string): CodeLang | null;
 export declare function extractCodeStructure(text: string, lang: CodeLang): string;
-export declare function preprocessForTool(text: string, toolName: string, pressure?: number): string;
+export declare function preprocessForTool(text: string, toolName: string, pressure?: number, query?: string): string;

package/dist/deterministic.js

@@ -27,7 +27,7 @@
  *      Glob — compact large file listings into directory summary
  */
 import { storeOriginal, storeSegments } from './expand.js';
-import { crushJsonArrays } from './jsonCrush.js';
+import { crushEmbeddedJson } from './contentRouter.js';
 import { crushText } from './textCrusher.js';
 // ── Pattern hit tracking (for squeezr discover) ───────────────────────────────
 export const detPatternHits = {};
@@ -928,13 +928,13 @@ function extractGenericErrors(text) {
 // (collapse near-duplicate spam, keep head/tail + high-signal lines) over blind
 // tail-keep — deterministic and cache-safe. Falls back to tail-keep if the crusher
 // can't beat it (e.g. all lines unique and low-signal).
-function truncateLongOutput(text, pressure = 0) {
+function truncateLongOutput(text, pressure = 0, query = '') {
     const threshold = pressure >= 0.9 ? 50 : 80;
     const keepLines = pressure >= 0.9 ? 30 : 50;
     const lines = text.split('\n');
     if (lines.length <= threshold)
         return text;
-    const crushed = crushText(text, { maxLines: keepLines, headKeep: 5, tailKeep: 8 });
+    const crushed = crushText(text, { maxLines: keepLines, headKeep: 5, tailKeep: 8, query });
     if (crushed.dropped > 0 && crushed.text.length < text.length) {
         hit('textCrush');
         return crushed.text;
@@ -959,7 +959,7 @@ function compactFileListing(text) {
     const summary = Object.entries(byDir).sort(([, a], [, b]) => b - a).map(([d, n]) => `${d}/ (${n} files)`).join('\n');
     return `${lines.length} files total:\n${summary}`;
 }
-function applyBashPatterns(text, pressure = 0) {
+function applyBashPatterns(text, pressure = 0, query = '') {
     if (looksLikeGitDiff(text)) {
         hit('gitDiff');
         return compactGitDiff(text, pressure);
@@ -1083,7 +1083,7 @@ function applyBashPatterns(text, pressure = 0) {
         return errExtracted;
     }
     hit('truncated');
-    return truncateLongOutput(text, pressure);
+    return truncateLongOutput(text, pressure, query);
 }
 // ── Grep tool ─────────────────────────────────────────────────────────────────
 // Group matches by file, cap at MAX_PER_FILE lines each, truncate if too many files
@@ -1344,7 +1344,7 @@ function makeRecoverable(original, compacted) {
     const id = storeOriginal(original);
     return `${compacted}\n[squeezr_expand("${id}") — full untruncated output]`;
 }
-export function preprocessForTool(text, toolName, pressure = 0) {
+export function preprocessForTool(text, toolName, pressure = 0, query = '') {
     const tool = toolName.toLowerCase();
     // NEVER compress the result of an expand call. The whole point of squeezr_expand
     // is to deliver the FULL original back to the model; re-compacting it here (or
@@ -1354,19 +1354,20 @@ export function preprocessForTool(text, toolName, pressure = 0) {
     if (tool === 'read') {
         return makeRecoverable(text, compactReadOutput(preprocessRead(text)));
     }
-    // JSON array crush (deterministic, cache-safe, reversible). Fires only when the whole
-    // result is an array of homogeneous objects (gh api / curl / MCP record lists / kubectl
-    // -o json…): the repeated per-row keys collapse to a single header. Runs BEFORE the
-    // base pipeline so line-dedup/JSON-minify never touch the structured data. Self-recoverable
-    // (its marker already carries squeezr_expand), so we return it directly.
-    const crushed = crushJsonArrays(text);
+    // JSON array crush (deterministic, cache-safe, reversible). Now via the content router,
+    // so it fires on arrays of homogeneous objects whether they are the WHOLE result or
+    // EMBEDDED inside a larger one (gh api with a header, MCP envelopes, bash echo + json…).
+    // Repeated per-row keys collapse to a single header; each crushed span is self-recoverable
+    // (its marker carries squeezr_expand). Runs BEFORE the base pipeline so line-dedup/JSON-minify
+    // never touch the structured data; returned directly.
+    const crushed = crushEmbeddedJson(text);
     if (crushed.savedChars > 0) {
         hit('jsonTableCrush');
         return crushed.text;
     }
     let t = preprocess(text);
     if (tool === 'bash') {
-        t = applyBashPatterns(t, pressure);
+        t = applyBashPatterns(t, pressure, query);
     }
     else if (tool === 'grep') {
         const before = t;

package/dist/doctor.d.ts

@@ -18,11 +18,16 @@ type Health = {
     version?: string;
     bypassed?: boolean;
     port?: number;
+    compression?: {
+        requests?: number;
+        savings_pct?: number;
+    };
 } | null;
 export declare function checkProxy(health: Health): CheckResult;
 export declare function checkVersion(installed: string, running: string | null): CheckResult;
 export declare function checkEnv(baseUrl: string | undefined, port: number): CheckResult;
 export declare function checkBypass(bypassed: boolean | undefined): CheckResult;
+export declare function checkSavings(requests: number | undefined, savingsPct: number | undefined): CheckResult;
 export declare function computeExitCode(checks: CheckResult[]): number;
 export declare function formatDoctor(checks: CheckResult[], exitCode: number): string;
 export declare function runDoctor(): Promise<{

package/dist/doctor.js

@@ -42,6 +42,16 @@ export function checkBypass(bypassed) {
         return { name: 'Bypass', status: 'warn', detail: 'bypass is ON — compression is disabled (squeezr bypass --off to resume)' };
     return { name: 'Bypass', status: 'pass', detail: 'compression active' };
 }
+export function checkSavings(requests, savingsPct) {
+    if (requests === undefined)
+        return { name: 'Savings', status: 'skip', detail: 'proxy not running' };
+    if (requests === 0)
+        return { name: 'Savings', status: 'skip', detail: 'no traffic yet' };
+    if (!savingsPct || savingsPct <= 0) {
+        return { name: 'Savings', status: 'warn', detail: `${requests} requests seen but ~0% saved — bypass on? stale build? all content incompressible?` };
+    }
+    return { name: 'Savings', status: 'pass', detail: `~${Math.round(savingsPct)}% saved over ${requests} requests` };
+}
 export function computeExitCode(checks) {
     if (checks.some(c => c.status === 'fail'))
         return 2;
@@ -96,6 +106,7 @@ export async function runDoctor() {
         checkVersion(installedVersion(), runningVersion),
         checkEnv(process.env.ANTHROPIC_BASE_URL, health?.port ?? port),
         checkBypass(health?.bypassed),
+        checkSavings(health?.compression?.requests, health?.compression?.savings_pct),
     ];
     const exitCode = computeExitCode(checks);
     return { checks, exitCode, report: formatDoctor(checks, exitCode) };

package/dist/index.js

@@ -46,7 +46,10 @@ if (PORT !== config.port) {
 const httpServer = createAdaptorServer({ fetch: app.fetch });
 // Persist caches every 60s so a crash doesn't lose more than a minute of work
 setInterval(() => { persistSessionCache(); persistExpandStore(); persistHistory(); }, 60_000).unref();
-httpServer.listen(PORT, () => {
+// Bind to loopback only: the proxy holds the user's API keys and exposes
+// state-changing control endpoints, so it must never be reachable from the LAN.
+// Matches the desktop proxy / MITM listeners, which already bind 127.0.0.1.
+httpServer.listen(PORT, '127.0.0.1', () => {
     console.log(`Squeezr v${VERSION} listening on http://localhost:${PORT}`);
     console.log(`Mode: ${config.dryRun ? 'dry-run' : 'active'}`);
     if (config.disabled)

package/dist/jsonCrush.d.ts

@@ -20,6 +20,10 @@
  * (the lossy SmartCrusher path) is intentionally NOT done here.
  */
 export declare const TABLE_MARKER_RE: RegExp;
+/** 32-bit SimHash fingerprint of a text (bit-voting over its word tokens). Two texts
+ *  that share most tokens get fingerprints a small Hamming distance apart. */
+export declare function simhash32(text: string): number;
+export declare function hammingDistance(a: number, b: number): number;
 /**
  * If `text` is (entirely) a crushable JSON array of objects, return the table form plus
  * the char count saved. Otherwise return the input unchanged with savedChars = 0.

package/dist/jsonCrush.js

@@ -23,11 +23,73 @@ import { storeOriginal } from './expand.js';
 const MIN_ITEMS = 5;
 const MIN_OBJECT_FRACTION = 0.8; // ≥80% of elements must be plain objects
 const MIN_SAVINGS_RATIO = 0.15; // only crush if it shaves ≥15% of the chars
-// Recognises the table header this module emits. Group 1 = the expand id.
-export const TABLE_MARKER_RE = /^\[squeezr:table ([0-9a-f]{6,}(?:~\d+)?) —/;
+const LOSSY_ROW_THRESHOLD = 50; // above this many rows, drop near-duplicates (SimHash)
+const HAMMING_NEAR_DUP = 3; // rows within this 32-bit Hamming distance are "the same"
+const ROW_SIGNAL_RE = /\b(error|err|fail|failed|failure|exception|fatal|panic|denied|refused|timeout|critical|crash|oom|unhealthy|degraded)\b/i;
+// Recognises the table header this module emits (anywhere in the text, since the table
+// may be embedded in a larger tool result). Group 1 = the expand id.
+export const TABLE_MARKER_RE = /\[squeezr:table ([0-9a-f]{6,}(?:~\d+)?) —/;
 function isPlainObject(v) {
     return typeof v === 'object' && v !== null && !Array.isArray(v);
 }
+// ── SimHash near-duplicate detection (deterministic, dependency-free) ─────────
+/** FNV-1a 32-bit hash of a token. */
+function fnv1a32(s) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < s.length; i++) {
+        h ^= s.charCodeAt(i);
+        h = Math.imul(h, 0x01000193);
+    }
+    return h >>> 0;
+}
+/** 32-bit SimHash fingerprint of a text (bit-voting over its word tokens). Two texts
+ *  that share most tokens get fingerprints a small Hamming distance apart. */
+export function simhash32(text) {
+    const tokens = text.toLowerCase().match(/[a-z0-9_]+/g) ?? [];
+    if (tokens.length === 0)
+        return 0;
+    const bits = new Array(32).fill(0);
+    for (const t of tokens) {
+        const h = fnv1a32(t);
+        for (let b = 0; b < 32; b++)
+            bits[b] += ((h >>> b) & 1) ? 1 : -1;
+    }
+    let f = 0;
+    for (let b = 0; b < 32; b++)
+        if (bits[b] > 0)
+            f |= (1 << b);
+    return f >>> 0;
+}
+export function hammingDistance(a, b) {
+    let x = (a ^ b) >>> 0;
+    let c = 0;
+    while (x) {
+        c += x & 1;
+        x >>>= 1;
+    }
+    return c;
+}
+/**
+ * Given the per-row rendered strings, return the indices to KEEP: one representative per
+ * SimHash cluster (near-duplicates dropped), plus every row that carries an error/anomaly
+ * signal (always kept). Order preserved. Deterministic.
+ */
+function selectRepresentativeRows(rowStrings) {
+    const reps = []; // simhash fingerprints of chosen representatives
+    const kept = [];
+    for (let i = 0; i < rowStrings.length; i++) {
+        if (ROW_SIGNAL_RE.test(rowStrings[i])) {
+            kept.push(i);
+            continue;
+        } // anomaly → always keep
+        const sig = simhash32(rowStrings[i]);
+        if (reps.some(r => hammingDistance(r, sig) <= HAMMING_NEAR_DUP))
+            continue; // near-dup → drop
+        reps.push(sig);
+        kept.push(i);
+    }
+    return kept;
+}
 /** Union of keys across rows, in first-seen order (deterministic). */
 function collectColumns(rows) {
     const cols = [];
@@ -86,27 +148,30 @@ export function crushJsonArrays(text) {
     if (cols.length === 0)
         return { text, savedChars: 0 };
     const header = cols.join(CELL_DELIM);
-    const bodyLines = [];
-    for (const row of rows) {
-        if (isPlainObject(row)) {
-            bodyLines.push(cols.map(c => cell(row[c], c in row)).join(CELL_DELIM));
-        }
-        else {
-            // Non-object element inside a mostly-object array: keep it verbatim as JSON on
-            // its own line so nothing is lost (recoverable in full via expand anyway).
-            bodyLines.push(JSON.stringify(row));
+    const rowLines = rows.map(row => isPlainObject(row)
+        ? cols.map(c => cell(row[c], c in row)).join(CELL_DELIM)
+        // Non-object element inside a mostly-object array: keep it verbatim as JSON so
+        // nothing is lost (recoverable in full via expand anyway).
+        : JSON.stringify(row));
+    // Lossy row-drop for LARGE arrays: collapse near-duplicate rows (SimHash), always
+    // keeping anomaly/error rows. Deterministic → cache-safe. The full original is in the
+    // expand store, so dropped rows are recoverable. Small arrays keep every row.
+    let bodyLines = rowLines;
+    let omitted = 0;
+    if (rows.length > LOSSY_ROW_THRESHOLD) {
+        const kept = selectRepresentativeRows(rowLines);
+        if (kept.length < rowLines.length) {
+            bodyLines = kept.map(i => rowLines[i]);
+            omitted = rowLines.length - kept.length;
         }
     }
-    // Build the (id-less) body first to measure real savings before we pay for an id.
-    const provisional = `${header}\n${bodyLines.join('\n')}`;
-    const saved = text.length - provisional.length;
-    if (saved <= 0 || saved / text.length < MIN_SAVINGS_RATIO)
-        return { text, savedChars: 0 };
     const id = storeOriginal(text);
-    const marker = `[squeezr:table ${id} — ${rows.length} rows × ${cols.length} cols; squeezr_expand("${id}") for original JSON]`;
+    const marker = omitted > 0
+        ? `[squeezr:table ${id} — showing ${bodyLines.length} of ${rows.length} rows × ${cols.length} cols; ${omitted} near-duplicate rows omitted; squeezr_expand("${id}") for original JSON]`
+        : `[squeezr:table ${id} — ${rows.length} rows × ${cols.length} cols; squeezr_expand("${id}") for original JSON]`;
     const out = `${marker}\n${header}\n${bodyLines.join('\n')}`;
     const savedChars = text.length - out.length;
-    if (savedChars <= 0)
+    if (savedChars <= 0 || savedChars / text.length < MIN_SAVINGS_RATIO)
         return { text, savedChars: 0 };
     return { text: out, savedChars };
 }

package/dist/outputSavings.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Output-savings signals — Squeezr's honest, no-counterfactual take on headroom's output
+ * measurement. The fully honest number (headroom's holdout A/B) needs a control arm; that
+ * is deferred. This ships the tier that needs NO counterfactual: the ECHO RATIO — how much
+ * of what the model just wrote merely restated context it was already given.
+ *
+ * High echo = wasted output tokens = exactly what verbosity steering (1.83) targets. Reading
+ * the response to measure this never changes the bytes we forward, so it's cache-safe.
+ */
+/** Word n-grams (set) of a text. */
+export declare function wordNgrams(text: string, n?: number): Set<string>;
+/**
+ * Fraction of the OUTPUT's n-grams that also appear in the CONTEXT (0..1). 1 = the model
+ * only restated what it was already shown; 0 = fully novel output.
+ */
+export declare function echoRatio(output: string, context: string, n?: number): number;
+/** Pull the assistant's text out of an Anthropic SSE stream (concatenated text_delta). */
+export declare function extractAssistantTextFromSse(sse: string): string;
+/** Assistant text from a non-streaming Anthropic response body's content blocks. */
+export declare function extractAssistantTextFromContent(content: unknown): string;

package/dist/outputSavings.js

@@ -0,0 +1,52 @@
+/**
+ * Output-savings signals — Squeezr's honest, no-counterfactual take on headroom's output
+ * measurement. The fully honest number (headroom's holdout A/B) needs a control arm; that
+ * is deferred. This ships the tier that needs NO counterfactual: the ECHO RATIO — how much
+ * of what the model just wrote merely restated context it was already given.
+ *
+ * High echo = wasted output tokens = exactly what verbosity steering (1.83) targets. Reading
+ * the response to measure this never changes the bytes we forward, so it's cache-safe.
+ */
+/** Word n-grams (set) of a text. */
+export function wordNgrams(text, n = 3) {
+    const words = text.toLowerCase().match(/[a-z0-9_]+/g) ?? [];
+    const set = new Set();
+    for (let i = 0; i + n <= words.length; i++)
+        set.add(words.slice(i, i + n).join(' '));
+    return set;
+}
+/**
+ * Fraction of the OUTPUT's n-grams that also appear in the CONTEXT (0..1). 1 = the model
+ * only restated what it was already shown; 0 = fully novel output.
+ */
+export function echoRatio(output, context, n = 3) {
+    const out = wordNgrams(output, n);
+    if (out.size === 0)
+        return 0;
+    const ctx = wordNgrams(context, n);
+    let echoed = 0;
+    for (const g of out)
+        if (ctx.has(g))
+            echoed++;
+    return echoed / out.size;
+}
+/** Pull the assistant's text out of an Anthropic SSE stream (concatenated text_delta). */
+export function extractAssistantTextFromSse(sse) {
+    let out = '';
+    for (const m of sse.matchAll(/"type":"text_delta","text":"((?:[^"\\]|\\.)*)"/g)) {
+        try {
+            out += JSON.parse(`"${m[1]}"`);
+        }
+        catch { /* skip malformed delta */ }
+    }
+    return out;
+}
+/** Assistant text from a non-streaming Anthropic response body's content blocks. */
+export function extractAssistantTextFromContent(content) {
+    if (!Array.isArray(content))
+        return '';
+    return content
+        .filter(b => b && b.type === 'text' && typeof b.text === 'string')
+        .map(b => b.text)
+        .join('');
+}

package/dist/relevance.d.ts

@@ -0,0 +1,27 @@
+/**
+ * BM25 relevance scorer — a shared primitive so Squeezr's compressors can keep content
+ * by RELEVANCE to the current task, not just by position (head/tail) or recency.
+ *
+ * This is the single highest-leverage idea copied from headroom: their SmartCrusher,
+ * TextCrusher and CodeCompressor all route "what to keep" through one relevance scorer.
+ * Squeezr, until now, kept blindly. This module is that scorer — pure TS, zero deps,
+ * deterministic (→ cache-safe when used to drive byte-stable compression).
+ *
+ * BM25 (Okapi) is the standard, well-understood ranking function: term frequency with
+ * saturation (k1) and document-length normalization (b), weighted by inverse document
+ * frequency so rare query terms (an error code, a filename) count far more than common
+ * words. The idf uses +0.5 smoothing so it is always positive.
+ */
+export interface BM25Options {
+    k1?: number;
+    b?: number;
+}
+/** Lowercase, split on non-alphanumerics, keep tokens of length >= 2. */
+export declare function tokenize(text: string): string[];
+/**
+ * BM25 score of each doc in `docs` against `query`. One score per doc, in order.
+ * A doc with no query-term overlap scores exactly 0.
+ */
+export declare function bm25Scores(query: string, docs: string[], opts?: BM25Options): number[];
+/** Document indices sorted by descending relevance (ties keep original order). */
+export declare function rankByRelevance(query: string, docs: string[], opts?: BM25Options): number[];

package/dist/relevance.js

@@ -0,0 +1,78 @@
+/**
+ * BM25 relevance scorer — a shared primitive so Squeezr's compressors can keep content
+ * by RELEVANCE to the current task, not just by position (head/tail) or recency.
+ *
+ * This is the single highest-leverage idea copied from headroom: their SmartCrusher,
+ * TextCrusher and CodeCompressor all route "what to keep" through one relevance scorer.
+ * Squeezr, until now, kept blindly. This module is that scorer — pure TS, zero deps,
+ * deterministic (→ cache-safe when used to drive byte-stable compression).
+ *
+ * BM25 (Okapi) is the standard, well-understood ranking function: term frequency with
+ * saturation (k1) and document-length normalization (b), weighted by inverse document
+ * frequency so rare query terms (an error code, a filename) count far more than common
+ * words. The idf uses +0.5 smoothing so it is always positive.
+ */
+const DEFAULT_K1 = 1.5;
+const DEFAULT_B = 0.75;
+/** Lowercase, split on non-alphanumerics, keep tokens of length >= 2. */
+export function tokenize(text) {
+    const out = [];
+    for (const m of text.toLowerCase().matchAll(/[a-z0-9_]+/g)) {
+        if (m[0].length >= 2)
+            out.push(m[0]);
+    }
+    return out;
+}
+/**
+ * BM25 score of each doc in `docs` against `query`. One score per doc, in order.
+ * A doc with no query-term overlap scores exactly 0.
+ */
+export function bm25Scores(query, docs, opts = {}) {
+    const k1 = opts.k1 ?? DEFAULT_K1;
+    const b = opts.b ?? DEFAULT_B;
+    const queryTerms = [...new Set(tokenize(query))];
+    const N = docs.length;
+    if (queryTerms.length === 0 || N === 0)
+        return new Array(N).fill(0);
+    // Tokenize docs once; compute lengths and average length.
+    const docTokens = docs.map(tokenize);
+    const docLen = docTokens.map(t => t.length);
+    const avgdl = docLen.reduce((s, l) => s + l, 0) / N || 1;
+    // Per-doc term-frequency maps.
+    const tf = docTokens.map(tokens => {
+        const m = new Map();
+        for (const t of tokens)
+            m.set(t, (m.get(t) ?? 0) + 1);
+        return m;
+    });
+    // Document frequency + idf per query term.
+    const idf = new Map();
+    for (const term of queryTerms) {
+        let df = 0;
+        for (const m of tf)
+            if (m.has(term))
+                df++;
+        // Always-positive smoothed idf.
+        idf.set(term, Math.log(1 + (N - df + 0.5) / (df + 0.5)));
+    }
+    return docTokens.map((_, i) => {
+        let score = 0;
+        const len = docLen[i];
+        for (const term of queryTerms) {
+            const f = tf[i].get(term) ?? 0;
+            if (f === 0)
+                continue;
+            const denom = f + k1 * (1 - b + (b * len) / avgdl);
+            score += (idf.get(term) ?? 0) * ((f * (k1 + 1)) / denom);
+        }
+        return score;
+    });
+}
+/** Document indices sorted by descending relevance (ties keep original order). */
+export function rankByRelevance(query, docs, opts) {
+    const scores = bm25Scores(query, docs, opts);
+    return scores
+        .map((score, i) => ({ score, i }))
+        .sort((a, b) => (b.score - a.score) || (a.i - b.i))
+        .map(x => x.i);
+}