pi-mega-compact 0.6.7 → 0.7.0

package/dist/src/wordpiece.js

@@ -0,0 +1,129 @@
+/**
+ * wordpiece.ts — a self-contained WordPiece tokenizer for BERT/MiniLM.
+ *
+ * Loads the canonical `vocab.txt` (bert-base-uncased, ~30K tokens) from disk and
+ * implements the standard uncased BERT preprocessing + greedy longest-match
+ * WordPiece segmentation. No native dependency, no network — the vocab file is a
+ * local artifact fetched once by scripts/setup-minilm.mjs (PREVENT-PI-004).
+ *
+ * This mirrors HuggingFace `BertTokenizer` closely enough for sentence-embedding
+ * use: lowercase, strip accents, split on whitespace + punctuation, then
+ * WordPiece each token with the `##` continuation convention. Special tokens
+ * [CLS]/[SEP] are added by the caller's encode().
+ */
+import { readFileSync, existsSync } from "node:fs";
+const UNK = "[UNK]";
+const CLS = "[CLS]";
+const SEP = "[SEP]";
+const PAD = "[PAD]";
+const MAX_INPUT_CHARS_PER_WORD = 200;
+export class WordPieceTokenizer {
+    vocab;
+    clsId;
+    sepId;
+    padId;
+    unkId;
+    constructor(vocab) {
+        this.vocab = vocab;
+        this.clsId = vocab.get(CLS) ?? 101;
+        this.sepId = vocab.get(SEP) ?? 102;
+        this.padId = vocab.get(PAD) ?? 0;
+        this.unkId = vocab.get(UNK) ?? 100;
+    }
+    /** Build a tokenizer from a vocab.txt file (one token per line, index = line). */
+    static fromVocabFile(path) {
+        if (!existsSync(path)) {
+            throw new Error(`WordPiece vocab not found at ${path}. Run: node scripts/setup-minilm.mjs`);
+        }
+        const lines = readFileSync(path, "utf-8").split("\n");
+        const vocab = new Map();
+        for (let i = 0; i < lines.length; i++) {
+            const tok = lines[i].replace(/\r$/, "");
+            if (tok.length > 0 || i < lines.length - 1)
+                vocab.set(tok, i);
+        }
+        return new WordPieceTokenizer(vocab);
+    }
+    /** Uncased BERT basic tokenization: lowercase, strip accents, split on ws+punct. */
+    basicTokenize(text) {
+        // NFD + strip combining marks (accent removal), then lowercase.
+        const cleaned = text
+            .normalize("NFD")
+            .replace(/[̀-ͯ]/g, "")
+            .toLowerCase();
+        const tokens = [];
+        let buf = "";
+        const flush = () => {
+            if (buf.length > 0) {
+                tokens.push(buf);
+                buf = "";
+            }
+        };
+        for (const ch of cleaned) {
+            if (/\s/.test(ch)) {
+                flush();
+            }
+            else if (/[!-/:-@[-`{-~¡-¿]/.test(ch)) {
+                // Punctuation becomes its own token.
+                flush();
+                tokens.push(ch);
+            }
+            else {
+                buf += ch;
+            }
+        }
+        flush();
+        return tokens;
+    }
+    /** Greedy longest-match WordPiece for a single word. */
+    wordpiece(word) {
+        if (word.length > MAX_INPUT_CHARS_PER_WORD)
+            return [UNK];
+        const pieces = [];
+        let start = 0;
+        while (start < word.length) {
+            let end = word.length;
+            let cur = null;
+            while (start < end) {
+                let sub = word.slice(start, end);
+                if (start > 0)
+                    sub = "##" + sub;
+                if (this.vocab.has(sub)) {
+                    cur = sub;
+                    break;
+                }
+                end--;
+            }
+            if (cur === null)
+                return [UNK]; // any unmatchable piece → whole word is UNK
+            pieces.push(cur);
+            start = end;
+        }
+        return pieces;
+    }
+    /** Tokenize text into WordPiece token strings (no special tokens). */
+    tokenize(text) {
+        const out = [];
+        for (const word of this.basicTokenize(text)) {
+            for (const piece of this.wordpiece(word))
+                out.push(piece);
+        }
+        return out;
+    }
+    /**
+     * Encode text into model inputs with [CLS]…[SEP], truncated to `maxLen`.
+     * attention_mask is all 1s (no padding for single-sequence inference).
+     */
+    encode(text, maxLen = 256) {
+        const pieces = this.tokenize(text).slice(0, Math.max(0, maxLen - 2));
+        const inputIds = [this.clsId];
+        for (const p of pieces)
+            inputIds.push(this.vocab.get(p) ?? this.unkId);
+        inputIds.push(this.sepId);
+        return {
+            inputIds,
+            attentionMask: inputIds.map(() => 1),
+            tokenTypeIds: inputIds.map(() => 0),
+        };
+    }
+}

package/extensions/dashboard-server.ts

@@ -70,6 +70,17 @@ interface IndexRepo {
   inputRate: number | null;
   outputRate: number | null;
   lastSeen: number;
+  // Per-repo token + model detail (S25 dashboard enrichment), read directly
+  // from each repo's node:sqlite store at stateDir. tokensKept = Σ stored
+  // summary tokens ("out"); tokensDropped = Σ original region tokens ("in");
+  // sessions = distinct sessions with a checkpoint; contextWindow/maxTokens/
+  // reasoning come from the latest model_snapshots row for that repo.
+  tokensKept: number;
+  tokensDropped: number;
+  sessions: number;
+  contextWindow: number | null;
+  maxTokens: number | null;
+  reasoning: boolean | null;
 }
 
 interface IndexSummary {
@@ -107,7 +118,54 @@ function readIndex(): IndexIndex | null {
       inputRate: (r.input_rate as number | null) ?? null,
       outputRate: (r.output_rate as number | null) ?? null,
       lastSeen: Number(r.last_seen ?? 0),
+      // Defaults — enriched below from each repo's own store.
+      tokensKept: 0,
+      tokensDropped: 0,
+      sessions: 0,
+      contextWindow: null,
+      maxTokens: null,
+      reasoning: null,
     }));
+    // Enrich each repo with per-store token + model detail read directly via
+    // node:sqlite (same zero-dependency invariant as readIndex; no store graph
+    // import). Best-effort: a missing/corrupt store degrades to the defaults
+    // above so the dashboard never fails to render.
+    for (const repo of mapped) {
+      try {
+        const storePath = join(repo.stateDir, "sqlite.db");
+        if (existsSync(storePath)) {
+          const sdb = new DatabaseSync(storePath, { readOnly: true });
+          try {
+            const tok = sdb
+              .prepare(
+                `SELECT COALESCE(SUM(token_estimate),0) AS kept,
+                        COALESCE(SUM(original_token_estimate),0) AS dropped,
+                        COUNT(DISTINCT session_id) AS sess
+                 FROM context_chunks WHERE dedup_status != 'removed'`,
+              )
+              .get() as { kept: number; dropped: number; sess: number };
+            repo.tokensKept = Number(tok.kept ?? 0);
+            repo.tokensDropped = Number(tok.dropped ?? 0);
+            repo.sessions = Number(tok.sess ?? 0);
+            const mrow = sdb
+              .prepare(
+                `SELECT context_window, max_tokens, reasoning
+                 FROM model_snapshots ORDER BY captured_at DESC LIMIT 1`,
+              )
+              .get() as { context_window: number; max_tokens: number; reasoning: number } | undefined;
+            if (mrow) {
+              repo.contextWindow = Number(mrow.context_window ?? 0) || null;
+              repo.maxTokens = Number(mrow.max_tokens ?? 0) || null;
+              repo.reasoning = Number(mrow.reasoning ?? 0) === 1;
+            }
+          } finally {
+            sdb.close();
+          }
+        }
+      } catch {
+        /* best-effort — keep the defaults */
+      }
+    }
     // Defensive display hygiene (belt-and-suspenders — the real fix is that
     // tests now isolate via MEGACOMPACT_INDEX_DIR): drop transient test/temp
     // paths that should never have been real repos, and collapse duplicate
@@ -236,14 +294,15 @@ function readSnapshot(snapshotPath: string) {
       tier: "unknown",
       presetTier: "unknown",
       pressure: 0,
-      config: { fastGatePct: 80, thresholdTokens: 100_000, anchorUserMessages: 1, preserveRecent: 2, auto: true, autoInlineK: 3 },
+      config: { fastGatePct: 80, thresholdTokens: 100_000, tierPct: null, effectiveThresholdPct: null, anchorUserMessages: 1, preserveRecent: 2, auto: true, autoInlineK: 3 },
       session: { id: null, state: null, persistedThisSession: false, lastCheckpointId: null, lastCompactedFrom: 0 },
       context: { tokens: null, percent: null, contextWindow: 0 },
-      trigger: { armed: false, ready: false, currentTokens: null, thresholdTokens: 100_000, fastGatePct: 80 },
+      trigger: { armed: false, ready: false, currentTokens: null, thresholdTokens: 100_000, fastGatePct: 80, tierPct: null, effectiveThresholdPct: null },
       store: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, injectedCount: 0, dedupHitRate: 0, storageDedupRate: 0, dedupCollapsed: 0 },
       crew: { activeAgents: 0, currentTurn: 0 },
       repo: { checkpointCount: 0, totalTokenEstimate: 0, originalTokens: 0, tokensSaved: 0, sessionCount: 0, dedupAttempts: 0, dedupCollapsed: 0, storageDedupRate: 0 },
       integrity: { regionsRetained: 0, compressedOriginalBytes: 0, duplicatesCollapsed: 0, bytesPermanentlyDeleted: 0 },
+      compression: { session: { tokensIn: 0, tokensOut: 0, tokensFreed: 0, compressionPct: 0, dedupPct: 0 }, repo: { tokensIn: 0, tokensOut: 0, tokensFreed: 0, compressionPct: 0, dedupPct: 0 } },
       model: undefined,
     } as Snapshot;
   }
@@ -430,9 +489,9 @@ function dashboardHtml(tierName: string): string {
     <div class="conf-grid">
       <span class="label" title="Live pressure band — climbs low→mega as context fills the window.">Tier (live)</span><span class="value" id="cf-tier">${tierName}</span>
       <span class="label" title="The env-resolved base compaction preset (low/medium/high/ultra/mega) that set the token threshold.">Preset</span><span class="value" id="cf-preset">—</span>
-      <span class="label" title="Live pressure = currentTokens / thresholdTokens (0–100%).">Pressure</span><span class="value" id="cf-pressure">—</span>
-      <span class="label">Threshold</span><span class="value" id="cf-threshold">—</span>
-      <span class="label">Fast Gate</span><span class="value" id="cf-gate">—</span>
+      <span class="label" title="Live pressure = currentTokens / threshold — % of the model context window (threshold fires at the tier's % of window).">Pressure</span><span class="value" id="cf-pressure">—</span>
+      <span class="label" title="Compaction threshold = tierPct × model context window — mega-compact trims BELOW pi's native ~80% auto-compact for any model size.">Threshold</span><span class="value" id="cf-threshold">—</span>
+      <span class="label" title="Fast-gate arming floor — the live trim arms once context passes this % of the window.">Fast Gate</span><span class="value" id="cf-gate">—</span>
       <span class="label">Auto</span><span class="value" id="cf-auto">—</span>
       <span class="label">Anchor</span><span class="value" id="cf-anchor">—</span>
     </div>
@@ -541,15 +600,23 @@ function dashboardHtml(tierName: string): string {
     <thead>
       <tr>
         <th>Model</th><th>Provider</th>
-        <th style="text-align:right">Repos</th>
+        <th style="text-align:right" title="Tokens dropped from context by compaction (the input reclaimed)">Tokens In</th>
+        <th style="text-align:right" title="Tokens kept as compacted summaries still in context (the output retained)">Tokens Out</th>
+        <th style="text-align:right">Freed</th>
+        <th style="text-align:right" title="Model context window (max input tokens the model accepts)">Ctx Window</th>
+        <th style="text-align:right" title="Model max output tokens per turn">Max Out</th>
+        <th style="text-align:right" title="Reasoning-capable model">Reas.</th>
+        <th style="text-align:right" title="Distinct sessions with at least one checkpoint">Sessions</th>
         <th style="text-align:right">Checkpoints</th>
-        <th style="text-align:right">Tokens Saved</th>
+        <th style="text-align:right" title="USD per input token">In $/tok</th>
+        <th style="text-align:right" title="USD per output token">Out $/tok</th>
         <th style="text-align:right">$ Saved</th>
         <th style="text-align:right">Last Used</th>
       </tr>
     </thead>
-    <tbody id="bm-rows"><tr><td colspan="7" class="repo-none">loading…</td></tr></tbody>
+    <tbody id="bm-rows"><tr><td colspan="14" class="repo-none">loading…</td></tr></tbody>
   </table>
+  <p class="legend-note" style="margin-top:8px">Tokens In = Σ original region tokens dropped by compaction. Tokens Out = Σ compacted summary tokens still retained in context. Freed = Tokens In − Tokens Out (net context reclaimed). Ctx Window / Max Out / Reas. come from the latest captured model snapshot for each repo.</p>
 
   <div class="updated" id="sm-updated"></div>
 </div>
@@ -645,7 +712,17 @@ function dashboardHtml(tierName: string): string {
     document.getElementById('cf-tier').textContent = d.tier + ' (live)';
     document.getElementById('cf-preset').textContent = d.presetTier;
     document.getElementById('cf-pressure').textContent = Math.round((d.pressure || 0) * 100) + '%';
-    document.getElementById('cf-threshold').textContent = d.config.thresholdTokens.toLocaleString();
+    // (b) Threshold: show the effective token threshold AND the % of the model
+    // context window it represents (percentage-based tiers). d.config.tierPct
+    // is present on the live snapshot written by the runtime (Phase-1/2a).
+    var cfgPct = d.config.tierPct;
+    var cw = d.context.contextWindow || 0;
+    var thresholdTxt = d.config.thresholdTokens.toLocaleString();
+    if (cfgPct != null && cw > 0) {
+      thresholdTxt += ' (' + Math.round(cfgPct * 100) + '% of ' + cw.toLocaleString() + ')';
+    }
+    document.getElementById('cf-threshold').textContent = thresholdTxt;
+    // (c) Fast Gate: arming floor — live trim arms once context passes this %.
     document.getElementById('cf-gate').textContent = d.config.fastGatePct + '%';
     document.getElementById('cf-auto').textContent = d.config.auto ? 'enabled' : 'disabled';
     document.getElementById('cf-anchor').textContent = d.config.anchorUserMessages;
@@ -771,33 +848,67 @@ function dashboardHtml(tierName: string): string {
     var rows = document.getElementById('bm-rows');
     if (!rows) return;
     if (!repos || !repos.length) {
-      rows.innerHTML = '<tr><td colspan="7" class="repo-none">No repositories registered yet.</td></tr>';
+      rows.innerHTML = '<tr><td colspan="14" class="repo-none">No repositories registered yet.</td></tr>';
       return;
     }
     var groups = {};
     for (var i = 0; i < repos.length; i++) {
       var r = repos[i];
       var key = (r.modelName && String(r.modelName).trim()) || '(unknown)';
-      if (!groups[key]) groups[key] = { model: key, provider: r.providerName || r.provider || '—', repos: 0, checkpoints: 0, tokensSaved: 0, usd: 0, lastAt: 0, rates: [] };
+      if (!groups[key]) groups[key] = {
+        model: key, provider: r.providerName || r.provider || '—', repos: 0, checkpoints: 0,
+        tokensSaved: 0, tokensIn: 0, tokensOut: 0, sessions: 0, usd: 0, lastAt: 0,
+        inRates: [], outRates: [], ctxWindows: [], maxTokens: [], reasoning: null,
+      };
       var g = groups[key];
       g.repos++;
       g.checkpoints += (r.checkpointCount || 0);
       g.tokensSaved += (r.tokensSaved || 0);
-      if (r.inputRate) { g.usd += (r.tokensSaved || 0) * r.inputRate; g.rates.push(r.inputRate); }
+      g.tokensIn += (r.tokensDropped || 0);
+      g.tokensOut += (r.tokensKept || 0);
+      g.sessions += (r.sessions || 0);
+      if (r.inputRate) { g.usd += (r.tokensSaved || 0) * r.inputRate; g.inRates.push(r.inputRate); }
+      if (r.outputRate) g.outRates.push(r.outputRate);
+      if (r.contextWindow) g.ctxWindows.push(r.contextWindow);
+      if (r.maxTokens) g.maxTokens.push(r.maxTokens);
+      if (r.reasoning != null) g.reasoning = r.reasoning;
       if (r.lastCompactedAt && r.lastCompactedAt > g.lastAt) g.lastAt = r.lastCompactedAt;
     }
     var arr = [];
     for (var k in groups) { if (Object.prototype.hasOwnProperty.call(groups, k)) arr.push(groups[k]); }
     arr.sort(function(a, b) { return b.tokensSaved - a.tokensSaved; });
+    // Helper: a set of numeric samples collapses to a single value when all
+    // repos in the group agree, otherwise shows the range (min–max) so the
+    // user can see mixed-config model groups at a glance.
+    function collapseNum(samples) {
+      if (!samples || !samples.length) return '—';
+      var lo = Math.min.apply(null, samples), hi = Math.max.apply(null, samples);
+      return lo === hi ? lo.toLocaleString() : lo.toLocaleString() + '–' + hi.toLocaleString();
+    }
+    function collapseRate(samples) {
+      if (!samples || !samples.length) return '—';
+      var lo = Math.min.apply(null, samples), hi = Math.max.apply(null, samples);
+      var fmt = function(v) { return '$' + v.toFixed(6); };
+      return lo === hi ? fmt(lo) : fmt(lo) + '–' + fmt(hi);
+    }
     rows.innerHTML = arr.map(function(g) {
+      var freed = (g.tokensIn || 0) - (g.tokensOut || 0);
       var usd = g.usd > 0 ? '$' + g.usd.toFixed(4) : '—';
       var when = g.lastAt ? new Date(g.lastAt).toLocaleString() : '—';
+      var reas = g.reasoning == null ? '—' : (g.reasoning ? 'yes' : 'no');
       return '<tr>' +
         '<td><span class="repo-model">' + sanitize(g.model) + '</span></td>' +
         '<td>' + sanitize(g.provider) + '</td>' +
-        '<td class="num">' + g.repos.toLocaleString() + '</td>' +
+        '<td class="num">' + (g.tokensIn || 0).toLocaleString() + '</td>' +
+        '<td class="num">' + (g.tokensOut || 0).toLocaleString() + '</td>' +
+        '<td class="num">' + freed.toLocaleString() + '</td>' +
+        '<td class="num">' + collapseNum(g.ctxWindows) + '</td>' +
+        '<td class="num">' + collapseNum(g.maxTokens) + '</td>' +
+        '<td class="num">' + reas + '</td>' +
+        '<td class="num">' + g.sessions.toLocaleString() + '</td>' +
         '<td class="num">' + g.checkpoints.toLocaleString() + '</td>' +
-        '<td class="num">' + g.tokensSaved.toLocaleString() + '</td>' +
+        '<td class="num">' + collapseRate(g.inRates) + '</td>' +
+        '<td class="num">' + collapseRate(g.outRates) + '</td>' +
         '<td class="num">' + sanitize(usd) + '</td>' +
         '<td class="num">' + sanitize(when) + '</td>' +
       '</tr>';

package/extensions/mega-commands.ts

@@ -125,10 +125,21 @@ export function registerCommands(pi: ExtensionAPI, runtime: MegaRuntime, config:
         repoCount = listRepoRegistry(process.env.MEGACOMPACT_INDEX_DIR).length;
       } catch { /* non-fatal */ }
       const crossRepoStr = `${crossRepoInjections} cross-repo injections recorded · ${repoCount} repos indexed`;
+      // Effective compaction threshold = tierPct × model context window (kept
+      // BELOW pi's native ~80% auto-compact for any model size). Falls back to
+      // the boot token value when the window is unknown (custom tier / pre-
+      // model-select). Display matches the dashboard's percentage-based view.
+      const effThreshold = config.tierPct != null && ctxWindow > 0
+        ? Math.round(config.tierPct * ctxWindow)
+        : config.thresholdTokens;
+      const winStr = ctxWindow > 0
+        ? (ctxWindow >= 1_000_000 ? `${Math.round(ctxWindow / 1_000_000)}M` : `${Math.round(ctxWindow / 1_000)}k`)
+        : "?";
+      const tierPctStr = config.tierPct != null ? `${Math.round(config.tierPct * 100)}%` : "n/a";
       ctx.ui.notify(
         `[mega-compact] pct=${pct} tokens=${tokens} tier=${runtime.pressureBand} (live) preset=${config.tier} ` +
           `pressure=${Math.round(runtime.pressure * 100)}% fastGate=${config.fastGatePct}% ` +
-          `threshold=${config.thresholdTokens} auto=${config.auto} autoInline=${config.autoInline}\n` +
+          `threshold=${effThreshold.toLocaleString()} (${tierPctStr} of ${winStr} window) tierPct=${config.tierPct != null ? config.tierPct.toFixed(2) : "n/a"} auto=${config.auto} autoInline=${config.autoInline}\n` +
           `[mega-compact] store: ${st.checkpointCount} chkpt · ` +
           `${st.totalTokenEstimate} tok · last=${st.lastCheckpointId ?? "—"} · ` +
           `injected=${st.injectedCount} · dedup=${(st.dedupHitRate * 100).toFixed(0)}%\n` +