ziprin-context-optimizer 8.0.0

package/src/extension.js

@@ -0,0 +1,322 @@
+'use strict';
+
+/**
+ * Ziprin Context Inspector — observer/dashboard only.
+ * Does not start MCP, run optimize, or own lifecycle. Native Cursor Chat is the UI.
+ */
+
+const path = require('path');
+const { listContextProfiles } = require('./context-profiles');
+const { getConfig } = require('./config');
+const { installTokenBudget } = require('./supervisor');
+const {
+  clearSessions,
+  listSessions,
+  getSession,
+  historySummary,
+  SESSIONS_REL,
+} = require('./session-store');
+const {
+  EVENTS_REL,
+  readEvents,
+  servicesFromEvents,
+} = require('./mcp-event-stream');
+const {
+  runHealthCheck,
+  loadHealth,
+  HEALTH_REL,
+} = require('./mcp-lifecycle-manager');
+
+let vscode;
+try {
+  vscode = require('vscode');
+} catch {
+  vscode = null;
+}
+
+const { InspectorViewProvider } = require('./inspector-panel');
+
+let provider;
+let refreshTimer = null;
+let sessionPollTimer = null;
+let lastHistoryFingerprint = '';
+let lastEventsFingerprint = '';
+let lastMcpServices = null;
+
+function workspaceRoot() {
+  if (!vscode || !vscode.workspace.workspaceFolders?.length) return null;
+  return vscode.workspace.workspaceFolders[0].uri.fsPath;
+}
+
+function cfg() {
+  return getConfig(vscode);
+}
+
+function enrichHistoryEntry(session) {
+  const h = historySummary(session);
+  const before = Number(h.tokensBeforePrune) || 0;
+  const after = Number(h.tokensAfterPrune) || 0;
+  const reductionPct =
+    session.tokenAnalysis?.reductionPct != null
+      ? session.tokenAnalysis.reductionPct
+      : before > 0
+        ? Math.round((1 - after / before) * 100)
+        : null;
+  const status =
+    session.qualityReport?.ok === false || /weak context/i.test(String(session.result || ''))
+      ? 'weak'
+      : 'ok';
+  const decisionSummary =
+    session.decisionReport?.topSelected?.[0]?.path ||
+    session.intent?.action ||
+    String(session.task || '').slice(0, 48);
+  return { ...h, reductionPct, status, decisionSummary, intent: session.intent };
+}
+
+function connectionState(root, history) {
+  if (!root) return { state: 'no-workspace' };
+  return {
+    state: 'observing',
+    lastSessionAt: history[0]?.timestamp || null,
+    sessionCount: history.length,
+  };
+}
+
+/** Derive MCP Services from event stream + optional cached health (observe only). */
+function refreshMcpServicesFromDisk() {
+  const root = workspaceRoot();
+  if (!root) {
+    lastMcpServices = servicesFromEvents([], null);
+    return lastMcpServices;
+  }
+  const events = readEvents(root, 120);
+  const health = loadHealth(root);
+  lastMcpServices = servicesFromEvents(events, health);
+  lastEventsFingerprint = eventsFingerprint(events);
+  return lastMcpServices;
+}
+
+function eventsFingerprint(events) {
+  const last = events[events.length - 1];
+  return `${events.length}:${last?.timestamp || ''}:${last?.type || ''}`;
+}
+
+function pushModel() {
+  if (!provider) return;
+  const root = workspaceRoot();
+  const sessions = root ? listSessions(root, 40) : [];
+  const history = sessions.map(enrichHistoryEntry);
+  const latest = sessions[0] ? getSession(root, sessions[0].id) : null;
+  lastHistoryFingerprint = history.map((h) => h.id).join(',') + ':' + (latest?.timestamp || '');
+  if (!lastMcpServices) refreshMcpServicesFromDisk();
+  provider.setModel({
+    type: 'model',
+    profiles: listContextProfiles(),
+    history,
+    profileId: cfg().profile,
+    latestSession: latest,
+    connection: connectionState(root, history),
+    mcpServices: lastMcpServices,
+  });
+}
+
+function refreshHistoryFromDisk() {
+  pushModel();
+  const root = workspaceRoot();
+  if (!root || !provider) return;
+  const listed = listSessions(root, 1);
+  if (listed[0]) {
+    const session = getSession(root, listed[0].id);
+    if (session) provider.showSession(session);
+  }
+}
+
+function pollLiveArtifacts() {
+  const root = workspaceRoot();
+  if (!root) return;
+  const sessions = listSessions(root, 40);
+  const fp = sessions.map((s) => s.id).join(',') + ':' + (sessions[0]?.timestamp || '');
+  if (fp !== lastHistoryFingerprint) {
+    refreshHistoryFromDisk();
+  }
+  const events = readEvents(root, 120);
+  const efp = eventsFingerprint(events);
+  if (efp !== lastEventsFingerprint) {
+    lastMcpServices = servicesFromEvents(events, loadHealth(root));
+    lastEventsFingerprint = efp;
+    pushModel();
+  }
+}
+
+function scheduleLive() {
+  if (!vscode || !cfg().liveRefresh) return;
+  if (refreshTimer) clearTimeout(refreshTimer);
+  refreshTimer = setTimeout(() => {
+    refreshTimer = null;
+    refreshMcpServicesFromDisk();
+    pushModel();
+  }, Math.max(120, Number(cfg().refreshDebounceMs) || 500));
+}
+
+function activate(context) {
+  if (!vscode) return;
+
+  provider = new InspectorViewProvider({
+    setProfile: async (id) => {
+      await vscode.workspace
+        .getConfiguration('ziprinContext')
+        .update('profile', id, vscode.ConfigurationTarget.Workspace);
+      scheduleLive();
+    },
+    openSession: (id) => {
+      const root = workspaceRoot();
+      if (!root) return;
+      const session = getSession(root, id);
+      if (session) provider.showSession(session);
+    },
+    clear: async () => {
+      const root = workspaceRoot();
+      if (!root) return;
+      const ok = await vscode.window.showWarningMessage(
+        'Clear all Context Inspector sessions?',
+        'Clear',
+        'Cancel'
+      );
+      if (ok !== 'Clear') return;
+      clearSessions(root);
+      pushModel();
+    },
+  });
+
+  context.subscriptions.push(
+    vscode.window.registerWebviewViewProvider('ziprinContext.inspector', provider, {
+      webviewOptions: { retainContextWhenHidden: true },
+    })
+  );
+
+  context.subscriptions.push(
+    vscode.commands.registerCommand('ziprinContext.openInspector', () =>
+      vscode.commands.executeCommand('workbench.view.extension.ziprinContext')
+    )
+  );
+
+  context.subscriptions.push(
+    vscode.commands.registerCommand('ziprinContext.refreshDashboard', () => {
+      refreshMcpServicesFromDisk();
+      refreshHistoryFromDisk();
+    })
+  );
+
+  // One-shot diagnostic probe — does not start MCP or replace Cursor lifecycle.
+  context.subscriptions.push(
+    vscode.commands.registerCommand('ziprinContext.diagnoseMcp', () => {
+      const root = workspaceRoot();
+      if (!root) {
+        vscode.window.showWarningMessage('Open a workspace to diagnose MCP.');
+        return;
+      }
+      try {
+        runHealthCheck(root, { persist: true });
+      } catch (err) {
+        vscode.window.showErrorMessage(`MCP diagnose failed: ${err.message}`);
+      }
+      const services = refreshMcpServicesFromDisk();
+      pushModel();
+      vscode.window.showInformationMessage(
+        `Ziprin MCP health: ${services?.overall || 'unknown'}. See Context Inspector → MCP Services.`
+      );
+    })
+  );
+
+  context.subscriptions.push(
+    vscode.commands.registerCommand('ziprinContext.installTokenBudget', async () => {
+      const root = workspaceRoot();
+      if (!root) return;
+      const choice = await vscode.window.showWarningMessage(
+        'Install token budget: write supervisor stub and strip extra .mdc files.',
+        'ادامه',
+        'انصراف'
+      );
+      if (choice !== 'ادامه') return;
+      const r = installTokenBudget(root, { write: true, stripOtherMdc: cfg().stripOtherMdc });
+      vscode.window.showInformationMessage(
+        `Context Inspector: stub written, stripped ${r.stripped.length}, flipped ${r.flipped.length}.`
+      );
+      scheduleLive();
+    })
+  );
+
+  context.subscriptions.push(
+    vscode.workspace.onDidChangeConfiguration((e) => {
+      if (e.affectsConfiguration('ziprinContext')) scheduleLive();
+    })
+  );
+  context.subscriptions.push(
+    vscode.workspace.onDidChangeWorkspaceFolders(() => {
+      refreshMcpServicesFromDisk();
+      pushModel();
+    })
+  );
+
+  const root = workspaceRoot();
+  if (root) {
+    const sessionWatch = vscode.workspace.createFileSystemWatcher(
+      new vscode.RelativePattern(root, SESSIONS_REL)
+    );
+    sessionWatch.onDidChange(refreshHistoryFromDisk);
+    sessionWatch.onDidCreate(refreshHistoryFromDisk);
+    sessionWatch.onDidDelete(pushModel);
+    context.subscriptions.push(sessionWatch);
+
+    const eventsWatch = vscode.workspace.createFileSystemWatcher(
+      new vscode.RelativePattern(root, EVENTS_REL)
+    );
+    const onEvents = () => {
+      refreshMcpServicesFromDisk();
+      pushModel();
+    };
+    eventsWatch.onDidChange(onEvents);
+    eventsWatch.onDidCreate(onEvents);
+    eventsWatch.onDidDelete(onEvents);
+    context.subscriptions.push(eventsWatch);
+
+    const healthWatch = vscode.workspace.createFileSystemWatcher(
+      new vscode.RelativePattern(root, HEALTH_REL)
+    );
+    healthWatch.onDidChange(() => {
+      refreshMcpServicesFromDisk();
+      pushModel();
+    });
+    context.subscriptions.push(healthWatch);
+
+    context.subscriptions.push(
+      vscode.workspace.onDidSaveTextDocument((doc) => {
+        try {
+          const { patchFile } = require('./fts-index');
+          const rel = path.relative(root, doc.fileName).replace(/\\/g, '/');
+          if (rel && !rel.startsWith('..')) patchFile(root, rel);
+        } catch {
+          /* observer-side index patch; ignore */
+        }
+      })
+    );
+  }
+
+  // Backup poll when FS watchers miss appends (JSONL).
+  sessionPollTimer = setInterval(pollLiveArtifacts, 2000);
+  context.subscriptions.push({ dispose: () => clearInterval(sessionPollTimer) });
+
+  refreshMcpServicesFromDisk();
+  pushModel();
+  refreshHistoryFromDisk();
+  if (cfg().autoRevealPanel) {
+    vscode.commands.executeCommand('workbench.view.extension.ziprinContext');
+  }
+}
+
+function deactivate() {
+  if (sessionPollTimer) clearInterval(sessionPollTimer);
+  if (refreshTimer) clearTimeout(refreshTimer);
+}
+
+module.exports = { activate, deactivate };

package/src/firewall.js

@@ -0,0 +1,35 @@
+'use strict';
+
+/**
+ * Context Firewall — block / summarize / allow classification.
+ */
+
+const BLOCK = [
+  /node_modules\//,
+  /\/dist\//,
+  /\/build\//,
+  /\.next\//,
+  /\/reports\//,
+  /\.cursor\/(agents|skills|_tmp)/,
+  /project-index\.json$/,
+  /project-must-follow-catalog\.json$/,
+  /\.vsix$/,
+];
+
+function firewallDecision(rel, bytes = 0) {
+  const r = String(rel || '').replace(/\\/g, '/');
+  for (const re of BLOCK) {
+    if (re.test(r)) {
+      return { action: 'block', reason: 'blocked by context firewall', path: r };
+    }
+  }
+  if (bytes > 40_000 || (/service|documentation|\.md$/i.test(r) && bytes > 12_000)) {
+    return { action: 'summarize', reason: 'large file — compress instead of full dump', path: r };
+  }
+  if (/\/src\/|packages\/(ui|shared)\/|AGENTS\.md$|\.policy\.json$|\.mdc$/.test(r)) {
+    return { action: 'allow', reason: 'task-relevant / architecture source', path: r };
+  }
+  return { action: 'allow', reason: 'candidate', path: r };
+}
+
+module.exports = { firewallDecision, BLOCK };

package/src/fts-index.js

@@ -0,0 +1,319 @@
+'use strict';
+
+/**
+ * Persistent FTS5 unicode61 index at .ziprin-context/index.db
+ * Incremental by mtime. Column weights: path×8, body×1, symbols×12.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { tokenizePath, splitIdentifier } = require('./identifier');
+const { tokenizeBody, scoreDocsBm25F } = require('./bm25');
+const { isTokenBomb } = require('./pruner');
+
+const SOURCE_EXT = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.css', '.md', '.mdc']);
+const SKIP_DIR = new Set([
+  'node_modules', '.git', 'dist', 'build', '.next', 'coverage', '.turbo',
+  '.ziprin-context', 'reports', '__mocks__', 'generated',
+]);
+const EXPORT_RE = /^export\s+(?:async\s+)?(?:function|class|const|type|interface|enum)\s+([A-Za-z0-9_]+)/gm;
+
+const openDbs = new Map();
+const memFallback = new Map();
+
+function indexDir(root) {
+  return path.join(root, '.ziprin-context');
+}
+
+function dbPath(root) {
+  return path.join(indexDir(root), 'index.db');
+}
+
+function extractSymbols(text) {
+  const out = [];
+  EXPORT_RE.lastIndex = 0;
+  let m;
+  while ((m = EXPORT_RE.exec(text))) out.push(m[1]);
+  return out;
+}
+
+function walkSource(root, acc = [], max = 4000) {
+  function walk(abs) {
+    if (acc.length >= max) return;
+    let entries;
+    try {
+      entries = fs.readdirSync(abs, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const ent of entries) {
+      if (acc.length >= max) return;
+      const full = path.join(abs, ent.name);
+      if (ent.isDirectory()) {
+        if (SKIP_DIR.has(ent.name)) continue;
+        walk(full);
+      } else {
+        const rel = path.relative(root, full).replace(/\\/g, '/');
+        if (!SOURCE_EXT.has(path.extname(rel))) continue;
+        if (isTokenBomb(rel)) continue;
+        acc.push(rel);
+      }
+    }
+  }
+  walk(root);
+  return acc;
+}
+
+function openDb(root) {
+  if (openDbs.has(root)) return openDbs.get(root);
+  let DatabaseSync;
+  try {
+    ({ DatabaseSync } = require('node:sqlite'));
+  } catch {
+    return null;
+  }
+  try {
+    fs.mkdirSync(indexDir(root), { recursive: true });
+    const db = new DatabaseSync(dbPath(root));
+    db.exec('PRAGMA busy_timeout=5000');
+    db.exec('PRAGMA journal_mode=WAL');
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS files (
+        path TEXT PRIMARY KEY,
+        mtime INTEGER NOT NULL,
+        size INTEGER NOT NULL
+      );
+      CREATE VIRTUAL TABLE IF NOT EXISTS docs USING fts5(
+        path,
+        body,
+        symbols,
+        tokenize = 'unicode61'
+      );
+      CREATE TABLE IF NOT EXISTS meta (
+        k TEXT PRIMARY KEY,
+        v TEXT
+      );
+    `);
+    openDbs.set(root, db);
+    return db;
+  } catch {
+    return null;
+  }
+}
+
+function ftsQuery(tokens) {
+  const cleaned = [...new Set((tokens || [])
+    .map((t) => String(t).replace(/["']/g, '').trim())
+    .filter((t) => t.length > 1 && !/^[-+*(){}]+$/.test(t)))]
+    .slice(0, 20);
+  if (!cleaned.length) return '';
+  return cleaned.map((t) => `"${t}"`).join(' OR ');
+}
+
+function upsertMem(root, rel, rec) {
+  if (!memFallback.has(root)) memFallback.set(root, new Map());
+  memFallback.get(root).set(rel, rec);
+}
+
+function indexFileRecord(root, rel) {
+  const abs = path.join(root, rel);
+  let st;
+  try {
+    st = fs.statSync(abs);
+  } catch {
+    return null;
+  }
+  let text = '';
+  try {
+    text = fs.readFileSync(abs, 'utf8').slice(0, 24000);
+  } catch {
+    return null;
+  }
+  const symbols = extractSymbols(text);
+  return {
+    path: rel,
+    mtime: Math.round(st.mtimeMs),
+    size: st.size,
+    body: text,
+    symbols: symbols.join(' '),
+    pathTerms: tokenizePath(rel),
+    bodyTerms: tokenizeBody(text),
+    symbolTerms: symbols.flatMap((s) => [s.toLowerCase(), ...splitIdentifier(s)]),
+  };
+}
+
+function ensureIndex(root, opts = {}) {
+  const files = walkSource(root, [], opts.maxFiles || 4000);
+  const db = openDb(root);
+  let updated = 0;
+  let fromCache = 0;
+
+  if (db) {
+    const getMeta = db.prepare('SELECT mtime, size FROM files WHERE path = ?');
+    const insFile = db.prepare('INSERT OR REPLACE INTO files(path, mtime, size) VALUES (?, ?, ?)');
+    const delDocs = db.prepare('DELETE FROM docs WHERE path = ?');
+    const insDocs = db.prepare('INSERT INTO docs(path, body, symbols) VALUES (?, ?, ?)');
+    db.exec('BEGIN');
+    try {
+      const seen = new Set();
+      for (const rel of files) {
+        seen.add(rel);
+        const abs = path.join(root, rel);
+        let st;
+        try {
+          st = fs.statSync(abs);
+        } catch {
+          continue;
+        }
+        const mtime = Math.round(st.mtimeMs);
+        const row = getMeta.get(rel);
+        if (row && Number(row.mtime) === mtime && Number(row.size) === st.size) {
+          fromCache++;
+          continue;
+        }
+        const rec = indexFileRecord(root, rel);
+        if (!rec) continue;
+        delDocs.run(rel);
+        insDocs.run(rec.path, rec.body, rec.symbols);
+        insFile.run(rel, rec.mtime, rec.size);
+        upsertMem(root, rel, rec);
+        updated++;
+      }
+      const stale = db.prepare('SELECT path FROM files').all();
+      for (const row of stale) {
+        if (!seen.has(row.path)) {
+          db.prepare('DELETE FROM files WHERE path = ?').run(row.path);
+          delDocs.run(row.path);
+          memFallback.get(root)?.delete(row.path);
+        }
+      }
+      db.prepare("INSERT OR REPLACE INTO meta(k, v) VALUES ('builtAt', ?)").run(new Date().toISOString());
+      db.exec('COMMIT');
+    } catch (err) {
+      try { db.exec('ROLLBACK'); } catch { /* ignore */ }
+      throw err;
+    }
+  } else {
+    for (const rel of files) {
+      const rec = indexFileRecord(root, rel);
+      if (rec) {
+        upsertMem(root, rel, rec);
+        updated++;
+      }
+    }
+  }
+
+  return { files: files.length, updated, fromCache, backend: db ? 'fts5' : 'memory' };
+}
+
+function searchFts(root, tokens, opts = {}) {
+  const limit = opts.limit || 40;
+  const q = ftsQuery(tokens);
+  const db = openDb(root);
+  if (db && q) {
+    try {
+      const stmt = db.prepare(`
+        SELECT path, bm25(docs, 8.0, 1.0, 12.0) AS rank
+        FROM docs
+        WHERE docs MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      `);
+      const rows = stmt.all(q, limit);
+      return rows.map((r, i) => ({
+        path: r.path,
+        score: -Number(r.rank) || (limit - i),
+        source: 'fts5',
+      }));
+    } catch {
+      /* fall through to memory */
+    }
+  }
+  const bag = memFallback.get(root);
+  if (!bag || !bag.size) {
+    ensureIndex(root);
+  }
+  const docs = [...(memFallback.get(root)?.values() || [])];
+  return scoreDocsBm25F(docs, tokens).filter((x) => x.score > 0).slice(0, limit)
+    .map((x) => ({ ...x, source: 'bm25f' }));
+}
+
+function patchFile(root, rel) {
+  if (!rel || isTokenBomb(rel)) return false;
+  if (!SOURCE_EXT.has(path.extname(rel))) return false;
+  const rec = indexFileRecord(root, rel);
+  if (!rec) {
+    const db = openDb(root);
+    if (db) {
+      try {
+        db.prepare('DELETE FROM files WHERE path = ?').run(rel);
+        db.prepare('DELETE FROM docs WHERE path = ?').run(rel);
+      } catch { /* ignore */ }
+    }
+    memFallback.get(root)?.delete(rel);
+    return false;
+  }
+  const db = openDb(root);
+  if (db) {
+    try {
+      db.exec('BEGIN');
+      db.prepare('DELETE FROM docs WHERE path = ?').run(rel);
+      db.prepare('INSERT INTO docs(path, body, symbols) VALUES (?, ?, ?)').run(rec.path, rec.body, rec.symbols);
+      db.prepare('INSERT OR REPLACE INTO files(path, mtime, size) VALUES (?, ?, ?)').run(rel, rec.mtime, rec.size);
+      db.exec('COMMIT');
+    } catch {
+      try { db.exec('ROLLBACK'); } catch { /* ignore */ }
+    }
+  }
+  upsertMem(root, rel, rec);
+  return true;
+}
+
+function indexStats(root) {
+  const db = openDb(root);
+  let fileCount = 0;
+  let builtAt = null;
+  let bytes = 0;
+  if (db) {
+    try {
+      fileCount = Number(db.prepare('SELECT COUNT(*) AS n FROM files').get()?.n || 0);
+      builtAt = db.prepare("SELECT v FROM meta WHERE k = 'builtAt'").get()?.v || null;
+    } catch { /* ignore */ }
+    try {
+      bytes = fs.statSync(dbPath(root)).size;
+    } catch { /* ignore */ }
+  } else {
+    fileCount = memFallback.get(root)?.size || 0;
+  }
+  return {
+    fileCount,
+    builtAt,
+    bytes,
+    backend: db ? 'fts5' : 'memory',
+    dbPath: db ? dbPath(root) : null,
+  };
+}
+
+function closeIndex(root) {
+  const db = openDbs.get(root);
+  if (db) {
+    try {
+      db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
+      db.close();
+    } catch { /* ignore */ }
+    openDbs.delete(root);
+  }
+  memFallback.delete(root);
+}
+
+module.exports = {
+  ensureIndex,
+  searchFts,
+  patchFile,
+  indexStats,
+  closeIndex,
+  walkSource,
+  ftsQuery,
+  extractSymbols,
+  dbPath,
+};