ziprin-context-optimizer 8.0.0

package/src/retrieve.js

@@ -0,0 +1,350 @@
+'use strict';
+
+/**
+ * Hybrid retrieval v8:
+ * FTS5 BM25F + path BM25 + git dirty/recency + Personalized PageRank
+ * → weighted RRF → cascade (skip grep when top is clear) → MMR.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { isTokenBomb } = require('./pruner');
+const { bm25Path } = require('./bm25');
+const { splitMany, tokenizePath } = require('./identifier');
+const { ensureIndex, searchFts } = require('./fts-index');
+const { rankGraph } = require('./pagerank');
+const { mmrSelect } = require('./mmr');
+const { gitRecency } = require('./git-recency');
+
+const SOURCE_EXT = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.css', '.md', '.mdc']);
+const SKIP_DIR = new Set(['node_modules', '.git', 'dist', 'build', '.next', 'coverage', '.turbo', '.ziprin-context']);
+
+function rrf(lists, k = 60) {
+  const scores = new Map();
+  for (const list of lists) {
+    list.forEach((item, i) => {
+      const p = typeof item === 'string' ? item : item.path;
+      if (!p) return;
+      scores.set(p, (scores.get(p) || 0) + 1 / (k + i + 1));
+    });
+  }
+  return [...scores.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .map(([p, score]) => ({ path: p, rrf: score }));
+}
+
+function weightedRrf(weightedLists, k = 60) {
+  const scores = new Map();
+  for (const { list, weight } of weightedLists || []) {
+    const w = weight == null ? 1 : weight;
+    (list || []).forEach((item, i) => {
+      const p = typeof item === 'string' ? item : item.path;
+      if (!p) return;
+      scores.set(p, (scores.get(p) || 0) + w / (k + i + 1));
+    });
+  }
+  return [...scores.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .map(([p, score]) => ({ path: p, rrf: score }));
+}
+
+function gitDirty(root) {
+  try {
+    const r = spawnSync('git', ['-C', root, 'status', '--porcelain', '-uall'], {
+      encoding: 'utf8',
+      timeout: 4000,
+    });
+    if (r.status !== 0) return [];
+    return String(r.stdout || '')
+      .split('\n')
+      .map((line) => line.slice(3).trim().replace(/\\/g, '/'))
+      .filter((p) => p && SOURCE_EXT.has(path.extname(p)) && !isTokenBomb(p));
+  } catch {
+    return [];
+  }
+}
+
+function gitDiffNames(root) {
+  try {
+    const r = spawnSync('git', ['-C', root, 'diff', '--name-only', 'HEAD'], {
+      encoding: 'utf8',
+      timeout: 4000,
+    });
+    if (r.status !== 0) return [];
+    return String(r.stdout || '')
+      .split('\n')
+      .map((p) => p.trim().replace(/\\/g, '/'))
+      .filter((p) => p && SOURCE_EXT.has(path.extname(p)) && !isTokenBomb(p));
+  } catch {
+    return [];
+  }
+}
+
+function walkLimited(abs, root, acc, max, hints) {
+  if (acc.length >= max) return;
+  let entries;
+  try {
+    entries = fs.readdirSync(abs, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  const ranked = entries.slice().sort((a, b) => {
+    const ha = hints.some((h) => a.name.includes(h)) ? 0 : 1;
+    const hb = hints.some((h) => b.name.includes(h)) ? 0 : 1;
+    if (ha !== hb) return ha - hb;
+    return a.name.localeCompare(b.name);
+  });
+  for (const ent of ranked) {
+    if (acc.length >= max) return;
+    if (ent.isDirectory()) {
+      if (SKIP_DIR.has(ent.name)) continue;
+      walkLimited(path.join(abs, ent.name), root, acc, max, hints);
+    } else {
+      const rel = path.relative(root, path.join(abs, ent.name)).replace(/\\/g, '/');
+      if (!SOURCE_EXT.has(path.extname(rel))) continue;
+      if (isTokenBomb(rel)) continue;
+      acc.push(rel);
+    }
+  }
+}
+
+function collectPathUniverse(root, pathHints, domain, max = 80) {
+  const seeds = [];
+  if (domain?.priorityPaths) seeds.push(...domain.priorityPaths);
+  for (const h of pathHints || []) {
+    if (h.includes('/')) seeds.push(h);
+    seeds.push(`apps/dashboard/bss/src/portals/vendor/${h}`);
+    seeds.push(`apps/marketplace/src/features/${h}`);
+  }
+  seeds.push('apps/dashboard/bss/src', 'apps/marketplace/src', 'packages/shared/src', 'packages/ui/src');
+
+  const files = [];
+  const seen = new Set();
+  for (const seed of seeds) {
+    const abs = path.join(root, seed);
+    if (!fs.existsSync(abs)) continue;
+    const bucket = [];
+    if (fs.statSync(abs).isFile()) bucket.push(seed.replace(/\\/g, '/'));
+    else walkLimited(abs, root, bucket, 50, pathHints || []);
+    for (const p of bucket) {
+      if (seen.has(p)) continue;
+      seen.add(p);
+      files.push(p);
+    }
+    if (files.length >= max) break;
+  }
+  return files;
+}
+
+function queryTerms(analysis) {
+  const bag = [
+    ...(analysis.expanded || analysis.tokens || []),
+    ...(analysis.symbols || []),
+    ...(analysis.pathHints || []),
+    ...splitMany(analysis.symbols || []),
+    ...splitMany(analysis.pathHints || []),
+  ];
+  return [...new Set(bag.map((t) => String(t).toLowerCase()).filter((t) => t.length > 1))];
+}
+
+function pathRank(root, tokens, pathHints, domain, max = 80) {
+  const files = collectPathUniverse(root, pathHints, domain, max);
+  const q = [
+    ...(tokens || []).filter((t) => t.length > 2),
+    ...(pathHints || []),
+    ...splitMany(pathHints || []),
+  ].map((t) => String(t).toLowerCase());
+  const ranked = bm25Path(files, q);
+  const boosted = ranked.map((r) => {
+    const low = r.path.toLowerCase();
+    let s = r.score;
+    for (const h of pathHints || []) if (low.includes(h)) s += 2.5;
+    if (domain?.priorityPaths?.some((d) => low.startsWith(d.toLowerCase()) || low.includes(d.toLowerCase()))) s += 1.2;
+    return { path: r.path, score: s };
+  });
+  return boosted.sort((a, b) => b.score - a.score).slice(0, max);
+}
+
+function rgAvailable() {
+  try {
+    const r = spawnSync('rg', ['--version'], { encoding: 'utf8', timeout: 1500 });
+    return r.status === 0;
+  } catch {
+    return false;
+  }
+}
+
+function contentGrepRg(root, needles, maxHits = 40) {
+  const n = (needles || []).filter((x) => x && x.length > 2).slice(0, 8);
+  if (!n.length) return [];
+  try {
+    const pattern = n.map((x) => x.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|');
+    const r = spawnSync(
+      'rg',
+      ['--json', '-i', '-l', '-g', '!node_modules', '-g', '!.git', '-g', '!dist', pattern, root],
+      { encoding: 'utf8', timeout: 4000, maxBuffer: 2_000_000 },
+    );
+    const hits = [];
+    for (const line of String(r.stdout || '').split('\n')) {
+      if (!line.trim()) continue;
+      try {
+        const j = JSON.parse(line);
+        if (j.type !== 'path' && !(j.data && j.data.path)) continue;
+        const p = (j.data.path.text || '').replace(/\\/g, '/');
+        const rel = path.relative(root, p).replace(/\\/g, '/');
+        if (rel && SOURCE_EXT.has(path.extname(rel)) && !isTokenBomb(rel)) {
+          hits.push({ path: rel, score: 1 });
+        }
+      } catch {
+        continue;
+      }
+      if (hits.length >= maxHits) break;
+    }
+    return hits;
+  } catch {
+    return [];
+  }
+}
+
+function contentGrep(root, files, needles, maxHits = 40) {
+  if (rgAvailable()) {
+    const rgHits = contentGrepRg(root, needles, maxHits);
+    if (rgHits.length) return rgHits;
+  }
+  const hits = [];
+  const n = (needles || []).filter((x) => x && x.length > 2).slice(0, 12);
+  if (!n.length) return hits;
+  for (const rel of files.slice(0, 120)) {
+    const abs = path.join(root, rel);
+    let text = '';
+    try {
+      text = fs.readFileSync(abs, 'utf8').slice(0, 16000).toLowerCase();
+    } catch {
+      continue;
+    }
+    let score = 0;
+    for (const needle of n) {
+      if (text.includes(needle.toLowerCase())) score += 1;
+    }
+    if (score) hits.push({ path: rel, score });
+    if (hits.length >= maxHits) break;
+  }
+  return hits.sort((a, b) => b.score - a.score);
+}
+
+function shouldCascadeSkip(ftsHits, pathList) {
+  const a = ftsHits[0]?.score || 0;
+  const b = ftsHits[1]?.score || 0;
+  const p0 = pathList[0]?.score || 0;
+  const p1 = pathList[1]?.score || 0;
+  const ftsClear = a > 0 && a >= 2 * Math.max(b, 1e-9);
+  const pathClear = p0 > 0 && p0 >= 2 * Math.max(p1, 1e-9);
+  const sameTop =
+    ftsHits[0]?.path && pathList[0]?.path && ftsHits[0].path === pathList[0].path;
+  return (ftsClear || pathClear) && (sameTop || ftsClear);
+}
+
+function retrieveCandidates(root, analysis, opts = {}) {
+  const t0 = Date.now();
+  const stages = {};
+  const dirty = [...new Set([...gitDirty(root), ...gitDiffNames(root), ...(opts.openFiles || [])])];
+  stages.gitMs = Date.now() - t0;
+
+  const tIndex = Date.now();
+  let ftsMeta = { backend: 'none', updated: 0 };
+  try {
+    ftsMeta = ensureIndex(root, { maxFiles: opts.maxIndex || 4000 });
+  } catch {
+    ftsMeta = { backend: 'error', updated: 0 };
+  }
+  stages.indexMs = Date.now() - tIndex;
+
+  const q = queryTerms(analysis);
+  const tFts = Date.now();
+  const ftsHits = searchFts(root, q, { limit: opts.max || 60 });
+  stages.ftsMs = Date.now() - tFts;
+
+  const pathList = pathRank(
+    root,
+    analysis.expanded || analysis.tokens,
+    analysis.pathHints,
+    analysis.domain,
+    opts.max || 80,
+  );
+
+  const cascadeSkip = shouldCascadeSkip(ftsHits, pathList);
+  let content = [];
+  const tGrep = Date.now();
+  if (!cascadeSkip) {
+    const grepNeedles = [
+      ...(analysis.symbols || []),
+      ...(analysis.pathHints || []),
+      ...(analysis.expanded || []).filter((t) => /[a-z]/i.test(t)).slice(0, 8),
+    ];
+    content = contentGrep(root, pathList.map((x) => x.path), grepNeedles);
+  }
+  stages.grepMs = Date.now() - tGrep;
+  stages.cascadeSkip = cascadeSkip;
+
+  const recency = gitRecency(root);
+
+  const tPr = Date.now();
+  const seeds = [
+    ...dirty,
+    ...ftsHits.slice(0, 12).map((x) => x.path),
+    ...pathList.slice(0, 12).map((x) => x.path),
+    ...(opts.openFiles || []),
+  ];
+  let prRanked = [];
+  try {
+    const pr = rankGraph(root, seeds.slice(0, 40), seeds.slice(0, 16), { maxFiles: 60 });
+    prRanked = pr.ranked.filter((x) => x.score > 0).slice(0, 40);
+    stages.graphCached = Boolean(pr.graph?.cached);
+    stages.graphFingerprint = pr.graph?.fingerprint || null;
+  } catch {
+    prRanked = [];
+  }
+  stages.pagerankMs = Date.now() - tPr;
+
+  const fusedRaw = weightedRrf([
+    { list: dirty.map((p) => ({ path: p })), weight: 1.4 },
+    { list: ftsHits, weight: 1.25 },
+    { list: pathList, weight: 1.0 },
+    { list: prRanked, weight: 1.1 },
+    { list: recency, weight: 0.8 },
+    { list: content, weight: 0.55 },
+  ]);
+
+  const diversified = mmrSelect(fusedRaw.map((x) => ({ path: x.path, score: x.rrf })), {
+    k: opts.max || 60,
+    lambda: 0.72,
+  }).map((x) => ({ path: x.path, rrf: x.score, mmr: x.mmr }));
+
+  stages.totalMs = Date.now() - t0;
+
+  return {
+    dirty,
+    pathRanked: pathList.map((x) => x.path),
+    contentHits: content.map((x) => x.path),
+    ftsHits: ftsHits.map((x) => x.path),
+    pagerank: prRanked.slice(0, 12).map((x) => x.path),
+    fused: diversified.slice(0, opts.max || 60),
+    cascadeSkip,
+    ftsMeta,
+    stages,
+  };
+}
+
+module.exports = {
+  rrf,
+  weightedRrf,
+  gitDirty,
+  gitDiffNames,
+  pathRank,
+  contentGrep,
+  retrieveCandidates,
+  shouldCascadeSkip,
+  queryTerms,
+  tokenizePath,
+};

package/src/serena.js

@@ -0,0 +1,126 @@
+'use strict';
+
+/**
+ * Serena-first: emit exact find_symbol / overview calls + local regex fallback
+ * so the Agent does not have to read_file the whole module.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { splitIdentifier } = require('./identifier');
+
+const SYMBOL_RE = /(?:export\s+)?(?:async\s+)?(?:function|class|const|type|interface|enum)\s+([A-Za-z_][A-Za-z0-9_]*)/g;
+
+function emitFindSymbol(name, opts = {}) {
+  return {
+    tool: 'find_symbol',
+    name_path_pattern: name,
+    relative_path: opts.relative_path || '',
+    include_body: opts.include_body === true,
+    depth: opts.depth || 0,
+  };
+}
+
+function emitOverview(relativePath) {
+  return {
+    tool: 'get_symbols_overview',
+    relative_path: relativePath,
+    max_answer_chars: 2000,
+  };
+}
+
+function emitReferencing(name) {
+  return {
+    tool: 'find_referencing_symbols',
+    name_path_pattern: name,
+  };
+}
+
+function emitForTask(symbols, selectedPaths) {
+  const calls = [];
+  for (const s of (symbols || []).slice(0, 8)) {
+    calls.push(emitFindSymbol(s, { include_body: false }));
+  }
+  for (const p of (selectedPaths || []).slice(0, 3)) {
+    calls.push(emitOverview(p));
+  }
+  return calls;
+}
+
+function scanFileForSymbol(abs, name) {
+  let text = '';
+  try {
+    text = fs.readFileSync(abs, 'utf8');
+  } catch {
+    return null;
+  }
+  const lines = text.split(/\r?\n/);
+  const want = String(name || '');
+  const wantLow = want.toLowerCase();
+  const parts = splitIdentifier(want);
+  const hits = [];
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    SYMBOL_RE.lastIndex = 0;
+    let m;
+    const re = /(?:export\s+)?(?:async\s+)?(?:function|class|const|type|interface|enum)\s+([A-Za-z_][A-Za-z0-9_]*)/g;
+    while ((m = re.exec(line))) {
+      const n = m[1];
+      if (n === want || n.toLowerCase() === wantLow || parts.every((p) => n.toLowerCase().includes(p))) {
+        hits.push({ name: n, line: i + 1, snippet: line.trim().slice(0, 160) });
+      }
+    }
+  }
+  return hits.length ? { hits, bytes: text.length } : null;
+}
+
+function localFindSymbol(root, name, searchPaths = []) {
+  const hits = [];
+  const seen = new Set();
+  const queue = [...(searchPaths || [])];
+  while (queue.length && hits.length < 12) {
+    const rel = queue.shift();
+    if (!rel || seen.has(rel)) continue;
+    seen.add(rel);
+    const abs = path.join(root, rel);
+    let st;
+    try {
+      st = fs.statSync(abs);
+    } catch {
+      continue;
+    }
+    if (st.isDirectory()) {
+      let ents = [];
+      try {
+        ents = fs.readdirSync(abs);
+      } catch {
+        continue;
+      }
+      for (const e of ents) {
+        if (/\.(tsx?|jsx?)$/.test(e)) queue.push(path.posix.join(rel.replace(/\\/g, '/'), e));
+      }
+      continue;
+    }
+    const found = scanFileForSymbol(abs, name);
+    if (found) {
+      for (const h of found.hits) {
+        hits.push({ path: rel.replace(/\\/g, '/'), ...h });
+      }
+    }
+  }
+  return {
+    name,
+    source: 'local',
+    hits,
+    serena: emitFindSymbol(name),
+  };
+}
+
+module.exports = {
+  emitFindSymbol,
+  emitOverview,
+  emitReferencing,
+  emitForTask,
+  localFindSymbol,
+  scanFileForSymbol,
+};

package/src/session-ledger.js

@@ -0,0 +1,83 @@
+'use strict';
+
+/**
+ * Per-session alreadyInContext ledger so token cost falls across turns.
+ * readNow = fresh files only; already delivered stay in alreadyInContext.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const LEDGER_REL = '.ziprin-context/ledger.json';
+
+function ledgerPath(root) {
+  return path.join(root, LEDGER_REL);
+}
+
+function loadLedger(root) {
+  const file = ledgerPath(root);
+  if (!fs.existsSync(file)) return { version: 1, sessions: {} };
+  try {
+    const j = JSON.parse(fs.readFileSync(file, 'utf8'));
+    return { version: 1, sessions: j.sessions || {} };
+  } catch {
+    return { version: 1, sessions: {} };
+  }
+}
+
+function saveLedger(root, ledger) {
+  const file = ledgerPath(root);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, JSON.stringify(ledger), 'utf8');
+}
+
+function splitFresh(selected, ledger, sessionKey = 'default') {
+  const files = selected || [];
+  const prev = new Set(ledger?.sessions?.[sessionKey]?.files || []);
+  const fresh = [];
+  const alreadyInContext = [];
+  for (const p of files) {
+    if (prev.has(p)) alreadyInContext.push(p);
+    else fresh.push(p);
+  }
+  return { fresh, alreadyInContext };
+}
+
+function remember(root, sessionKey, files) {
+  const key = sessionKey || 'default';
+  const ledger = loadLedger(root);
+  const prev = new Set(ledger.sessions[key]?.files || []);
+  for (const f of files || []) prev.add(f);
+  ledger.sessions[key] = {
+    files: [...prev],
+    ts: new Date().toISOString(),
+  };
+  // cap sessions
+  const keys = Object.keys(ledger.sessions);
+  if (keys.length > 40) {
+    const oldest = keys
+      .map((k) => ({ k, ts: ledger.sessions[k].ts || '' }))
+      .sort((a, b) => a.ts.localeCompare(b.ts))
+      .slice(0, keys.length - 40);
+    for (const o of oldest) delete ledger.sessions[o.k];
+  }
+  saveLedger(root, ledger);
+  return ledger;
+}
+
+function clearLedger(root, sessionKey) {
+  const ledger = loadLedger(root);
+  if (sessionKey) delete ledger.sessions[sessionKey];
+  else ledger.sessions = {};
+  saveLedger(root, ledger);
+  return ledger;
+}
+
+module.exports = {
+  LEDGER_REL,
+  loadLedger,
+  saveLedger,
+  splitFresh,
+  remember,
+  clearLedger,
+};

package/src/session-store.js

@@ -0,0 +1,78 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const SESSIONS_REL = '.cursor/context-inspector-sessions.jsonl';
+
+function sessionsPath(root) {
+  return path.join(root, SESSIONS_REL);
+}
+
+function readSessions(root) {
+  const file = sessionsPath(root);
+  if (!fs.existsSync(file)) return [];
+  const out = [];
+  for (const line of fs.readFileSync(file, 'utf8').split(/\r?\n/)) {
+    if (!line.trim()) continue;
+    try {
+      out.push(JSON.parse(line));
+    } catch {
+      /* skip */
+    }
+  }
+  return out;
+}
+
+function writeSessions(root, sessions) {
+  const file = sessionsPath(root);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  const body = sessions.map((s) => JSON.stringify(s)).join('\n');
+  fs.writeFileSync(file, body ? body + '\n' : '', 'utf8');
+}
+
+function appendSession(root, session, maxEntries = 80) {
+  if (!root || !session) return session;
+  const all = readSessions(root);
+  all.push(session);
+  writeSessions(root, all.slice(-Math.max(1, maxEntries)));
+  return session;
+}
+
+function clearSessions(root) {
+  const file = sessionsPath(root);
+  if (fs.existsSync(file)) fs.unlinkSync(file);
+}
+
+function listSessions(root, limit = 40) {
+  return readSessions(root).slice(-limit).reverse();
+}
+
+function getSession(root, id) {
+  return readSessions(root).find((s) => s.id === id) || null;
+}
+
+function historySummary(session) {
+  return {
+    id: session.id,
+    timestamp: session.timestamp,
+    task: session.task,
+    promptPreview: String(session.prompt || '').slice(0, 80),
+    filesIncludedCount: (session.filesIncluded || []).length,
+    filesExcludedCount: (session.filesExcluded || []).length,
+    tokensBeforePrune: session.tokensBeforePrune,
+    tokensAfterPrune: session.tokensAfterPrune,
+    profileId: session.profileId,
+  };
+}
+
+module.exports = {
+  SESSIONS_REL,
+  sessionsPath,
+  readSessions,
+  appendSession,
+  clearSessions,
+  listSessions,
+  getSession,
+  historySummary,
+};