ziprin-context-optimizer 8.0.0

package/src/bm25.js

@@ -0,0 +1,132 @@
+'use strict';
+
+/**
+ * BM25 / BM25F (path ×8, body ×1, symbols ×12). Pure JS fallback when FTS5 is cold.
+ */
+
+const { tokenizePath, splitIdentifier } = require('./identifier');
+
+const K1 = 1.2;
+const B = 0.75;
+const FIELD_WEIGHTS = { path: 8, body: 1, symbols: 12 };
+
+function idf(N, df) {
+  return Math.log(1 + (N - df + 0.5) / (df + 0.5));
+}
+
+function tfComponent(tf, dl, avgdl, k1 = K1, b = B) {
+  if (tf <= 0) return 0;
+  return (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * (dl / Math.max(avgdl, 1))));
+}
+
+function termFreq(terms) {
+  const tf = new Map();
+  for (const t of terms) tf.set(t, (tf.get(t) || 0) + 1);
+  return tf;
+}
+
+function buildFieldCorpus(docs, field) {
+  const df = new Map();
+  let totalLen = 0;
+  for (const d of docs) {
+    const terms = d[field] || [];
+    totalLen += terms.length;
+    for (const t of new Set(terms)) df.set(t, (df.get(t) || 0) + 1);
+  }
+  return {
+    df,
+    N: docs.length,
+    avgdl: docs.length ? totalLen / docs.length : 1,
+  };
+}
+
+function bm25F(doc, queryTerms, corpora) {
+  let score = 0;
+  const seen = new Set();
+  for (const q of queryTerms) {
+    if (!q || seen.has(q)) continue;
+    seen.add(q);
+    for (const field of ['path', 'body', 'symbols']) {
+      const corp = corpora[field];
+      if (!corp || !corp.N) continue;
+      const tfMap = termFreq(doc[field] || []);
+      const tf = tfMap.get(q) || 0;
+      if (!tf) continue;
+      const w = FIELD_WEIGHTS[field] || 1;
+      score += w * idf(corp.N, corp.df.get(q) || 0) * tfComponent(tf, (doc[field] || []).length, corp.avgdl);
+    }
+  }
+  return score;
+}
+
+function bm25Path(paths, queryTerms) {
+  const docs = (paths || []).map((p) => ({
+    path: p,
+    terms: tokenizePath(p),
+  }));
+  const df = new Map();
+  let total = 0;
+  for (const d of docs) {
+    total += d.terms.length;
+    for (const t of new Set(d.terms)) df.set(t, (df.get(t) || 0) + 1);
+  }
+  const corpus = { df, N: docs.length, avgdl: docs.length ? total / docs.length : 1 };
+  const q = (queryTerms || []).map((t) => String(t).toLowerCase()).filter((t) => t.length > 1);
+  return docs
+    .map((d) => {
+      let s = 0;
+      const tf = termFreq(d.terms);
+      const seen = new Set();
+      for (const term of q) {
+        if (seen.has(term)) continue;
+        seen.add(term);
+        const f = tf.get(term) || 0;
+        if (!f) continue;
+        s += idf(corpus.N, df.get(term) || 0) * tfComponent(f, d.terms.length, corpus.avgdl);
+      }
+      return { path: d.path, score: s };
+    })
+    .sort((a, b) => b.score - a.score);
+}
+
+function tokenizeBody(text, maxTerms = 800) {
+  const raw = String(text || '')
+    .slice(0, 16000)
+    .split(/[^\p{L}\p{N}_]+/u)
+    .map((t) => t.toLowerCase())
+    .filter((t) => t.length > 1);
+  const extra = [];
+  for (const t of raw.slice(0, 120)) extra.push(...splitIdentifier(t));
+  return [...raw, ...extra].slice(0, maxTerms);
+}
+
+function scoreDocsBm25F(docs, queryTerms) {
+  if (!docs.length) return [];
+  const corpora = {
+    path: buildFieldCorpus(docs, 'pathTerms'),
+    body: buildFieldCorpus(docs, 'bodyTerms'),
+    symbols: buildFieldCorpus(docs, 'symbolTerms'),
+  };
+  const q = (queryTerms || []).map((t) => String(t).toLowerCase()).filter((t) => t.length > 1);
+  return docs
+    .map((d) => ({
+      path: d.path,
+      score: bm25F(
+        { path: d.pathTerms, body: d.bodyTerms, symbols: d.symbolTerms },
+        q,
+        corpora,
+      ),
+    }))
+    .sort((a, b) => b.score - a.score);
+}
+
+module.exports = {
+  FIELD_WEIGHTS,
+  idf,
+  tfComponent,
+  bm25Path,
+  bm25F,
+  scoreDocsBm25F,
+  tokenizeBody,
+  termFreq,
+};

package/src/collector.js

@@ -0,0 +1,343 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { tokens, safeSize } = require('./estimate');
+const { isTokenBomb } = require('./pruner');
+const { matchAny, rootFromInclude } = require('./glob');
+const { getContextProfile } = require('./context-profiles');
+const { splitIdentifier } = require('./identifier');
+
+const SKIP_DIRS = new Set([
+  'node_modules',
+  '.git',
+  'dist',
+  'build',
+  '.next',
+  'coverage',
+  'public',
+  '.turbo',
+  'reports',
+  'fixtures',
+  'mock-data',
+  '__mocks__',
+  'seed',
+  'generated',
+  '.ziprin-context',
+]);
+
+const SOURCE_EXT = new Set(['.ts', '.tsx', '.js', '.jsx', '.css', '.md', '.mdc']);
+const MAX_FILES_TOTAL = 220;
+const MAX_FILES_PER_ROOT = 90;
+const MAX_SEED_FILES = 40;
+
+function isSourceFile(rel) {
+  const ext = path.extname(String(rel || '')).toLowerCase();
+  return SOURCE_EXT.has(ext);
+}
+
+function pascalToKebab(s) {
+  return String(s).replace(/([a-z])([A-Z])/g, '$1-$2').toLowerCase();
+}
+
+function collectRules(root) {
+  const dir = path.join(root, '.cursor', 'rules');
+  if (!fs.existsSync(dir)) return [];
+  let names;
+  try {
+    names = fs.readdirSync(dir);
+  } catch {
+    return [];
+  }
+  const rules = [];
+  for (const name of names) {
+    if (!name.endsWith('.mdc')) continue;
+    const rel = `.cursor/rules/${name}`;
+    const abs = path.join(root, rel);
+    let text = '';
+    try {
+      text = fs.readFileSync(abs, 'utf8').slice(0, 400);
+    } catch {
+      continue;
+    }
+    rules.push({
+      path: rel,
+      alwaysApply: /alwaysApply:\s*true/.test(text),
+      excerpt: text.split('\n').slice(0, 8).join('\n'),
+    });
+  }
+  return rules;
+}
+
+/** Hint-first only. No hardcoded product-status vs add-product rank. */
+function sortEntries(entries, pathHints = []) {
+  return entries.sort((a, b) => {
+    const hintRank = (n) => {
+      if (pathHints.some((h) => n.includes(h) || h.includes(n))) return 0;
+      if (n === 'src') return 1;
+      if (n === 'public') return 9;
+      if (n === 'docs') return 8;
+      return 5;
+    };
+    const ha = hintRank(a.name);
+    const hb = hintRank(b.name);
+    if (ha !== hb) return ha - hb;
+    return a.name.localeCompare(b.name);
+  });
+}
+
+function walkSource(absDir, root, onFile, maxMatches, pathHints = []) {
+  let count = 0;
+  function walk(dir) {
+    if (count >= maxMatches) return;
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const ent of sortEntries(entries, pathHints)) {
+      if (count >= maxMatches) return;
+      if (ent.isDirectory()) {
+        if (SKIP_DIRS.has(ent.name)) continue;
+        walk(path.join(dir, ent.name));
+      } else {
+        const rel = path.relative(root, path.join(dir, ent.name)).replace(/\\/g, '/');
+        if (!isSourceFile(rel)) continue;
+        onFile(rel);
+        count++;
+      }
+    }
+  }
+  walk(absDir);
+}
+
+function buildSeedDirs(root, collectOpts = {}) {
+  const seeds = new Set();
+  const { domain, pathHints = [] } = collectOpts;
+  if (domain?.priorityPaths) {
+    for (const dp of domain.priorityPaths) seeds.add(dp.replace(/\\/g, '/'));
+  }
+  for (const hint of pathHints) {
+    for (const base of [
+      'apps/dashboard/bss/src/portals/vendor',
+      'apps/dashboard/bss/src',
+      'apps/marketplace/src',
+      'apps/dashboard',
+    ]) {
+      seeds.add(`${base}/${hint}`);
+    }
+    if (hint.includes('/')) {
+      seeds.add(`apps/dashboard/bss/src/portals/${hint}`);
+    }
+  }
+  return [...seeds].filter((s) => {
+    const abs = path.join(root, s);
+    return fs.existsSync(abs);
+  });
+}
+
+function findFilesByBasename(root, relRoot, basenames, includeGlobs, excludeGlobs, limit = 10) {
+  const found = [];
+  const abs = path.join(root, relRoot);
+  if (!fs.existsSync(abs)) return found;
+  const want = new Set(basenames.map((b) => b.toLowerCase()));
+  walkSource(abs, root, (rel) => {
+    if (found.length >= limit) return;
+    const base = path.basename(rel).toLowerCase();
+    const stem = base.replace(/\.(tsx|ts|jsx|js)$/, '');
+    if (!want.has(base) && !want.has(stem)) return;
+    if (!matchAny(rel, includeGlobs)) return;
+    if (matchAny(rel, excludeGlobs || [])) return;
+    if (!found.includes(rel)) found.push(rel);
+  }, limit * 20, basenames);
+  return found;
+}
+
+function findSymbolFiles(root, symbols, includeGlobs, excludeGlobs, limit = 20) {
+  const found = [];
+  if (!symbols.length) return found;
+
+  for (const sym of symbols) {
+    const kebab = pascalToKebab(sym);
+    const parts = splitIdentifier(sym);
+    const basenames = [
+      `${kebab}.tsx`,
+      `${kebab}.ts`,
+      `${sym.toLowerCase()}.tsx`,
+    ];
+    if (parts.includes('product') && parts.includes('status')) {
+      basenames.push('product-status-screen.tsx', 'product-status.api.ts');
+    }
+    for (const sr of [
+      'apps/dashboard/bss/src',
+      'apps/marketplace/src',
+      'packages/ui/src',
+    ]) {
+      for (const rel of findFilesByBasename(root, sr, basenames, includeGlobs, excludeGlobs, limit)) {
+        if (!found.includes(rel)) found.push(rel);
+      }
+    }
+
+    const variants = [sym.toLowerCase(), kebab, kebab.replace(/-screen$/, ''), ...parts.filter((p) => p.length > 3)];
+    for (const sr of ['apps/dashboard/bss/src', 'apps/marketplace/src', 'packages/ui/src']) {
+      const abs = path.join(root, sr);
+      if (!fs.existsSync(abs)) continue;
+      walkSource(
+        abs,
+        root,
+        (rel) => {
+          if (found.length >= limit) return;
+          const lower = rel.toLowerCase();
+          if (!variants.some((v) => v.length > 2 && lower.includes(v))) return;
+          if (!matchAny(rel, includeGlobs)) return;
+          if (matchAny(rel, excludeGlobs || [])) return;
+          if (!found.includes(rel)) found.push(rel);
+        },
+        limit * 8,
+        variants,
+      );
+    }
+  }
+  return found.slice(0, limit);
+}
+
+function collectFromSeeds(root, seedDirs, includeGlobs, excludeGlobs, limit, pathHints = []) {
+  const bag = [];
+  for (const relDir of seedDirs) {
+    if (bag.length >= limit) break;
+    const abs = path.join(root, relDir);
+    if (!fs.existsSync(abs)) continue;
+    const st = fs.statSync(abs);
+    if (st.isFile()) {
+      const rel = relDir.replace(/\\/g, '/');
+      if (isSourceFile(rel) && matchAny(rel, includeGlobs) && !matchAny(rel, excludeGlobs || [])) {
+        bag.push(rel);
+      }
+      continue;
+    }
+    walkSource(abs, root, (rel) => {
+      if (bag.length >= limit) return;
+      if (!matchAny(rel, includeGlobs)) return;
+      if (matchAny(rel, excludeGlobs || [])) return;
+      if (!bag.includes(rel)) bag.push(rel);
+    }, limit - bag.length, pathHints);
+  }
+  return bag;
+}
+
+function addFileEntry(root, rel, seen, files, ignored, profile, charsPerToken, maxTotal) {
+  if (seen.has(rel)) return false;
+  seen.add(rel);
+  const bytes = safeSize(path.join(root, rel));
+  const tok = tokens(bytes, charsPerToken);
+
+  if (matchAny(rel, profile.exclude || [])) {
+    ignored.push({ file: rel, reason: 'ignored by project rules', tokenSaving: tok });
+    return false;
+  }
+  if (isTokenBomb(rel)) {
+    ignored.push({ file: rel, reason: 'token bomb / never-index', tokenSaving: tok });
+    return false;
+  }
+  files.push({ path: rel, bytes, tokens: tok, seeded: true });
+  return files.length < maxTotal;
+}
+
+/**
+ * Collect workspace files for scoring. Hint-first, no folder rank bias (v8).
+ */
+function collectWorkspace(root, profileId, charsPerToken = 4, collectOpts = {}) {
+  const profile = getContextProfile(profileId);
+  const seen = new Set();
+  const files = [];
+  const ignored = [];
+  let truncated = false;
+
+  const seedDirs = buildSeedDirs(root, collectOpts);
+  const symbolFiles = findSymbolFiles(
+    root,
+    collectOpts.symbols || [],
+    profile.include,
+    profile.exclude,
+    MAX_SEED_FILES,
+  );
+
+  const seedBag = [
+    ...symbolFiles,
+    ...collectFromSeeds(root, seedDirs, profile.include, profile.exclude, MAX_SEED_FILES, collectOpts.pathHints || []),
+  ];
+  for (const rel of seedBag) {
+    if (files.length >= MAX_FILES_TOTAL) {
+      truncated = true;
+      break;
+    }
+    addFileEntry(root, rel, seen, files, ignored, profile, charsPerToken, MAX_FILES_TOTAL);
+  }
+
+  const roots = [...new Set(profile.include.map(rootFromInclude))];
+  for (const relRoot of roots) {
+    if (files.length >= MAX_FILES_TOTAL) {
+      truncated = true;
+      break;
+    }
+    const abs = relRoot === '.' ? root : path.join(root, relRoot);
+    if (!fs.existsSync(abs)) continue;
+    const st = fs.statSync(abs);
+    const perRootLimit = MAX_FILES_PER_ROOT;
+    const bag = [];
+    if (st.isFile()) {
+      const rel = relRoot.replace(/\\/g, '/');
+      if (isSourceFile(rel)) bag.push(rel);
+    } else {
+      walkSource(abs, root, (rel) => {
+        if (bag.length >= perRootLimit) return;
+        if (!matchAny(rel, profile.include)) return;
+        if (matchAny(rel, profile.exclude || [])) return;
+        if (!bag.includes(rel)) bag.push(rel);
+      }, perRootLimit, collectOpts.pathHints || []);
+    }
+    if (bag.length >= perRootLimit) truncated = true;
+
+    for (const rel of bag) {
+      if (files.length >= MAX_FILES_TOTAL) {
+        truncated = true;
+        break;
+      }
+      if (seen.has(rel)) continue;
+      seen.add(rel);
+      const bytes = safeSize(path.join(root, rel));
+      const tok = tokens(bytes, charsPerToken);
+      if (matchAny(rel, profile.exclude || [])) {
+        ignored.push({ file: rel, reason: 'ignored by project rules', tokenSaving: tok });
+        continue;
+      }
+      if (isTokenBomb(rel)) {
+        ignored.push({ file: rel, reason: 'token bomb / never-index', tokenSaving: tok });
+        continue;
+      }
+      files.push({ path: rel, bytes, tokens: tok });
+    }
+  }
+
+  return {
+    profileId: profile.id,
+    files,
+    ignored,
+    rules: collectRules(root),
+    truncated,
+    seedDirs,
+    symbolFiles,
+  };
+}
+
+module.exports = {
+  collectWorkspace,
+  collectRules,
+  MAX_FILES: MAX_FILES_TOTAL,
+  SKIP_DIRS,
+  isSourceFile,
+  buildSeedDirs,
+  findSymbolFiles,
+  sortEntries,
+};

package/src/compression.js

@@ -0,0 +1,67 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { skeletonFile, extractSignatures } = require('./skeleton');
+
+/**
+ * Smart compression v8 — function skeletons instead of head-55.
+ */
+
+const EXPORT_RE = /^export\s+(?:async\s+)?(?:function|class|const|type|interface|enum)\s+([A-Za-z0-9_]+)/gm;
+const FUNC_RE = /^(?:export\s+)?(?:async\s+)?function\s+([A-Za-z0-9_]+)/gm;
+const TYPE_RE = /^export\s+(?:type|interface)\s+([A-Za-z0-9_]+)/gm;
+
+function estimateTokensFromText(text) {
+  const chars = String(text || '').length;
+  const idents = (String(text || '').match(/[A-Za-z_][A-Za-z0-9_]*/g) || []).length;
+  return Math.ceil(chars / 3.6 + idents * 0.12);
+}
+
+function extractExports(text) {
+  const exports = new Set();
+  let m;
+  for (const re of [EXPORT_RE, FUNC_RE, TYPE_RE]) {
+    re.lastIndex = 0;
+    while ((m = re.exec(text))) exports.add(m[1]);
+  }
+  return exports;
+}
+
+function compressFile(root, rel, maxChars = 1800) {
+  const abs = path.join(root, rel);
+  let text = '';
+  try {
+    text = fs.readFileSync(abs, 'utf8');
+  } catch {
+    return { path: rel, mode: 'missing', text: '', tokenEstimate: 0, exports: [] };
+  }
+
+  const exports = extractExports(text);
+  const sigs = extractSignatures(text);
+
+  if (text.length <= 8000) {
+    return {
+      path: rel,
+      mode: 'full',
+      text,
+      tokenEstimate: estimateTokensFromText(text),
+      exports: [...exports].slice(0, 60),
+      signatures: sigs,
+    };
+  }
+
+  const skel = skeletonFile(root, rel);
+  const clipped = skel.text.slice(0, Math.max(maxChars, 1200));
+  return {
+    path: rel,
+    mode: text.length <= 40000 ? 'skeleton' : 'compressed',
+    text: clipped,
+    tokenEstimate: estimateTokensFromText(clipped),
+    exports: [...exports].slice(0, 60),
+    signatures: skel.signatures,
+    removedSections: `${text.split(/\r?\n/).length} lines → ${skel.signatures.length} signatures`,
+  };
+}
+
+module.exports = { compressFile, extractExports, estimateTokensFromText };

package/src/config.js

@@ -0,0 +1,28 @@
+'use strict';
+
+function getConfig(vscode) {
+  const defaults = {
+    profile: 'standard',
+    charsPerToken: 4,
+    stripOtherMdc: true,
+    liveRefresh: true,
+    refreshDebounceMs: 500,
+    autoRevealPanel: true,
+    maxSessions: 80,
+    recordOnStartOnly: true,
+  };
+  if (!vscode) return defaults;
+  const c = vscode.workspace.getConfiguration('ziprinContext');
+  return {
+    profile: c.get('profile', defaults.profile),
+    charsPerToken: c.get('charsPerToken', defaults.charsPerToken),
+    stripOtherMdc: c.get('stripOtherMdc', defaults.stripOtherMdc),
+    liveRefresh: c.get('liveRefresh', defaults.liveRefresh),
+    refreshDebounceMs: c.get('refreshDebounceMs', defaults.refreshDebounceMs),
+    autoRevealPanel: c.get('autoRevealPanel', defaults.autoRevealPanel),
+    maxSessions: c.get('maxSessions', defaults.maxSessions),
+    recordOnStartOnly: c.get('recordOnStartOnly', defaults.recordOnStartOnly),
+  };
+}
+
+module.exports = { getConfig };