sweet-search 2.4.2 → 2.5.1

package/core/cli.js

@@ -20,15 +20,29 @@ if (args[0] === 'init') {
 } else if (args[0] === 'prewarm-vocab') {
   const { handlePrewarmVocabCli } = await import('./vocabulary/index.js');
   await handlePrewarmVocabCli(args.slice(1));
+} else if (args[0] === 'read') {
+  // Filesystem-grounded reader; runs in JS (no native equivalent yet).
+  const { handleReadCli } = await import('./search/search-read.js');
+  await handleReadCli(args.slice(1));
+} else if (args[0] === 'read-semantic') {
+  // Hybrid span-selection reader; runs in JS (depends on LI index + ranking).
+  const { handleReadSemanticCli } = await import('./search/search-read-semantic.js');
+  await handleReadSemanticCli(args.slice(1));
+} else if (args[0] === '--serve' || args[0] === '--stop') {
+  // Warm search server lifecycle is implemented in JS.
+  const { runCli } = await import('./search/index.js');
+  await runCli(args);
 } else if (args[0] === '--help' || args[0] === '-h' || args.length === 0) {
   console.log(`sweet-search — hybrid code search engine
 
 Usage:
-  sweet-search <query>                Search the indexed codebase
-  sweet-search init [options]         Set up runtime assets and models
-  sweet-search uninstall [opts]       Remove local state created by init
-  sweet-search prewarm-vocab [file]   Pre-warm vocabulary cache with terms
-  sweet-search --help                 Show this help
+  sweet-search <query>                  Search the indexed codebase
+  sweet-search read <file...>           Filesystem-grounded read (1-20 files)
+  sweet-search read-semantic <f> <q>    Return only file spans relevant to a query
+  sweet-search init [options]           Set up runtime assets and models
+  sweet-search uninstall [opts]         Remove local state created by init
+  sweet-search prewarm-vocab [file]     Pre-warm vocabulary cache with terms
+  sweet-search --help                   Show this help
 
 Options:
   --mode <mode>     Search mode: auto, lexical, semantic, hybrid, pattern

package/core/embedding/embedding-cache.js

@@ -60,6 +60,67 @@ export class LRUCache {
   }
 }
 
+// =============================================================================
+// ATOMIC JSON WRITER — serialised tmp-file-and-rename
+// =============================================================================
+//
+// Both Vocabulary.save() and QueryStats.save() are fired as background
+// promises from the embedding hot path (`vocabulary.save().catch(()=>{})`
+// inside getEmbedding). Under concurrent benchmarks (12+ in-flight queries)
+// multiple save() calls overlap on the same file. The previous direct
+// `fs.writeFile` was non-atomic — interleaving writes produced invalid JSON,
+// which then poisoned every subsequent `.load()` call and silently degraded
+// retrieval quality to near-zero.
+//
+// `writeJsonAtomic` writes to a unique temp file then atomically renames it
+// into place. `serialiseAtomicWrite` chains an instance's writes so at most
+// one is in flight at a time — and at most one extra coalesced write is
+// queued behind the in-flight one. Bursts of N saves collapse into 2 writes,
+// each producing a fully-formed file.
+
+async function writeJsonAtomic(targetPath, json) {
+  await fs.mkdir(path.dirname(targetPath), { recursive: true });
+  const tmpPath = `${targetPath}.tmp.${process.pid}.${Date.now().toString(36)}.${Math.random().toString(36).slice(2, 8)}`;
+  try {
+    await fs.writeFile(tmpPath, json);
+    await fs.rename(tmpPath, targetPath);
+  } catch (err) {
+    try { await fs.unlink(tmpPath); } catch { /* may not exist */ }
+    throw err;
+  }
+}
+
+/**
+ * Serialise calls to `produce()` for a given owner. At most one write is
+ * in flight; bursts coalesce so callers arriving during an in-flight save
+ * all share a single follow-up save, but no further waste accumulates.
+ *
+ * @param {object} owner - instance with mutable `_atomicInFlight` / `_atomicPending` slots
+ * @param {() => Promise<void>} produce - async writer that captures the
+ *   current state at call time and writes it atomically
+ * @returns {Promise<void>}
+ */
+function serialiseAtomicWrite(owner, produce) {
+  if (owner._atomicPending) return owner._atomicPending;
+
+  const previous = owner._atomicInFlight || Promise.resolve();
+  owner._atomicPending = previous
+    .catch(() => { /* a previous save's failure must not block the next one */ })
+    .then(() => {
+      // Move from "pending" to "in-flight" before doing the actual write,
+      // so additional save() callers arriving during the write start a new
+      // pending entry rather than piggy-backing on this one (otherwise they
+      // would not see their latest state on disk).
+      owner._atomicPending = null;
+      const inFlight = produce();
+      owner._atomicInFlight = inFlight.finally(() => {
+        if (owner._atomicInFlight === inFlight) owner._atomicInFlight = null;
+      });
+      return owner._atomicInFlight;
+    });
+  return owner._atomicPending;
+}
+
 // =============================================================================
 // QUERY STATS (Cross-session usage tracking)
 // =============================================================================
@@ -89,10 +150,20 @@ export class QueryStats {
 
   async save() {
     if (!this.dirty) return;
-    const data = { queries: Object.fromEntries(this.stats), lastUpdated: new Date().toISOString() };
-    await fs.mkdir(path.dirname(this.statsPath), { recursive: true });
-    await fs.writeFile(this.statsPath, JSON.stringify(data));
-    this.dirty = false;
+    return serialiseAtomicWrite(this, async () => {
+      // Re-check dirty inside the queued task: if a coalesced earlier write
+      // already persisted everything, there is nothing left to do.
+      if (!this.dirty) return;
+      const data = { queries: Object.fromEntries(this.stats), lastUpdated: new Date().toISOString() };
+      this.dirty = false;
+      try {
+        await writeJsonAtomic(this.statsPath, JSON.stringify(data));
+      } catch (err) {
+        // Re-mark dirty so a future save retries this state
+        this.dirty = true;
+        throw err;
+      }
+    });
   }
 
   increment(query) {
@@ -112,24 +183,92 @@ export class QueryStats {
 // VOCABULARY
 // =============================================================================
 
+// Schema version for the persisted vocabulary file. Bump when the on-disk
+// shape changes in a way that should invalidate previously-saved files.
+//   v2: { metadata: { created, lastUpdated, version, provider }, terms: {...} }
+//   v3: { metadata: { ..., model, dimension, schemaVersion: 3 }, terms: {...} }
+//        — adds full embedding fingerprint so a cache produced under one
+//        model is not silently served when a different model is active.
+const VOCAB_SCHEMA_VERSION = 3;
+
+/** Build the embedding-fingerprint we expect a vocabulary file to match. */
+function currentVocabFingerprint() {
+  return {
+    schemaVersion: VOCAB_SCHEMA_VERSION,
+    provider: EMBEDDING_CONFIG.provider,
+    model: EMBEDDING_CONFIG.model,
+    dimension: EMBEDDING_CONFIG.dimension,
+  };
+}
+
+/**
+ * Decide whether a persisted vocabulary file's metadata is compatible
+ * with the active embedding configuration. Returns `{ compatible: bool,
+ * reason?: string }`. Stale `provider` / `model` / `dimension` /
+ * schema-version mismatches are explicitly rejected; a missing file is
+ * treated as a fresh start (compatible by definition).
+ */
+export function isVocabFingerprintCompatible(metadata, fingerprint = currentVocabFingerprint()) {
+  if (!metadata || typeof metadata !== 'object') {
+    return { compatible: false, reason: 'missing-metadata' };
+  }
+  // Pre-v3 files persist `version` (not `schemaVersion`) and never recorded
+  // `model` / `dimension`. Treat those as incompatible — we cannot prove the
+  // cached embeddings came from the active model.
+  const persistedSchema = metadata.schemaVersion ?? metadata.version;
+  if (persistedSchema !== fingerprint.schemaVersion) {
+    return { compatible: false, reason: `schema-version (file=${persistedSchema} expected=${fingerprint.schemaVersion})` };
+  }
+  if (metadata.provider && metadata.provider !== fingerprint.provider) {
+    return { compatible: false, reason: `provider (file=${metadata.provider} expected=${fingerprint.provider})` };
+  }
+  if (metadata.model && metadata.model !== fingerprint.model) {
+    return { compatible: false, reason: `model (file=${metadata.model} expected=${fingerprint.model})` };
+  }
+  if (Number.isFinite(metadata.dimension)
+      && Number.isFinite(fingerprint.dimension)
+      && metadata.dimension !== fingerprint.dimension) {
+    return { compatible: false, reason: `dimension (file=${metadata.dimension} expected=${fingerprint.dimension})` };
+  }
+  return { compatible: true };
+}
+
 export class Vocabulary {
   constructor(vocabPath) {
     this.vocabPath = vocabPath;
     this.terms = new Map();
-    this.metadata = { created: null, lastUpdated: null, version: 2, provider: null };
+    this.metadata = {
+      created: null,
+      lastUpdated: null,
+      schemaVersion: VOCAB_SCHEMA_VERSION,
+      provider: null,
+      model: null,
+      dimension: null,
+    };
     this.loaded = false;
   }
 
+  /**
+   * Whether `getEmbedding` should consult this vocabulary at all.
+   * Reads from `EMBEDDING_CONFIG.cache.useVocabulary` at call time so
+   * tests / benchmarks that toggle the env var see the change without
+   * having to re-import the module.
+   */
+  static isEnabled() {
+    return EMBEDDING_CONFIG.cache?.useVocabulary !== false;
+  }
+
   async load() {
     if (this.loaded) return;
     try {
       if (existsSync(this.vocabPath)) {
         const data = JSON.parse(await fs.readFile(this.vocabPath, 'utf-8'));
-        if (data.metadata?.provider && data.metadata.provider !== EMBEDDING_CONFIG.provider) {
-          console.log(`Vocabulary: Provider changed (${data.metadata.provider} → ${EMBEDDING_CONFIG.provider}), clearing cache`);
+        const compat = isVocabFingerprintCompatible(data.metadata);
+        if (!compat.compatible) {
+          console.log(`Vocabulary: Ignoring incompatible cache (${compat.reason})`);
           this.terms.clear();
         } else {
-          this.metadata = data.metadata || this.metadata;
+          this.metadata = { ...this.metadata, ...(data.metadata || {}) };
           for (const [term, embedding] of Object.entries(data.terms || {})) {
             this.terms.set(term, embedding);
           }
@@ -143,20 +282,43 @@ export class Vocabulary {
   }
 
   async save() {
-    this.metadata.lastUpdated = new Date().toISOString();
-    this.metadata.provider = EMBEDDING_CONFIG.provider;
-    if (!this.metadata.created) this.metadata.created = this.metadata.lastUpdated;
-    const data = { metadata: this.metadata, terms: Object.fromEntries(this.terms) };
-    await fs.mkdir(path.dirname(this.vocabPath), { recursive: true });
-    await fs.writeFile(this.vocabPath, JSON.stringify(data, null, 2));
+    return serialiseAtomicWrite(this, async () => {
+      // Snapshot mutable state INSIDE the queued task so the file written
+      // here matches the latest set/has state at write time, not whatever
+      // it was when this save() was first scheduled.
+      this.metadata.lastUpdated = new Date().toISOString();
+      this.metadata.schemaVersion = VOCAB_SCHEMA_VERSION;
+      this.metadata.provider = EMBEDDING_CONFIG.provider;
+      this.metadata.model = EMBEDDING_CONFIG.model;
+      this.metadata.dimension = EMBEDDING_CONFIG.dimension;
+      if (!this.metadata.created) this.metadata.created = this.metadata.lastUpdated;
+      const data = { metadata: this.metadata, terms: Object.fromEntries(this.terms) };
+      await writeJsonAtomic(this.vocabPath, JSON.stringify(data, null, 2));
+    });
   }
 
-  get(term) { return this.terms.get(this.normalize(term)) || null; }
+  get(term) {
+    if (!Vocabulary.isEnabled()) return null;
+    return this.terms.get(this.normalize(term)) || null;
+  }
   set(term, embedding) { this.terms.set(this.normalize(term), embedding); }
   has(term) { return this.terms.has(this.normalize(term)); }
   normalize(term) { return term.toLowerCase().trim(); }
   size() { return this.terms.size; }
 
+  /**
+   * Whether the vocabulary is at or above the configured max-terms cap.
+   * Auto-expansion in `getEmbedding` is gated on this so a long-running
+   * benchmark cannot inflate the file unbounded. Explicit
+   * `addToVocabulary` / `expandVocabulary` calls (admin / pre-warm
+   * paths) bypass the cap.
+   */
+  isFull() {
+    const cap = EMBEDDING_CONFIG.cache?.maxTerms;
+    if (!Number.isFinite(cap) || cap <= 0) return false;
+    return this.terms.size >= cap;
+  }
+
   async addDefaultTerms(embedFn) {
     const defaultTerms = [
       'AuthService', 'EmployeeService', 'LoginService', 'UserService',

package/core/embedding/embedding-service.js

@@ -63,6 +63,9 @@ export { TimeWindowRateLimiter };
 // UNIFIED EMBEDDING SERVICE (hub functions)
 // =============================================================================
 
+// Process-scoped flag so the "vocab is full" message logs once, not per-query.
+let _vocabFullLogged = false;
+
 /** Generate embedding using the active provider with circuit breaker */
 async function generateEmbedding(text, provider = EMBEDDING_CONFIG.provider, isQuery = false) {
   const localText = isQuery ? applyLocalQueryPrefix(text) : text;
@@ -206,7 +209,7 @@ export async function getEmbedding(text, options = {}) {
       return { embedding: cached, cached: true, source: 'lru', latency_us: Math.round((performance.now() - start) * 1000) };
     }
 
-    if (isQuery) {
+    if (isQuery && EMBEDDING_CONFIG.cache?.useVocabulary !== false) {
       await vocabulary.load();
       const vocabHit = vocabulary.get(text);
       if (vocabHit) {
@@ -259,9 +262,20 @@ export async function getEmbedding(text, options = {}) {
       queryStats.save().catch(() => {});
       const threshold = EMBEDDING_CONFIG.cache?.expansionThreshold || 3;
       if (usageCount >= threshold && !vocabulary.has(text)) {
-        vocabulary.set(text, embedding);
-        vocabulary.save().catch(() => {});
-        console.log(`Vocabulary: Auto-added "${text}" (used ${usageCount}x)`);
+        if (vocabulary.isFull()) {
+          // Cap reached: skip auto-promotion and log once per batch (the
+          // queryStats counter still increments so we don't lose the
+          // signal — explicit `addToVocabulary` can still write through).
+          if (!_vocabFullLogged) {
+            const cap = EMBEDDING_CONFIG.cache?.maxTerms;
+            console.log(`Vocabulary: Auto-expand cap reached (${cap} terms); skipping further auto-promotion. Override via SWEET_SEARCH_VOCAB_MAX_TERMS.`);
+            _vocabFullLogged = true;
+          }
+        } else {
+          vocabulary.set(text, embedding);
+          vocabulary.save().catch(() => {});
+          console.log(`Vocabulary: Auto-added "${text}" (used ${usageCount}x)`);
+        }
       }
     }
   }

package/core/graph/graph-expansion.js

@@ -276,16 +276,26 @@ export function expandResults(db, results, options = {}) {
  */
 function collectSeedIds(db, results) {
   const seedIds = new Set();
+  const needsLineMatch = [];
+
+  // Distinguish chunk ids from entity ids by shape: chunk ids look like
+  // `path/to/file.ext:start-end:n` (always contain `:`), entity ids are
+  // hex hashes / opaque tokens that never contain `:`. Treating chunk ids
+  // as entity ids feeds them into the relationships SQL and yields zero
+  // neighbours — which silently disabled graph expansion on HNSW results.
+  const looksLikeEntityId = (s) => typeof s === 'string' && !s.includes(':');
 
   for (const r of results) {
     if (r.entity_id) seedIds.add(r.entity_id);
     else if (r.metadata?.entity_id) seedIds.add(r.metadata.entity_id);
-    else if (r.id) seedIds.add(r.id);
+    else if (r.is_expanded && r.id) seedIds.add(r.id);
+    else if (r.id && looksLikeEntityId(r.id)) seedIds.add(r.id);
+    else needsLineMatch.push(r);
   }
 
-  if (seedIds.size > 0) return seedIds;
+  if (needsLineMatch.length === 0) return seedIds;
 
-  // Fallback: match results to entities by file_path + line range
+  // Line-range fallback for chunk-id keyed results.
   let entityLookup;
   try {
     entityLookup = db.prepare(`
@@ -296,18 +306,48 @@ function collectSeedIds(db, results) {
     return seedIds;
   }
 
-  for (const r of results) {
-    const filePath = r.file_path || r.file || r.metadata?.file || r.metadata?.path;
-    const lineStart = r.start_line || r.startLine || r.metadata?.line_start || r.metadata?.startLine;
-    if (!filePath) continue;
+  // Chunk-id pattern: `path/to/file.ext:<start>-<end>:<n>`. When metadata
+  // doesn't carry file_path / line numbers (older indexes can be sparse),
+  // parse them out of the id itself.
+  const parseChunkId = (id) => {
+    if (typeof id !== 'string' || !id.includes(':')) return null;
+    const m = id.match(/^(.+):(\d+)-(\d+):(\d+)$/);
+    if (!m) return null;
+    return { file: m[1], startLine: parseInt(m[2], 10), endLine: parseInt(m[3], 10) };
+  };
 
-    for (const e of entityLookup) {
-      if (e.file_path === filePath && e.start_line != null && lineStart != null &&
-          e.start_line <= lineStart && e.end_line >= lineStart) {
-        seedIds.add(e.id);
-        break;
+  for (const r of needsLineMatch) {
+    let filePath = r.file_path || r.file || r.metadata?.file || r.metadata?.path;
+    let lineStart = r.start_line || r.startLine
+      || r.metadata?.line_start || r.metadata?.startLine || r.metadata?.start_line;
+    let lineEnd = r.end_line || r.endLine
+      || r.metadata?.line_end || r.metadata?.endLine || r.metadata?.end_line;
+    if (!filePath || lineStart == null || lineEnd == null) {
+      const parsed = parseChunkId(r.id);
+      if (parsed) {
+        filePath = filePath || parsed.file;
+        lineStart = lineStart ?? parsed.startLine;
+        lineEnd = lineEnd ?? parsed.endLine;
       }
     }
+    if (!filePath || lineStart == null) continue;
+    // If we still don't have an end line, treat the chunk as a single line.
+    if (lineEnd == null) lineEnd = lineStart;
+
+    // Find the SMALLEST entity that overlaps the chunk's [start, end] —
+    // smaller entities (functions/methods) are more meaningful seeds than
+    // file-level container entities. Cap to one seed per result to avoid
+    // unbounded seed-set blow-up that can break the relationships SQL.
+    let bestId = null;
+    let bestSize = Infinity;
+    for (const e of entityLookup) {
+      if (e.file_path !== filePath) continue;
+      if (e.start_line == null || e.end_line == null) continue;
+      if (e.start_line > lineEnd || e.end_line < lineStart) continue;
+      const size = (e.end_line - e.start_line) + 1;
+      if (size < bestSize) { bestSize = size; bestId = e.id; }
+    }
+    if (bestId) seedIds.add(bestId);
   }
 
   return seedIds;

package/core/graph/graph-extractor.js

@@ -253,9 +253,34 @@ export const GENERIC_RELATIONSHIP_MAPPING = Object.freeze({
   mixin: 'extends',
   with: 'extends',
   category: 'extends',
+  // TS: interface extends interface(s) is a true `extends` edge in
+  // the graph (separate pattern key because the registry regex needs
+  // to match on the `interface` keyword, not `class`).
+  interfaceExtends: 'extends',
   implements: 'implements',
   protocol: 'implements',
   implFor: 'implements',
+  // TS: type-only imports/re-exports are still module-level
+  // dependencies, so they map to the same `imports` edge.
+  typeImport: 'imports',
+  typeReexport: 'imports',
+  // TS: `<T extends Foo>` is a type reference, not an inheritance
+  // edge — emit it as a `uses` relationship (consistent with how
+  // decorators and method-of references are handled).
+  genericConstraint: 'uses',
+  // FOLLOW-UP (documented, NOT implemented): per-line type references
+  // in function/method/property signatures (e.g. `function foo(x: User):
+  // Result` → `uses` edges to User and Result; `field: Token` → `uses`
+  // edge to Token). Intentionally not added at the regex layer — the
+  // false-positive surface (matching identifiers in comments, strings,
+  // and unrelated positions) is too high. Two prerequisites before
+  // shipping:
+  //   1. AST-level type-reference extractor (walk `type_annotation` /
+  //      `parameter` / `return_type` nodes via tree-sitter, not regex)
+  //   2. Graph-density benchmark showing retrieval benefit without
+  //      precision loss (the new `uses` edges should improve graph
+  //      expansion recall without adding noise that hurts MRR).
+  // See May-2026 design discussion in chat history for details.
   decorator: 'uses',
   embed: 'uses',
   extend: 'uses',
@@ -274,6 +299,9 @@ const escapeRegexLiteral = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&
 // Module-scope constant to avoid per-call Set allocation.
 const MULTI_TARGET_TYPES = new Set([
   'plainImport', 'implements', 'inherit', 'protocol', 'with',
+  // TS: `interface Foo extends Bar, Baz<T>` — comma-separated
+  // parents, generics handled by expandRelationshipTargets.
+  'interfaceExtends',
 ]);
 
 export const TREE_SITTER_ENTITY_PRIORITY = Object.freeze({
@@ -1328,7 +1356,8 @@ export class GraphExtractor {
       return { targets: [source], filtered: false };
     }
 
-    if (isJsTs && (relType === 'require' || relType === 'reexport' || relType === 'dynamicImport')) {
+    if (isJsTs && (relType === 'require' || relType === 'reexport' || relType === 'dynamicImport'
+      || relType === 'typeImport' || relType === 'typeReexport')) {
       const source = match[1]?.trim();
       if (!source) return { targets: [], filtered: false };
       if (source.startsWith('.')) return { targets: [], filtered: true };