sweet-search 2.5.1 → 2.5.3

package/core/cli.js

@@ -20,6 +20,9 @@ if (args[0] === 'init') {
 } else if (args[0] === 'prewarm-vocab') {
   const { handlePrewarmVocabCli } = await import('./vocabulary/index.js');
   await handlePrewarmVocabCli(args.slice(1));
+} else if (args[0] === 'reconcile' || args[0] === 'rebuild') {
+  const { handleIncrementalCli } = await import('./incremental-indexing/application/operator-cli.mjs');
+  await handleIncrementalCli(args[0], args.slice(1));
 } else if (args[0] === 'read') {
   // Filesystem-grounded reader; runs in JS (no native equivalent yet).
   const { handleReadCli } = await import('./search/search-read.js');
@@ -28,6 +31,30 @@ if (args[0] === 'init') {
   // Hybrid span-selection reader; runs in JS (depends on LI index + ranking).
   const { handleReadSemanticCli } = await import('./search/search-read-semantic.js');
   await handleReadSemanticCli(args.slice(1));
+} else if (args[0] === 'trace') {
+  // Unified structural code context: callers, callees, and impact.
+  const { handleTraceCli } = await import('./search/search-trace.js');
+  await handleTraceCli(args.slice(1));
+} else if (args[0] === 'index') {
+  // Indexing pipeline. Forwarded to index-codebase-v21.js::main(), which
+  // reads its own flags via process.argv. Setting argv here is required
+  // because the indexer's parseArgs reads process.argv.slice(2) by default.
+  // Without this subcommand, npm-installed users had no way to invoke
+  // indexing — `node ./node_modules/sweet-search/core/indexing/index-codebase-v21.js`
+  // was a silent no-op (direct-run guard mismatched under symlinked installs)
+  // and the bin had no `index` entry at all. Forwards every argument after
+  // `index` so existing flag combos (--full / --graph-only / --vectors-only /
+  // --files-from-stdin / --late-interaction-model=… / etc.) all work.
+  const indexerArgs = args.slice(1);
+  const hasAddHint = indexerArgs.includes('--add') || indexerArgs.some((arg) => arg.startsWith('--add='));
+  if (hasAddHint) {
+    const { handleIndexAddCli } = await import('./incremental-indexing/application/operator-cli.mjs');
+    await handleIndexAddCli(indexerArgs);
+  } else {
+    process.argv = [process.argv[0], 'index-codebase-v21.js', ...indexerArgs];
+    const { main: runIndexer } = await import('./indexing/index-codebase-v21.js');
+    await runIndexer();
+  }
 } else if (args[0] === '--serve' || args[0] === '--stop') {
   // Warm search server lifecycle is implemented in JS.
   const { runCli } = await import('./search/index.js');
@@ -37,8 +64,16 @@ if (args[0] === 'init') {
 
 Usage:
   sweet-search <query>                  Search the indexed codebase
+  sweet-search trace <symbol>           Structural context: callers, callees, impact
   sweet-search read <file...>           Filesystem-grounded read (1-20 files)
   sweet-search read-semantic <f> <q>    Return only file spans relevant to a query
+  sweet-search index [options]          Build / update the codebase index
+  sweet-search index --add <path>       Hint a file as dirty
+  sweet-search reconcile status         Show incremental epoch and dirty status
+  sweet-search reconcile inspect <path> Explain why a file is dirty or clean
+  sweet-search reconcile pause|resume   Pause or resume automatic reconcile work
+  sweet-search rebuild status           Show incremental maintenance queue
+  sweet-search rebuild force <tier>     Queue maintenance for a tier
   sweet-search init [options]           Set up runtime assets and models
   sweet-search uninstall [opts]         Remove local state created by init
   sweet-search prewarm-vocab [file]     Pre-warm vocabulary cache with terms
@@ -50,6 +85,16 @@ Options:
   --json            Output results as JSON
   --cold            Force cold start (skip warm server)
 
+Indexing flags (sweet-search index ...):
+  --full            Full reindex from scratch
+  --graph-only      Build code graph only
+  --vectors-only    Build vectors + HNSW only (skips code graph)
+  --files-from-stdin  Read newline-delimited paths from stdin
+  --add <path>        Queue a dirty-file hint without running the indexer
+  --late-interaction-model=ID  Override the LI variant for this run
+  --no-late-interaction        Skip LI index build
+  --quiet | --verbose          Logging verbosity
+
 Run 'sweet-search init --help' or 'sweet-search uninstall --help' for subcommand options.`);
 } else {
   const { resolveNativeBinary } = await import('./infrastructure/index.js');

package/core/embedding/embedding-cache.js

@@ -45,6 +45,7 @@ export class LRUCache {
   }
 
   has(key) { return this.cache.has(key); }
+  delete(key) { this.hitCount.delete(key); return this.cache.delete(key); }
   getHitCount(key) { return this.hitCount.get(key) || 0; }
   size() { return this.cache.size; }
   clear() { this.cache.clear(); this.hitCount.clear(); }
@@ -191,6 +192,62 @@ export class QueryStats {
 //        model is not silently served when a different model is active.
 const VOCAB_SCHEMA_VERSION = 3;
 
+/**
+ * Coerce an input value into a Float32Array suitable for downstream embedding
+ * math (truncateForHNSW, late-interaction MaxSim, cosine similarity).
+ *
+ * Why this exists: persisted vocabularies are JSON-serialised. JSON.stringify
+ * on a Float32Array produces an indexed object `{"0": v0, "1": v1, ...}`,
+ * not an array. After `JSON.parse`, the value has `.length === undefined`,
+ * `.slice === undefined`, and crashes any downstream consumer that calls
+ * vector methods. This helper repairs the value at the cache boundary so
+ * the rest of the embedding pipeline can rely on a uniform vector contract.
+ *
+ * Accepted inputs:
+ *   - Float32Array → returned as-is
+ *   - Array<number> → wrapped in Float32Array
+ *   - Float64Array / Int*Array etc. → copied into Float32Array
+ *   - Plain object with stringly-keyed numeric indices ("0","1",...,"N-1")
+ *     → reconstructed as Float32Array of length N
+ *
+ * Returns null when the input cannot be sensibly interpreted as a vector
+ * (callers should drop the cache entry and re-derive).
+ *
+ * @param {*} value
+ * @returns {Float32Array|null}
+ */
+export function coerceToFloat32Vector(value) {
+  if (value == null) return null;
+  if (value instanceof Float32Array) return value;
+  if (Array.isArray(value)) return Float32Array.from(value);
+  // Other typed arrays: copy values into a Float32Array.
+  if (ArrayBuffer.isView(value) && typeof value.length === 'number') {
+    return Float32Array.from(value);
+  }
+  // Plain object form from JSON-deserialised Float32Array.
+  if (typeof value === 'object') {
+    const keys = Object.keys(value);
+    if (keys.length === 0) return null;
+    // All keys must be string-encoded non-negative integers and contiguous
+    // from 0 to length-1. (We do not try to "fill gaps" — that would silently
+    // mask a real bug.)
+    const indices = new Array(keys.length);
+    for (let i = 0; i < keys.length; i++) {
+      const k = keys[i];
+      // Reject anything that isn't an integer-shaped key.
+      if (!/^\d+$/.test(k)) return null;
+      const n = +k;
+      if (!Number.isInteger(n) || n < 0 || n >= keys.length) return null;
+      indices[n] = value[k];
+    }
+    for (let i = 0; i < indices.length; i++) {
+      if (typeof indices[i] !== 'number' || !Number.isFinite(indices[i])) return null;
+    }
+    return Float32Array.from(indices);
+  }
+  return null;
+}
+
 /** Build the embedding-fingerprint we expect a vocabulary file to match. */
 function currentVocabFingerprint() {
   return {
@@ -269,10 +326,27 @@ export class Vocabulary {
           this.terms.clear();
         } else {
           this.metadata = { ...this.metadata, ...(data.metadata || {}) };
-          for (const [term, embedding] of Object.entries(data.terms || {})) {
-            this.terms.set(term, embedding);
+          let normalized = 0;
+          let dropped = 0;
+          for (const [term, raw] of Object.entries(data.terms || {})) {
+            // Coerce to Float32Array. Persisted vocabs JSON-serialise typed
+            // arrays as indexed objects (`{"0": v0, ...}`), which otherwise
+            // crash downstream `embedding.slice(...)` calls (see
+            // `truncateForHNSW`). Reject any entry we cannot interpret as a
+            // vector — better to re-embed than to surface a corrupt vector.
+            const vec = coerceToFloat32Vector(raw);
+            if (vec) {
+              this.terms.set(term, vec);
+              normalized++;
+            } else {
+              dropped++;
+            }
+          }
+          if (dropped > 0) {
+            console.log(`Vocabulary: Loaded ${normalized} pre-computed embeddings (dropped ${dropped} unrecognised)`);
+          } else {
+            console.log(`Vocabulary: Loaded ${normalized} pre-computed embeddings`);
           }
-          console.log(`Vocabulary: Loaded ${this.terms.size} pre-computed embeddings`);
         }
       }
     } catch (err) {
@@ -292,7 +366,18 @@ export class Vocabulary {
       this.metadata.model = EMBEDDING_CONFIG.model;
       this.metadata.dimension = EMBEDDING_CONFIG.dimension;
       if (!this.metadata.created) this.metadata.created = this.metadata.lastUpdated;
-      const data = { metadata: this.metadata, terms: Object.fromEntries(this.terms) };
+      // Normalise to plain arrays so JSON.stringify produces a compact,
+      // round-trippable form. Float32Array would otherwise serialise as
+      // an indexed object ({"0": v0, "1": v1, ...}) which load() can read
+      // (via coerceToFloat32Vector) but which is wasteful and was the
+      // shape that originally caused the `embedding.slice` bug.
+      const termsOut = {};
+      for (const [term, vec] of this.terms.entries()) {
+        termsOut[term] = vec instanceof Float32Array || ArrayBuffer.isView(vec)
+          ? Array.from(vec)
+          : vec;
+      }
+      const data = { metadata: this.metadata, terms: termsOut };
       await writeJsonAtomic(this.vocabPath, JSON.stringify(data, null, 2));
     });
   }
@@ -303,6 +388,7 @@ export class Vocabulary {
   }
   set(term, embedding) { this.terms.set(this.normalize(term), embedding); }
   has(term) { return this.terms.has(this.normalize(term)); }
+  delete(term) { return this.terms.delete(this.normalize(term)); }
   normalize(term) { return term.toLowerCase().trim(); }
   size() { return this.terms.size; }
 

package/core/embedding/embedding-service.js

@@ -45,6 +45,7 @@ import {
   queryDeduplicator,
   queryStats,
   cacheStats,
+  coerceToFloat32Vector,
   getCacheStats as _getCacheStats,
   getSemanticCacheStats,
   clearCache,
@@ -205,17 +206,38 @@ export async function getEmbedding(text, options = {}) {
   if (useCache && EMBEDDING_CONFIG.cache?.enabled) {
     const cached = queryCache.get(cacheKey);
     if (cached) {
-      cacheStats.hits++;
-      return { embedding: cached, cached: true, source: 'lru', latency_us: Math.round((performance.now() - start) * 1000) };
+      // Defensive guard: a cache value MUST be a vector with .length and
+      // .slice. Persisted vocabularies that round-tripped through JSON
+      // produce indexed-object shapes which crash downstream consumers.
+      // Coerce; if unrecoverable, drop the entry and fall through.
+      const cachedVec = coerceToFloat32Vector(cached);
+      if (cachedVec) {
+        if (cachedVec !== cached) queryCache.set(cacheKey, cachedVec);
+        cacheStats.hits++;
+        return { embedding: cachedVec, cached: true, source: 'lru', latency_us: Math.round((performance.now() - start) * 1000) };
+      }
+      queryCache.delete?.(cacheKey);
+      console.warn(`[embedding] LRU cache held non-vector for "${cacheKey.slice(0, 60)}"; regenerating`);
     }
 
     if (isQuery && EMBEDDING_CONFIG.cache?.useVocabulary !== false) {
       await vocabulary.load();
       const vocabHit = vocabulary.get(text);
       if (vocabHit) {
-        cacheStats.vocabularyHits++;
-        queryCache.set(cacheKey, vocabHit);
-        return { embedding: vocabHit, cached: true, source: 'vocabulary', latency_us: Math.round((performance.now() - start) * 1000) };
+        const vocabVec = coerceToFloat32Vector(vocabHit);
+        if (vocabVec) {
+          // Backfill the in-memory vocab map with the typed-array form so
+          // subsequent hits skip re-coercion.
+          if (vocabVec !== vocabHit) vocabulary.set?.(text, vocabVec);
+          cacheStats.vocabularyHits++;
+          queryCache.set(cacheKey, vocabVec);
+          return { embedding: vocabVec, cached: true, source: 'vocabulary', latency_us: Math.round((performance.now() - start) * 1000) };
+        }
+        // Unrecoverable vocab entry — drop it and continue. (load() now
+        // normalises on read, so this branch should be unreachable in
+        // practice; it is the belt-and-braces for older code paths.)
+        vocabulary.delete?.(text);
+        console.warn(`[embedding] vocabulary held non-vector for "${text.slice(0, 60)}"; dropping and regenerating`);
       }
     }
   }