spec-gen-cli 1.2.1 → 1.2.3

package/dist/core/analyzer/type-inference-engine.js

@@ -0,0 +1,130 @@
+/**
+ * TypeInferenceEngine — lightweight regex-based type inference for 7 languages.
+ *
+ * Given the source text of a single function body, returns a map of
+ * { variableName → className } inferred from declarations, annotations,
+ * and constructor calls.
+ *
+ * Intentionally NOT a full type system — false positives are acceptable;
+ * false negatives (missing resolutions) are the only cost.  Only class names
+ * starting with an uppercase letter are tracked (conventional in all supported
+ * languages), which eliminates most false positives from primitive types.
+ */
+export function inferTypesFromSource(source, language) {
+    switch (language) {
+        case 'Python': return inferPython(source);
+        case 'C++': return inferCpp(source);
+        case 'TypeScript':
+        case 'JavaScript': return inferTypeScript(source);
+        case 'Go': return inferGo(source);
+        case 'Rust': return inferRust(source);
+        case 'Java': return inferJava(source);
+        case 'Ruby': return inferRuby(source);
+        default: return new Map();
+    }
+}
+// ---------------------------------------------------------------------------
+// Per-language inference rules
+// ---------------------------------------------------------------------------
+function inferPython(source) {
+    const result = new Map();
+    // var = ClassName(...)
+    for (const m of source.matchAll(/^\s*(\w+)\s*=\s*([A-Z]\w*)\s*\(/gm))
+        result.set(m[1], m[2]);
+    // var: ClassName = ...
+    for (const m of source.matchAll(/^\s*(\w+)\s*:\s*([A-Z]\w*)\s*=/gm))
+        result.set(m[1], m[2]);
+    // param: ClassName in signatures
+    for (const m of source.matchAll(/\b(\w+)\s*:\s*([A-Z]\w*)\b/g))
+        result.set(m[1], m[2]);
+    return result;
+}
+function inferCpp(source) {
+    const result = new Map();
+    // ClassName var;  or  ClassName var(...)
+    for (const m of source.matchAll(/\b([A-Z]\w*)\s+(\w+)\s*[;({]/g))
+        result.set(m[2], m[1]);
+    // ClassName* var = new ClassName(...)
+    for (const m of source.matchAll(/\b([A-Z]\w*)\s*\*\s*(\w+)\s*=\s*new\s+\1/g))
+        result.set(m[2], m[1]);
+    // auto var = make_shared<ClassName>(...)  /  make_unique<ClassName>(...)
+    for (const m of source.matchAll(/auto\s+(\w+)\s*=\s*(?:make_shared|make_unique)<([A-Z]\w*)>/g))
+        result.set(m[1], m[2]);
+    // shared_ptr<ClassName> var  /  unique_ptr  /  weak_ptr
+    for (const m of source.matchAll(/(?:shared_ptr|unique_ptr|weak_ptr)<([A-Z]\w*)>\s+(\w+)/g))
+        result.set(m[2], m[1]);
+    return result;
+}
+function inferTypeScript(source) {
+    const result = new Map();
+    // const var = new ClassName(...)
+    for (const m of source.matchAll(/\bconst\s+(\w+)\s*=\s*new\s+([A-Z]\w*)\s*\(/g))
+        result.set(m[1], m[2]);
+    // let/var/const var: ClassName =
+    for (const m of source.matchAll(/\b(?:let|var|const)\s+(\w+)\s*:\s*([A-Z]\w*)\s*=/g))
+        result.set(m[1], m[2]);
+    // param: ClassName in signatures
+    for (const m of source.matchAll(/\b(\w+)\s*:\s*([A-Z]\w*)\b/g))
+        result.set(m[1], m[2]);
+    return result;
+}
+function inferGo(source) {
+    const result = new Map();
+    // var svc *MyService
+    for (const m of source.matchAll(/\bvar\s+(\w+)\s+\*?([A-Z]\w*)\b/g))
+        result.set(m[1], m[2]);
+    // svc := MyService{...}  or  NewMyService(...)
+    for (const m of source.matchAll(/\b(\w+)\s*:=\s*(?:New)?([A-Z]\w*)[{(]/g))
+        result.set(m[1], m[2]);
+    // svc := &MyService{...}
+    for (const m of source.matchAll(/\b(\w+)\s*:=\s*&([A-Z]\w*)\s*{/g))
+        result.set(m[1], m[2]);
+    // func f(svc *MyService) — parameter annotations
+    for (const m of source.matchAll(/\b(\w+)\s+\*?([A-Z]\w*)\b/g))
+        result.set(m[1], m[2]);
+    return result;
+}
+function inferRust(source) {
+    const result = new Map();
+    // let svc: MyService = ...
+    for (const m of source.matchAll(/\blet\s+(?:mut\s+)?(\w+)\s*:\s*([A-Z]\w*)\b/g))
+        result.set(m[1], m[2]);
+    // let svc = MyService::new(...)  /  MyService::default()
+    for (const m of source.matchAll(/\blet\s+(?:mut\s+)?(\w+)\s*=\s*([A-Z]\w*)::(?:new|default)\s*\(/g))
+        result.set(m[1], m[2]);
+    // let svc = Box::new(MyService::new(...))
+    for (const m of source.matchAll(/\blet\s+(?:mut\s+)?(\w+)\s*=\s*Box::new\(([A-Z]\w*)::new/g))
+        result.set(m[1], m[2]);
+    return result;
+}
+function inferJava(source) {
+    const result = new Map();
+    // ClassName var = ...  or  ClassName var;
+    for (const m of source.matchAll(/\b([A-Z]\w*)\s+(\w+)\s*(?:=|;)/g))
+        result.set(m[2], m[1]);
+    // Interface var = new ConcreteClass(...)  — prefer the concrete type
+    for (const m of source.matchAll(/\b([A-Z]\w*)\s+(\w+)\s*=\s*new\s+([A-Z]\w*)\s*\(/g))
+        result.set(m[2], m[3]);
+    return result;
+}
+function inferRuby(source) {
+    const result = new Map();
+    // svc = MyClass.new(...)
+    for (const m of source.matchAll(/\b(\w+)\s*=\s*([A-Z]\w*)\.new\b/g))
+        result.set(m[1], m[2]);
+    return result;
+}
+// ---------------------------------------------------------------------------
+// Common resolution helper
+// ---------------------------------------------------------------------------
+/**
+ * Given a receiver variable name and a method name, look up the inferred type
+ * of the receiver and resolve the method to a FunctionNode via the trie.
+ */
+export function resolveViaTypeInference(calleeObject, calleeName, inferredTypes, trie) {
+    const className = inferredTypes.get(calleeObject);
+    if (!className)
+        return undefined;
+    return trie.findByQualifiedName(className, calleeName)[0];
+}
+//# sourceMappingURL=type-inference-engine.js.map

package/dist/core/analyzer/type-inference-engine.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"type-inference-engine.js","sourceRoot":"","sources":["../../../src/core/analyzer/type-inference-engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAQH,MAAM,UAAU,oBAAoB,CAAC,MAAc,EAAE,QAAgB;IACnE,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ,CAAC,CAAK,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC;QAC9C,KAAK,KAAK,CAAC,CAAQ,OAAO,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC3C,KAAK,YAAY,CAAC;QAClB,KAAK,YAAY,CAAC,CAAC,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;QAClD,KAAK,IAAI,CAAC,CAAS,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1C,KAAK,MAAM,CAAC,CAAO,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5C,KAAK,MAAM,CAAC,CAAO,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5C,KAAK,MAAM,CAAC,CAAO,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC;QAC5C,OAAO,CAAC,CAAW,OAAO,IAAI,GAAG,EAAE,CAAC;IACtC,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,+BAA+B;AAC/B,8EAA8E;AAE9E,SAAS,WAAW,CAAC,MAAc;IACjC,MAAM,MAAM,GAAkB,IAAI,GAAG,EAAE,CAAC;IACxC,uBAAuB;IACvB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,mCAAmC,CAAC;QAClE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,uBAAuB;IACvB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,kCAAkC,CAAC;QACjE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,iCAAiC;IACjC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,6BAA6B,CAAC;QAC5D,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,MAAc;IAC9B,MAAM,MAAM,GAAkB,IAAI,GAAG,EAAE,CAAC;IACxC,yCAAyC;IACzC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,+BAA+B,CAAC;QAC9D,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,sCAAsC;IACtC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,2CAA2C,CAAC;QAC1E,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,yEAAyE;IACzE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,6DAA6D,CAAC;QAC5F,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,wDAAwD;IACxD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,yDAAyD,CAAC;QACxF,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,eAAe,CAAC,MAAc;IACrC,MAAM,MAAM,GAAkB,IAAI,GAAG,EAAE,CAAC;IACxC,iCAAiC;IACjC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,8CAA8C,CAAC;QAC7E,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,iCAAiC;IACjC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,mDAAmD,CAAC;QAClF,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,iCAAiC;IACjC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,6BAA6B,CAAC;QAC5D,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,OAAO,CAAC,MAAc;IAC7B,MAAM,MAAM,GAAkB,IAAI,GAAG,EAAE,CAAC;IACxC,qBAAqB;IACrB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,kCAAkC,CAAC;QACjE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,+CAA+C;IAC/C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,wCAAwC,CAAC;QACvE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,yBAAyB;IACzB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,iCAAiC,CAAC;QAChE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,iDAAiD;IACjD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,4BAA4B,CAAC;QAC3D,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,MAAc;IAC/B,MAAM,MAAM,GAAkB,IAAI,GAAG,EAAE,CAAC;IACxC,2BAA2B;IAC3B,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,8CAA8C,CAAC;QAC7E,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,yDAAyD;IACzD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,kEAAkE,CAAC;QACjG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,0CAA0C;IAC1C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,2DAA2D,CAAC;QAC1F,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,MAAc;IAC/B,MAAM,MAAM,GAAkB,IAAI,GAAG,EAAE,CAAC;IACxC,0CAA0C;IAC1C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,iCAAiC,CAAC;QAChE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,qEAAqE;IACrE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,mDAAmD,CAAC;QAClF,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,MAAc;IAC/B,MAAM,MAAM,GAAkB,IAAI,GAAG,EAAE,CAAC;IACxC,yBAAyB;IACzB,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,kCAAkC,CAAC;QACjE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,8EAA8E;AAC9E,2BAA2B;AAC3B,8EAA8E;AAE9E;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CACrC,YAAoB,EACpB,UAAkB,EAClB,aAA4B,EAC5B,IAA0B;IAE1B,MAAM,SAAS,GAAG,aAAa,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAClD,IAAI,CAAC,SAAS;QAAE,OAAO,SAAS,CAAC;IACjC,OAAO,IAAI,CAAC,mBAAmB,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5D,CAAC"}

package/dist/core/analyzer/vector-index.d.ts

@@ -37,24 +37,53 @@ export interface FunctionRecord {
 }
 export interface SearchResult {
     record: Omit<FunctionRecord, 'vector'>;
-    /** Distance score (lower = more similar) */
+    /**
+     * Relevance score.  For hybrid search (default): RRF score, higher = more relevant.
+     * For dense-only search: cosine distance from LanceDB, lower = more similar.
+     */
     score: number;
 }
 export declare class VectorIndex {
     /**
      * Build (or rebuild) the vector index from call graph nodes + signatures.
-     * Overwrites any existing index.
+     *
+     * When `incremental` is true and an existing index is found, only functions
+     * whose text has changed since the last build are re-embedded.  Unchanged
+     * functions reuse their cached vectors.  Pass `incremental: false` (or omit
+     * when no index exists) to do a full rebuild.
+     *
+     * Returns a summary of how many functions were embedded vs reused.
      */
-    static build(outputDir: string, nodes: FunctionNode[], signatures: FileSignatureMap[], hubIds: Set<string>, entryPointIds: Set<string>, embedSvc: EmbeddingService): Promise<void>;
+    static build(outputDir: string, nodes: FunctionNode[], signatures: FileSignatureMap[], hubIds: Set<string>, entryPointIds: Set<string>, embedSvc: EmbeddingService, 
+    /** Optional map of filePath → source content for skeleton-based body indexing */
+    fileContents?: Map<string, string>, 
+    /** When true, reuse cached vectors for unchanged functions */
+    incremental?: boolean): Promise<{
+        embedded: number;
+        reused: number;
+    }>;
     /**
-     * Semantic search over the index.
-     * Returns up to `limit` results sorted by similarity (closest first).
+     * Hybrid search over the index: dense (ANN) + sparse (BM25) merged via RRF.
+     *
+     * Dense recall fetches top `limit*5` candidates from the vector index.
+     * Sparse recall scores the full corpus with BM25 (cached per session).
+     * Reciprocal Rank Fusion (RRF) combines both rankings into a single list.
+     *
+     * Set `hybrid: false` to use dense-only search (original behaviour).
+     * Returns up to `limit` results sorted by relevance (highest first).
      */
-    static search(outputDir: string, query: string, embedSvc: EmbeddingService, opts?: {
+    static search(outputDir: string, query: string, embedSvc: EmbeddingService | null | undefined, opts?: {
         limit?: number;
         language?: string;
         minFanIn?: number;
+        /** Enable hybrid dense+sparse retrieval via RRF (default: true when embedSvc available) */
+        hybrid?: boolean;
     }): Promise<SearchResult[]>;
+    /**
+     * BM25-only search: used when no embedding service is available.
+     * Scores the full corpus with BM25 and returns the top `limit` results.
+     */
+    private static _bm25Only;
     /**
      * Returns true if a vector index has been built for this output directory.
      */

package/dist/core/analyzer/vector-index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"vector-index.d.ts","sourceRoot":"","sources":["../../../src/core/analyzer/vector-index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAM/D,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;IACtB,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,uBAAuB;IACvB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;IACvC,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;CACf;AAiED,qBAAa,WAAW;IACtB;;;OAGG;WACU,KAAK,CAChB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,YAAY,EAAE,EACrB,UAAU,EAAE,gBAAgB,EAAE,EAC9B,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,EACnB,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,EAC1B,QAAQ,EAAE,gBAAgB,GACzB,OAAO,CAAC,IAAI,CAAC;IA6EhB;;;OAGG;WACU,MAAM,CACjB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,gBAAgB,EAC1B,IAAI,GAAE;QACJ,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACd,GACL,OAAO,CAAC,YAAY,EAAE,CAAC;IAmD1B;;OAEG;IACH,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;CAG1C"}
+{"version":3,"file":"vector-index.d.ts","sourceRoot":"","sources":["../../../src/core/analyzer/vector-index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAO/D,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;IACtB,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,uBAAuB;IACvB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;IACvC;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;CACf;AAuKD,qBAAa,WAAW;IACtB;;;;;;;;;OASG;WACU,KAAK,CAChB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,YAAY,EAAE,EACrB,UAAU,EAAE,gBAAgB,EAAE,EAC9B,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,EACnB,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,EAC1B,QAAQ,EAAE,gBAAgB;IAC1B,iFAAiF;IACjF,YAAY,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAClC,8DAA8D;IAC9D,WAAW,UAAQ,GAClB,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAsIhD;;;;;;;;;OASG;WACU,MAAM,CACjB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,gBAAgB,GAAG,IAAI,GAAG,SAAS,EAC7C,IAAI,GAAE;QACJ,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,2FAA2F;QAC3F,MAAM,CAAC,EAAE,OAAO,CAAC;KACb,GACL,OAAO,CAAC,YAAY,EAAE,CAAC;IAuH1B;;;OAGG;mBACkB,SAAS;IAoD9B;;OAEG;IACH,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO;CAG1C"}

package/dist/core/analyzer/vector-index.js

@@ -17,19 +17,84 @@
  */
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
+import { getSkeletonContent, isSkeletonWorthIncluding } from './code-shaper.js';
 // ============================================================================
 // CONSTANTS
 // ============================================================================
 const DB_FOLDER = 'vector-index';
 const TABLE_NAME = 'functions';
+/** Convert a raw LanceDB row to a FunctionRecord (without the vector field). */
+function rowToRecord(row) {
+    return {
+        id: row.id,
+        name: row.name,
+        filePath: row.filePath,
+        className: row.className,
+        language: row.language,
+        signature: row.signature,
+        docstring: row.docstring,
+        fanIn: row.fanIn,
+        fanOut: row.fanOut,
+        isHub: row.isHub,
+        isEntryPoint: row.isEntryPoint,
+        text: row.text,
+    };
+}
+function tokenize(text) {
+    // Split on non-alphanumeric, keep tokens longer than 1 char
+    return text.toLowerCase().split(/[^a-z0-9]+/).filter(t => t.length > 1);
+}
+function buildBm25Corpus(records) {
+    const docs = [];
+    const df = new Map();
+    let totalLen = 0;
+    for (const r of records) {
+        const tokens = tokenize(r.text);
+        const tfMap = new Map();
+        for (const t of tokens)
+            tfMap.set(t, (tfMap.get(t) ?? 0) + 1);
+        docs.push({ id: r.id, tfMap, length: tokens.length });
+        totalLen += tokens.length;
+        for (const t of tfMap.keys())
+            df.set(t, (df.get(t) ?? 0) + 1);
+    }
+    return { docs, df, avgLength: docs.length > 0 ? totalLen / docs.length : 1, N: docs.length };
+}
+const BM25_K1 = 1.2;
+const BM25_B = 0.75;
+function bm25Score(corpus, queryTokens, docIdx) {
+    const doc = corpus.docs[docIdx];
+    let score = 0;
+    for (const q of queryTokens) {
+        const df = corpus.df.get(q) ?? 0;
+        if (df === 0)
+            continue;
+        const idf = Math.log((corpus.N - df + 0.5) / (df + 0.5) + 1);
+        const tf = doc.tfMap.get(q) ?? 0;
+        const tfNorm = (tf * (BM25_K1 + 1)) /
+            (tf + BM25_K1 * (1 - BM25_B + BM25_B * (doc.length / corpus.avgLength)));
+        score += idf * tfNorm;
+    }
+    return score;
+}
+/**
+ * Reciprocal Rank Fusion: merges two ranked lists into a single relevance score.
+ * k=60 is the standard parameter (Cormack et al., 2009).
+ */
+function rrfScore(rankDense, rankSparse, k = 60) {
+    return 1 / (k + rankDense + 1) + 1 / (k + rankSparse + 1);
+}
+// Module-level BM25 corpus cache: avoids a full table scan on every search call
+// when the index hasn't changed.  Keyed by dbPath; invalidated when row count changes.
+const _bm25Cache = new Map();
 // ============================================================================
 // HELPERS
 // ============================================================================
 /**
  * Build the text to embed for a function.
- * Combines language, path, qualified name, signature, and docstring.
+ * Combines language, path, qualified name, signature, docstring, and skeleton body.
  */
-function buildText(node, signature, docstring) {
+function buildText(node, signature, docstring, fileContents) {
     const qualifiedName = node.className
         ? `${node.className}.${node.name}`
         : node.name;
@@ -38,6 +103,22 @@ function buildText(node, signature, docstring) {
         parts.push(signature);
     if (docstring)
         parts.push(docstring);
+    // Append skeleton body when file contents are available.
+    // The skeleton strips noise (logs, comments) while preserving business-logic signals
+    // (variable names, control flow, calls, return/throw). Only included when it provides
+    // meaningful reduction over the raw body (≥20% smaller).
+    if (fileContents && node.startIndex < node.endIndex) {
+        const src = fileContents.get(node.filePath);
+        if (src) {
+            const body = src.slice(node.startIndex, node.endIndex);
+            if (body.trim()) {
+                const skeleton = getSkeletonContent(body, node.language);
+                if (isSkeletonWorthIncluding(body, skeleton)) {
+                    parts.push(skeleton);
+                }
+            }
+        }
+    }
     return parts.join('\n');
 }
 /**
@@ -69,18 +150,36 @@ function findSignatureEntry(node, sigIndex) {
 export class VectorIndex {
     /**
      * Build (or rebuild) the vector index from call graph nodes + signatures.
-     * Overwrites any existing index.
+     *
+     * When `incremental` is true and an existing index is found, only functions
+     * whose text has changed since the last build are re-embedded.  Unchanged
+     * functions reuse their cached vectors.  Pass `incremental: false` (or omit
+     * when no index exists) to do a full rebuild.
+     *
+     * Returns a summary of how many functions were embedded vs reused.
      */
-    static async build(outputDir, nodes, signatures, hubIds, entryPointIds, embedSvc) {
+    static async build(outputDir, nodes, signatures, hubIds, entryPointIds, embedSvc, 
+    /** Optional map of filePath → source content for skeleton-based body indexing */
+    fileContents, 
+    /** When true, reuse cached vectors for unchanged functions */
+    incremental = false) {
         const { connect } = await import('@lancedb/lancedb');
         if (nodes.length === 0) {
             throw new Error('No functions to index');
         }
         const sigIndex = buildSignatureIndex(signatures);
-        // Build records from call graph nodes
+        // Build candidate records (without vectors)
         const nodeIds = new Set(nodes.map(n => n.id));
-        const records = nodes.map(node => {
-            const { signature, docstring } = findSignatureEntry(node, sigIndex);
+        const candidates = nodes.map(node => {
+            const cgDoc = node.docstring ?? '';
+            const cgSig = node.signature ?? '';
+            // Always check regex index as fallback — CG may miss docstrings when
+            // startIndex points inside an export_statement (past the `export` keyword),
+            // causing extractDocstringBefore to scan into the export keyword instead of
+            // reaching the JSDoc block above it.
+            const { signature: regexSig, docstring: regexDoc } = findSignatureEntry(node, sigIndex);
+            const signature = cgSig || regexSig;
+            const docstring = cgDoc || regexDoc;
             return {
                 id: node.id,
                 name: node.name,
@@ -93,7 +192,7 @@ export class VectorIndex {
                 fanOut: node.fanOut,
                 isHub: hubIds.has(node.id),
                 isEntryPoint: entryPointIds.has(node.id),
-                text: buildText(node, signature, docstring),
+                text: buildText(node, signature, docstring, fileContents),
             };
         });
         // Also index signature entries that have no call graph node (constants, type aliases, etc.)
@@ -107,7 +206,7 @@ export class VectorIndex {
                     continue;
                 const sig = entry.signature ?? '';
                 const doc = entry.docstring ?? '';
-                records.push({
+                candidates.push({
                     id: syntheticId,
                     name: entry.name,
                     filePath: fsm.path,
@@ -123,69 +222,224 @@ export class VectorIndex {
                 });
             }
         }
-        // Batch-embed all texts
-        const texts = records.map(r => r.text);
-        const vectors = await embedSvc.embed(texts);
-        if (vectors.length !== records.length) {
-            throw new Error(`Embedding count mismatch: expected ${records.length}, got ${vectors.length}`);
-        }
-        // Assemble final records with vectors
-        const fullRecords = records.map((r, i) => ({
-            ...r,
-            vector: vectors[i],
-        }));
-        // Connect to LanceDB and write table
+        // ── Incremental cache lookup ─────────────────────────────────────────────
         const dbPath = join(outputDir, DB_FOLDER);
+        let cachedVectors = new Map(); // id → vector
+        if (incremental && VectorIndex.exists(outputDir)) {
+            try {
+                const db = await connect(dbPath);
+                const table = await db.openTable(TABLE_NAME);
+                // Full table scan to load existing vectors
+                const existing = await table.query().toArray();
+                for (const row of existing) {
+                    const id = row.id;
+                    const text = row.text;
+                    // Convert Arrow typed arrays (Float32Array etc.) to plain number[]
+                    // so LanceDB can re-infer the schema when writing back
+                    const vector = Array.from(row.vector);
+                    // Cache the vector keyed by "id::text" so a text change invalidates it
+                    cachedVectors.set(`${id}::${text}`, vector);
+                }
+            }
+            catch {
+                // Existing index unreadable — fall back to full build
+                cachedVectors = new Map();
+            }
+        }
+        // ── Split into cached vs needs-embedding ────────────────────────────────
+        const toEmbed = [];
+        const toEmbedIdx = []; // index into `candidates`
+        const cachedIdx = [];
+        for (let i = 0; i < candidates.length; i++) {
+            const r = candidates[i];
+            const cacheKey = `${r.id}::${r.text}`;
+            if (cachedVectors.has(cacheKey)) {
+                cachedIdx.push(i);
+            }
+            else {
+                toEmbed.push(r);
+                toEmbedIdx.push(i);
+            }
+        }
+        // ── Embed only changed / new functions ───────────────────────────────────
+        let newVectors = [];
+        if (toEmbed.length > 0) {
+            newVectors = await embedSvc.embed(toEmbed.map(r => r.text));
+            if (newVectors.length !== toEmbed.length) {
+                throw new Error(`Embedding count mismatch: expected ${toEmbed.length}, got ${newVectors.length}`);
+            }
+        }
+        // ── Assemble final records ───────────────────────────────────────────────
+        const fullRecords = new Array(candidates.length);
+        for (let i = 0; i < cachedIdx.length; i++) {
+            const idx = cachedIdx[i];
+            const r = candidates[idx];
+            fullRecords[idx] = { ...r, vector: cachedVectors.get(`${r.id}::${r.text}`) };
+        }
+        for (let i = 0; i < toEmbedIdx.length; i++) {
+            const idx = toEmbedIdx[i];
+            fullRecords[idx] = { ...candidates[idx], vector: newVectors[i] };
+        }
+        // ── Write table ──────────────────────────────────────────────────────────
         const db = await connect(dbPath);
         await db.createTable(TABLE_NAME, fullRecords, { mode: 'overwrite' });
+        return { embedded: toEmbed.length, reused: cachedIdx.length };
     }
     /**
-     * Semantic search over the index.
-     * Returns up to `limit` results sorted by similarity (closest first).
+     * Hybrid search over the index: dense (ANN) + sparse (BM25) merged via RRF.
+     *
+     * Dense recall fetches top `limit*5` candidates from the vector index.
+     * Sparse recall scores the full corpus with BM25 (cached per session).
+     * Reciprocal Rank Fusion (RRF) combines both rankings into a single list.
+     *
+     * Set `hybrid: false` to use dense-only search (original behaviour).
+     * Returns up to `limit` results sorted by relevance (highest first).
      */
     static async search(outputDir, query, embedSvc, opts = {}) {
         const { connect } = await import('@lancedb/lancedb');
-        const { limit = 10, language, minFanIn } = opts;
-        // Embed the query
-        const [queryVector] = await embedSvc.embed([query]);
-        if (!queryVector) {
-            throw new Error('Failed to embed query');
-        }
-        const dbPath = join(outputDir, DB_FOLDER);
+        const { limit = 10, language, minFanIn, hybrid = true } = opts;
         if (!VectorIndex.exists(outputDir)) {
             throw new Error('Vector index not found. Run "spec-gen analyze --embed" first.');
         }
+        const dbPath = join(outputDir, DB_FOLDER);
         const db = await connect(dbPath);
         const table = await db.openTable(TABLE_NAME);
-        // Fetch more candidates than requested so post-filtering still yields `limit` results.
-        // We over-fetch by 10x (capped at 1000) to compensate for filtered-out rows.
-        const fetchLimit = Math.min(limit * 10, 1000);
-        const rows = await table.query().nearestTo(queryVector).limit(fetchLimit).toArray();
-        // Post-filter in JavaScript (avoids SQL case-sensitivity issues with column names)
-        const filtered = rows.filter(row => {
+        // ── BM25-only path (no embedding service available) ───────────────────────
+        if (!embedSvc) {
+            return VectorIndex._bm25Only(table, dbPath, query, limit, language, minFanIn);
+        }
+        // ── Dense recall ──────────────────────────────────────────────────────────
+        let queryVector;
+        try {
+            [queryVector] = await embedSvc.embed([query]);
+        }
+        catch {
+            // Embedding server unreachable — fall back to BM25
+            return VectorIndex._bm25Only(table, dbPath, query, limit, language, minFanIn);
+        }
+        if (!queryVector)
+            throw new Error('Failed to embed query');
+        const denseFetch = hybrid ? Math.min(limit * 5, 500) : Math.min(limit * 10, 1000);
+        const denseRows = await table.query().nearestTo(queryVector).limit(denseFetch).toArray();
+        const passesFilters = (row) => {
+            if (language && row.language !== language)
+                return false;
+            if (minFanIn !== undefined && minFanIn > 0 && row.fanIn < minFanIn)
+                return false;
+            return true;
+        };
+        // ── Dense-only path ───────────────────────────────────────────────────────
+        if (!hybrid) {
+            return denseRows
+                .filter(passesFilters)
+                .slice(0, limit)
+                .map(row => ({ record: rowToRecord(row), score: row._distance }));
+        }
+        // ── Sparse recall (BM25 over full corpus) ─────────────────────────────────
+        let cachedEntry = _bm25Cache.get(dbPath);
+        let allRows;
+        if (!cachedEntry) {
+            allRows = await table.query().toArray();
+            const corpus = buildBm25Corpus(allRows.map(r => ({ id: r.id, text: r.text })));
+            cachedEntry = { corpus, rowCount: allRows.length };
+            _bm25Cache.set(dbPath, cachedEntry);
+        }
+        else {
+            // Lightweight cache validation: re-scan only if row count has changed
+            allRows = await table.query().toArray();
+            if (allRows.length !== cachedEntry.rowCount) {
+                const corpus = buildBm25Corpus(allRows.map(r => ({ id: r.id, text: r.text })));
+                cachedEntry = { corpus, rowCount: allRows.length };
+                _bm25Cache.set(dbPath, cachedEntry);
+            }
+        }
+        const { corpus } = cachedEntry;
+        const queryTokens = tokenize(query);
+        // Score all corpus documents with BM25
+        const sparseScored = corpus.docs
+            .map((_, i) => ({ idx: i, score: bm25Score(corpus, queryTokens, i) }))
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit * 5);
+        // Build id→row map from allRows for sparse candidates
+        const rowById = new Map(allRows.map(r => [r.id, r]));
+        // ── RRF merge ────────────────────────────────────────────────────────────
+        const rrfMap = new Map();
+        denseRows.forEach((row, rank) => {
+            const id = row.id;
+            const entry = rrfMap.get(id) ?? { row, score: 0 };
+            entry.score += rrfScore(rank, Infinity); // sparse rank = Infinity if not in sparse list
+            rrfMap.set(id, entry);
+        });
+        sparseScored.forEach(({ idx, score: bm25 }, rank) => {
+            if (bm25 === 0)
+                return; // no BM25 signal — skip
+            const id = corpus.docs[idx].id;
+            const row = rowById.get(id);
+            if (!row)
+                return;
+            const entry = rrfMap.get(id) ?? { row, score: 0 };
+            entry.score += 1 / (60 + rank + 1);
+            rrfMap.set(id, entry);
+        });
+        // Fix dense ranks now that we know the full picture
+        // Re-compute proper RRF scores with both ranks available
+        const denseRankById = new Map(denseRows.map((r, i) => [r.id, i]));
+        const sparseRankById = new Map(sparseScored.map(({ idx }, i) => [corpus.docs[idx].id, i]));
+        const merged = [...rrfMap.values()].map(({ row }) => {
+            const id = row.id;
+            const dr = denseRankById.get(id) ?? Infinity;
+            const sr = sparseRankById.get(id) ?? Infinity;
+            return { row, score: rrfScore(dr, sr) };
+        });
+        return merged
+            .sort((a, b) => b.score - a.score)
+            .filter(({ row }) => passesFilters(row))
+            .slice(0, limit)
+            .map(({ row, score }) => ({ record: rowToRecord(row), score }));
+    }
+    /**
+     * BM25-only search: used when no embedding service is available.
+     * Scores the full corpus with BM25 and returns the top `limit` results.
+     */
+    static async _bm25Only(table, dbPath, query, limit, language, minFanIn) {
+        let cachedEntry = _bm25Cache.get(dbPath);
+        let allRows;
+        if (!cachedEntry) {
+            allRows = await table.query().toArray();
+            const corpus = buildBm25Corpus(allRows.map(r => ({ id: r.id, text: r.text })));
+            cachedEntry = { corpus, rowCount: allRows.length };
+            _bm25Cache.set(dbPath, cachedEntry);
+        }
+        else {
+            allRows = await table.query().toArray();
+            if (allRows.length !== cachedEntry.rowCount) {
+                const corpus = buildBm25Corpus(allRows.map(r => ({ id: r.id, text: r.text })));
+                cachedEntry = { corpus, rowCount: allRows.length };
+                _bm25Cache.set(dbPath, cachedEntry);
+            }
+        }
+        const { corpus } = cachedEntry;
+        const queryTokens = tokenize(query);
+        const rowById = new Map(allRows.map(r => [r.id, r]));
+        return corpus.docs
+            .map((_, i) => ({ idx: i, score: bm25Score(corpus, queryTokens, i) }))
+            .filter(({ score }) => score > 0)
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit * 3) // oversample before filtering
+            .map(({ idx, score }) => {
+            const row = rowById.get(corpus.docs[idx].id);
+            return row ? { row, score } : null;
+        })
+            .filter((x) => x !== null)
+            .filter(({ row }) => {
             if (language && row.language !== language)
                 return false;
             if (minFanIn !== undefined && minFanIn > 0 && row.fanIn < minFanIn)
                 return false;
             return true;
-        }).slice(0, limit);
-        return filtered.map(row => ({
-            record: {
-                id: row.id,
-                name: row.name,
-                filePath: row.filePath,
-                className: row.className,
-                language: row.language,
-                signature: row.signature,
-                docstring: row.docstring,
-                fanIn: row.fanIn,
-                fanOut: row.fanOut,
-                isHub: row.isHub,
-                isEntryPoint: row.isEntryPoint,
-                text: row.text,
-            },
-            score: row._distance,
-        }));
+        })
+            .slice(0, limit)
+            .map(({ row, score }) => ({ record: rowToRecord(row), score }));
     }
     /**
      * Returns true if a vector index has been built for this output directory.