sweet-search 2.5.2 → 2.5.4

package/core/graph/graph-extractor.js

@@ -14,13 +14,80 @@ import { createHash } from 'crypto';
 import path from 'path';
 import fs from 'fs/promises';
 import { GRAPH_CONFIG, DB_PATHS } from '../infrastructure/config/index.js';
-import { getLanguageByPath } from '../infrastructure/language-patterns.js';
+import { getLanguageByPath, resolveLanguage } from '../infrastructure/language-patterns.js';
 import { getTreeSitterProvider } from '../infrastructure/tree-sitter-provider.js';
 
 // Schema version - increment when schema changes require full reindex
 // Users should run `/index-codebase --full` after upgrading
 export const SCHEMA_VERSION = 2;
 
+/**
+ * Sentinel `end_line` clamp (2026-05-13). Lua-specific by design.
+ *
+ * Background: Lua's regex extractor uses `findEndLineKeyword` to find the
+ * `end` keyword that closes a function body. The helper tracks nesting
+ * depth across `if`/`while`/`for`/`function`/`do` keywords and decrements
+ * on `end`. When the depth counter mis-balances (control-flow keywords
+ * sharing line context with the closing `end`), the helper falls through
+ * to `return lines.length` (the file's last line), producing entities
+ * with end_line = EOF that structurally span half the file (LU-003:
+ * tablex.deepcopy at 118-120 got rendered as 98-999 because its
+ * preceding sibling cycle_aware_copy had bogus end_line=999).
+ *
+ * Gated to language='lua' EXPLICITLY because:
+ *   - Tree-sitter languages (Java, Python, JS, TS, Go, Rust, C, C++,
+ *     Ruby, etc.) get accurate end_lines from grammar-driven extraction —
+ *     this clamp's pattern doesn't apply.
+ *   - Other regex-path languages (zig, scala, kotlin, dart, elixir, php)
+ *     may have similar bugs but haven't been audited. Apply only after
+ *     per-language validation.
+ *
+ * Clamp condition (BOTH required):
+ *   1. cur.type ∈ NON_CONTAINER_TYPES (function-shaped — these cannot
+ *      legitimately contain a same-level sibling that starts inside them)
+ *   2. cur.end_line >= file_line_count (ends at-or-past EOF)
+ *   3. A later entity starts after cur.start_line and before cur.end_line.
+ *
+ * Clamp target: next entity's start_line - 1. Mutates in place.
+ *
+ * The container-type gate (NON_CONTAINER_TYPES) is a defence-in-depth
+ * even within Lua — Lua doesn't really have classes, but if a future
+ * change adds 'module' or 'class' types via metatable detection, they
+ * stay protected.
+ */
+const NON_CONTAINER_TYPES = new Set([
+  'function', 'method', 'arrowFunction', 'variable', 'const', 'field',
+  'decorator', 'assignedFunc', 'component', 'typeAlias',
+]);
+
+const LUA_CLAMP_ALLOWED_LANGUAGES = new Set(['lua']);
+
+export function clampSentinelEndLines(entities, fileLineCount, language) {
+  if (!Array.isArray(entities) || entities.length < 2) return entities;
+  if (fileLineCount == null || fileLineCount <= 0) return entities;
+  if (!LUA_CLAMP_ALLOWED_LANGUAGES.has(language)) return entities;
+  for (let i = 0; i < entities.length - 1; i++) {
+    const cur = entities[i];
+    if (!NON_CONTAINER_TYPES.has(cur?.type)) continue;
+    const curEnd = Number(cur?.end_line ?? 0);
+    if (!Number.isFinite(curEnd) || curEnd < fileLineCount) continue;
+    for (let j = i + 1; j < entities.length; j++) {
+      const next = entities[j];
+      const nextStart = Number(next?.start_line ?? 0);
+      const curStart = Number(cur?.start_line ?? 0);
+      if (!Number.isFinite(nextStart) || nextStart <= curStart) continue;
+      if (nextStart >= curEnd) break;
+      // Sentinel detected: clamp.
+      if (nextStart - 1 >= curStart) {
+        cur.end_line = nextStart - 1;
+      }
+      break;
+    }
+  }
+  return entities;
+}
+
+
 /**
  * Normalize an identifier into searchable alias tokens.
  * Splits camelCase, PascalCase, snake_case, digits and emits both
@@ -317,9 +384,34 @@ export const TREE_SITTER_ENTITY_PRIORITY = Object.freeze({
   struct: 30,
   record: 30,
   module: 25,
+  // `variable` is intentionally lowest: when an `export const X = memo(...)`
+  // matches BOTH the @component (rank 40) and @variable rules, component wins.
+  // When `export const handler = async () => {}` matches BOTH @arrow (rank 20)
+  // and @variable, arrowFunction wins. Plain `export const FOO = "bar"` only
+  // matches @variable so it lands at rank 5 (kept).
+  variable: 5,
   trait: 25,
   impl: 20,
   decorator: 15,
+  // Rust macro_rules! definitions — same rank as function/struct/impl since
+  // they're top-level definitions with similar discoverability needs.
+  macro: 30,
+  // Java enum constants (FieldNamingPolicy.UPPER_CAMEL_CASE) — fine-grained
+  // anchor inside the enclosing enum class, but worth surfacing for
+  // symbol-anchored probes. Rank between decorator and arrow: low enough
+  // to not steal the enum's primary anchor when both match, high enough
+  // to win over plain variables in disambiguation.
+  enum_constant: 10,
+  // Java field declarations (static finals like TypeAdapters.BIT_SET that
+  // initialize anonymous inner-class subclasses). Same priority story as
+  // enum_constant — useful for anchoring, not primary.
+  field: 10,
+  // C# property declarations (`public RespCommand Command { get; init; }`) —
+  // first-class members per the C# spec, but lower in retrieval priority
+  // than methods/classes when both could anchor a result. Same rank as
+  // arrowFunction/interface/enum (20): high enough to win over enum_constant
+  // when both match, low enough to never overshadow the owning class.
+  property: 20,
 });
 
 // Module-scope constants for extractJavaScript() — avoid per-call/per-line allocation.
@@ -396,7 +488,10 @@ export class GraphExtractor {
   async extractFromFile(filePath, content) {
     this.currentFile = filePath;
     const lines = content.split('\n');
-    const langInfo = getLanguageByPath(filePath);
+    // resolveLanguage handles per-file disambiguation of ambiguous extensions
+    // (today: `.h` → c-vs-cpp) so header-only C++ libraries get parsed by
+    // tree-sitter-cpp rather than tree-sitter-c.
+    const langInfo = resolveLanguage(filePath, content);
 
     if (!langInfo) {
       return { entities: [], relationships: [] };
@@ -516,12 +611,64 @@ export class GraphExtractor {
     let currentClass = null;
     let braceDepth = 0;
     let classStartDepth = 0;
+    // Track whether we are inside a `/* ... */` or `/** ... */` block
+    // comment. Without this, every entity-emission regex below also
+    // matches Javadoc `<pre>` examples ("public class MyClass { ... }"),
+    // creating phantom classes/methods/calls in the graph. Verified on
+    // gson SerializedName.java / Since.java / Until.java where phantom
+    // `MyClass`/`User`/`Gson`/`fromJson` entities were polluting
+    // search-time symbol attribution via findFirstEntityInRange.
+    // The state is a per-line boolean: true if the line BEGINS inside
+    // a block comment (and we therefore skip all regex extractions and
+    // brace counting on that line). State transitions on the first
+    // `/*` open and the first `*/` close encountered, scanned left-to-
+    // right. Inline `/* ... */` on a single line is treated as the
+    // line containing both open and close — the line ends OUT of the
+    // comment, so extraction runs as normal (a rare but harmless edge:
+    // identifiers on the same line as a closing `*/` could still be
+    // picked up; this matches existing whole-file regex behaviour).
+    let inBlockComment = false;
 
     for (let i = 0; i < lines.length; i++) {
       const line = lines[i];
       const lineNum = i + 1;
 
-      // Track brace depth
+      const enteredAtStart = inBlockComment;
+      // Update inBlockComment state from this line's `/*` opens and
+      // `*/` closes. We scan character-by-character but cheaply: a
+      // single pass with two indexOf-style searches per iteration.
+      {
+        let scan = 0;
+        while (scan < line.length) {
+          if (inBlockComment) {
+            const close = line.indexOf('*/', scan);
+            if (close < 0) { scan = line.length; break; }
+            inBlockComment = false;
+            scan = close + 2;
+          } else {
+            const open = line.indexOf('/*', scan);
+            if (open < 0) { scan = line.length; break; }
+            // Inline line-comment `//` before `/*` on the same line:
+            // // /* not really a block */ — treat the `//` as wins.
+            const lineCom = line.indexOf('//', scan);
+            if (lineCom >= 0 && lineCom < open) { scan = line.length; break; }
+            inBlockComment = true;
+            scan = open + 2;
+          }
+        }
+      }
+
+      // Skip lines that are entirely inside a block comment (including
+      // the case where the line opens AND stays inside — entered false,
+      // ends true: line has no executable code AFTER the `/*`).
+      const lineWasFullyInComment = enteredAtStart && inBlockComment;
+      if (lineWasFullyInComment) {
+        // Don't count braces, don't run extraction regexes.
+        continue;
+      }
+
+      // Track brace depth (raw-line approximation, matches pre-fix
+      // behaviour for non-comment lines).
       braceDepth += (line.match(/{/g) || []).length;
       braceDepth -= (line.match(/}/g) || []).length;
 
@@ -530,6 +677,15 @@ export class GraphExtractor {
         currentClass = null;
       }
 
+      // If we OPENED a block comment on this line, code BEFORE the
+      // `/*` is still real — run extraction on the line as usual; the
+      // Javadoc body that follows starts on the next iteration with
+      // inBlockComment=true. Same for lines that close a block comment
+      // (we already cleared the state above by the time we get here).
+      // Defensive: if the line is mostly Javadoc but has trailing code
+      // after `*/`, the regex will still capture; that mirrors the
+      // existing 99% case (real `public class Foo {` lines).
+
       // Class declarations
       const classMatch = line.match(/(?:public|private|protected)?\s*(?:static)?\s*(?:final|abstract)?\s*class\s+(\w+)(?:\s+extends\s+(\w+))?(?:\s+implements\s+([\w,\s]+))?/);
       if (classMatch) {
@@ -1112,6 +1268,17 @@ export class GraphExtractor {
       }
     }
 
+    // Sentinel clamp (2026-05-13): Lua-only. The regex `findEndLineKeyword`
+    // falls through to `return lines.length` when the `end` keyword counter
+    // mis-balances (control-flow keywords sharing line context), producing
+    // entities with end_line=EOF that swallow subsequent siblings (LU-003:
+    // tablex.deepcopy at 118-120 was being rendered as 98-999 because the
+    // preceding sibling cycle_aware_copy had bogus end_line=999). The
+    // language gate inside clampSentinelEndLines is explicit — other
+    // regex-path languages (zig, scala, kotlin, etc.) are unaffected and
+    // would need per-language validation before opt-in.
+    clampSentinelEndLines(entities, lines.length, langInfo?.id);
+
     return { entities, relationships };
   }
 
@@ -1305,15 +1472,20 @@ export class GraphExtractor {
       .filter(Boolean);
   }
 
+  _clampSentinelEndLines(entities, fileLineCount) {
+    return clampSentinelEndLines(entities, fileLineCount);
+  }
+
   _normalizeTreeSitterEntities(filePath, symbols, language) {
     const dedupedBySymbolAndLine = new Map();
 
     for (const sym of symbols) {
       if (!sym?.name || !sym?.type) continue;
       const normalizedType = this._normalizeTreeSitterSymbolType(sym.type, sym.name);
-      if ((language === 'javascript' || language === 'typescript') && normalizedType === 'variable') {
-        continue;
-      }
+      // Note: previously dropped 'variable' for js/ts to avoid noise from
+      // every internal `let x = 1`. The current TS/TSX/JS tag query scopes
+      // @variable.definition to `(export_statement (lexical_declaration ...))`
+      // so only EXPORTED top-level consts reach this point — keep them.
       const startLine = Number.isInteger(sym.startLine) ? sym.startLine : 0;
       const endLine = Number.isInteger(sym.endLine) ? sym.endLine : startLine;
       const rank = TREE_SITTER_ENTITY_PRIORITY[normalizedType] || 0;
@@ -1334,9 +1506,14 @@ export class GraphExtractor {
       }
     }
 
-    return Array.from(dedupedBySymbolAndLine.values())
-      .sort((a, b) => a.start_line - b.start_line)
-      .map(({ rank, ...entity }) => entity);
+    const sorted = Array.from(dedupedBySymbolAndLine.values())
+      .sort((a, b) => a.start_line - b.start_line);
+    // Tree-sitter path: NO sentinel clamp. Tree-sitter parsers return
+    // accurate end_lines via grammar-driven extraction; the regex-path
+    // `findEndLineKeyword` fall-through is the only known source of the
+    // bogus-EOF pattern, and only Lua currently goes through that path
+    // (Lua has no tree-sitter grammar registered).
+    return sorted.map(({ rank, ...entity }) => entity);
   }
 
   _normalizeTreeSitterSymbolType(type, name) {
@@ -1347,7 +1524,7 @@ export class GraphExtractor {
   }
 
   _resolveRelationshipTargets(relType, match, language) {
-    const isJsTs = language === 'javascript' || language === 'typescript';
+    const isJsTs = language === 'javascript' || language === 'typescript' || language === 'tsx';
 
     if (isJsTs && relType === 'import') {
       const source = match[3]?.trim();
@@ -1852,7 +2029,8 @@ export function createGraphSchema(db) {
       hierarchy_level INTEGER DEFAULT 0,
       code TEXT,
       name_alias TEXT,
-      stale_since INTEGER DEFAULT NULL
+      stale_since INTEGER DEFAULT NULL,
+      page_rank REAL DEFAULT 0
     )
   `);
 
@@ -1869,6 +2047,11 @@ export function createGraphSchema(db) {
       db.exec('ALTER TABLE entities ADD COLUMN name_alias TEXT');
       console.log('  Migrated: added name_alias column to entities table');
     }
+    const hasPageRankColumn = columns.some(col => col.name === 'page_rank');
+    if (!hasPageRankColumn) {
+      db.exec('ALTER TABLE entities ADD COLUMN page_rank REAL DEFAULT 0');
+      console.log('  Migrated: added page_rank column to entities table');
+    }
   } catch (err) {
     // Ignore errors - column might already exist or table not created yet
   }
@@ -1936,6 +2119,8 @@ export function createGraphSchema(db) {
   db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_rel_unique ON relationships(source_id, target_id, type, target_name) WHERE source_id IS NOT NULL`);
   // Index on target_id for efficient reverse lookups ("what calls X")
   db.exec(`CREATE INDEX IF NOT EXISTS idx_rel_target_id ON relationships(target_id) WHERE target_id IS NOT NULL`);
+  // Index supports `page_rank DESC` lookups for ss-trace ranking and ranking probes.
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_entities_page_rank ON entities(page_rank) WHERE stale_since IS NULL`);
 
   setSchemaVersion(db);