sweet-search 2.5.2 → 2.5.3

package/core/indexing/ast-chunker.js

@@ -12,7 +12,7 @@ import { createHash } from 'crypto';
 import path from 'path';
 import fs from 'fs/promises';
 import { detectProjectBoundary } from '../infrastructure/project-detector.js';
-import { getLanguageByPath } from '../infrastructure/language-patterns.js';
+import { getLanguageByPath, resolveLanguage } from '../infrastructure/language-patterns.js';
 import { DocumentChunker } from './document-chunker.js';
 
 const MAX_CHUNK_SIZE = 2000;
@@ -151,12 +151,25 @@ function buildEmbeddingText({ variant: variantOverride, content, relativePath, l
   const pathLine = relativePath ? `# ${relativePath}` : null;
   const parentLine = hierarchyInfo?.parentSymbol
     ? `# Parent: ${hierarchyInfo.parentType} ${hierarchyInfo.parentSymbol}` : null;
-  const symbolLine = (symbol && symbol !== 'unknown')
+  // Ruby method chunks keep metadata anchors, but omit method-name header
+  // text so top-level Ruby snippets stay aligned with the pre-method-boundary
+  // embedding surface.
+  const isRubyMethodChunk = language === 'ruby'
+    && (chunkType === 'method' || chunkType === 'singleton_method');
+  const symbolLine = (symbol && symbol !== 'unknown' && !isRubyMethodChunk)
     ? `# ${chunkType}: ${symbol}` : null;
   const langLine = (language && language !== 'text')
     ? `# Language: ${language}` : null;
   const signatureLine = hierarchyInfo?.signature
     ? `# Signature: ${hierarchyInfo.signature}` : null;
+  // Multi-symbol header: when the cAST sibling-merge collapses several
+  // top-level boundaries into one chunk (small Rust files with adjacent
+  // free-standing fns, e.g. packaging.rs:is_package + detect_package_root),
+  // the bi-encoder otherwise only sees the FIRST boundary's name. Adding
+  // an `# Additional:` line surfaces sibling symbol names to the encoder
+  // without changing chunk count or chunk text.
+  const additionalLine = (hierarchyInfo?.additionalSymbols && hierarchyInfo.additionalSymbols.length > 0)
+    ? `# Additional: ${hierarchyInfo.additionalSymbols.join(', ')}` : null;
 
   switch (variant) {
     case 'no_path':
@@ -223,6 +236,7 @@ function buildEmbeddingText({ variant: variantOverride, content, relativePath, l
       if (pathLine) parts.push(pathLine);
       if (parentLine) parts.push(parentLine);
       if (symbolLine) parts.push(symbolLine);
+      if (additionalLine) parts.push(additionalLine);
       if (langLine) parts.push(langLine);
       break;
   }
@@ -279,9 +293,17 @@ function buildLiText({ content, relativePath, language, chunkType, symbol, hiera
   if (hierarchyInfo?.parentSymbol) {
     lines.push(`# Parent: ${hierarchyInfo.parentType} ${hierarchyInfo.parentSymbol}`);
   }
-  if (symbol && symbol !== 'unknown') {
+  // Mirror the Ruby method header carve-out used by embedding_text.
+  const isRubyMethodChunk = language === 'ruby'
+    && (chunkType === 'method' || chunkType === 'singleton_method');
+  if (symbol && symbol !== 'unknown' && !isRubyMethodChunk) {
     lines.push(`# ${chunkType}: ${symbol}`);
   }
+  // Mirror embedding_text: surface sibling symbol names so the LI MaxSim
+  // stage's input includes the same context as the bi-encoder embedding.
+  if (hierarchyInfo?.additionalSymbols && hierarchyInfo.additionalSymbols.length > 0) {
+    lines.push(`# Additional: ${hierarchyInfo.additionalSymbols.join(', ')}`);
+  }
   if (language && language !== 'text') {
     lines.push(`# Language: ${language}`);
   }
@@ -321,7 +343,11 @@ export class ASTChunker {
       return this._docChunker.parseFile(filePath, content);
     }
 
-    const langInfo = getLanguageByPath(filePath);
+    // resolveLanguage handles per-file disambiguation of ambiguous extensions
+    // (today: `.h` → c-vs-cpp) using a content scan for C++-only tokens.
+    // Header-only C++ libraries (highway, Eigen, …) are routed to cpp so the
+    // chunker uses tree-sitter-cpp instead of tree-sitter-c.
+    const langInfo = resolveLanguage(filePath, content);
     if (!langInfo || !langInfo.chunker) {
       return this.parseGenericFile(filePath, content);
     }
@@ -361,20 +387,27 @@ export class ASTChunker {
     const tsChunks = await provider.parseFileToChunks(content, langInfo.id);
     if (!tsChunks || tsChunks.length === 0) return null;
 
-    return tsChunks.map(chunk =>
-      this.buildChunk(
-        chunk.text, filePath, langInfo.id, chunk.type, chunk.name,
-        chunk.startLine, chunk.endLine,
-        {
-          chunkId: chunk.chunkId,
-          parentChunkId: chunk.parentChunkId,
-          parentSymbol: chunk.parentSymbol,
-          parentType: chunk.parentType,
-          signature: chunk.signature || null,
-        }
-      )
-    );
-  }
+    return tsChunks.map(chunk => {
+      const isTopLevelRubyMethod = langInfo.id === 'ruby'
+        && (chunk.type === 'method' || chunk.type === 'singleton_method')
+        && !chunk.parentSymbol;
+
+      return this.buildChunk(
+        chunk.text, filePath, langInfo.id,
+        isTopLevelRubyMethod ? 'code' : chunk.type,
+        isTopLevelRubyMethod ? null : chunk.name,
+        chunk.startLine, chunk.endLine,
+        {
+          chunkId: chunk.chunkId,
+          parentChunkId: chunk.parentChunkId,
+          parentSymbol: chunk.parentSymbol,
+          parentType: chunk.parentType,
+          signature: chunk.signature || null,
+          additionalSymbols: chunk.additionalSymbols || null,
+        }
+      );
+    });
+  }
 
   parseBraceBasedFile(filePath, content, language, patterns, comment, multiLine) {
     const chunks = [];
@@ -388,12 +421,30 @@ export class ASTChunker {
 
     for (let i = 0; i < lines.length; i++) {
       const line = lines[i];
+      // Capture the comment state BEFORE _stripNonCode mutates it so we
+      // can tell whether this line entered the iteration inside a block
+      // comment (e.g. mid-Javadoc). Boundary detection must be skipped
+      // for such lines — otherwise `_matchBoundary` happily matches
+      // `public class MyClass {` inside a Java `/** ... <pre> ... */`
+      // example and emits a phantom class chunk. Verified on gson
+      // SerializedName.java where the regex fallback path emits a
+      // 25-82 chunk attributed to "class MyClass" sourced from the
+      // Javadoc body. See the matching extractJava() block-comment
+      // skip in core/graph/graph-extractor.js for the symmetric fix.
+      const inBlockCommentAtStart = stripState.inBlockComment;
       const stripped = this._stripNonCode(line, stripState, comment, hasTemplateInterpolation);
 
       braceDepth += (stripped.match(/{/g) || []).length;
       braceDepth -= (stripped.match(/}/g) || []).length;
 
-      const { name: matched, type: matchType, joinedLines } = this._matchBoundary(line, patterns, language, lines, i, multiLine);
+      // When the line is entirely inside a block comment (entered as
+      // such AND still inside on exit), there's nothing executable to
+      // match — skip boundary detection entirely. The stripped output
+      // is already empty/whitespace so brace-depth tracking is a no-op.
+      const lineFullyInComment = inBlockCommentAtStart && stripState.inBlockComment;
+      const { name: matched, type: matchType, joinedLines } = lineFullyInComment
+        ? { name: null, type: null, joinedLines: 0 }
+        : this._matchBoundary(stripped, patterns, language, lines, i, multiLine);
 
       if ((matched && currentChunk) || (braceDepth === 0 && currentChunk)) {
         const chunkContent = lines.slice(chunkStart, i + 1).join('\n');
@@ -805,13 +856,20 @@ export class ASTChunker {
     for (let i = 0; i < lines.length; i++) {
       const line = lines[i];
       const lineSize = line.length + 1; // +1 for newline
+      const inBlockCommentAtStart = stripState.inBlockComment;
       const stripped = this._stripNonCode(line, stripState, comment, hasTemplateInterpolation);
 
       braceDepth += (stripped.match(/{/g) || []).length;
       braceDepth -= (stripped.match(/}/g) || []).length;
 
-      // Check if this line is a sub-boundary (a new construct starting at depth >= 1)
-      const { name: matched, type: matchType } = this._matchBoundary(line, patterns, language, lines, i, false);
+      // Check if this line is a sub-boundary (a new construct starting at depth >= 1).
+      // Use the comment-stripped line so Javadoc `<pre>public class Foo {...}</pre>`
+      // examples don't get matched as real sub-boundaries (parallels
+      // parseBraceBasedFile fix above).
+      const lineFullyInComment = inBlockCommentAtStart && stripState.inBlockComment;
+      const { name: matched, type: matchType } = lineFullyInComment
+        ? { name: null, type: null }
+        : this._matchBoundary(stripped, patterns, language, lines, i, false);
       const isSubBoundary = matched && i > segStart;
 
       // Split condition: at a sub-boundary, or accumulated segment exceeds max
@@ -938,6 +996,11 @@ export class ASTChunker {
       metadata.parent_symbol = hierarchyInfo.parentSymbol;
       metadata.parent_type = hierarchyInfo.parentType;
     }
+    // Carry sibling-symbol context into metadata so enrichEmbeddingText()
+    // can rebuild the multi-symbol header during post-chunk enrichment.
+    if (hierarchyInfo?.additionalSymbols && hierarchyInfo.additionalSymbols.length > 0) {
+      metadata.additional_symbols = hierarchyInfo.additionalSymbols;
+    }
 
     return {
       text: content.trim(),
@@ -967,15 +1030,29 @@ export class ASTChunker {
     const parts = [];
     parts.push(`# ${chunk.metadata.path}`);
 
-    if (scopeChain && scopeChain.length > 0) {
-      parts.push(`# Scope: ${scopeChain.join(' > ')}`);
-    } else if (chunk.metadata.parent_symbol) {
-      // Preserve cAST parent context when no scope chain from code graph
-      parts.push(`# Parent: ${chunk.metadata.parent_type} ${chunk.metadata.parent_symbol}`);
-    }
-
-    if (chunk.metadata.symbol && chunk.metadata.symbol !== 'unknown') {
-      parts.push(`# Defines: ${chunk.metadata.chunk_type} ${chunk.metadata.symbol}`);
+    const isRubyMethodChunk = chunk.metadata.language === 'ruby'
+      && (chunk.metadata.chunk_type === 'method'
+        || chunk.metadata.chunk_type === 'singleton_method');
+    const hasOnlySelfScope = scopeChain
+      && scopeChain.length === 1
+      && scopeChain[0] === chunk.metadata.symbol
+      && !chunk.metadata.parent_symbol;
+
+    if (scopeChain && scopeChain.length > 0 && !(isRubyMethodChunk && hasOnlySelfScope)) {
+      parts.push(`# Scope: ${scopeChain.join(' > ')}`);
+    } else if (chunk.metadata.parent_symbol) {
+      // Preserve cAST parent context when no scope chain from code graph
+      parts.push(`# Parent: ${chunk.metadata.parent_type} ${chunk.metadata.parent_symbol}`);
+    }
+    // Keep Ruby method metadata, but avoid injecting the method name into
+    // the production embedding/LI text. Top-level Ruby method chunks also
+    // skip self-only scope above.
+    if (chunk.metadata.symbol && chunk.metadata.symbol !== 'unknown' && !isRubyMethodChunk) {
+      parts.push(`# Defines: ${chunk.metadata.chunk_type} ${chunk.metadata.symbol}`);
+    }
+
+    if (chunk.metadata.additional_symbols && chunk.metadata.additional_symbols.length > 0) {
+      parts.push(`# Additional: ${chunk.metadata.additional_symbols.join(', ')}`);
     }
 
     if (imports && imports.length > 0) {

package/core/indexing/dedup/exemplar-selector.js

@@ -16,7 +16,25 @@ function lengthOf(chunk) {
 }
 
 function pathOf(chunk) {
-  return chunk.file || chunk.metadata?.path || '';
+  return firstSafeRelativePath(
+    chunk.metadata?.relative_path,
+    chunk.metadata?.path,
+    chunk.metadata?.file_path,
+    chunk.file,
+    chunk.metadata?.file,
+  ) || '';
+}
+
+function firstSafeRelativePath(...candidates) {
+  for (const candidate of candidates) {
+    if (typeof candidate !== 'string') continue;
+    const normalized = candidate.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+/g, '/');
+    if (!normalized || normalized === '.' || normalized.startsWith('/')) continue;
+    if (/^[A-Za-z]:\//.test(normalized)) continue;
+    if (normalized === '..' || normalized.startsWith('../') || normalized.includes('/../')) continue;
+    return normalized;
+  }
+  return null;
 }
 
 function hashOf(chunk) {

package/core/indexing/gitignore-filter.js

@@ -0,0 +1,223 @@
+/**
+ * Gitignore alignment for file admission.
+ *
+ * Extracted from indexer-utils.js so both full indexing (`discoverFiles`) and
+ * the shared admission policy (`admission-policy.js`, used by incremental
+ * indexing) run the *same* `.gitignore` logic. The only behavioural change vs
+ * the original is that the project root is a parameter instead of the global
+ * `PROJECT_ROOT` constant, so the incremental maintainer can align gitignore
+ * against the worktree it actually reconciles. Full indexing keeps passing
+ * `PROJECT_ROOT` (the default), so its behaviour is unchanged.
+ */
+
+import fs from 'fs/promises';
+import { existsSync } from 'fs';
+import { spawn } from 'child_process';
+import path from 'path';
+
+import { PROJECT_ROOT, AGENTIC_GITIGNORE_ALLOWLIST } from '../infrastructure/config/index.js';
+
+export function toPosixPath(filePath) {
+  return filePath.replace(/\\/g, '/');
+}
+
+function logError(message) {
+  console.error(`[indexer] ${message}`);
+}
+
+/**
+ * Agentic tooling paths stay indexable even when listed in `.gitignore`
+ * (local AI workflow files). Mirrors AGENTIC_GITIGNORE_ALLOWLIST.
+ */
+export function isGitignoreAllowlistedAgenticPath(relativePath) {
+  const normalized = toPosixPath(relativePath).replace(/^\.\//, '');
+  const basename = path.posix.basename(normalized);
+
+  if (AGENTIC_GITIGNORE_ALLOWLIST.files.includes(basename)) {
+    return true;
+  }
+
+  if (AGENTIC_GITIGNORE_ALLOWLIST.filePrefixes.some(prefix => basename.startsWith(prefix))) {
+    return true;
+  }
+
+  return AGENTIC_GITIGNORE_ALLOWLIST.directories.some(dirPrefix =>
+    normalized.startsWith(dirPrefix) || normalized.includes(`/${dirPrefix}`)
+  );
+}
+
+/**
+ * Run `git check-ignore` on a single batch of paths.
+ * Returns a list of ignored paths, or null on fatal error.
+ */
+function checkIgnoreBatch(batch, projectRoot, reportError) {
+  return new Promise((resolve) => {
+    const ignoredChunks = [];
+    let settled = false;
+
+    const git = spawn('git', ['check-ignore', '-z', '--stdin'], { cwd: projectRoot });
+
+    git.stdout.on('data', chunk => ignoredChunks.push(chunk));
+    git.stderr.on('data', () => {}); // Suppress — batched caller handles partial failures
+
+    git.on('error', (err) => {
+      if (settled) return;
+      settled = true;
+      reportError(`WARN: Unable to run git check-ignore (${err.message})`);
+      resolve(null);
+    });
+
+    git.on('close', (code) => {
+      if (settled) return;
+      settled = true;
+
+      // code 0 = some ignored, code 1 = none ignored, both valid.
+      // code 128 = fatal (e.g. path beyond symlink) — still use partial stdout.
+      if (code !== 0 && code !== 1 && ignoredChunks.length === 0) {
+        resolve(null);
+        return;
+      }
+
+      const ignored = Buffer.concat(ignoredChunks)
+        .toString('utf8')
+        .split('\0')
+        .filter(Boolean)
+        .map(toPosixPath);
+
+      resolve(ignored);
+    });
+
+    const stdinPayload = `${batch.map(toPosixPath).join('\0')}\0`;
+    git.stdin.on('error', () => {}); // Suppress EPIPE if git exits early
+    git.stdin.end(stdinPayload);
+  });
+}
+
+const CHECK_IGNORE_BATCH_SIZE = 5000;
+
+/**
+ * Find directory components that are symlinks, so we can filter out paths
+ * that traverse them (git check-ignore fatals on "beyond a symbolic link").
+ */
+async function findSymlinkDirs(paths, projectRoot) {
+  const checked = new Map();
+  const symlinkPrefixes = [];
+
+  for (const p of paths) {
+    const parts = p.split('/');
+    let dir = '';
+    for (let i = 0; i < parts.length - 1; i++) {
+      dir = dir ? `${dir}/${parts[i]}` : parts[i];
+      if (checked.has(dir)) continue;
+      try {
+        const stat = await fs.lstat(path.join(projectRoot, dir));
+        const isLink = stat.isSymbolicLink();
+        checked.set(dir, isLink);
+        if (isLink) symlinkPrefixes.push(dir + '/');
+      } catch {
+        checked.set(dir, false);
+      }
+    }
+  }
+
+  return symlinkPrefixes;
+}
+
+export async function getGitIgnoredPathSet(paths, options = {}) {
+  const projectRoot = options.projectRoot || PROJECT_ROOT;
+  const silent = options.silent ?? false;
+  const reportError = silent ? () => {} : logError;
+
+  if (paths.length === 0) {
+    return new Set();
+  }
+
+  const ignored = new Set();
+
+  // Pre-filter paths that traverse symlinks — git check-ignore fatals on these.
+  // Files beyond symlinks are also checked: if the symlink dir itself is ignored,
+  // all files under it are treated as ignored too.
+  const symlinkPrefixes = await findSymlinkDirs(paths, projectRoot);
+  let safePaths = paths;
+  if (symlinkPrefixes.length > 0) {
+    // Check if the symlink directories themselves are ignored
+    const symlinkDirs = symlinkPrefixes.map(p => p.slice(0, -1)); // remove trailing /
+    const symlinkIgnored = await checkIgnoreBatch(symlinkDirs, projectRoot, reportError);
+    const ignoredSymlinks = new Set(symlinkIgnored || []);
+
+    safePaths = [];
+    for (const p of paths) {
+      const matchedPrefix = symlinkPrefixes.find(prefix => p.startsWith(prefix));
+      if (matchedPrefix) {
+        // Path traverses a symlink — check if symlink dir is gitignored
+        const dir = matchedPrefix.slice(0, -1);
+        if (ignoredSymlinks.has(dir)) {
+          ignored.add(toPosixPath(p)); // inherit parent's ignored status
+        }
+        // Either way, skip git check-ignore (would fatal)
+      } else {
+        safePaths.push(p);
+      }
+    }
+  }
+
+  let failedBatches = 0;
+
+  for (let i = 0; i < safePaths.length; i += CHECK_IGNORE_BATCH_SIZE) {
+    const batch = safePaths.slice(i, i + CHECK_IGNORE_BATCH_SIZE);
+    const result = await checkIgnoreBatch(batch, projectRoot, reportError);
+    if (result) {
+      for (const p of result) ignored.add(p);
+    } else {
+      failedBatches++;
+    }
+  }
+
+  const totalBatches = Math.ceil(safePaths.length / CHECK_IGNORE_BATCH_SIZE);
+  if (failedBatches === totalBatches && totalBatches > 0) {
+    reportError('WARN: git check-ignore failed on all batches — gitignore filtering disabled');
+    return null;
+  }
+
+  return ignored;
+}
+
+export async function applyGitignoreAlignment(files, respectGitignore, options = {}) {
+  const projectRoot = options.projectRoot || PROJECT_ROOT;
+  if (!respectGitignore || !existsSync(path.join(projectRoot, '.git'))) {
+    return { files, gitignored: 0 };
+  }
+
+  const bypassGitignore = new Set();
+  const candidates = [];
+  for (const file of files) {
+    if (isGitignoreAllowlistedAgenticPath(file)) {
+      bypassGitignore.add(file);
+    } else {
+      candidates.push(file);
+    }
+  }
+
+  const ignoredSet = await getGitIgnoredPathSet(candidates, { projectRoot, silent: options.silent });
+  if (!ignoredSet) {
+    return { files, gitignored: 0 };
+  }
+
+  const kept = [];
+  let gitignored = 0;
+  for (const file of files) {
+    if (bypassGitignore.has(file)) {
+      kept.push(file);
+      continue;
+    }
+
+    const normalized = toPosixPath(file);
+    if (ignoredSet.has(normalized)) {
+      gitignored++;
+      continue;
+    }
+    kept.push(file);
+  }
+
+  return { files: kept, gitignored };
+}