raggrep 0.9.0 → 0.10.1

package/dist/infrastructure/index.d.ts

@@ -9,3 +9,4 @@ export { TransformersEmbeddingProvider, getCacheDir, isModelCached, } from "./em
 export { FileIndexStorage, SymbolicIndex, getSymbolicPath } from "./storage";
 export { DEFAULT_CONFIG, EMBEDDING_MODELS, getRaggrepDir, getModuleIndexPath, getModuleManifestPath, getGlobalManifestPath, getConfigPath, loadConfig, saveConfig, getModuleConfig, getEmbeddingConfigFromModule, } from "./config";
 export { ConsoleLogger, InlineProgressLogger, SilentLogger, createLogger, createInlineLogger, createSilentLogger, } from "./logger";
+export { TypeScriptParser, TreeSitterParser, GrammarManager, getGrammarManager, createParserForFile, createParserForLanguage, detectLanguage, detectLanguagesFromFiles, isFileSupported, getSupportedExtensions, getSupportedLanguages, } from "./parsing";

package/dist/infrastructure/introspection/IntrospectionIndex.d.ts

@@ -24,8 +24,12 @@ export declare class IntrospectionIndex {
     getStructure(): ProjectStructure | null;
     /**
      * Introspect a file and add to index.
+     *
+     * @param filepath - Relative file path
+     * @param content - Optional file content for framework detection
+     * @param enableReadmeContext - Whether to look for README files (default: true)
      */
-    addFile(filepath: string, content?: string): FileIntrospection;
+    addFile(filepath: string, content?: string, enableReadmeContext?: boolean): FileIntrospection;
     /**
      * Get introspection for a file.
      */

package/dist/infrastructure/parsing/grammarManager.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Grammar Manager
+ *
+ * Manages tree-sitter grammar installation and loading.
+ * Uses Bun for dynamic package installation on first use.
+ *
+ * This is an infrastructure component that handles:
+ * - Grammar package installation via Bun
+ * - Grammar caching in memory
+ * - Thread-safe concurrent installation handling
+ */
+import type { IGrammarManager, GrammarStatus, ParserLanguage } from "../../domain/ports/parser";
+import type { Logger } from "../../domain/ports/logger";
+/**
+ * Singleton Grammar Manager for tree-sitter grammars.
+ *
+ * Handles:
+ * - Checking if grammars are installed
+ * - Installing grammars via Bun on demand
+ * - Caching loaded grammars
+ * - Thread-safe concurrent installation
+ */
+export declare class GrammarManager implements IGrammarManager {
+    private static instance;
+    /** Cache of loaded grammars */
+    private grammarCache;
+    /** Track installation status */
+    private installationStatus;
+    /** Track pending installations to prevent duplicate installs */
+    private pendingInstalls;
+    /** Optional logger for progress reporting */
+    private logger?;
+    private constructor();
+    /**
+     * Get the singleton instance.
+     */
+    static getInstance(): GrammarManager;
+    /**
+     * Set a logger for progress reporting.
+     */
+    setLogger(logger: Logger): void;
+    /**
+     * Check if a grammar package is installed.
+     */
+    isInstalled(language: ParserLanguage): Promise<boolean>;
+    /**
+     * Install a grammar for a language using Bun.
+     *
+     * Thread-safe: concurrent calls for the same language will wait
+     * for the first installation to complete.
+     */
+    install(language: ParserLanguage): Promise<GrammarStatus>;
+    /**
+     * Perform the actual installation using Bun.
+     */
+    private doInstall;
+    /**
+     * Get the status of all supported grammars.
+     */
+    getStatus(): Promise<GrammarStatus[]>;
+    /**
+     * Pre-install grammars for a batch of languages.
+     * Shows overall progress and handles failures gracefully.
+     */
+    preInstallBatch(languages: ParserLanguage[]): Promise<GrammarStatus[]>;
+    /**
+     * Load a grammar module.
+     * Returns the grammar if installed, null otherwise.
+     */
+    loadGrammar(language: ParserLanguage): Promise<unknown | null>;
+    /**
+     * Get the package name for a language.
+     */
+    getPackageName(language: ParserLanguage): string | undefined;
+    /**
+     * Clear cached grammars (mainly for testing).
+     */
+    clearCache(): void;
+}
+/**
+ * Get the singleton grammar manager instance.
+ */
+export declare function getGrammarManager(): GrammarManager;

package/dist/infrastructure/parsing/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Parsing Infrastructure
+ *
+ * Provides parser implementations for different programming languages.
+ *
+ * Exports:
+ * - TypeScriptParser: Uses TypeScript Compiler API for TS/JS
+ * - TreeSitterParser: Uses tree-sitter for Python, Go, Rust, Java
+ * - GrammarManager: Manages tree-sitter grammar installation
+ * - Parser factory functions for automatic parser selection
+ */
+export { TypeScriptParser, createTypeScriptParser } from "./typescriptParser";
+export { TreeSitterParser, createTreeSitterParser } from "./treeSitterParser";
+export { GrammarManager, getGrammarManager } from "./grammarManager";
+export { createParserForFile, createParserForLanguage, detectLanguage, detectLanguagesFromFiles, isFileSupported, getSupportedExtensions, getSupportedLanguages, } from "./parserFactory";

package/dist/infrastructure/parsing/parserFactory.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Parser Factory
+ *
+ * Creates the appropriate parser for a given file or language.
+ * Implements the strategy pattern for parser selection.
+ *
+ * Strategy:
+ * - TypeScript/JavaScript: Use TypeScriptParser (TypeScript Compiler API)
+ * - Python, Go, Rust, Java: Use TreeSitterParser
+ * - Unknown: Fall back to TreeSitterParser or null
+ */
+import type { IParser, ParserLanguage } from "../../domain/ports/parser";
+/**
+ * Create a parser for the given file.
+ *
+ * @param filepath - The file path to get a parser for
+ * @returns The appropriate parser, or null if no parser supports the file
+ */
+export declare function createParserForFile(filepath: string): IParser | null;
+/**
+ * Create a parser for the given language.
+ *
+ * @param language - The language to get a parser for
+ * @returns The appropriate parser
+ */
+export declare function createParserForLanguage(language: ParserLanguage): IParser;
+/**
+ * Detect the language from a file path.
+ *
+ * @param filepath - The file path to detect language from
+ * @returns The detected language, or null if unknown
+ */
+export declare function detectLanguage(filepath: string): ParserLanguage | null;
+/**
+ * Detect languages from a list of file paths.
+ * Returns unique languages found.
+ *
+ * @param filepaths - Array of file paths
+ * @returns Set of detected languages
+ */
+export declare function detectLanguagesFromFiles(filepaths: string[]): Set<ParserLanguage>;
+/**
+ * Check if a file is supported by any parser.
+ *
+ * @param filepath - The file path to check
+ * @returns True if a parser can handle this file
+ */
+export declare function isFileSupported(filepath: string): boolean;
+/**
+ * Get all supported file extensions.
+ */
+export declare function getSupportedExtensions(): string[];
+/**
+ * Get all supported languages.
+ */
+export declare function getSupportedLanguages(): ParserLanguage[];

package/dist/infrastructure/parsing/parsing.test.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Parser Infrastructure Tests
+ *
+ * Tests for the parser infrastructure including:
+ * - TypeScriptParser (TypeScript Compiler API wrapper)
+ * - TreeSitterParser (web-tree-sitter based)
+ * - Parser factory functions
+ * - Language detection
+ */
+export {};

package/dist/infrastructure/parsing/treeSitterParser.d.ts

@@ -0,0 +1,103 @@
+/**
+ * Tree-sitter Parser
+ *
+ * Uses web-tree-sitter (WebAssembly) for parsing multiple languages.
+ * This parser supports Python, Go, Rust, Java, and can act as a fallback for TS/JS.
+ *
+ * Web-tree-sitter provides:
+ * - Cross-platform compatibility (no native compilation)
+ * - Fast, incremental parsing
+ * - Support for many languages via WASM grammars
+ * - Consistent AST structure
+ */
+import type { IParser, ParseResult, ParserConfig, ParserLanguage } from "../../domain/ports/parser";
+/**
+ * Tree-sitter Parser implementation using web-tree-sitter (WebAssembly).
+ *
+ * Supports multiple languages through WASM grammar files.
+ * Falls back to basic chunking if grammar is not available.
+ */
+export declare class TreeSitterParser implements IParser {
+    readonly supportedLanguages: ParserLanguage[];
+    private grammarManager;
+    private parserInstance;
+    private loadedLanguages;
+    private initPromise;
+    /**
+     * Parse source code into semantic chunks using tree-sitter.
+     */
+    parse(content: string, filepath: string, config?: ParserConfig): Promise<ParseResult>;
+    /**
+     * Check if this parser can handle the given file.
+     */
+    canParse(filepath: string): boolean;
+    /**
+     * Detect language from file extension.
+     */
+    private detectLanguage;
+    /**
+     * Ensure tree-sitter is initialized.
+     */
+    private ensureInitialized;
+    /**
+     * Initialize web-tree-sitter.
+     */
+    private initializeTreeSitter;
+    /**
+     * Load a language module for tree-sitter.
+     * Returns null if the language is not available.
+     */
+    private loadLanguage;
+    /**
+     * Parse source code with tree-sitter.
+     */
+    private parseWithTreeSitter;
+    /**
+     * Extract chunks from Python AST.
+     */
+    private extractPythonChunks;
+    /**
+     * Extract chunks from Go AST.
+     */
+    private extractGoChunks;
+    /**
+     * Extract chunks from Rust AST.
+     */
+    private extractRustChunks;
+    /**
+     * Extract chunks from Java AST.
+     */
+    private extractJavaChunks;
+    /**
+     * Extract generic chunks.
+     */
+    private extractGenericChunks;
+    /**
+     * Create a chunk from a tree-sitter node.
+     */
+    private createChunkFromNode;
+    /**
+     * Get the text content of a tree-sitter node.
+     */
+    private getNodeText;
+    /**
+     * Find a preceding comment (for Go-style comments).
+     */
+    private findPrecedingComment;
+    /**
+     * Find Rust doc comments (/// or //!).
+     */
+    private findRustDocComment;
+    /**
+     * Find Javadoc comments.
+     */
+    private findJavadoc;
+    /**
+     * Fall back to basic line-based parsing.
+     */
+    private fallbackParse;
+}
+/**
+ * Create a new tree-sitter parser instance.
+ */
+export declare function createTreeSitterParser(): TreeSitterParser;

package/dist/infrastructure/parsing/typescriptParser.d.ts

@@ -0,0 +1,43 @@
+/**
+ * TypeScript Parser
+ *
+ * Wraps the existing TypeScript Compiler API-based parser in the IParser interface.
+ * This provides superior type information and JSDoc parsing for TS/JS files.
+ *
+ * For TypeScript and JavaScript files, this parser is preferred over tree-sitter
+ * because it has access to full type information and proven quality.
+ */
+import type { IParser, ParseResult, ParserConfig, ParserLanguage } from "../../domain/ports/parser";
+/**
+ * TypeScript Parser implementation using TypeScript Compiler API.
+ *
+ * This is the primary parser for TypeScript and JavaScript files.
+ * It provides:
+ * - Accurate AST parsing
+ * - Full JSDoc extraction
+ * - Export detection
+ * - Type information
+ */
+export declare class TypeScriptParser implements IParser {
+    readonly supportedLanguages: ParserLanguage[];
+    /**
+     * Parse TypeScript/JavaScript source code into semantic chunks.
+     */
+    parse(content: string, filepath: string, config?: ParserConfig): Promise<ParseResult>;
+    /**
+     * Check if this parser can handle the given file.
+     */
+    canParse(filepath: string): boolean;
+    /**
+     * Convert from the existing ParsedChunk format to the domain ParsedChunk format.
+     */
+    private convertChunk;
+    /**
+     * Detect language from file extension.
+     */
+    private detectLanguage;
+}
+/**
+ * Create a new TypeScript parser instance.
+ */
+export declare function createTypeScriptParser(): TypeScriptParser;

package/dist/infrastructure/storage/literalIndex.d.ts

@@ -8,12 +8,16 @@
  *   .raggrep/index/<module>/literals/
  *   └── _index.json    (literal → chunk mappings)
  */
-import type { ExtractedLiteral, DetectedLiteral, LiteralMatch } from "../../domain/entities/literal";
+import type { ExtractedLiteral, DetectedLiteral, LiteralMatch, LiteralIndexEntry } from "../../domain/entities/literal";
 /**
  * Literal Index Manager
  *
  * Manages the literal index for exact-match boosting.
  * Provides O(1) lookup for literal matches.
+ *
+ * Now also supports vocabulary-based search for partial matching:
+ * - Store vocabulary words extracted from literals
+ * - Search by vocabulary words to find partial matches
  */
 export declare class LiteralIndex {
     private indexPath;
@@ -22,6 +26,11 @@ export declare class LiteralIndex {
      * In-memory index: lowercase literal value → entries
      */
     private entries;
+    /**
+     * Vocabulary index: vocabulary word → literal values that contain it
+     * Used for partial matching (e.g., "user" → ["getUserById", "fetchUserData"])
+     */
+    private vocabularyIndex;
     /**
      * Schema version for compatibility checking.
      */
@@ -32,6 +41,11 @@ export declare class LiteralIndex {
      * Attempts to load from disk, creates empty index if not found.
      */
     initialize(): Promise<void>;
+    /**
+     * Rebuild the vocabulary index from entries.
+     * Called after loading from disk.
+     */
+    private rebuildVocabularyIndex;
     /**
      * Add literals from a chunk to the index.
      *
@@ -68,6 +82,24 @@ export declare class LiteralIndex {
      * @returns Array of chunk IDs
      */
     getChunksForLiteral(literal: string): string[];
+    /**
+     * Find literals that contain a specific vocabulary word.
+     *
+     * @param word - The vocabulary word to search for
+     * @returns Array of literal entries that contain this word
+     */
+    findByVocabulary(word: string): LiteralIndexEntry[];
+    /**
+     * Find matches for query by vocabulary words.
+     * Returns literals that contain any of the given vocabulary words.
+     *
+     * @param vocabularyWords - Words to search for
+     * @returns Array of matches with overlap information
+     */
+    findByVocabularyWords(vocabularyWords: string[]): Array<{
+        entry: LiteralIndexEntry;
+        matchedWords: string[];
+    }>;
     /**
      * Save the index to disk.
      */

package/dist/modules/language/go/index.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Go Language Index Module
+ *
+ * Provides Go-aware code search using:
+ * - AST parsing via tree-sitter (with regex fallback)
+ * - Local text embeddings for semantic similarity
+ * - BM25 keyword matching for fast filtering
+ *
+ * Supported file types: .go
+ *
+ * Index location: .raggrep/index/language/go/
+ */
+import { IndexModule, IndexContext, SearchContext, SearchOptions, FileIndex, SearchResult, ModuleConfig } from "../../../types";
+/** Default minimum similarity score for search results */
+export declare const DEFAULT_MIN_SCORE = 0.15;
+/** Default number of results to return */
+export declare const DEFAULT_TOP_K = 10;
+/** File extensions supported by this module */
+export declare const GO_EXTENSIONS: string[];
+/**
+ * Check if a file is supported by this module.
+ */
+export declare function isGoFile(filepath: string): boolean;
+export declare const supportsFile: typeof isGoFile;
+/**
+ * Module-specific data stored alongside file index.
+ */
+export interface GoModuleData {
+    embeddings: number[][];
+    embeddingModel: string;
+    [key: string]: unknown;
+}
+export declare class GoModule implements IndexModule {
+    readonly id = "language/go";
+    readonly name = "Go Search";
+    readonly description = "Go-aware code search with AST parsing and semantic embeddings";
+    readonly version = "1.0.0";
+    supportsFile(filepath: string): boolean;
+    private embeddingConfig;
+    private symbolicIndex;
+    private literalIndex;
+    private pendingSummaries;
+    private pendingLiterals;
+    private rootDir;
+    private logger;
+    initialize(config: ModuleConfig): Promise<void>;
+    indexFile(filepath: string, content: string, ctx: IndexContext): Promise<FileIndex | null>;
+    /**
+     * Regex-based fallback parser for Go.
+     */
+    private parseGoRegex;
+    /**
+     * Create file index from parsed chunks.
+     */
+    private createFileIndex;
+    finalize(ctx: IndexContext): Promise<void>;
+    search(query: string, ctx: SearchContext, options?: SearchOptions): Promise<SearchResult[]>;
+}

package/dist/modules/language/go/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/modules/language/python/index.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Python Language Index Module
+ *
+ * Provides Python-aware code search using:
+ * - AST parsing via tree-sitter (with regex fallback)
+ * - Local text embeddings for semantic similarity
+ * - BM25 keyword matching for fast filtering
+ *
+ * Supported file types: .py, .pyw
+ *
+ * Index location: .raggrep/index/language/python/
+ */
+import { IndexModule, IndexContext, SearchContext, SearchOptions, FileIndex, SearchResult, ModuleConfig } from "../../../types";
+/** Default minimum similarity score for search results */
+export declare const DEFAULT_MIN_SCORE = 0.15;
+/** Default number of results to return */
+export declare const DEFAULT_TOP_K = 10;
+/** File extensions supported by this module */
+export declare const PYTHON_EXTENSIONS: string[];
+/**
+ * Check if a file is supported by this module.
+ */
+export declare function isPythonFile(filepath: string): boolean;
+export declare const supportsFile: typeof isPythonFile;
+/**
+ * Module-specific data stored alongside file index.
+ */
+export interface PythonModuleData {
+    embeddings: number[][];
+    embeddingModel: string;
+    [key: string]: unknown;
+}
+export declare class PythonModule implements IndexModule {
+    readonly id = "language/python";
+    readonly name = "Python Search";
+    readonly description = "Python-aware code search with AST parsing and semantic embeddings";
+    readonly version = "1.0.0";
+    supportsFile(filepath: string): boolean;
+    private embeddingConfig;
+    private symbolicIndex;
+    private literalIndex;
+    private pendingSummaries;
+    private pendingLiterals;
+    private rootDir;
+    private logger;
+    initialize(config: ModuleConfig): Promise<void>;
+    indexFile(filepath: string, content: string, ctx: IndexContext): Promise<FileIndex | null>;
+    /**
+     * Regex-based fallback parser for Python.
+     * Used when tree-sitter is not available.
+     */
+    private parsePythonRegex;
+    /**
+     * Create file index from parsed chunks.
+     */
+    private createFileIndex;
+    finalize(ctx: IndexContext): Promise<void>;
+    search(query: string, ctx: SearchContext, options?: SearchOptions): Promise<SearchResult[]>;
+}

package/dist/modules/language/rust/index.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Rust Language Index Module
+ *
+ * Provides Rust-aware code search using:
+ * - AST parsing via tree-sitter (with regex fallback)
+ * - Local text embeddings for semantic similarity
+ * - BM25 keyword matching for fast filtering
+ *
+ * Supported file types: .rs
+ *
+ * Index location: .raggrep/index/language/rust/
+ */
+import { IndexModule, IndexContext, SearchContext, SearchOptions, FileIndex, SearchResult, ModuleConfig } from "../../../types";
+/** Default minimum similarity score for search results */
+export declare const DEFAULT_MIN_SCORE = 0.15;
+/** Default number of results to return */
+export declare const DEFAULT_TOP_K = 10;
+/** File extensions supported by this module */
+export declare const RUST_EXTENSIONS: string[];
+/**
+ * Check if a file is supported by this module.
+ */
+export declare function isRustFile(filepath: string): boolean;
+export declare const supportsFile: typeof isRustFile;
+/**
+ * Module-specific data stored alongside file index.
+ */
+export interface RustModuleData {
+    embeddings: number[][];
+    embeddingModel: string;
+    [key: string]: unknown;
+}
+export declare class RustModule implements IndexModule {
+    readonly id = "language/rust";
+    readonly name = "Rust Search";
+    readonly description = "Rust-aware code search with AST parsing and semantic embeddings";
+    readonly version = "1.0.0";
+    supportsFile(filepath: string): boolean;
+    private embeddingConfig;
+    private symbolicIndex;
+    private literalIndex;
+    private pendingSummaries;
+    private pendingLiterals;
+    private rootDir;
+    private logger;
+    initialize(config: ModuleConfig): Promise<void>;
+    indexFile(filepath: string, content: string, ctx: IndexContext): Promise<FileIndex | null>;
+    /**
+     * Regex-based fallback parser for Rust.
+     */
+    private parseRustRegex;
+    /**
+     * Create file index from parsed chunks.
+     */
+    private createFileIndex;
+    finalize(ctx: IndexContext): Promise<void>;
+    search(query: string, ctx: SearchContext, options?: SearchOptions): Promise<SearchResult[]>;
+}

package/dist/modules/language/rust/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/tests/vocabulary.test.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Vocabulary Matching Tests
+ *
+ * Tests for vocabulary extraction and matching capabilities.
+ * These tests verify that:
+ * - Vocabulary is correctly extracted from symbol names
+ * - Partial vocabulary matches work in search
+ * - Scoring tiers are applied correctly
+ */
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "raggrep",
-  "version": "0.9.0",
+  "version": "0.10.1",
   "description": "Local filesystem-based RAG system for codebases - semantic search using local embeddings",
   "type": "module",
   "main": "./dist/index.js",
@@ -58,7 +58,8 @@
     "@xenova/transformers": "^2.17.0",
     "chokidar": "^5.0.0",
     "glob": "^10.0.0",
-    "typescript": "^5.0.0"
+    "typescript": "^5.0.0",
+    "web-tree-sitter": "^0.26.3"
   },
   "devDependencies": {
     "@types/bun": "latest",