viberag 0.1.0

package/dist/rag/gitignore/index.js

@@ -0,0 +1,178 @@
+/**
+ * Gitignore-based file filtering.
+ *
+ * Uses the `ignore` package to parse .gitignore files and filter paths.
+ * This replaces the hardcoded excludePatterns approach.
+ */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { createRequire } from 'node:module';
+// ignore is a CJS module, use createRequire to import it
+const require = createRequire(import.meta.url);
+const ignore = require('ignore');
+/**
+ * Patterns that should always be ignored, regardless of .gitignore.
+ * These are internal/system directories that should never be indexed.
+ */
+const ALWAYS_IGNORED = [
+    '.git',
+    '.viberag',
+    'node_modules', // Fallback in case not in .gitignore
+];
+/**
+ * Cache of Ignore instances per project root.
+ */
+const ignoreCache = new Map();
+/**
+ * Load and parse .gitignore file from project root.
+ * Returns an Ignore instance that can filter paths.
+ *
+ * @param projectRoot - Project root directory
+ * @returns Ignore instance for filtering
+ */
+export async function loadGitignore(projectRoot) {
+    // Check cache first
+    const cached = ignoreCache.get(projectRoot);
+    if (cached) {
+        return cached;
+    }
+    const ig = ignore();
+    // Add always-ignored patterns
+    ig.add(ALWAYS_IGNORED);
+    // Try to load .gitignore
+    const gitignorePath = path.join(projectRoot, '.gitignore');
+    try {
+        const content = await fs.readFile(gitignorePath, 'utf-8');
+        ig.add(content);
+    }
+    catch {
+        // .gitignore doesn't exist, that's fine
+    }
+    // Cache the instance
+    ignoreCache.set(projectRoot, ig);
+    return ig;
+}
+/**
+ * Check if a path should be ignored based on .gitignore rules.
+ *
+ * @param projectRoot - Project root directory
+ * @param relativePath - Path relative to project root
+ * @returns true if the path should be ignored
+ */
+export async function shouldIgnore(projectRoot, relativePath) {
+    const ig = await loadGitignore(projectRoot);
+    return ig.ignores(relativePath);
+}
+/**
+ * Create a filter function for use with file listing.
+ * The filter returns true for files that should be INCLUDED (not ignored).
+ *
+ * @param projectRoot - Project root directory
+ * @returns Filter function that returns true for non-ignored files
+ */
+export async function createGitignoreFilter(projectRoot) {
+    const ig = await loadGitignore(projectRoot);
+    return (relativePath) => !ig.ignores(relativePath);
+}
+/**
+ * Clear the cache for a specific project root.
+ * Call this if .gitignore has been modified.
+ *
+ * @param projectRoot - Project root directory
+ */
+export function clearGitignoreCache(projectRoot) {
+    ignoreCache.delete(projectRoot);
+}
+/**
+ * Clear all cached Ignore instances.
+ */
+export function clearAllGitignoreCache() {
+    ignoreCache.clear();
+}
+/**
+ * Convert gitignore patterns to fast-glob ignore patterns.
+ * This allows fast-glob to skip directories upfront instead of
+ * scanning them and filtering later.
+ *
+ * @param projectRoot - Project root directory
+ * @returns Array of fast-glob compatible ignore patterns
+ */
+export async function getGlobIgnorePatterns(projectRoot) {
+    const patterns = [];
+    // Always exclude these (same as ALWAYS_IGNORED)
+    patterns.push('**/.git/**', '**/.viberag/**', '**/node_modules/**');
+    // Try to load .gitignore
+    const gitignorePath = path.join(projectRoot, '.gitignore');
+    try {
+        const content = await fs.readFile(gitignorePath, 'utf-8');
+        const lines = content.split('\n');
+        for (const line of lines) {
+            const trimmed = line.trim();
+            // Skip empty lines and comments
+            if (!trimmed || trimmed.startsWith('#')) {
+                continue;
+            }
+            // Skip negation patterns (fast-glob handles these differently)
+            if (trimmed.startsWith('!')) {
+                continue;
+            }
+            // Convert gitignore pattern to fast-glob pattern
+            const globPattern = gitignoreToGlob(trimmed);
+            if (globPattern) {
+                patterns.push(globPattern);
+            }
+        }
+    }
+    catch {
+        // .gitignore doesn't exist, that's fine
+    }
+    return patterns;
+}
+/**
+ * Convert a single gitignore pattern to a fast-glob pattern.
+ *
+ * Gitignore patterns:
+ * - `foo` matches `foo` anywhere
+ * - `foo/` matches directory `foo` anywhere
+ * - `/foo` matches `foo` only at root
+ * - `*.log` matches `*.log` anywhere
+ *
+ * Fast-glob patterns:
+ * - Need `**/ ` prefix to match anywhere
+ * - Need `; /**` suffix to match directory contents
+*/
+function gitignoreToGlob(pattern) {
+    let result = pattern;
+    // Handle rooted patterns (start with /)
+    const isRooted = result.startsWith('/');
+    if (isRooted) {
+        result = result.slice(1);
+    }
+    // Handle directory patterns (end with /)
+    const isDirectory = result.endsWith('/');
+    if (isDirectory) {
+        result = result.slice(0, -1);
+    }
+    // Skip patterns that are already glob-like with **
+    const hasDoublestar = result.includes('**');
+    // Build the glob pattern
+    if (isRooted) {
+        // Rooted: match only at project root
+        result = isDirectory ? `${result}/**` : result;
+    }
+    else if (!hasDoublestar) {
+        // Non-rooted: match anywhere in tree
+        result = isDirectory ? `**/${result}/**` : `**/${result}`;
+        // If it doesn't look like a directory name (has extension or glob),
+        // don't add trailing /**
+        if (!isDirectory &&
+            (result.includes('.') || result.includes('*') || result.includes('?'))) {
+            // Keep as-is, it's likely a file pattern
+        }
+        else if (!isDirectory) {
+            // Bare name like "node_modules" - treat as directory
+            result = `**/${pattern}/**`;
+        }
+    }
+    return result;
+}

package/dist/rag/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * RAG Engine Core
+ *
+ * Local codebase indexing with hybrid search (vector + BM25).
+ */
+export { VIBERAG_DIR, getViberagDir, getConfigPath, getManifestPath, getLanceDbPath, getLogsDir, TABLE_NAMES, EXTENSION_TO_LANGUAGE, DEFAULT_EMBEDDING_DIMENSIONS, } from './constants.js';
+export { createLogger, createNullLogger, getLogPath, type Logger, type LogLevel, } from './logger/index.js';
+export { loadConfig, saveConfig, configExists, DEFAULT_CONFIG, DEFAULT_WATCH_CONFIG, PROVIDER_CONFIGS, type ViberagConfig, type WatchConfig, type EmbeddingProviderType, } from './config/index.js';
+export { loadManifest, saveManifest, manifestExists, createEmptyManifest, updateManifestStats, updateManifestTree, isSchemaVersionCurrent, getSchemaVersionInfo, type Manifest, type ManifestStats, } from './manifest/index.js';
+export { Storage, SCHEMA_VERSION, createCodeChunksSchema, createEmbeddingCacheSchema, chunkToRow, rowToChunk, embeddingToRow, rowToEmbedding, type CodeChunk, type CodeChunkRow, type CachedEmbedding, type CachedEmbeddingRow, type ChunkType, } from './storage/index.js';
+export { MerkleTree, compareTrees, createEmptyDiff, computeFileHash, computeStringHash, computeDirectoryHash, isBinaryFile, shouldExclude, hasValidExtension, serializeNode, deserializeNode, createFileNode, createDirectoryNode, type MerkleNode, type NodeType, type SerializedNode, type TreeDiff, type BuildStats, } from './merkle/index.js';
+export { Chunker, Indexer, createEmptyIndexStats, type Chunk, type IndexOptions, type IndexStats, type ProgressCallback, type SupportedLanguage, } from './indexer/index.js';
+export { GeminiEmbeddingProvider, MistralEmbeddingProvider, OpenAIEmbeddingProvider, type EmbeddingProvider, } from './embeddings/index.js';
+export { SearchEngine, vectorSearch, ftsSearch, ensureFtsIndex, hybridRerank, type SearchFilters, type SearchMode, type SearchOptions, type SearchResult, type SearchResults, } from './search/index.js';
+export { loadGitignore, shouldIgnore, createGitignoreFilter, clearGitignoreCache, clearAllGitignoreCache, } from './gitignore/index.js';

package/dist/rag/index.js

@@ -0,0 +1,25 @@
+/**
+ * RAG Engine Core
+ *
+ * Local codebase indexing with hybrid search (vector + BM25).
+ */
+// Constants
+export { VIBERAG_DIR, getViberagDir, getConfigPath, getManifestPath, getLanceDbPath, getLogsDir, TABLE_NAMES, EXTENSION_TO_LANGUAGE, DEFAULT_EMBEDDING_DIMENSIONS, } from './constants.js';
+// Logger
+export { createLogger, createNullLogger, getLogPath, } from './logger/index.js';
+// Config
+export { loadConfig, saveConfig, configExists, DEFAULT_CONFIG, DEFAULT_WATCH_CONFIG, PROVIDER_CONFIGS, } from './config/index.js';
+// Manifest
+export { loadManifest, saveManifest, manifestExists, createEmptyManifest, updateManifestStats, updateManifestTree, isSchemaVersionCurrent, getSchemaVersionInfo, } from './manifest/index.js';
+// Storage
+export { Storage, SCHEMA_VERSION, createCodeChunksSchema, createEmbeddingCacheSchema, chunkToRow, rowToChunk, embeddingToRow, rowToEmbedding, } from './storage/index.js';
+// Merkle Tree
+export { MerkleTree, compareTrees, createEmptyDiff, computeFileHash, computeStringHash, computeDirectoryHash, isBinaryFile, shouldExclude, hasValidExtension, serializeNode, deserializeNode, createFileNode, createDirectoryNode, } from './merkle/index.js';
+// Indexer (Chunking & Orchestration)
+export { Chunker, Indexer, createEmptyIndexStats, } from './indexer/index.js';
+// Embeddings
+export { GeminiEmbeddingProvider, MistralEmbeddingProvider, OpenAIEmbeddingProvider, } from './embeddings/index.js';
+// Search
+export { SearchEngine, vectorSearch, ftsSearch, ensureFtsIndex, hybridRerank, } from './search/index.js';
+// Gitignore
+export { loadGitignore, shouldIgnore, createGitignoreFilter, clearGitignoreCache, clearAllGitignoreCache, } from './gitignore/index.js';

package/dist/rag/indexer/chunker.d.ts

@@ -0,0 +1,129 @@
+import { type Chunk, type SupportedLanguage } from './types.js';
+/**
+ * Chunker that uses web-tree-sitter (WASM) to extract semantic code chunks.
+ * Provides 100% platform compatibility - no native compilation required.
+ */
+export declare class Chunker {
+    private parser;
+    private languages;
+    private initialized;
+    private wasmBasePath;
+    constructor();
+    /**
+     * Initialize web-tree-sitter and load language grammars.
+     * Must be called before using chunkFile().
+     */
+    initialize(): Promise<void>;
+    /**
+     * Get the language for a file extension.
+     */
+    getLanguageForExtension(ext: string): SupportedLanguage | null;
+    /**
+     * Check if a language is supported.
+     */
+    isLanguageSupported(lang: SupportedLanguage): boolean;
+    /**
+     * Check if a file is a markdown file.
+     */
+    private isMarkdownFile;
+    /**
+     * Extract chunks from a file.
+     *
+     * @param filepath - Path to the file (used for extension detection and context headers)
+     * @param content - File content to parse
+     * @param maxChunkSize - Maximum chunk size in characters (default: 2000)
+     * @returns Array of extracted chunks
+     */
+    chunkFile(filepath: string, content: string, maxChunkSize?: number): Chunk[];
+    /**
+     * Extract chunks from a syntax tree.
+     */
+    private extractChunks;
+    /**
+     * Recursively traverse nodes to extract chunks.
+     * Tracks parent context (class name) for context headers.
+     */
+    private traverseNode;
+    /**
+     * Convert a syntax node to a chunk.
+     */
+    private nodeToChunk;
+    /**
+     * Extract the signature line (first line of function/class declaration).
+     */
+    private extractSignature;
+    /**
+     * Extract docstring from a function/class node.
+     */
+    private extractDocstring;
+    /**
+     * Check if a node is exported/public.
+     */
+    private extractIsExported;
+    /**
+     * Helper to check for visibility modifiers in a node.
+     */
+    private hasVisibilityModifier;
+    /**
+     * Extract decorator names from a node.
+     */
+    private extractDecoratorNames;
+    /**
+     * Helper to find a child node of specific types.
+     */
+    private findChildOfType;
+    /**
+     * Build a context header for a chunk.
+     */
+    private buildContextHeader;
+    /**
+     * Extract the name of a function/class/method from its node.
+     */
+    private extractName;
+    /**
+     * Create a module-level chunk for the entire file.
+     */
+    private createModuleChunk;
+    /**
+     * Chunk markdown files with heading-aware splitting and overlap.
+     *
+     * Strategy:
+     * 1. Try to split at heading boundaries (# lines)
+     * 2. Use sliding window with overlap between chunks
+     * 3. Merge small final chunks to avoid orphans
+     */
+    private chunkMarkdown;
+    /**
+     * Create a chunk from markdown content.
+     */
+    private createMarkdownChunk;
+    /**
+     * Enforce size limits: split oversized chunks and merge tiny ones.
+     *
+     * @param overlapLines - Number of lines to overlap between chunks (for context continuity)
+     */
+    private enforceSizeLimits;
+    /**
+     * Split an oversized chunk by lines.
+     * Tries to split at natural boundaries (empty lines, statement ends).
+     *
+     * @param overlapLines - Number of lines from previous chunk to include for context
+     */
+    private splitChunkByLines;
+    /**
+     * Create a chunk from a split portion.
+     */
+    private createSplitChunk;
+    /**
+     * Extract class name from a context header string.
+     */
+    private extractClassFromContext;
+    /**
+     * Merge small adjacent chunks of the same type to avoid fragment explosion.
+     */
+    private mergeSmallChunks;
+    /**
+     * Close the parser and free resources.
+     */
+    close(): void;
+}