viberag 0.1.0

package/dist/rag/merkle/index.js

@@ -0,0 +1,298 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import fg from 'fast-glob';
+import { createDirectoryNode, createFileNode, deserializeNode, serializeNode, } from './node.js';
+import { computeDirectoryHash, computeFileHash, hasValidExtension, isBinaryFile, } from './hash.js';
+import { compareTrees } from './diff.js';
+import { loadGitignore, getGlobIgnorePatterns } from '../gitignore/index.js';
+export * from './node.js';
+export * from './hash.js';
+export * from './diff.js';
+/**
+ * A Merkle tree for efficient codebase change detection.
+ *
+ * The tree is content-addressed: if a file's content doesn't change,
+ * its hash stays the same. Directory hashes are computed from their
+ * children's hashes, so unchanged subtrees have unchanged hashes.
+ */
+export class MerkleTree {
+    constructor(root, fileCount, buildStats) {
+        /** Root node of the tree */
+        Object.defineProperty(this, "root", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** Total number of files in the tree */
+        Object.defineProperty(this, "fileCount", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** Build statistics (populated after build) */
+        Object.defineProperty(this, "buildStats", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.root = root;
+        this.fileCount = fileCount;
+        this.buildStats = buildStats;
+    }
+    /**
+     * Build a Merkle tree from the filesystem.
+     *
+     * Uses .gitignore for exclusions instead of hardcoded patterns.
+     * If extensions is provided, only files with those extensions are included.
+     * If extensions is empty/undefined, all text files are included.
+     *
+     * @param projectRoot - Absolute path to project root
+     * @param extensions - File extensions to include (e.g., [".py", ".ts"]), or empty for all
+     * @param _excludePatterns - DEPRECATED: Use .gitignore instead. This parameter is ignored.
+     * @param previousTree - Previous tree for mtime optimization
+     */
+    static async build(projectRoot, extensions, _excludePatterns, previousTree) {
+        // Build a lookup map from the previous tree for mtime optimization
+        const previousNodes = previousTree
+            ? buildNodeLookup(previousTree.root)
+            : new Map();
+        // Initialize build stats
+        const stats = {
+            filesScanned: 0,
+            filesIndexed: 0,
+            cacheHits: 0,
+            cacheMisses: 0,
+            filesSkipped: 0,
+        };
+        // Load gitignore rules for both fast-glob (upfront filtering) and post-filtering
+        const [gitignore, globIgnorePatterns] = await Promise.all([
+            loadGitignore(projectRoot),
+            getGlobIgnorePatterns(projectRoot),
+        ]);
+        // Find all files - use gitignore patterns upfront to avoid scanning excluded dirs
+        const pattern = '**/*';
+        const files = await fg(pattern, {
+            cwd: projectRoot,
+            dot: true,
+            onlyFiles: true,
+            followSymbolicLinks: false, // Skip symlinks
+            // Apply gitignore patterns upfront so fast-glob skips excluded directories
+            ignore: globIgnorePatterns,
+        });
+        stats.filesScanned = files.length;
+        // Filter using gitignore and optionally by extension
+        const validFiles = files.filter(relativePath => {
+            // Apply gitignore rules
+            if (gitignore.ignores(relativePath)) {
+                return false;
+            }
+            // If extensions specified, filter by extension
+            if (extensions.length > 0 &&
+                !hasValidExtension(relativePath, extensions)) {
+                return false;
+            }
+            return true;
+        });
+        // Build file nodes
+        const fileNodes = new Map();
+        for (const relativePath of validFiles) {
+            const absolutePath = path.join(projectRoot, relativePath);
+            try {
+                // Get file stats (use lstat to detect symlinks)
+                const fileStats = await fs.lstat(absolutePath);
+                // Skip symlinks
+                if (fileStats.isSymbolicLink()) {
+                    stats.filesSkipped++;
+                    continue;
+                }
+                // Check if it's a binary file
+                const binary = await isBinaryFile(absolutePath);
+                if (binary) {
+                    stats.filesSkipped++;
+                    continue;
+                }
+                const size = fileStats.size;
+                const mtime = fileStats.mtimeMs;
+                // Check if we can reuse hash from previous tree (mtime optimization)
+                let hash;
+                const prevNode = previousNodes.get(relativePath);
+                if (prevNode &&
+                    prevNode.type === 'file' &&
+                    prevNode.size === size &&
+                    prevNode.mtime === mtime) {
+                    // File unchanged - reuse cached hash
+                    hash = prevNode.hash;
+                    stats.cacheHits++;
+                }
+                else {
+                    // File is new or changed - compute hash
+                    hash = await computeFileHash(absolutePath);
+                    stats.cacheMisses++;
+                }
+                const node = createFileNode(relativePath, hash, size, mtime);
+                fileNodes.set(relativePath, node);
+                stats.filesIndexed++;
+            }
+            catch {
+                // Skip files we can't read
+                stats.filesSkipped++;
+                continue;
+            }
+        }
+        // Build directory structure
+        const root = buildDirectoryTree(fileNodes);
+        return new MerkleTree(root, stats.filesIndexed, stats);
+    }
+    /**
+     * Compare this tree with another tree.
+     *
+     * @param other - The other tree (usually the new/current tree)
+     * @returns TreeDiff with new, modified, and deleted files
+     */
+    compare(other) {
+        return compareTrees(this.root, other.root);
+    }
+    /**
+     * Serialize the tree to a plain object for JSON storage.
+     */
+    toJSON() {
+        if (!this.root) {
+            return null;
+        }
+        return serializeNode(this.root);
+    }
+    /**
+     * Deserialize a tree from a plain object.
+     */
+    static fromJSON(data) {
+        const emptyStats = {
+            filesScanned: 0,
+            filesIndexed: 0,
+            cacheHits: 0,
+            cacheMisses: 0,
+            filesSkipped: 0,
+        };
+        if (!data) {
+            return new MerkleTree(null, 0, emptyStats);
+        }
+        const root = deserializeNode(data);
+        const fileCount = countFiles(root);
+        return new MerkleTree(root, fileCount, emptyStats);
+    }
+    /**
+     * Create an empty tree.
+     */
+    static empty() {
+        return new MerkleTree(null, 0, {
+            filesScanned: 0,
+            filesIndexed: 0,
+            cacheHits: 0,
+            cacheMisses: 0,
+            filesSkipped: 0,
+        });
+    }
+}
+/**
+ * Build a lookup map from path to node for quick access.
+ */
+function buildNodeLookup(root) {
+    const lookup = new Map();
+    if (!root) {
+        return lookup;
+    }
+    function traverse(node) {
+        lookup.set(node.path, node);
+        if (node.children) {
+            for (const child of node.children.values()) {
+                traverse(child);
+            }
+        }
+    }
+    traverse(root);
+    return lookup;
+}
+/**
+ * Build a directory tree from a flat map of file nodes.
+ */
+function buildDirectoryTree(fileNodes) {
+    if (fileNodes.size === 0) {
+        return null;
+    }
+    // Build intermediate directory nodes
+    const allNodes = new Map(fileNodes);
+    const dirChildren = new Map();
+    // Collect children for each directory
+    for (const [filePath] of fileNodes) {
+        const parts = filePath.split('/');
+        // Build path to each ancestor directory
+        for (let i = 1; i <= parts.length; i++) {
+            // Parent directory path (empty string for root)
+            const dirPath = parts.slice(0, i - 1).join('/');
+            const childName = parts[i - 1];
+            const childPath = i === parts.length ? filePath : parts.slice(0, i).join('/');
+            if (!dirChildren.has(dirPath)) {
+                dirChildren.set(dirPath, new Map());
+            }
+            // Add the immediate child to its parent directory
+            const childNode = allNodes.get(childPath);
+            if (childNode) {
+                dirChildren.get(dirPath).set(childName, childNode);
+            }
+        }
+    }
+    // Build directory nodes bottom-up (deepest first)
+    const dirPaths = [...dirChildren.keys()].sort((a, b) => b.split('/').length - a.split('/').length);
+    for (const dirPath of dirPaths) {
+        if (dirPath === '')
+            continue; // Skip root for now
+        const children = dirChildren.get(dirPath);
+        // Collect actual child nodes (may include already-built directories)
+        const childNodes = new Map();
+        for (const [name, child] of children) {
+            // Check if there's a directory node we built
+            const builtDir = allNodes.get(child.path);
+            if (builtDir) {
+                childNodes.set(name, builtDir);
+            }
+        }
+        // Compute directory hash and create node
+        const hash = computeDirectoryHash(childNodes);
+        const dirNode = createDirectoryNode(dirPath, hash, childNodes);
+        allNodes.set(dirPath, dirNode);
+        // Update parent's reference to this directory
+        const parentPath = dirPath.includes('/')
+            ? dirPath.slice(0, dirPath.lastIndexOf('/'))
+            : '';
+        const dirName = dirPath.includes('/')
+            ? dirPath.slice(dirPath.lastIndexOf('/') + 1)
+            : dirPath;
+        if (dirChildren.has(parentPath)) {
+            dirChildren.get(parentPath).set(dirName, dirNode);
+        }
+    }
+    // Build root node
+    const rootChildren = dirChildren.get('');
+    if (!rootChildren || rootChildren.size === 0) {
+        return null;
+    }
+    const hash = computeDirectoryHash(rootChildren);
+    return createDirectoryNode('', hash, rootChildren);
+}
+/**
+ * Count the number of files in a tree.
+ */
+function countFiles(node) {
+    if (node.type === 'file') {
+        return 1;
+    }
+    let count = 0;
+    if (node.children) {
+        for (const child of node.children.values()) {
+            count += countFiles(child);
+        }
+    }
+    return count;
+}

package/dist/rag/merkle/node.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Type of a Merkle tree node.
+ */
+export type NodeType = 'file' | 'directory';
+/**
+ * A node in the Merkle tree.
+ *
+ * Files have hash = SHA256(content).
+ * Directories have hash = SHA256(sorted child name+hash pairs).
+ */
+export interface MerkleNode {
+    /** SHA256 hash of content (file) or children (directory) */
+    hash: string;
+    /** Node type */
+    type: NodeType;
+    /** Path relative to project root */
+    path: string;
+    /** Child nodes (directories only) */
+    children?: Map<string, MerkleNode>;
+    /** File size in bytes (files only) */
+    size?: number;
+    /** File modification time in Unix ms (files only) */
+    mtime?: number;
+}
+/**
+ * Serialized form of a MerkleNode for JSON storage.
+ */
+export interface SerializedNode {
+    hash: string;
+    type: NodeType;
+    path: string;
+    children?: Record<string, SerializedNode>;
+    size?: number;
+    mtime?: number;
+}
+/**
+ * Serialize a MerkleNode to a plain object for JSON storage.
+ */
+export declare function serializeNode(node: MerkleNode): SerializedNode;
+/**
+ * Deserialize a plain object back to a MerkleNode.
+ */
+export declare function deserializeNode(data: SerializedNode): MerkleNode;
+/**
+ * Create a file node.
+ */
+export declare function createFileNode(path: string, hash: string, size: number, mtime: number): MerkleNode;
+/**
+ * Create a directory node.
+ */
+export declare function createDirectoryNode(path: string, hash: string, children: Map<string, MerkleNode>): MerkleNode;

package/dist/rag/merkle/node.js

@@ -0,0 +1,69 @@
+/**
+ * Serialize a MerkleNode to a plain object for JSON storage.
+ */
+export function serializeNode(node) {
+    const serialized = {
+        hash: node.hash,
+        type: node.type,
+        path: node.path,
+    };
+    if (node.children) {
+        serialized.children = {};
+        for (const [name, child] of node.children) {
+            serialized.children[name] = serializeNode(child);
+        }
+    }
+    if (node.size !== undefined) {
+        serialized.size = node.size;
+    }
+    if (node.mtime !== undefined) {
+        serialized.mtime = node.mtime;
+    }
+    return serialized;
+}
+/**
+ * Deserialize a plain object back to a MerkleNode.
+ */
+export function deserializeNode(data) {
+    const node = {
+        hash: data.hash,
+        type: data.type,
+        path: data.path,
+    };
+    if (data.children) {
+        node.children = new Map();
+        for (const [name, childData] of Object.entries(data.children)) {
+            node.children.set(name, deserializeNode(childData));
+        }
+    }
+    if (data.size !== undefined) {
+        node.size = data.size;
+    }
+    if (data.mtime !== undefined) {
+        node.mtime = data.mtime;
+    }
+    return node;
+}
+/**
+ * Create a file node.
+ */
+export function createFileNode(path, hash, size, mtime) {
+    return {
+        hash,
+        type: 'file',
+        path,
+        size,
+        mtime,
+    };
+}
+/**
+ * Create a directory node.
+ */
+export function createDirectoryNode(path, hash, children) {
+    return {
+        hash,
+        type: 'directory',
+        path,
+        children,
+    };
+}

package/dist/rag/search/filters.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Filter builder for LanceDB WHERE clauses.
+ *
+ * Converts SearchFilters to LanceDB filter strings.
+ */
+import type { SearchFilters } from './types.js';
+/**
+ * Build a LanceDB WHERE clause from search filters.
+ *
+ * @param filters - Search filters to convert
+ * @returns Filter string for LanceDB WHERE clause, or undefined if no filters
+ */
+export declare function buildFilterClause(filters: SearchFilters | undefined): string | undefined;
+/**
+ * Build a name match filter for definition mode.
+ *
+ * @param symbolName - Symbol name to match
+ * @param typeFilter - Optional type filter
+ * @returns Filter string for LanceDB WHERE clause
+ */
+export declare function buildDefinitionFilter(symbolName: string, typeFilter?: ('function' | 'class' | 'method' | 'module')[]): string;

package/dist/rag/search/filters.js

@@ -0,0 +1,100 @@
+/**
+ * Filter builder for LanceDB WHERE clauses.
+ *
+ * Converts SearchFilters to LanceDB filter strings.
+ */
+/**
+ * Build a LanceDB WHERE clause from search filters.
+ *
+ * @param filters - Search filters to convert
+ * @returns Filter string for LanceDB WHERE clause, or undefined if no filters
+ */
+export function buildFilterClause(filters) {
+    if (!filters)
+        return undefined;
+    const conditions = [];
+    // Path prefix filter
+    if (filters.pathPrefix) {
+        const escaped = escapeSqlString(filters.pathPrefix);
+        conditions.push(`filepath LIKE '${escaped}%'`);
+    }
+    // Path contains (ALL must match)
+    if (filters.pathContains && filters.pathContains.length > 0) {
+        for (const str of filters.pathContains) {
+            const escaped = escapeSqlString(str);
+            conditions.push(`filepath LIKE '%${escaped}%'`);
+        }
+    }
+    // Path not contains (NONE must match)
+    if (filters.pathNotContains && filters.pathNotContains.length > 0) {
+        for (const str of filters.pathNotContains) {
+            const escaped = escapeSqlString(str);
+            conditions.push(`filepath NOT LIKE '%${escaped}%'`);
+        }
+    }
+    // Type filter (chunk type: function, class, method, module)
+    if (filters.type && filters.type.length > 0) {
+        const types = filters.type.map(t => `'${escapeSqlString(t)}'`).join(', ');
+        conditions.push(`type IN (${types})`);
+    }
+    // Extension filter
+    if (filters.extension && filters.extension.length > 0) {
+        const extensions = filters.extension
+            .map(e => `'${escapeSqlString(e)}'`)
+            .join(', ');
+        conditions.push(`extension IN (${extensions})`);
+    }
+    // Is exported filter
+    if (filters.isExported !== undefined) {
+        conditions.push(`is_exported = ${filters.isExported}`);
+    }
+    // Decorator contains filter
+    if (filters.decoratorContains) {
+        const escaped = escapeSqlString(filters.decoratorContains);
+        conditions.push(`decorator_names LIKE '%${escaped}%'`);
+    }
+    // Has docstring filter
+    if (filters.hasDocstring !== undefined) {
+        if (filters.hasDocstring) {
+            conditions.push(`docstring IS NOT NULL AND docstring != ''`);
+        }
+        else {
+            conditions.push(`(docstring IS NULL OR docstring = '')`);
+        }
+    }
+    if (conditions.length === 0) {
+        return undefined;
+    }
+    return conditions.join(' AND ');
+}
+/**
+ * Build a name match filter for definition mode.
+ *
+ * @param symbolName - Symbol name to match
+ * @param typeFilter - Optional type filter
+ * @returns Filter string for LanceDB WHERE clause
+ */
+export function buildDefinitionFilter(symbolName, typeFilter) {
+    const conditions = [];
+    // Exact name match
+    const escaped = escapeSqlString(symbolName);
+    conditions.push(`name = '${escaped}'`);
+    // Type filter for definitions (exclude module chunks)
+    if (typeFilter && typeFilter.length > 0) {
+        const types = typeFilter.map(t => `'${escapeSqlString(t)}'`).join(', ');
+        conditions.push(`type IN (${types})`);
+    }
+    else {
+        // Default: look for function, class, method definitions
+        conditions.push(`type IN ('function', 'class', 'method')`);
+    }
+    return conditions.join(' AND ');
+}
+/**
+ * Escape a string for use in SQL LIKE clause.
+ */
+function escapeSqlString(str) {
+    // Escape single quotes by doubling them
+    // Also escape % and _ which are LIKE wildcards
+    return str.replace(/'/g, "''").replace(/%/g, '\\%').replace(/_/g, '\\_');
+}

package/dist/rag/search/fts.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Full-text search (BM25) using LanceDB.
+ */
+import type { Table } from '@lancedb/lancedb';
+import type { SearchResult } from './types.js';
+/**
+ * Options for FTS search.
+ */
+export interface FtsSearchOptions {
+    /** Maximum number of results */
+    limit: number;
+    /** LanceDB WHERE clause filter */
+    filterClause?: string;
+    /** Minimum score threshold (0-1) */
+    minScore?: number;
+}
+/**
+ * Ensure FTS index exists on the text column.
+ * Creates the index if it doesn't exist.
+ *
+ * @param table - LanceDB table to index
+ */
+export declare function ensureFtsIndex(table: Table): Promise<void>;
+/**
+ * Perform full-text search using BM25.
+ *
+ * @param table - LanceDB table to search
+ * @param query - Search query string
+ * @param options - Search options
+ * @returns Array of search results with FTS scores
+ */
+export declare function ftsSearch(table: Table, query: string, options: FtsSearchOptions | number): Promise<SearchResult[]>;

package/dist/rag/search/fts.js

@@ -0,0 +1,61 @@
+/**
+ * Full-text search (BM25) using LanceDB.
+ */
+import * as lancedb from '@lancedb/lancedb';
+/**
+ * Ensure FTS index exists on the text column.
+ * Creates the index if it doesn't exist.
+ *
+ * @param table - LanceDB table to index
+ */
+export async function ensureFtsIndex(table) {
+    const indices = await table.listIndices();
+    const hasFtsIndex = indices.some(idx => idx.columns.includes('text') && idx.indexType === 'FTS');
+    if (!hasFtsIndex) {
+        await table.createIndex('text', {
+            config: lancedb.Index.fts(),
+        });
+    }
+}
+/**
+ * Perform full-text search using BM25.
+ *
+ * @param table - LanceDB table to search
+ * @param query - Search query string
+ * @param options - Search options
+ * @returns Array of search results with FTS scores
+ */
+export async function ftsSearch(table, query, options) {
+    // Support legacy signature: ftsSearch(table, query, limit)
+    const opts = typeof options === 'number' ? { limit: options } : options;
+    // Ensure FTS index exists
+    await ensureFtsIndex(table);
+    let searchQuery = table.search(query, 'fts').limit(opts.limit);
+    // Apply filter if provided
+    if (opts.filterClause) {
+        searchQuery = searchQuery.where(opts.filterClause);
+    }
+    const results = await searchQuery.toArray();
+    return results
+        .map((row, index) => {
+        const chunk = row;
+        // BM25 score (higher is better)
+        // Normalize by rank for consistent scoring
+        const ftsScore = chunk._score ?? 1 / (index + 1);
+        return {
+            id: chunk.id,
+            text: chunk.text,
+            filepath: chunk.filepath,
+            filename: chunk.filename,
+            name: chunk.name,
+            type: chunk.type,
+            startLine: chunk.start_line,
+            endLine: chunk.end_line,
+            score: ftsScore,
+            ftsScore,
+            signature: chunk.signature,
+            isExported: chunk.is_exported,
+        };
+    })
+        .filter(r => !opts.minScore || r.score >= opts.minScore);
+}

package/dist/rag/search/hybrid.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Hybrid search combining vector and FTS with RRF reranking.
+ */
+import type { SearchResult } from './types.js';
+/**
+ * Combine vector and FTS results using Reciprocal Rank Fusion.
+ *
+ * RRF formula: score = sum(1 / (k + rank))
+ * where k is a constant (typically 60) and rank is 1-indexed.
+ *
+ * @param vectorResults - Results from vector search
+ * @param ftsResults - Results from FTS search
+ * @param limit - Maximum number of results to return
+ * @param vectorWeight - Weight for vector results (0.0-1.0, default 0.7)
+ * @returns Combined and reranked results
+ */
+export declare function hybridRerank(vectorResults: SearchResult[], ftsResults: SearchResult[], limit: number, vectorWeight?: number): SearchResult[];