viberag 0.1.0

package/dist/rag/indexer/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Indexer module for code chunking and indexing.
+ */
+export { Chunker } from './chunker.js';
+export { Indexer, type IndexOptions } from './indexer.js';
+export { createEmptyIndexStats, type Chunk, type ChunkType, type IndexStats, type ProgressCallback, type SupportedLanguage, } from './types.js';

package/dist/rag/indexer/index.js

@@ -0,0 +1,6 @@
+/**
+ * Indexer module for code chunking and indexing.
+ */
+export { Chunker } from './chunker.js';
+export { Indexer } from './indexer.js';
+export { createEmptyIndexStats, } from './types.js';

package/dist/rag/indexer/indexer.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Indexer - Orchestrates the full indexing pipeline.
+ *
+ * Pipeline:
+ * 1. Load previous Merkle tree from manifest
+ * 2. Build current Merkle tree from filesystem
+ * 3. Compare trees → get new/modified/deleted files
+ * 4. Delete chunks for deleted files
+ * 5. For new/modified files:
+ *    - Chunk with tree-sitter
+ *    - Compute embeddings (with cache lookup)
+ *    - Upsert to LanceDB
+ * 6. Save updated manifest
+ */
+import type { Logger } from '../logger/index.js';
+import { type IndexStats, type ProgressCallback } from './types.js';
+/**
+ * Options for the index operation.
+ */
+export interface IndexOptions {
+    /** Force full reindex, ignoring Merkle tree diff */
+    force?: boolean;
+    /** Progress callback for UI updates */
+    progressCallback?: ProgressCallback;
+}
+/**
+ * Indexer class for orchestrating the full indexing pipeline.
+ */
+export declare class Indexer {
+    private readonly projectRoot;
+    private config;
+    private storage;
+    private chunker;
+    private embeddings;
+    private logger;
+    constructor(projectRoot: string, logger?: Logger);
+    /**
+     * Run the indexing pipeline.
+     */
+    index(options?: IndexOptions): Promise<IndexStats>;
+    /**
+     * Create a diff that treats all files as new (for force reindex).
+     */
+    private createForceDiff;
+    /**
+     * Recursively collect all file paths from a serialized Merkle tree node.
+     */
+    private collectAllFilesFromSerialized;
+    /**
+     * Process a batch of files: read, chunk, embed, and prepare CodeChunks.
+     */
+    private processFileBatch;
+    /**
+     * Get a friendly name for the embedding provider.
+     */
+    private getProviderDisplayName;
+    /**
+     * Initialize all components.
+     */
+    private initialize;
+    /**
+     * Create the appropriate embedding provider based on config.
+     */
+    private createEmbeddingProvider;
+    /**
+     * Log a message.
+     */
+    private log;
+    /**
+     * Close all resources.
+     */
+    close(): void;
+}

package/dist/rag/indexer/indexer.js

@@ -0,0 +1,356 @@
+/**
+ * Indexer - Orchestrates the full indexing pipeline.
+ *
+ * Pipeline:
+ * 1. Load previous Merkle tree from manifest
+ * 2. Build current Merkle tree from filesystem
+ * 3. Compare trees → get new/modified/deleted files
+ * 4. Delete chunks for deleted files
+ * 5. For new/modified files:
+ *    - Chunk with tree-sitter
+ *    - Compute embeddings (with cache lookup)
+ *    - Upsert to LanceDB
+ * 6. Save updated manifest
+ */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { loadConfig } from '../config/index.js';
+import { GeminiEmbeddingProvider, Local4BEmbeddingProvider, LocalEmbeddingProvider, MistralEmbeddingProvider, OpenAIEmbeddingProvider, } from '../embeddings/index.js';
+import { loadManifest, saveManifest, manifestExists, createEmptyManifest, updateManifestStats, updateManifestTree, } from '../manifest/index.js';
+import { MerkleTree } from '../merkle/index.js';
+import { Storage } from '../storage/index.js';
+import { Chunker } from './chunker.js';
+import { createEmptyIndexStats, } from './types.js';
+/**
+ * Indexer class for orchestrating the full indexing pipeline.
+ */
+export class Indexer {
+    constructor(projectRoot, logger) {
+        Object.defineProperty(this, "projectRoot", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "config", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "storage", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "chunker", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "embeddings", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "logger", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        this.projectRoot = projectRoot;
+        this.logger = logger ?? null;
+    }
+    /**
+     * Run the indexing pipeline.
+     */
+    async index(options = {}) {
+        const stats = createEmptyIndexStats();
+        const { force = false, progressCallback } = options;
+        try {
+            // Initialize components
+            await this.initialize(progressCallback);
+            const config = this.config;
+            const storage = this.storage;
+            const chunker = this.chunker;
+            const embeddings = this.embeddings;
+            // 1. Load previous manifest and Merkle tree
+            this.log('info', 'Loading manifest');
+            let manifest = (await manifestExists(this.projectRoot))
+                ? await loadManifest(this.projectRoot)
+                : createEmptyManifest();
+            const previousTree = manifest.tree
+                ? MerkleTree.fromJSON(manifest.tree)
+                : MerkleTree.empty();
+            // 2. Build current Merkle tree from filesystem
+            this.log('info', 'Building Merkle tree');
+            progressCallback?.(0, 100, 'Scanning files');
+            const currentTree = await MerkleTree.build(this.projectRoot, config.extensions, config.excludePatterns, previousTree);
+            stats.filesScanned = currentTree.buildStats.filesScanned;
+            this.log('info', `Scanned ${stats.filesScanned} files, indexed ${currentTree.fileCount}`);
+            // 3. Compare trees to get diff
+            const diff = force
+                ? this.createForceDiff(currentTree)
+                : previousTree.compare(currentTree);
+            stats.filesNew = diff.new.length;
+            stats.filesModified = diff.modified.length;
+            stats.filesDeleted = diff.deleted.length;
+            this.log('info', `Changes: ${diff.new.length} new, ${diff.modified.length} modified, ${diff.deleted.length} deleted`);
+            // Short-circuit if no changes
+            if (!diff.hasChanges && !force) {
+                this.log('info', 'No changes detected');
+                return stats;
+            }
+            // 4. Handle force reindex - drop and recreate table to avoid schema issues
+            if (force) {
+                this.log('info', 'Force reindex: resetting chunks table');
+                await storage.resetChunksTable();
+            }
+            // 5. Delete chunks for deleted files
+            if (diff.deleted.length > 0) {
+                this.log('info', `Deleting chunks for ${diff.deleted.length} files`);
+                stats.chunksDeleted = await storage.deleteChunksByFilepaths(diff.deleted);
+            }
+            // 6. Process new and modified files
+            const filesToProcess = [...diff.new, ...diff.modified];
+            const totalFiles = filesToProcess.length;
+            if (totalFiles > 0) {
+                this.log('info', `Processing ${totalFiles} files`);
+                // First, delete existing chunks for modified files
+                if (diff.modified.length > 0 && !force) {
+                    const deletedCount = await storage.deleteChunksByFilepaths(diff.modified);
+                    stats.chunksDeleted += deletedCount;
+                }
+                // Process files in batches
+                const batchSize = 10;
+                for (let i = 0; i < filesToProcess.length; i += batchSize) {
+                    const batch = filesToProcess.slice(i, i + batchSize);
+                    const batchChunks = await this.processFileBatch(batch, chunker, embeddings, storage, stats);
+                    if (batchChunks.length > 0) {
+                        // Use addChunks after table reset to avoid schema mismatch,
+                        // upsertChunks for normal incremental updates
+                        if (force) {
+                            await storage.addChunks(batchChunks);
+                        }
+                        else {
+                            await storage.upsertChunks(batchChunks);
+                        }
+                        stats.chunksAdded += batchChunks.length;
+                    }
+                    const progress = Math.round(((i + batch.length) / totalFiles) * 100);
+                    progressCallback?.(i + batch.length, totalFiles, 'Indexing files');
+                    this.log('debug', `Progress: ${progress}% (${i + batch.length}/${totalFiles})`);
+                }
+            }
+            // 7. Update manifest with new tree and stats
+            const chunkCount = await storage.countChunks();
+            manifest = updateManifestTree(manifest, currentTree.toJSON());
+            manifest = updateManifestStats(manifest, {
+                totalFiles: currentTree.fileCount,
+                totalChunks: chunkCount,
+            });
+            await saveManifest(this.projectRoot, manifest);
+            this.log('info', `Index complete: ${stats.chunksAdded} chunks added, ${stats.chunksDeleted} deleted`);
+            return stats;
+        }
+        catch (error) {
+            this.log('error', 'Indexing failed', error);
+            throw error;
+        }
+    }
+    /**
+     * Create a diff that treats all files as new (for force reindex).
+     */
+    createForceDiff(tree) {
+        const allFiles = [];
+        const serialized = tree.toJSON();
+        this.collectAllFilesFromSerialized(serialized, allFiles);
+        return {
+            new: allFiles,
+            modified: [],
+            deleted: [],
+            hasChanges: allFiles.length > 0,
+        };
+    }
+    /**
+     * Recursively collect all file paths from a serialized Merkle tree node.
+     */
+    collectAllFilesFromSerialized(node, files) {
+        if (!node)
+            return;
+        if (node.type === 'file') {
+            files.push(node.path);
+        }
+        else if (node.children) {
+            for (const child of Object.values(node.children)) {
+                this.collectAllFilesFromSerialized(child, files);
+            }
+        }
+    }
+    /**
+     * Process a batch of files: read, chunk, embed, and prepare CodeChunks.
+     */
+    async processFileBatch(filepaths, chunker, embeddings, storage, stats) {
+        const allChunks = [];
+        for (const filepath of filepaths) {
+            try {
+                const absolutePath = path.join(this.projectRoot, filepath);
+                const content = await fs.readFile(absolutePath, 'utf-8');
+                const fileHash = (await import('../merkle/hash.js')).computeStringHash(content);
+                // Chunk the file (with size limits from config)
+                const chunks = await chunker.chunkFile(filepath, content, this.config.chunkMaxSize);
+                // Check embedding cache for each chunk
+                const contentHashes = chunks.map(c => c.contentHash);
+                const cachedEmbeddings = await storage.getCachedEmbeddings(contentHashes);
+                // Compute embeddings for cache misses
+                const missingChunks = chunks.filter(c => !cachedEmbeddings.has(c.contentHash));
+                if (missingChunks.length > 0) {
+                    // Embed contextHeader + text for semantic relevance
+                    const texts = missingChunks.map(c => c.contextHeader ? `${c.contextHeader}\n${c.text}` : c.text);
+                    const newEmbeddings = await embeddings.embed(texts);
+                    stats.embeddingsComputed += missingChunks.length;
+                    // Cache the new embeddings
+                    const cacheEntries = missingChunks.map((chunk, i) => ({
+                        contentHash: chunk.contentHash,
+                        vector: newEmbeddings[i],
+                        createdAt: new Date().toISOString(),
+                    }));
+                    await storage.cacheEmbeddings(cacheEntries);
+                    // Add to cachedEmbeddings map
+                    missingChunks.forEach((chunk, i) => {
+                        cachedEmbeddings.set(chunk.contentHash, newEmbeddings[i]);
+                    });
+                }
+                stats.embeddingsCached += chunks.length - missingChunks.length;
+                // Build CodeChunk objects
+                const filename = path.basename(filepath);
+                const extension = path.extname(filepath);
+                for (const chunk of chunks) {
+                    const vector = cachedEmbeddings.get(chunk.contentHash);
+                    allChunks.push({
+                        id: `${filepath}:${chunk.startLine}`,
+                        vector,
+                        text: chunk.text,
+                        contentHash: chunk.contentHash,
+                        filepath,
+                        filename,
+                        extension,
+                        type: chunk.type,
+                        name: chunk.name,
+                        startLine: chunk.startLine,
+                        endLine: chunk.endLine,
+                        fileHash,
+                        // New metadata fields from schema v2
+                        signature: chunk.signature,
+                        docstring: chunk.docstring,
+                        isExported: chunk.isExported,
+                        decoratorNames: chunk.decoratorNames,
+                    });
+                }
+            }
+            catch (error) {
+                this.log('warn', `Failed to process file: ${filepath}`, error);
+                // Continue with other files
+            }
+        }
+        return allChunks;
+    }
+    /**
+     * Get a friendly name for the embedding provider.
+     */
+    getProviderDisplayName(provider) {
+        switch (provider) {
+            case 'local':
+                return 'Qwen3-0.6B';
+            case 'gemini':
+                return 'Gemini';
+            case 'mistral':
+                return 'Mistral';
+            case 'openai':
+                return 'OpenAI';
+            default:
+                return provider;
+        }
+    }
+    /**
+     * Initialize all components.
+     */
+    async initialize(progressCallback) {
+        // Load config
+        this.config = await loadConfig(this.projectRoot);
+        const providerName = this.getProviderDisplayName(this.config.embeddingProvider);
+        // Initialize storage
+        progressCallback?.(0, 0, 'Connecting to database');
+        this.storage = new Storage(this.projectRoot, this.config.embeddingDimensions);
+        await this.storage.connect();
+        // Initialize chunker (loads tree-sitter parsers)
+        progressCallback?.(0, 0, 'Loading parsers');
+        this.chunker = new Chunker();
+        await this.chunker.initialize();
+        // Initialize embeddings based on provider type
+        // For local models, this may download the model on first run
+        const isLocal = this.config.embeddingProvider === 'local';
+        progressCallback?.(0, 0, isLocal
+            ? `Loading ${providerName} model`
+            : `Connecting to ${providerName}`);
+        this.embeddings = this.createEmbeddingProvider(this.config);
+        // Pass model progress to the UI for local models
+        await this.embeddings.initialize(isLocal && progressCallback
+            ? (status, progress, _message) => {
+                if (status === 'downloading') {
+                    progressCallback(progress ?? 0, 100, `Downloading ${providerName} (${progress}%)`);
+                }
+                else if (status === 'loading') {
+                    progressCallback(0, 0, `Loading ${providerName} model`);
+                }
+            }
+            : undefined);
+        this.log('info', 'Indexer initialized');
+    }
+    /**
+     * Create the appropriate embedding provider based on config.
+     */
+    createEmbeddingProvider(config) {
+        const apiKey = config.apiKey;
+        switch (config.embeddingProvider) {
+            case 'local':
+                return new LocalEmbeddingProvider();
+            case 'local-4b':
+                return new Local4BEmbeddingProvider();
+            case 'gemini':
+                return new GeminiEmbeddingProvider(apiKey);
+            case 'mistral':
+                return new MistralEmbeddingProvider(apiKey);
+            case 'openai':
+                return new OpenAIEmbeddingProvider(apiKey);
+            default:
+                throw new Error(`Unknown embedding provider: ${config.embeddingProvider}`);
+        }
+    }
+    /**
+     * Log a message.
+     */
+    log(level, message, error) {
+        if (!this.logger)
+            return;
+        if (level === 'error') {
+            this.logger.error('Indexer', message, error);
+        }
+        else {
+            this.logger[level]('Indexer', message);
+        }
+    }
+    /**
+     * Close all resources.
+     */
+    close() {
+        this.storage?.close();
+        this.chunker?.close();
+        this.embeddings?.close();
+        this.log('info', 'Indexer closed');
+    }
+}

package/dist/rag/indexer/types.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Types of code chunks extracted by tree-sitter.
+ */
+export type ChunkType = 'function' | 'class' | 'method' | 'module';
+/**
+ * A raw code chunk extracted from a file before embedding.
+ */
+export interface Chunk {
+    /** The source code text */
+    text: string;
+    /** Context header for embedding (e.g., "// File: path.ts, Class: Foo") */
+    contextHeader: string;
+    /** Type of chunk */
+    type: ChunkType;
+    /** Symbol name (empty for module chunks) */
+    name: string;
+    /** Start line number (1-indexed) */
+    startLine: number;
+    /** End line number (1-indexed) */
+    endLine: number;
+    /** SHA256 hash of contextHeader + text */
+    contentHash: string;
+    /** Function/method signature line (null for module chunks) */
+    signature: string | null;
+    /** Extracted documentation (JSDoc, docstring, etc.) */
+    docstring: string | null;
+    /** Whether symbol has export modifier */
+    isExported: boolean;
+    /** Comma-separated decorator/annotation names (null if none) */
+    decoratorNames: string | null;
+}
+/**
+ * Supported languages for tree-sitter parsing.
+ */
+export type SupportedLanguage = 'python' | 'javascript' | 'typescript' | 'tsx' | 'go' | 'rust' | 'java' | 'csharp' | 'dart' | 'swift' | 'kotlin' | 'php';
+/**
+ * Map of file extensions to languages.
+ */
+export declare const EXTENSION_TO_LANGUAGE: Record<string, SupportedLanguage>;
+/**
+ * Statistics from indexing operations.
+ */
+export interface IndexStats {
+    /** Number of files scanned */
+    filesScanned: number;
+    /** Number of new files indexed */
+    filesNew: number;
+    /** Number of modified files re-indexed */
+    filesModified: number;
+    /** Number of deleted files removed from index */
+    filesDeleted: number;
+    /** Number of chunks added */
+    chunksAdded: number;
+    /** Number of chunks deleted */
+    chunksDeleted: number;
+    /** Number of embeddings computed (cache miss) */
+    embeddingsComputed: number;
+    /** Number of embeddings retrieved from cache */
+    embeddingsCached: number;
+}
+/**
+ * Progress callback for indexing operations.
+ */
+export type ProgressCallback = (current: number, total: number, stage: string) => void;
+/**
+ * Create empty index stats.
+ */
+export declare function createEmptyIndexStats(): IndexStats;

package/dist/rag/indexer/types.js

@@ -0,0 +1,47 @@
+/**
+ * Map of file extensions to languages.
+ */
+export const EXTENSION_TO_LANGUAGE = {
+    // JavaScript/TypeScript family
+    '.js': 'javascript',
+    '.mjs': 'javascript',
+    '.cjs': 'javascript',
+    '.ts': 'typescript',
+    '.mts': 'typescript',
+    '.cts': 'typescript',
+    '.tsx': 'tsx',
+    // Python
+    '.py': 'python',
+    // Go
+    '.go': 'go',
+    // Rust
+    '.rs': 'rust',
+    // Java
+    '.java': 'java',
+    // C#
+    '.cs': 'csharp',
+    // Dart
+    '.dart': 'dart',
+    // Swift
+    '.swift': 'swift',
+    // Kotlin
+    '.kt': 'kotlin',
+    '.kts': 'kotlin',
+    // PHP
+    '.php': 'php',
+};
+/**
+ * Create empty index stats.
+ */
+export function createEmptyIndexStats() {
+    return {
+        filesScanned: 0,
+        filesNew: 0,
+        filesModified: 0,
+        filesDeleted: 0,
+        chunksAdded: 0,
+        chunksDeleted: 0,
+        embeddingsComputed: 0,
+        embeddingsCached: 0,
+    };
+}

package/dist/rag/logger/index.d.ts

@@ -0,0 +1,20 @@
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
+export interface Logger {
+    debug(component: string, message: string, data?: object): void;
+    info(component: string, message: string, data?: object): void;
+    warn(component: string, message: string, data?: object): void;
+    error(component: string, message: string, error?: Error): void;
+}
+/**
+ * Get the path to today's log file.
+ */
+export declare function getLogPath(projectRoot: string): string;
+/**
+ * Create a logger that writes to daily log files.
+ * Log files are created in the .viberag/logs/ directory.
+ */
+export declare function createLogger(projectRoot: string): Logger;
+/**
+ * Create a no-op logger for testing or when logging is disabled.
+ */
+export declare function createNullLogger(): Logger;

package/dist/rag/logger/index.js

@@ -0,0 +1,75 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { getLogsDir } from '../constants.js';
+/**
+ * Get the path to today's log file.
+ */
+export function getLogPath(projectRoot) {
+    const logsDir = getLogsDir(projectRoot);
+    const date = new Date().toISOString().split('T')[0]; // YYYY-MM-DD
+    return path.join(logsDir, `${date}.log`);
+}
+/**
+ * Format a log entry.
+ */
+function formatEntry(level, component, message, extra) {
+    const timestamp = new Date().toISOString();
+    const levelStr = level.toUpperCase().padEnd(5);
+    let entry = `[${timestamp}] [${levelStr}] ${component}: ${message}`;
+    if (extra) {
+        if (extra instanceof Error) {
+            entry += `\n  Error: ${extra.message}`;
+            if (extra.stack) {
+                entry += `\n  Stack: ${extra.stack}`;
+            }
+        }
+        else {
+            entry += `\n  ${JSON.stringify(extra)}`;
+        }
+    }
+    return entry;
+}
+/**
+ * Create a logger that writes to daily log files.
+ * Log files are created in the .viberag/logs/ directory.
+ */
+export function createLogger(projectRoot) {
+    const logsDir = getLogsDir(projectRoot);
+    let initialized = false;
+    function ensureDir() {
+        if (!initialized) {
+            fs.mkdirSync(logsDir, { recursive: true });
+            initialized = true;
+        }
+    }
+    function write(entry) {
+        ensureDir();
+        const logPath = getLogPath(projectRoot);
+        fs.appendFileSync(logPath, entry + '\n');
+    }
+    return {
+        debug(component, message, data) {
+            write(formatEntry('debug', component, message, data));
+        },
+        info(component, message, data) {
+            write(formatEntry('info', component, message, data));
+        },
+        warn(component, message, data) {
+            write(formatEntry('warn', component, message, data));
+        },
+        error(component, message, error) {
+            write(formatEntry('error', component, message, error));
+        },
+    };
+}
+/**
+ * Create a no-op logger for testing or when logging is disabled.
+ */
+export function createNullLogger() {
+    return {
+        debug() { },
+        info() { },
+        warn() { },
+        error() { },
+    };
+}