viberag 0.2.0 → 0.3.0

package/dist/rag/embeddings/validate.js

@@ -0,0 +1,148 @@
+/**
+ * API key validation for cloud embedding providers.
+ *
+ * Makes a minimal test embedding call to verify the API key is valid
+ * before proceeding with indexing.
+ */
+/**
+ * API endpoints for each cloud provider.
+ */
+const ENDPOINTS = {
+    gemini: 'https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:embedContent',
+    mistral: 'https://api.mistral.ai/v1/embeddings',
+    openai: 'https://api.openai.com/v1/embeddings',
+};
+/**
+ * Safely parse JSON response, logging parse failures instead of swallowing them.
+ */
+async function safeParseJson(response, provider) {
+    try {
+        return (await response.json());
+    }
+    catch (error) {
+        console.warn(`[${provider}] Failed to parse error response:`, error instanceof Error ? error.message : String(error));
+        return {};
+    }
+}
+/**
+ * Validate an API key by making a minimal test embedding call.
+ *
+ * @param provider - The embedding provider type
+ * @param apiKey - The API key to validate
+ * @returns Validation result with error message if invalid
+ */
+export async function validateApiKey(provider, apiKey) {
+    // Local providers don't need API key validation
+    if (provider === 'local' || provider === 'local-4b') {
+        return { valid: true };
+    }
+    if (!apiKey || apiKey.trim() === '') {
+        return { valid: false, error: 'API key is required' };
+    }
+    try {
+        switch (provider) {
+            case 'gemini':
+                return await validateGeminiKey(apiKey);
+            case 'mistral':
+                return await validateMistralKey(apiKey);
+            case 'openai':
+                return await validateOpenAIKey(apiKey);
+            default:
+                return { valid: false, error: `Unknown provider: ${provider}` };
+        }
+    }
+    catch (err) {
+        return {
+            valid: false,
+            error: err instanceof Error ? err.message : String(err),
+        };
+    }
+}
+/**
+ * Validate Gemini API key.
+ */
+async function validateGeminiKey(apiKey) {
+    const response = await fetch(ENDPOINTS.gemini, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'x-goog-api-key': apiKey,
+        },
+        body: JSON.stringify({
+            content: { parts: [{ text: 'test' }] },
+        }),
+    });
+    if (response.ok) {
+        return { valid: true };
+    }
+    const data = await safeParseJson(response, 'gemini');
+    const message = data?.error?.message ||
+        `HTTP ${response.status}`;
+    if (response.status === 400 && message.includes('API key')) {
+        return { valid: false, error: 'Invalid API key' };
+    }
+    if (response.status === 403) {
+        return { valid: false, error: 'API key not authorized for this API' };
+    }
+    return { valid: false, error: message };
+}
+/**
+ * Validate Mistral API key.
+ */
+async function validateMistralKey(apiKey) {
+    const response = await fetch(ENDPOINTS.mistral, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${apiKey}`,
+        },
+        body: JSON.stringify({
+            model: 'codestral-embed',
+            input: ['test'],
+        }),
+    });
+    if (response.ok) {
+        return { valid: true };
+    }
+    const data = await safeParseJson(response, 'mistral');
+    const message = data?.message ||
+        data?.detail ||
+        `HTTP ${response.status}`;
+    if (response.status === 401) {
+        return { valid: false, error: 'Invalid API key' };
+    }
+    return { valid: false, error: message };
+}
+/**
+ * Validate OpenAI API key.
+ */
+async function validateOpenAIKey(apiKey) {
+    const response = await fetch(ENDPOINTS.openai, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${apiKey}`,
+        },
+        body: JSON.stringify({
+            model: 'text-embedding-3-small',
+            input: ['test'],
+        }),
+    });
+    if (response.ok) {
+        return { valid: true };
+    }
+    const data = await safeParseJson(response, 'openai');
+    const error = data?.error;
+    if (response.status === 401) {
+        return { valid: false, error: 'Invalid API key' };
+    }
+    if (error?.message) {
+        // Truncate long error messages
+        const msg = error.message;
+        return {
+            valid: false,
+            error: msg.length > 100 ? msg.slice(0, 100) + '...' : msg,
+        };
+    }
+    return { valid: false, error: `HTTP ${response.status}` };
+}

package/dist/rag/index.d.ts

@@ -10,6 +10,6 @@ export { loadManifest, saveManifest, manifestExists, createEmptyManifest, update
 export { Storage, SCHEMA_VERSION, createCodeChunksSchema, createEmbeddingCacheSchema, chunkToRow, rowToChunk, embeddingToRow, rowToEmbedding, type CodeChunk, type CodeChunkRow, type CachedEmbedding, type CachedEmbeddingRow, type ChunkType, } from './storage/index.js';
 export { MerkleTree, compareTrees, createEmptyDiff, computeFileHash, computeStringHash, computeDirectoryHash, isBinaryFile, shouldExclude, hasValidExtension, serializeNode, deserializeNode, createFileNode, createDirectoryNode, type MerkleNode, type NodeType, type SerializedNode, type TreeDiff, type BuildStats, } from './merkle/index.js';
 export { Chunker, Indexer, createEmptyIndexStats, type Chunk, type IndexOptions, type IndexStats, type ProgressCallback, type SupportedLanguage, } from './indexer/index.js';
-export { GeminiEmbeddingProvider, MistralEmbeddingProvider, OpenAIEmbeddingProvider, type EmbeddingProvider, } from './embeddings/index.js';
+export { GeminiEmbeddingProvider, MistralEmbeddingProvider, OpenAIEmbeddingProvider, validateApiKey, type EmbeddingProvider, type ValidationResult, } from './embeddings/index.js';
 export { SearchEngine, vectorSearch, ftsSearch, ensureFtsIndex, hybridRerank, type SearchFilters, type SearchMode, type SearchOptions, type SearchResult, type SearchResults, } from './search/index.js';
 export { loadGitignore, shouldIgnore, createGitignoreFilter, clearGitignoreCache, clearAllGitignoreCache, } from './gitignore/index.js';

package/dist/rag/index.js

@@ -18,7 +18,7 @@ export { MerkleTree, compareTrees, createEmptyDiff, computeFileHash, computeStri
 // Indexer (Chunking & Orchestration)
 export { Chunker, Indexer, createEmptyIndexStats, } from './indexer/index.js';
 // Embeddings
-export { GeminiEmbeddingProvider, MistralEmbeddingProvider, OpenAIEmbeddingProvider, } from './embeddings/index.js';
+export { GeminiEmbeddingProvider, MistralEmbeddingProvider, OpenAIEmbeddingProvider, validateApiKey, } from './embeddings/index.js';
 // Search
 export { SearchEngine, vectorSearch, ftsSearch, ensureFtsIndex, hybridRerank, } from './search/index.js';
 // Gitignore

package/dist/rag/indexer/chunker.js

@@ -188,28 +188,40 @@ export class Chunker {
         await Parser.init();
         // Create parser instance after init
         this.parser = new Parser();
-        // Resolve the path to tree-sitter-wasms/out/
-        const wasmPackagePath = require.resolve('tree-sitter-wasms/package.json');
-        this.wasmBasePath = path.join(path.dirname(wasmPackagePath), 'out');
-        // Load all language grammars sequentially (skip null entries like Dart)
-        // IMPORTANT: Must be sequential - web-tree-sitter has global state that
-        // gets corrupted when loading multiple WASM modules in parallel.
-        for (const [lang, wasmFile] of Object.entries(LANGUAGE_WASM_FILES)) {
-            if (!wasmFile) {
-                // Language temporarily disabled (e.g., Dart due to version mismatch)
-                continue;
-            }
-            try {
-                const wasmPath = path.join(this.wasmBasePath, wasmFile);
-                const language = await Parser.Language.load(wasmPath);
-                this.languages.set(lang, language);
+        try {
+            // Resolve the path to tree-sitter-wasms/out/
+            const wasmPackagePath = require.resolve('tree-sitter-wasms/package.json');
+            this.wasmBasePath = path.join(path.dirname(wasmPackagePath), 'out');
+            // Load all language grammars sequentially (skip null entries like Dart)
+            // IMPORTANT: Must be sequential - web-tree-sitter has global state that
+            // gets corrupted when loading multiple WASM modules in parallel.
+            for (const [lang, wasmFile] of Object.entries(LANGUAGE_WASM_FILES)) {
+                if (!wasmFile) {
+                    // Language temporarily disabled (e.g., Dart due to version mismatch)
+                    continue;
+                }
+                try {
+                    const wasmPath = path.join(this.wasmBasePath, wasmFile);
+                    const language = await Parser.Language.load(wasmPath);
+                    this.languages.set(lang, language);
+                }
+                catch (error) {
+                    // Log but don't fail - we can still work with other languages
+                    console.error(`Failed to load ${lang} grammar:`, error);
+                }
             }
-            catch (error) {
-                // Log but don't fail - we can still work with other languages
-                console.error(`Failed to load ${lang} grammar:`, error);
+            this.initialized = true;
+        }
+        catch (error) {
+            // Cleanup parser on failure to prevent resource leak
+            if (this.parser) {
+                this.parser.delete();
+                this.parser = null;
             }
+            this.wasmBasePath = null;
+            this.languages.clear();
+            throw error;
         }
-        this.initialized = true;
     }
     /**
      * Get the language for a file extension.

package/dist/rag/indexer/indexer.d.ts

@@ -33,11 +33,17 @@ export declare class Indexer {
     private chunker;
     private embeddings;
     private logger;
+    private indexPromise;
     constructor(projectRoot: string, logger?: Logger);
     /**
      * Run the indexing pipeline.
+     * Uses mutex to prevent concurrent index operations.
      */
     index(options?: IndexOptions): Promise<IndexStats>;
+    /**
+     * Perform the actual indexing operation.
+     */
+    private doIndex;
     /**
      * Create a diff that treats all files as new (for force reindex).
      */
@@ -48,6 +54,10 @@ export declare class Indexer {
     private collectAllFilesFromSerialized;
     /**
      * Process a batch of files: read, chunk, embed, and prepare CodeChunks.
+     *
+     * Error handling strategy:
+     * - File read/parse errors: Log and continue (file-specific, recoverable)
+     * - Embedding/storage errors: Let propagate (fatal, affects all files)
      */
     private processFileBatch;
     /**

package/dist/rag/indexer/indexer.js

@@ -62,13 +62,37 @@ export class Indexer {
             writable: true,
             value: null
         });
+        Object.defineProperty(this, "indexPromise", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
         this.projectRoot = projectRoot;
         this.logger = logger ?? null;
     }
     /**
      * Run the indexing pipeline.
+     * Uses mutex to prevent concurrent index operations.
      */
     async index(options = {}) {
+        // If indexing is already in progress, wait for it
+        if (this.indexPromise) {
+            this.log('warn', 'Index already in progress, waiting for completion');
+            return this.indexPromise;
+        }
+        this.indexPromise = this.doIndex(options);
+        try {
+            return await this.indexPromise;
+        }
+        finally {
+            this.indexPromise = null;
+        }
+    }
+    /**
+     * Perform the actual indexing operation.
+     */
+    async doIndex(options = {}) {
         const stats = createEmptyIndexStats();
         const { force = false, progressCallback } = options;
         try {
@@ -193,68 +217,79 @@ export class Indexer {
     }
     /**
      * Process a batch of files: read, chunk, embed, and prepare CodeChunks.
+     *
+     * Error handling strategy:
+     * - File read/parse errors: Log and continue (file-specific, recoverable)
+     * - Embedding/storage errors: Let propagate (fatal, affects all files)
      */
     async processFileBatch(filepaths, chunker, embeddings, storage, stats) {
         const allChunks = [];
         for (const filepath of filepaths) {
+            // Phase 1: File reading and chunking (recoverable errors)
+            let content;
+            let fileHash;
+            let chunks;
             try {
                 const absolutePath = path.join(this.projectRoot, filepath);
-                const content = await fs.readFile(absolutePath, 'utf-8');
-                const fileHash = (await import('../merkle/hash.js')).computeStringHash(content);
+                content = await fs.readFile(absolutePath, 'utf-8');
+                fileHash = (await import('../merkle/hash.js')).computeStringHash(content);
                 // Chunk the file (with size limits from config)
-                const chunks = await chunker.chunkFile(filepath, content, this.config.chunkMaxSize);
-                // Check embedding cache for each chunk
-                const contentHashes = chunks.map(c => c.contentHash);
-                const cachedEmbeddings = await storage.getCachedEmbeddings(contentHashes);
-                // Compute embeddings for cache misses
-                const missingChunks = chunks.filter(c => !cachedEmbeddings.has(c.contentHash));
-                if (missingChunks.length > 0) {
-                    // Embed contextHeader + text for semantic relevance
-                    const texts = missingChunks.map(c => c.contextHeader ? `${c.contextHeader}\n${c.text}` : c.text);
-                    const newEmbeddings = await embeddings.embed(texts);
-                    stats.embeddingsComputed += missingChunks.length;
-                    // Cache the new embeddings
-                    const cacheEntries = missingChunks.map((chunk, i) => ({
-                        contentHash: chunk.contentHash,
-                        vector: newEmbeddings[i],
-                        createdAt: new Date().toISOString(),
-                    }));
-                    await storage.cacheEmbeddings(cacheEntries);
-                    // Add to cachedEmbeddings map
-                    missingChunks.forEach((chunk, i) => {
-                        cachedEmbeddings.set(chunk.contentHash, newEmbeddings[i]);
-                    });
-                }
-                stats.embeddingsCached += chunks.length - missingChunks.length;
-                // Build CodeChunk objects
-                const filename = path.basename(filepath);
-                const extension = path.extname(filepath);
-                for (const chunk of chunks) {
-                    const vector = cachedEmbeddings.get(chunk.contentHash);
-                    allChunks.push({
-                        id: `${filepath}:${chunk.startLine}`,
-                        vector,
-                        text: chunk.text,
-                        contentHash: chunk.contentHash,
-                        filepath,
-                        filename,
-                        extension,
-                        type: chunk.type,
-                        name: chunk.name,
-                        startLine: chunk.startLine,
-                        endLine: chunk.endLine,
-                        fileHash,
-                        // New metadata fields from schema v2
-                        signature: chunk.signature,
-                        docstring: chunk.docstring,
-                        isExported: chunk.isExported,
-                        decoratorNames: chunk.decoratorNames,
-                    });
-                }
+                chunks = await chunker.chunkFile(filepath, content, this.config.chunkMaxSize);
             }
             catch (error) {
-                this.log('warn', `Failed to process file: ${filepath}`, error);
-                // Continue with other files
+                // File-specific error (read/parse) - log and continue with other files
+                this.log('warn', `Failed to read/parse file: ${filepath}`, error);
+                continue;
+            }
+            // Phase 2: Embedding and storage (fatal errors - let propagate)
+            // NO try-catch here - API/storage errors should stop indexing
+            // Check embedding cache for each chunk
+            const contentHashes = chunks.map(c => c.contentHash);
+            const cachedEmbeddings = await storage.getCachedEmbeddings(contentHashes);
+            // Compute embeddings for cache misses
+            const missingChunks = chunks.filter(c => !cachedEmbeddings.has(c.contentHash));
+            if (missingChunks.length > 0) {
+                // Embed contextHeader + text for semantic relevance
+                const texts = missingChunks.map(c => c.contextHeader ? `${c.contextHeader}\n${c.text}` : c.text);
+                const newEmbeddings = await embeddings.embed(texts);
+                stats.embeddingsComputed += missingChunks.length;
+                // Cache the new embeddings
+                const cacheEntries = missingChunks.map((chunk, i) => ({
+                    contentHash: chunk.contentHash,
+                    vector: newEmbeddings[i],
+                    createdAt: new Date().toISOString(),
+                }));
+                await storage.cacheEmbeddings(cacheEntries);
+                // Add to cachedEmbeddings map
+                missingChunks.forEach((chunk, i) => {
+                    cachedEmbeddings.set(chunk.contentHash, newEmbeddings[i]);
+                });
+            }
+            stats.embeddingsCached += chunks.length - missingChunks.length;
+            // Build CodeChunk objects
+            const filename = path.basename(filepath);
+            const extension = path.extname(filepath);
+            for (const chunk of chunks) {
+                const vector = cachedEmbeddings.get(chunk.contentHash);
+                allChunks.push({
+                    id: `${filepath}:${chunk.startLine}`,
+                    vector,
+                    text: chunk.text,
+                    contentHash: chunk.contentHash,
+                    filepath,
+                    filename,
+                    extension,
+                    type: chunk.type,
+                    name: chunk.name,
+                    startLine: chunk.startLine,
+                    endLine: chunk.endLine,
+                    fileHash,
+                    // New metadata fields from schema v2
+                    signature: chunk.signature,
+                    docstring: chunk.docstring,
+                    isExported: chunk.isExported,
+                    decoratorNames: chunk.decoratorNames,
+                });
             }
         }
         return allChunks;

package/dist/rag/search/index.d.ts

@@ -24,6 +24,7 @@ export declare class SearchEngine {
     private embeddings;
     private logger;
     private initialized;
+    private initPromise;
     constructor(projectRoot: string, logger?: Logger);
     /**
      * Primary search method. Dispatches to appropriate search mode.
@@ -74,8 +75,13 @@ export declare class SearchEngine {
     warmup(): Promise<void>;
     /**
      * Initialize the search engine.
+     * Uses idempotent promise pattern to prevent race conditions.
      */
     private ensureInitialized;
+    /**
+     * Perform actual initialization.
+     */
+    private doInitialize;
     /**
      * Create the appropriate embedding provider based on config.
      */

package/dist/rag/search/index.js

@@ -64,6 +64,12 @@ export class SearchEngine {
             writable: true,
             value: false
         });
+        Object.defineProperty(this, "initPromise", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
         this.projectRoot = projectRoot;
         this.logger = logger ?? null;
     }
@@ -315,19 +321,39 @@ export class SearchEngine {
     }
     /**
      * Initialize the search engine.
+     * Uses idempotent promise pattern to prevent race conditions.
      */
     async ensureInitialized() {
+        // Fast path: already initialized
         if (this.initialized)
             return;
-        const config = await loadConfig(this.projectRoot);
-        // Initialize storage
-        this.storage = new Storage(this.projectRoot, config.embeddingDimensions);
-        await this.storage.connect();
-        // Initialize embeddings with config (includes apiKey for cloud providers)
-        this.embeddings = this.createEmbeddingProvider(config);
-        await this.embeddings.initialize();
-        this.initialized = true;
-        this.log('info', 'SearchEngine initialized');
+        // Idempotent: return existing promise if initialization in progress
+        if (this.initPromise)
+            return this.initPromise;
+        // Start initialization and store promise
+        this.initPromise = this.doInitialize();
+        return this.initPromise;
+    }
+    /**
+     * Perform actual initialization.
+     */
+    async doInitialize() {
+        try {
+            const config = await loadConfig(this.projectRoot);
+            // Initialize storage
+            this.storage = new Storage(this.projectRoot, config.embeddingDimensions);
+            await this.storage.connect();
+            // Initialize embeddings with config (includes apiKey for cloud providers)
+            this.embeddings = this.createEmbeddingProvider(config);
+            await this.embeddings.initialize();
+            this.initialized = true;
+            this.log('info', 'SearchEngine initialized');
+        }
+        catch (error) {
+            // Reset promise on failure to allow retry
+            this.initPromise = null;
+            throw error;
+        }
     }
     /**
      * Create the appropriate embedding provider based on config.

package/dist/rag/storage/index.d.ts

@@ -15,8 +15,14 @@ export declare class Storage {
     /**
      * Connect to the LanceDB database.
      * Creates tables if they don't exist.
+     * Validates that existing tables have matching dimensions.
      */
     connect(): Promise<void>;
+    /**
+     * Get the vector column dimensions from a table schema.
+     * Returns null if vector column not found.
+     */
+    private getTableVectorDimensions;
     /**
      * Close the database connection.
      */
@@ -26,6 +32,14 @@ export declare class Storage {
      * Note: chunksTable may be null after resetChunksTable().
      */
     private ensureConnected;
+    /**
+     * Get the database connection with a clear error if not connected.
+     */
+    private getDb;
+    /**
+     * Get the cache table with a clear error if not connected.
+     */
+    private getCacheTable;
     /**
      * Upsert chunks into the database.
      * Uses merge insert to update existing chunks or add new ones.
@@ -34,7 +48,6 @@ export declare class Storage {
     /**
      * Add chunks to the database (no merge, just insert).
      * Use this after resetChunksTable() to avoid schema mismatch issues.
-     * Creates the table from data if it doesn't exist.
      */
     addChunks(chunks: CodeChunk[]): Promise<void>;
     /**
@@ -87,6 +100,7 @@ export declare class Storage {
     clearCache(): Promise<void>;
     /**
      * Get the chunks table for direct querying (e.g., search).
+     * @throws Error if table doesn't exist (not indexed yet)
      */
     getChunksTable(): Table;
 }