viberag 0.3.2 → 0.4.0

package/dist/daemon/providers/mistral.js

@@ -0,0 +1,160 @@
+/**
+ * Mistral embedding provider using Mistral AI API.
+ *
+ * Uses codestral-embed model (1536 dimensions).
+ * Optimized for code and technical content.
+ */
+import { chunk, processBatchesWithLimit, withRetry, } from './api-utils.js';
+const MISTRAL_API_BASE = 'https://api.mistral.ai/v1';
+const MODEL = 'codestral-embed';
+// Mistral limits: 8,192 tokens/text, 16,000 tokens/batch TOTAL
+// Chunks are ~2000 chars but token count varies (code can be 1.5-2x tokens/char)
+// 8 chunks × ~1500 tokens worst case = 12,000 tokens (75% margin under 16k limit)
+const BATCH_SIZE = 8;
+/**
+ * Mistral embedding provider.
+ * Uses codestral-embed model via Mistral AI API.
+ */
+export class MistralEmbeddingProvider {
+    constructor(apiKey) {
+        Object.defineProperty(this, "dimensions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 1536
+        });
+        Object.defineProperty(this, "apiKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "initialized", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        // Callback for rate limit throttling - message or null to clear
+        Object.defineProperty(this, "onThrottle", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        // Callback for batch progress - (processed, total) chunks
+        Object.defineProperty(this, "onBatchProgress", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        // Slot progress callbacks (wired by daemon owner)
+        Object.defineProperty(this, "onSlotProcessing", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "onSlotRateLimited", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "onSlotIdle", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "onSlotFailure", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "onResetSlots", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        // Trim the key to remove any accidental whitespace
+        this.apiKey = (apiKey ?? '').trim();
+    }
+    async initialize(_onProgress) {
+        if (!this.apiKey) {
+            throw new Error('Mistral API key required. Run /init to configure your API key.');
+        }
+        this.initialized = true;
+    }
+    async embed(texts, options) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        if (texts.length === 0) {
+            return [];
+        }
+        const batches = chunk(texts, BATCH_SIZE);
+        const callbacks = {
+            onThrottle: this.onThrottle,
+            onBatchProgress: this.onBatchProgress,
+            onSlotProcessing: this.onSlotProcessing,
+            onSlotRateLimited: this.onSlotRateLimited,
+            onSlotIdle: this.onSlotIdle,
+            onSlotFailure: this.onSlotFailure,
+            onResetSlots: this.onResetSlots,
+        };
+        // Convert chunk metadata to batch metadata if provided
+        let batchMetadata;
+        if (options?.chunkMetadata) {
+            const metaBatches = chunk(options.chunkMetadata, BATCH_SIZE);
+            batchMetadata = metaBatches.map(metaBatch => ({
+                filepaths: metaBatch.map(m => m.filepath),
+                lineRanges: metaBatch.map(m => ({ start: m.startLine, end: m.endLine })),
+                sizes: metaBatch.map(m => m.size),
+            }));
+        }
+        return processBatchesWithLimit(batches, (batch, onRetrying) => withRetry(() => this.embedBatch(batch), callbacks, onRetrying), callbacks, BATCH_SIZE, batchMetadata, options?.logger, options?.chunkOffset ?? 0);
+    }
+    async embedBatch(texts) {
+        const response = await fetch(`${MISTRAL_API_BASE}/embeddings`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+            body: JSON.stringify({
+                model: MODEL,
+                input: texts,
+            }),
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            let errorMessage;
+            try {
+                const errorJson = JSON.parse(errorText);
+                errorMessage = errorJson.message || errorJson.detail || errorText;
+            }
+            catch {
+                errorMessage = errorText;
+            }
+            if (response.status === 401) {
+                throw new Error(`Mistral API authentication failed (401). ` +
+                    `Verify your API key at https://console.mistral.ai/api-keys. Error: ${errorMessage}`);
+            }
+            throw new Error(`Mistral API error (${response.status}): ${errorMessage}`);
+        }
+        const data = (await response.json());
+        // Sort by index to ensure correct order
+        return data.data.sort((a, b) => a.index - b.index).map(d => d.embedding);
+    }
+    async embedSingle(text) {
+        const results = await this.embed([text]);
+        return results[0];
+    }
+    close() {
+        this.initialized = false;
+    }
+}

package/dist/daemon/providers/mock.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Mock embedding provider for testing.
+ *
+ * Generates deterministic hash-based embeddings that:
+ * - Run instantly (no model loading)
+ * - Are deterministic (same input = same output)
+ * - Normalized to unit length
+ * - Support any dimension count
+ *
+ * Usage:
+ * - Unit tests that need embeddings but don't need semantic quality
+ * - Testing search infrastructure without ONNX overhead
+ * - CI pipeline fast checks
+ */
+import type { EmbeddingProvider, ModelProgressCallback, EmbedOptions } from './types.js';
+/**
+ * Mock embedding provider using deterministic hash-based vectors.
+ */
+export declare class MockEmbeddingProvider implements EmbeddingProvider {
+    readonly dimensions: number;
+    constructor(dimensions?: number);
+    initialize(_onProgress?: ModelProgressCallback): Promise<void>;
+    embed(texts: string[], _options?: EmbedOptions): Promise<number[][]>;
+    embedSingle(text: string): Promise<number[]>;
+    /**
+     * Convert text to a deterministic unit vector.
+     * Uses a simple hash-based approach to generate pseudo-random but repeatable values.
+     */
+    private hashToVector;
+    /**
+     * Simple string hash function (djb2).
+     */
+    private hash;
+    close(): void;
+}

package/dist/daemon/providers/mock.js

@@ -0,0 +1,69 @@
+/**
+ * Mock embedding provider for testing.
+ *
+ * Generates deterministic hash-based embeddings that:
+ * - Run instantly (no model loading)
+ * - Are deterministic (same input = same output)
+ * - Normalized to unit length
+ * - Support any dimension count
+ *
+ * Usage:
+ * - Unit tests that need embeddings but don't need semantic quality
+ * - Testing search infrastructure without ONNX overhead
+ * - CI pipeline fast checks
+ */
+const DEFAULT_DIMENSIONS = 1024;
+/**
+ * Mock embedding provider using deterministic hash-based vectors.
+ */
+export class MockEmbeddingProvider {
+    constructor(dimensions = DEFAULT_DIMENSIONS) {
+        Object.defineProperty(this, "dimensions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.dimensions = dimensions;
+    }
+    async initialize(_onProgress) {
+        // No initialization needed - instant startup
+    }
+    async embed(texts, _options) {
+        return texts.map(t => this.hashToVector(t));
+    }
+    async embedSingle(text) {
+        return this.hashToVector(text);
+    }
+    /**
+     * Convert text to a deterministic unit vector.
+     * Uses a simple hash-based approach to generate pseudo-random but repeatable values.
+     */
+    hashToVector(text) {
+        const seed = this.hash(text);
+        // Generate deterministic pseudo-random values
+        const vec = new Array(this.dimensions).fill(0).map((_, i) => {
+            // LCG-like pseudo-random based on seed and index
+            const state = (((seed * (i + 1) * 1103515245 + 12345) >>> 0) % 0x7fffffff) /
+                0x7fffffff;
+            return state * 2 - 1; // Range [-1, 1]
+        });
+        // Normalize to unit length
+        const magnitude = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0));
+        return vec.map(v => (magnitude > 0 ? v / magnitude : 0));
+    }
+    /**
+     * Simple string hash function (djb2).
+     */
+    hash(str) {
+        let h = 5381;
+        for (let i = 0; i < str.length; i++) {
+            h = (h * 33) ^ str.charCodeAt(i);
+            h = h >>> 0; // Convert to unsigned 32-bit
+        }
+        return h;
+    }
+    close() {
+        // Nothing to close
+    }
+}

package/dist/daemon/providers/openai.d.ts

@@ -0,0 +1,41 @@
+/**
+ * OpenAI embedding provider using OpenAI API.
+ *
+ * Uses text-embedding-3-large model with reduced dimensions (1536).
+ * High quality embeddings with fast API responses ($0.13/1M tokens).
+ */
+import type { EmbeddingProvider, ModelProgressCallback, EmbedOptions } from './types.js';
+/**
+ * OpenAI embedding provider.
+ * Uses text-embedding-3-large model via OpenAI API with reduced dimensions.
+ *
+ * Supports regional endpoints for corporate accounts with data residency:
+ * - Default: https://api.openai.com/v1
+ * - US: https://us.api.openai.com/v1
+ * - EU: https://eu.api.openai.com/v1
+ */
+export declare class OpenAIEmbeddingProvider implements EmbeddingProvider {
+    readonly dimensions = 1536;
+    private apiKey;
+    private apiBase;
+    private initialized;
+    onThrottle: ((message: string | null) => void) | undefined;
+    onBatchProgress: ((processed: number, total: number) => void) | undefined;
+    onSlotProcessing: ((index: number, batchInfo: string) => void) | undefined;
+    onSlotRateLimited: ((index: number, batchInfo: string, retryInfo: string) => void) | undefined;
+    onSlotIdle: ((index: number) => void) | undefined;
+    onSlotFailure: ((data: {
+        batchInfo: string;
+        files: string[];
+        chunkCount: number;
+        error: string;
+        timestamp: string;
+    }) => void) | undefined;
+    onResetSlots: (() => void) | undefined;
+    constructor(apiKey?: string, baseUrl?: string);
+    initialize(_onProgress?: ModelProgressCallback): Promise<void>;
+    embed(texts: string[], options?: EmbedOptions): Promise<number[][]>;
+    private embedBatch;
+    embedSingle(text: string): Promise<number[]>;
+    close(): void;
+}

package/dist/daemon/providers/openai.js

@@ -0,0 +1,190 @@
+/**
+ * OpenAI embedding provider using OpenAI API.
+ *
+ * Uses text-embedding-3-large model with reduced dimensions (1536).
+ * High quality embeddings with fast API responses ($0.13/1M tokens).
+ */
+import { chunk, processBatchesWithLimit, withRetry, } from './api-utils.js';
+const DEFAULT_API_BASE = 'https://api.openai.com/v1';
+const MODEL = 'text-embedding-3-large';
+const DIMENSIONS = 1536; // Reduced from 3072 for storage efficiency
+// OpenAI limits: 8,191 tokens/text, 300,000 tokens/batch, 2,048 texts/batch
+// Chunks are ~2000 chars + context header ≈ 800-1000 tokens each
+// 32 chunks × 1000 tokens = 32,000 tokens (well under 300k limit)
+// Smaller batches = more progress visibility with 5 concurrent slots
+const BATCH_SIZE = 32;
+/**
+ * OpenAI embedding provider.
+ * Uses text-embedding-3-large model via OpenAI API with reduced dimensions.
+ *
+ * Supports regional endpoints for corporate accounts with data residency:
+ * - Default: https://api.openai.com/v1
+ * - US: https://us.api.openai.com/v1
+ * - EU: https://eu.api.openai.com/v1
+ */
+export class OpenAIEmbeddingProvider {
+    constructor(apiKey, baseUrl) {
+        Object.defineProperty(this, "dimensions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 1536
+        });
+        Object.defineProperty(this, "apiKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "apiBase", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "initialized", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        // Callback for rate limit throttling - message or null to clear
+        Object.defineProperty(this, "onThrottle", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        // Callback for batch progress - (processed, total) chunks
+        Object.defineProperty(this, "onBatchProgress", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        // Slot progress callbacks (wired by daemon owner)
+        Object.defineProperty(this, "onSlotProcessing", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "onSlotRateLimited", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "onSlotIdle", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "onSlotFailure", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "onResetSlots", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        // Trim the key to remove any accidental whitespace
+        this.apiKey = (apiKey ?? '').trim();
+        this.apiBase = baseUrl ?? DEFAULT_API_BASE;
+    }
+    async initialize(_onProgress) {
+        if (!this.apiKey) {
+            throw new Error('OpenAI API key required. Run /init to configure your API key.');
+        }
+        // Validate key format (should start with sk-)
+        if (!this.apiKey.startsWith('sk-')) {
+            throw new Error(`Invalid OpenAI API key format. Key should start with "sk-" but got "${this.apiKey.slice(0, 3)}..."`);
+        }
+        this.initialized = true;
+    }
+    async embed(texts, options) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        if (texts.length === 0) {
+            return [];
+        }
+        const batches = chunk(texts, BATCH_SIZE);
+        const callbacks = {
+            onThrottle: this.onThrottle,
+            onBatchProgress: this.onBatchProgress,
+            onSlotProcessing: this.onSlotProcessing,
+            onSlotRateLimited: this.onSlotRateLimited,
+            onSlotIdle: this.onSlotIdle,
+            onSlotFailure: this.onSlotFailure,
+            onResetSlots: this.onResetSlots,
+        };
+        // Convert chunk metadata to batch metadata if provided
+        let batchMetadata;
+        if (options?.chunkMetadata) {
+            const metaBatches = chunk(options.chunkMetadata, BATCH_SIZE);
+            batchMetadata = metaBatches.map(metaBatch => ({
+                filepaths: metaBatch.map(m => m.filepath),
+                lineRanges: metaBatch.map(m => ({ start: m.startLine, end: m.endLine })),
+                sizes: metaBatch.map(m => m.size),
+            }));
+        }
+        return processBatchesWithLimit(batches, (batch, onRetrying) => withRetry(() => this.embedBatch(batch), callbacks, onRetrying), callbacks, BATCH_SIZE, batchMetadata, options?.logger, options?.chunkOffset ?? 0);
+    }
+    async embedBatch(texts) {
+        const response = await fetch(`${this.apiBase}/embeddings`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+            body: JSON.stringify({
+                model: MODEL,
+                input: texts,
+                dimensions: DIMENSIONS,
+            }),
+        });
+        if (!response.ok) {
+            const errorText = await response.text();
+            let errorMessage;
+            try {
+                const errorJson = JSON.parse(errorText);
+                errorMessage = errorJson.error?.message || errorText;
+            }
+            catch {
+                errorMessage = errorText;
+            }
+            // Provide helpful context for common errors
+            if (response.status === 401) {
+                const keyPreview = `${this.apiKey.slice(0, 7)}...${this.apiKey.slice(-4)}`;
+                // Check for regional endpoint mismatch
+                if (errorMessage.includes('incorrect regional hostname')) {
+                    // Extract the required region from the error message if present
+                    const regionMatch = errorMessage.match(/make your request to (\w+\.api\.openai\.com)/);
+                    const requiredEndpoint = regionMatch?.[1] ?? 'the correct regional endpoint';
+                    throw new Error(`OpenAI API regional endpoint mismatch. Your account requires ${requiredEndpoint}. ` +
+                        `Run /init again and select the matching region (US or EU) instead of Default. ` +
+                        `Key: ${keyPreview}`);
+                }
+                throw new Error(`OpenAI API authentication failed (401). Key format: ${keyPreview}. ` +
+                    `Verify your API key at https://platform.openai.com/api-keys. Error: ${errorMessage}`);
+            }
+            throw new Error(`OpenAI API error (${response.status}): ${errorMessage}`);
+        }
+        const data = (await response.json());
+        // Sort by index to ensure correct order
+        return data.data.sort((a, b) => a.index - b.index).map(d => d.embedding);
+    }
+    async embedSingle(text) {
+        const results = await this.embed([text]);
+        return results[0];
+    }
+    close() {
+        this.initialized = false;
+    }
+}

package/dist/daemon/providers/types.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Embedding Provider Types.
+ *
+ * Types for embedding providers that generate vector embeddings from text.
+ */
+import type { Logger } from '../lib/logger.js';
+/**
+ * Progress callback for model loading/downloading.
+ * @param status - Current status: 'downloading', 'loading', 'ready'
+ * @param progress - Download progress 0-100 (only for 'downloading')
+ * @param message - Optional message (e.g., file being downloaded)
+ */
+export type ModelProgressCallback = (status: 'downloading' | 'loading' | 'ready', progress?: number, message?: string) => void;
+/**
+ * Metadata for a single chunk, used for detailed failure logging.
+ */
+export interface ChunkMetadata {
+    /** File path for this chunk */
+    filepath: string;
+    /** Start line number (1-indexed) */
+    startLine: number;
+    /** End line number (1-indexed) */
+    endLine: number;
+    /** Text size in characters */
+    size: number;
+}
+/**
+ * Options for embedding operations.
+ */
+export interface EmbedOptions {
+    /** Metadata for each chunk being embedded (parallel array to texts) */
+    chunkMetadata?: ChunkMetadata[];
+    /** Logger for debug output on failures */
+    logger?: Logger;
+    /** Offset for cumulative chunk numbering in progress display */
+    chunkOffset?: number;
+}
+/**
+ * Embedding provider interface for generating vector embeddings.
+ */
+export interface EmbeddingProvider {
+    /** Number of dimensions in the embedding vectors */
+    readonly dimensions: number;
+    /**
+     * Initialize the provider (load model, etc.)
+     * Must be called before using embed() or embedSingle().
+     * @param onProgress - Optional callback for download/loading progress
+     */
+    initialize(onProgress?: ModelProgressCallback): Promise<void>;
+    /**
+     * Generate embeddings for multiple texts.
+     * @param texts - Array of text strings to embed
+     * @param options - Optional settings for logging and metadata
+     * @returns Array of embedding vectors (one per text)
+     */
+    embed(texts: string[], options?: EmbedOptions): Promise<number[][]>;
+    /**
+     * Generate embedding for a single text.
+     * Optimized for query embedding.
+     * @param text - Text string to embed
+     * @returns Embedding vector
+     */
+    embedSingle(text: string): Promise<number[]>;
+    /**
+     * Close the provider and free resources.
+     */
+    close(): void;
+}

package/dist/daemon/providers/types.js

@@ -0,0 +1,6 @@
+/**
+ * Embedding Provider Types.
+ *
+ * Types for embedding providers that generate vector embeddings from text.
+ */
+export {};

package/dist/daemon/providers/validate.d.ts

@@ -0,0 +1,30 @@
+/**
+ * API key validation for cloud embedding providers.
+ *
+ * Makes a minimal test embedding call to verify the API key is valid
+ * before proceeding with indexing.
+ */
+import type { EmbeddingProviderType } from '../../common/types.js';
+/**
+ * Result of API key validation.
+ */
+export interface ValidationResult {
+    valid: boolean;
+    error?: string;
+}
+/**
+ * Options for API key validation.
+ */
+export interface ValidateApiKeyOptions {
+    /** OpenAI base URL for regional endpoints (e.g., https://us.api.openai.com/v1) */
+    openaiBaseUrl?: string;
+}
+/**
+ * Validate an API key by making a minimal test embedding call.
+ *
+ * @param provider - The embedding provider type
+ * @param apiKey - The API key to validate
+ * @param options - Optional configuration (e.g., openaiBaseUrl for regional endpoints)
+ * @returns Validation result with error message if invalid
+ */
+export declare function validateApiKey(provider: EmbeddingProviderType, apiKey: string, options?: ValidateApiKeyOptions): Promise<ValidationResult>;