viberag 0.1.0

package/dist/rag/embeddings/gemini.js

@@ -0,0 +1,116 @@
+/**
+ * Gemini embedding provider using Google's Generative AI API.
+ *
+ * Uses gemini-embedding-001 model with 1536 dimensions.
+ * Note: The model defaults to 3072 dims but we explicitly request 1536 for:
+ * - Good balance of quality and storage
+ * - Matches OpenAI text-embedding-3-small dimensions
+ *
+ * Free tier available with generous limits.
+ */
+const GEMINI_API_BASE = 'https://generativelanguage.googleapis.com/v1beta/models';
+const MODEL = 'gemini-embedding-001';
+const BATCH_SIZE = 100; // Gemini supports up to 100 texts per request
+/**
+ * Gemini embedding provider.
+ * Uses gemini-embedding-001 model via Google's Generative AI API.
+ */
+export class GeminiEmbeddingProvider {
+    constructor(apiKey) {
+        Object.defineProperty(this, "dimensions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 1536
+        });
+        Object.defineProperty(this, "apiKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "initialized", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        this.apiKey = apiKey ?? '';
+    }
+    async initialize(_onProgress) {
+        if (!this.apiKey) {
+            throw new Error('Gemini API key required. Run /init to configure your API key.');
+        }
+        this.initialized = true;
+    }
+    async embed(texts) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        if (texts.length === 0) {
+            return [];
+        }
+        const results = [];
+        // Process in batches
+        for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+            const batch = texts.slice(i, i + BATCH_SIZE);
+            const batchResults = await this.embedBatch(batch);
+            results.push(...batchResults);
+        }
+        return results;
+    }
+    async embedBatch(texts) {
+        const url = `${GEMINI_API_BASE}/${MODEL}:batchEmbedContents?key=${this.apiKey}`;
+        const response = await fetch(url, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                requests: texts.map(text => ({
+                    model: `models/${MODEL}`,
+                    content: {
+                        parts: [{ text }],
+                    },
+                    taskType: 'RETRIEVAL_DOCUMENT',
+                    outputDimensionality: this.dimensions,
+                })),
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Gemini API error: ${response.status} - ${error}`);
+        }
+        const data = (await response.json());
+        return data.embeddings.map(e => e.values);
+    }
+    async embedSingle(text) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        const url = `${GEMINI_API_BASE}/${MODEL}:embedContent?key=${this.apiKey}`;
+        const response = await fetch(url, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+            },
+            body: JSON.stringify({
+                model: `models/${MODEL}`,
+                content: {
+                    parts: [{ text }],
+                },
+                taskType: 'RETRIEVAL_QUERY',
+                outputDimensionality: this.dimensions,
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Gemini API error: ${response.status} - ${error}`);
+        }
+        const data = (await response.json());
+        return data.embedding.values;
+    }
+    close() {
+        this.initialized = false;
+    }
+}

package/dist/rag/embeddings/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Embeddings module for generating vector embeddings.
+ * Supports both local (ONNX) and cloud API providers.
+ */
+export { GeminiEmbeddingProvider } from './gemini.js';
+export { Local4BEmbeddingProvider } from './local-4b.js';
+export { LocalEmbeddingProvider } from './local.js';
+export { MistralEmbeddingProvider } from './mistral.js';
+export { OpenAIEmbeddingProvider } from './openai.js';
+export type { EmbeddingProvider, ModelProgressCallback } from './types.js';

package/dist/rag/embeddings/index.js

@@ -0,0 +1,9 @@
+/**
+ * Embeddings module for generating vector embeddings.
+ * Supports both local (ONNX) and cloud API providers.
+ */
+export { GeminiEmbeddingProvider } from './gemini.js';
+export { Local4BEmbeddingProvider } from './local-4b.js';
+export { LocalEmbeddingProvider } from './local.js';
+export { MistralEmbeddingProvider } from './mistral.js';
+export { OpenAIEmbeddingProvider } from './openai.js';

package/dist/rag/embeddings/local-4b.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Local embedding provider using Qwen3-Embedding-4B.
+ *
+ * ⚠️  NOT CURRENTLY AVAILABLE
+ *
+ * No transformers.js-compatible ONNX version exists yet.
+ * The zhiqing/Qwen3-Embedding-4B-ONNX model has files in root instead of onnx/ subfolder.
+ * Waiting for onnx-community to release a properly structured version.
+ *
+ * When available:
+ * - 2560 dimensions
+ * - ~8GB download (full precision)
+ * - ~8GB RAM usage
+ * - 32K context window
+ * - +5 MTEB points over 0.6B (69.45 vs 64.33)
+ */
+import type { EmbeddingProvider, ModelProgressCallback } from './types.js';
+/**
+ * Local embedding provider using Qwen3-Embedding-4B FP32.
+ * Currently throws an error - no compatible ONNX model available.
+ */
+export declare class Local4BEmbeddingProvider implements EmbeddingProvider {
+    readonly dimensions = 2560;
+    initialize(_onProgress?: ModelProgressCallback): Promise<void>;
+    embed(_texts: string[]): Promise<number[][]>;
+    embedSingle(_text: string): Promise<number[]>;
+    close(): void;
+}

package/dist/rag/embeddings/local-4b.js

@@ -0,0 +1,51 @@
+/**
+ * Local embedding provider using Qwen3-Embedding-4B.
+ *
+ * ⚠️  NOT CURRENTLY AVAILABLE
+ *
+ * No transformers.js-compatible ONNX version exists yet.
+ * The zhiqing/Qwen3-Embedding-4B-ONNX model has files in root instead of onnx/ subfolder.
+ * Waiting for onnx-community to release a properly structured version.
+ *
+ * When available:
+ * - 2560 dimensions
+ * - ~8GB download (full precision)
+ * - ~8GB RAM usage
+ * - 32K context window
+ * - +5 MTEB points over 0.6B (69.45 vs 64.33)
+ */
+const DIMENSIONS = 2560;
+const NOT_AVAILABLE_ERROR = 'local-4b is not available yet.\n\n' +
+    'No transformers.js-compatible ONNX version of Qwen3-Embedding-4B exists.\n' +
+    'The zhiqing/Qwen3-Embedding-4B-ONNX model has incorrect file structure.\n\n' +
+    'Options:\n' +
+    '  1. Use "local" (0.6B Q8) - works now, ~1.2GB RAM\n' +
+    '  2. Use "gemini" - free API, best quality\n' +
+    '  3. Wait for onnx-community to release 4B version\n\n' +
+    'Run "viberag /init" to choose a different provider.';
+/**
+ * Local embedding provider using Qwen3-Embedding-4B FP32.
+ * Currently throws an error - no compatible ONNX model available.
+ */
+export class Local4BEmbeddingProvider {
+    constructor() {
+        Object.defineProperty(this, "dimensions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: DIMENSIONS
+        });
+    }
+    async initialize(_onProgress) {
+        throw new Error(NOT_AVAILABLE_ERROR);
+    }
+    async embed(_texts) {
+        throw new Error(NOT_AVAILABLE_ERROR);
+    }
+    async embedSingle(_text) {
+        throw new Error(NOT_AVAILABLE_ERROR);
+    }
+    close() {
+        // Nothing to close
+    }
+}

package/dist/rag/embeddings/local.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Local embedding provider using Qwen3-Embedding-0.6B.
+ *
+ * Uses Qwen3-Embedding-0.6B Q8 via @huggingface/transformers (ONNX Runtime).
+ * - 1024 dimensions
+ * - ~700MB download (Q8 quantized)
+ * - ~10GB RAM usage
+ * - 32K context window
+ *
+ * Benefits:
+ * - Works completely offline
+ * - No API key required
+ * - No per-token costs
+ * - Data never leaves your machine
+ */
+import type { EmbeddingProvider, ModelProgressCallback } from './types.js';
+/**
+ * Local embedding provider using Qwen3-Embedding-0.6B Q8.
+ */
+export declare class LocalEmbeddingProvider implements EmbeddingProvider {
+    readonly dimensions = 1024;
+    private extractor;
+    private initialized;
+    initialize(onProgress?: ModelProgressCallback): Promise<void>;
+    embed(texts: string[]): Promise<number[][]>;
+    private embedBatch;
+    embedSingle(text: string): Promise<number[]>;
+    close(): void;
+}

package/dist/rag/embeddings/local.js

@@ -0,0 +1,119 @@
+/**
+ * Local embedding provider using Qwen3-Embedding-0.6B.
+ *
+ * Uses Qwen3-Embedding-0.6B Q8 via @huggingface/transformers (ONNX Runtime).
+ * - 1024 dimensions
+ * - ~700MB download (Q8 quantized)
+ * - ~10GB RAM usage
+ * - 32K context window
+ *
+ * Benefits:
+ * - Works completely offline
+ * - No API key required
+ * - No per-token costs
+ * - Data never leaves your machine
+ */
+import { pipeline } from '@huggingface/transformers';
+const MODEL_NAME = 'onnx-community/Qwen3-Embedding-0.6B-ONNX';
+const DIMENSIONS = 1024;
+const BATCH_SIZE = 8;
+/**
+ * Local embedding provider using Qwen3-Embedding-0.6B Q8.
+ */
+export class LocalEmbeddingProvider {
+    constructor() {
+        Object.defineProperty(this, "dimensions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: DIMENSIONS
+        });
+        Object.defineProperty(this, "extractor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "initialized", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+    }
+    async initialize(onProgress) {
+        if (this.initialized)
+            return;
+        // Track download progress for the model files
+        let lastProgress = 0;
+        const progressCallback = onProgress
+            ? (progress) => {
+                if (progress.status === 'progress' &&
+                    progress.progress !== undefined) {
+                    // Round to avoid too many updates
+                    const pct = Math.round(progress.progress);
+                    if (pct !== lastProgress) {
+                        lastProgress = pct;
+                        onProgress('downloading', pct, progress.file);
+                    }
+                }
+                else if (progress.status === 'ready') {
+                    onProgress('loading');
+                }
+            }
+            : undefined;
+        // Notify loading is starting
+        onProgress?.('loading');
+        // Load the model with q8 (int8) quantization for smaller size and faster inference
+        // First load will download the model (~700MB)
+        this.extractor = await pipeline('feature-extraction', MODEL_NAME, {
+            dtype: 'q8', // int8 quantization
+            progress_callback: progressCallback,
+        });
+        onProgress?.('ready');
+        this.initialized = true;
+    }
+    async embed(texts) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        if (texts.length === 0) {
+            return [];
+        }
+        const results = [];
+        // Process in batches for memory efficiency
+        for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+            const batch = texts.slice(i, i + BATCH_SIZE);
+            const batchResults = await this.embedBatch(batch);
+            results.push(...batchResults);
+        }
+        return results;
+    }
+    async embedBatch(texts) {
+        const results = [];
+        for (const text of texts) {
+            const output = await this.extractor(text, {
+                pooling: 'mean',
+                normalize: true,
+            });
+            // Extract embedding from output tensor
+            const embedding = Array.from(output.data);
+            results.push(embedding);
+        }
+        return results;
+    }
+    async embedSingle(text) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        const output = await this.extractor(text, {
+            pooling: 'mean',
+            normalize: true,
+        });
+        return Array.from(output.data);
+    }
+    close() {
+        this.extractor = null;
+        this.initialized = false;
+    }
+}

package/dist/rag/embeddings/mistral.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Mistral embedding provider using Mistral AI API.
+ *
+ * Uses codestral-embed model (1024 dimensions).
+ * Optimized for code and technical content.
+ */
+import type { EmbeddingProvider, ModelProgressCallback } from './types.js';
+/**
+ * Mistral embedding provider.
+ * Uses codestral-embed model via Mistral AI API.
+ */
+export declare class MistralEmbeddingProvider implements EmbeddingProvider {
+    readonly dimensions = 1024;
+    private apiKey;
+    private initialized;
+    constructor(apiKey?: string);
+    initialize(_onProgress?: ModelProgressCallback): Promise<void>;
+    embed(texts: string[]): Promise<number[][]>;
+    private embedBatch;
+    embedSingle(text: string): Promise<number[]>;
+    close(): void;
+}

package/dist/rag/embeddings/mistral.js

@@ -0,0 +1,85 @@
+/**
+ * Mistral embedding provider using Mistral AI API.
+ *
+ * Uses codestral-embed model (1024 dimensions).
+ * Optimized for code and technical content.
+ */
+const MISTRAL_API_BASE = 'https://api.mistral.ai/v1';
+const MODEL = 'codestral-embed';
+const BATCH_SIZE = 64; // Mistral supports batching
+/**
+ * Mistral embedding provider.
+ * Uses codestral-embed model via Mistral AI API.
+ */
+export class MistralEmbeddingProvider {
+    constructor(apiKey) {
+        Object.defineProperty(this, "dimensions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 1024
+        });
+        Object.defineProperty(this, "apiKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "initialized", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        this.apiKey = apiKey ?? '';
+    }
+    async initialize(_onProgress) {
+        if (!this.apiKey) {
+            throw new Error('Mistral API key required. Run /init to configure your API key.');
+        }
+        this.initialized = true;
+    }
+    async embed(texts) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        if (texts.length === 0) {
+            return [];
+        }
+        const results = [];
+        // Process in batches
+        for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+            const batch = texts.slice(i, i + BATCH_SIZE);
+            const batchResults = await this.embedBatch(batch);
+            results.push(...batchResults);
+        }
+        return results;
+    }
+    async embedBatch(texts) {
+        const response = await fetch(`${MISTRAL_API_BASE}/embeddings`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+            body: JSON.stringify({
+                model: MODEL,
+                input: texts,
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Mistral API error: ${response.status} - ${error}`);
+        }
+        const data = (await response.json());
+        // Sort by index to ensure correct order
+        return data.data.sort((a, b) => a.index - b.index).map(d => d.embedding);
+    }
+    async embedSingle(text) {
+        const results = await this.embed([text]);
+        return results[0];
+    }
+    close() {
+        this.initialized = false;
+    }
+}

package/dist/rag/embeddings/openai.d.ts

@@ -0,0 +1,22 @@
+/**
+ * OpenAI embedding provider using OpenAI API.
+ *
+ * Uses text-embedding-3-small model (1536 dimensions).
+ * Good quality with fast API responses and low cost ($0.02/1M tokens).
+ */
+import type { EmbeddingProvider, ModelProgressCallback } from './types.js';
+/**
+ * OpenAI embedding provider.
+ * Uses text-embedding-3-small model via OpenAI API.
+ */
+export declare class OpenAIEmbeddingProvider implements EmbeddingProvider {
+    readonly dimensions = 1536;
+    private apiKey;
+    private initialized;
+    constructor(apiKey?: string);
+    initialize(_onProgress?: ModelProgressCallback): Promise<void>;
+    embed(texts: string[]): Promise<number[][]>;
+    private embedBatch;
+    embedSingle(text: string): Promise<number[]>;
+    close(): void;
+}

package/dist/rag/embeddings/openai.js

@@ -0,0 +1,85 @@
+/**
+ * OpenAI embedding provider using OpenAI API.
+ *
+ * Uses text-embedding-3-small model (1536 dimensions).
+ * Good quality with fast API responses and low cost ($0.02/1M tokens).
+ */
+const OPENAI_API_BASE = 'https://api.openai.com/v1';
+const MODEL = 'text-embedding-3-small';
+const BATCH_SIZE = 2048; // OpenAI supports up to 2048 texts per request
+/**
+ * OpenAI embedding provider.
+ * Uses text-embedding-3-small model via OpenAI API.
+ */
+export class OpenAIEmbeddingProvider {
+    constructor(apiKey) {
+        Object.defineProperty(this, "dimensions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 1536
+        });
+        Object.defineProperty(this, "apiKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "initialized", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        this.apiKey = apiKey ?? '';
+    }
+    async initialize(_onProgress) {
+        if (!this.apiKey) {
+            throw new Error('OpenAI API key required. Run /init to configure your API key.');
+        }
+        this.initialized = true;
+    }
+    async embed(texts) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        if (texts.length === 0) {
+            return [];
+        }
+        const results = [];
+        // Process in batches
+        for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+            const batch = texts.slice(i, i + BATCH_SIZE);
+            const batchResults = await this.embedBatch(batch);
+            results.push(...batchResults);
+        }
+        return results;
+    }
+    async embedBatch(texts) {
+        const response = await fetch(`${OPENAI_API_BASE}/embeddings`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${this.apiKey}`,
+            },
+            body: JSON.stringify({
+                model: MODEL,
+                input: texts,
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`OpenAI API error: ${response.status} - ${error}`);
+        }
+        const data = (await response.json());
+        // Sort by index to ensure correct order
+        return data.data.sort((a, b) => a.index - b.index).map(d => d.embedding);
+    }
+    async embedSingle(text) {
+        const results = await this.embed([text]);
+        return results[0];
+    }
+    close() {
+        this.initialized = false;
+    }
+}

package/dist/rag/embeddings/types.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Progress callback for model loading/downloading.
+ * @param status - Current status: 'downloading', 'loading', 'ready'
+ * @param progress - Download progress 0-100 (only for 'downloading')
+ * @param message - Optional message (e.g., file being downloaded)
+ */
+export type ModelProgressCallback = (status: 'downloading' | 'loading' | 'ready', progress?: number, message?: string) => void;
+/**
+ * Embedding provider interface for generating vector embeddings.
+ */
+export interface EmbeddingProvider {
+    /** Number of dimensions in the embedding vectors */
+    readonly dimensions: number;
+    /**
+     * Initialize the provider (load model, etc.)
+     * Must be called before using embed() or embedSingle().
+     * @param onProgress - Optional callback for download/loading progress
+     */
+    initialize(onProgress?: ModelProgressCallback): Promise<void>;
+    /**
+     * Generate embeddings for multiple texts.
+     * @param texts - Array of text strings to embed
+     * @returns Array of embedding vectors (one per text)
+     */
+    embed(texts: string[]): Promise<number[][]>;
+    /**
+     * Generate embedding for a single text.
+     * Optimized for query embedding.
+     * @param text - Text string to embed
+     * @returns Embedding vector
+     */
+    embedSingle(text: string): Promise<number[]>;
+    /**
+     * Close the provider and free resources.
+     */
+    close(): void;
+}

package/dist/rag/embeddings/types.js

@@ -0,0 +1 @@
+export {};

package/dist/rag/gitignore/index.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Gitignore-based file filtering.
+ *
+ * Uses the `ignore` package to parse .gitignore files and filter paths.
+ * This replaces the hardcoded excludePatterns approach.
+ */
+interface Ignore {
+    add(patterns: string | string[]): this;
+    ignores(pathname: string): boolean;
+    filter(pathnames: string[]): string[];
+    createFilter(): (pathname: string) => boolean;
+}
+/**
+ * Load and parse .gitignore file from project root.
+ * Returns an Ignore instance that can filter paths.
+ *
+ * @param projectRoot - Project root directory
+ * @returns Ignore instance for filtering
+ */
+export declare function loadGitignore(projectRoot: string): Promise<Ignore>;
+/**
+ * Check if a path should be ignored based on .gitignore rules.
+ *
+ * @param projectRoot - Project root directory
+ * @param relativePath - Path relative to project root
+ * @returns true if the path should be ignored
+ */
+export declare function shouldIgnore(projectRoot: string, relativePath: string): Promise<boolean>;
+/**
+ * Create a filter function for use with file listing.
+ * The filter returns true for files that should be INCLUDED (not ignored).
+ *
+ * @param projectRoot - Project root directory
+ * @returns Filter function that returns true for non-ignored files
+ */
+export declare function createGitignoreFilter(projectRoot: string): Promise<(relativePath: string) => boolean>;
+/**
+ * Clear the cache for a specific project root.
+ * Call this if .gitignore has been modified.
+ *
+ * @param projectRoot - Project root directory
+ */
+export declare function clearGitignoreCache(projectRoot: string): void;
+/**
+ * Clear all cached Ignore instances.
+ */
+export declare function clearAllGitignoreCache(): void;
+/**
+ * Convert gitignore patterns to fast-glob ignore patterns.
+ * This allows fast-glob to skip directories upfront instead of
+ * scanning them and filtering later.
+ *
+ * @param projectRoot - Project root directory
+ * @returns Array of fast-glob compatible ignore patterns
+ */
+export declare function getGlobIgnorePatterns(projectRoot: string): Promise<string[]>;
+export {};