viberag 0.1.0

package/dist/rag/search/vector.js

@@ -0,0 +1,44 @@
+/**
+ * Vector similarity search using LanceDB.
+ */
+/**
+ * Perform vector similarity search.
+ *
+ * @param table - LanceDB table to search
+ * @param queryVector - Query embedding vector
+ * @param options - Search options
+ * @returns Array of search results with vector scores
+ */
+export async function vectorSearch(table, queryVector, options) {
+    // Support legacy signature: vectorSearch(table, vector, limit)
+    const opts = typeof options === 'number' ? { limit: options } : options;
+    let query = table.search(queryVector).limit(opts.limit);
+    // Apply filter if provided
+    if (opts.filterClause) {
+        query = query.where(opts.filterClause);
+    }
+    const results = await query.toArray();
+    return results
+        .map(row => {
+        const chunk = row;
+        // LanceDB returns _distance (lower is better for L2/cosine)
+        // Convert to similarity score (higher is better)
+        const distance = chunk._distance ?? 0;
+        const vectorScore = 1 / (1 + distance);
+        return {
+            id: chunk.id,
+            text: chunk.text,
+            filepath: chunk.filepath,
+            filename: chunk.filename,
+            name: chunk.name,
+            type: chunk.type,
+            startLine: chunk.start_line,
+            endLine: chunk.end_line,
+            score: vectorScore,
+            vectorScore,
+            signature: chunk.signature,
+            isExported: chunk.is_exported,
+        };
+    })
+        .filter(r => !opts.minScore || r.score >= opts.minScore);
+}

package/dist/rag/storage/index.d.ts

@@ -0,0 +1,92 @@
+import type { Table } from '@lancedb/lancedb';
+import { type CachedEmbedding, type CodeChunk } from './types.js';
+export * from './types.js';
+export * from './schema.js';
+/**
+ * Storage layer wrapping LanceDB for code chunks and embedding cache.
+ */
+export declare class Storage {
+    private readonly projectRoot;
+    private readonly dimensions;
+    private db;
+    private chunksTable;
+    private cacheTable;
+    constructor(projectRoot: string, dimensions?: number);
+    /**
+     * Connect to the LanceDB database.
+     * Creates tables if they don't exist.
+     */
+    connect(): Promise<void>;
+    /**
+     * Close the database connection.
+     */
+    close(): void;
+    /**
+     * Ensure we're connected.
+     * Note: chunksTable may be null after resetChunksTable().
+     */
+    private ensureConnected;
+    /**
+     * Upsert chunks into the database.
+     * Uses merge insert to update existing chunks or add new ones.
+     */
+    upsertChunks(chunks: CodeChunk[]): Promise<void>;
+    /**
+     * Add chunks to the database (no merge, just insert).
+     * Use this after resetChunksTable() to avoid schema mismatch issues.
+     * Creates the table from data if it doesn't exist.
+     */
+    addChunks(chunks: CodeChunk[]): Promise<void>;
+    /**
+     * Delete all chunks for a specific file.
+     * @returns Number of chunks deleted
+     */
+    deleteChunksByFilepath(filepath: string): Promise<number>;
+    /**
+     * Delete all chunks for multiple files.
+     * @returns Number of chunks deleted
+     */
+    deleteChunksByFilepaths(filepaths: string[]): Promise<number>;
+    /**
+     * Get all chunks for a specific file.
+     */
+    getChunksByFilepath(filepath: string): Promise<CodeChunk[]>;
+    /**
+     * Get all unique filepaths in the database.
+     */
+    getAllFilepaths(): Promise<Set<string>>;
+    /**
+     * Count total number of chunks.
+     */
+    countChunks(): Promise<number>;
+    /**
+     * Get cached embeddings for a list of content hashes.
+     * @returns Map from content hash to vector
+     */
+    getCachedEmbeddings(hashes: string[]): Promise<Map<string, number[]>>;
+    /**
+     * Cache embeddings for future use.
+     */
+    cacheEmbeddings(entries: CachedEmbedding[]): Promise<void>;
+    /**
+     * Count total number of cached embeddings.
+     */
+    countCachedEmbeddings(): Promise<number>;
+    /**
+     * Clear all chunks but keep the embedding cache.
+     */
+    clearAll(): Promise<void>;
+    /**
+     * Drop and recreate the chunks table.
+     * Use this for force reindex to avoid schema mismatch issues.
+     */
+    resetChunksTable(): Promise<void>;
+    /**
+     * Clear the embedding cache.
+     */
+    clearCache(): Promise<void>;
+    /**
+     * Get the chunks table for direct querying (e.g., search).
+     */
+    getChunksTable(): Table;
+}

package/dist/rag/storage/index.js

@@ -0,0 +1,287 @@
+import * as lancedb from '@lancedb/lancedb';
+import { makeArrowTable } from '@lancedb/lancedb';
+import { getLanceDbPath, TABLE_NAMES } from '../constants.js';
+import { createCodeChunksSchema, createEmbeddingCacheSchema } from './schema.js';
+import { chunkToRow, embeddingToRow, rowToChunk, } from './types.js';
+export * from './types.js';
+export * from './schema.js';
+/**
+ * Storage layer wrapping LanceDB for code chunks and embedding cache.
+ */
+export class Storage {
+    constructor(projectRoot, dimensions = 768) {
+        Object.defineProperty(this, "projectRoot", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "dimensions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "db", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "chunksTable", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "cacheTable", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        this.projectRoot = projectRoot;
+        this.dimensions = dimensions;
+    }
+    /**
+     * Connect to the LanceDB database.
+     * Creates tables if they don't exist.
+     */
+    async connect() {
+        const dbPath = getLanceDbPath(this.projectRoot);
+        this.db = await lancedb.connect(dbPath);
+        // Get existing table names
+        const tableNames = await this.db.tableNames();
+        // Open or create code_chunks table
+        if (tableNames.includes(TABLE_NAMES.CODE_CHUNKS)) {
+            this.chunksTable = await this.db.openTable(TABLE_NAMES.CODE_CHUNKS);
+        }
+        else {
+            const schema = createCodeChunksSchema(this.dimensions);
+            this.chunksTable = await this.db.createEmptyTable(TABLE_NAMES.CODE_CHUNKS, schema);
+        }
+        // Open or create embedding_cache table
+        if (tableNames.includes(TABLE_NAMES.EMBEDDING_CACHE)) {
+            this.cacheTable = await this.db.openTable(TABLE_NAMES.EMBEDDING_CACHE);
+        }
+        else {
+            const schema = createEmbeddingCacheSchema(this.dimensions);
+            this.cacheTable = await this.db.createEmptyTable(TABLE_NAMES.EMBEDDING_CACHE, schema);
+        }
+    }
+    /**
+     * Close the database connection.
+     */
+    close() {
+        // LanceDB connections don't need explicit closing in the JS SDK
+        this.db = null;
+        this.chunksTable = null;
+        this.cacheTable = null;
+    }
+    /**
+     * Ensure we're connected.
+     * Note: chunksTable may be null after resetChunksTable().
+     */
+    ensureConnected() {
+        if (!this.db || !this.cacheTable) {
+            throw new Error('Storage not connected. Call connect() first.');
+        }
+    }
+    // ============================================================
+    // Chunk Operations
+    // ============================================================
+    /**
+     * Upsert chunks into the database.
+     * Uses merge insert to update existing chunks or add new ones.
+     */
+    async upsertChunks(chunks) {
+        this.ensureConnected();
+        if (chunks.length === 0)
+            return;
+        const rows = chunks.map(chunkToRow);
+        // Use merge insert for upsert behavior
+        await this.chunksTable.mergeInsert('id')
+            .whenMatchedUpdateAll()
+            .whenNotMatchedInsertAll()
+            .execute(rows);
+    }
+    /**
+     * Add chunks to the database (no merge, just insert).
+     * Use this after resetChunksTable() to avoid schema mismatch issues.
+     * Creates the table from data if it doesn't exist.
+     */
+    async addChunks(chunks) {
+        this.ensureConnected();
+        if (chunks.length === 0)
+            return;
+        const rows = chunks.map(chunkToRow);
+        const schema = createCodeChunksSchema(this.dimensions);
+        // Use makeArrowTable to properly convert data with schema
+        const arrowTable = makeArrowTable(rows, { schema });
+        // If table was reset (null), create from Arrow table
+        if (!this.chunksTable) {
+            this.chunksTable = await this.db.createTable(TABLE_NAMES.CODE_CHUNKS, arrowTable);
+        }
+        else {
+            await this.chunksTable.add(arrowTable);
+        }
+    }
+    /**
+     * Delete all chunks for a specific file.
+     * @returns Number of chunks deleted
+     */
+    async deleteChunksByFilepath(filepath) {
+        this.ensureConnected();
+        const countBefore = await this.chunksTable.countRows();
+        await this.chunksTable.delete(`filepath = '${escapeString(filepath)}'`);
+        const countAfter = await this.chunksTable.countRows();
+        return countBefore - countAfter;
+    }
+    /**
+     * Delete all chunks for multiple files.
+     * @returns Number of chunks deleted
+     */
+    async deleteChunksByFilepaths(filepaths) {
+        this.ensureConnected();
+        if (filepaths.length === 0)
+            return 0;
+        const countBefore = await this.chunksTable.countRows();
+        // Build IN clause with escaped strings
+        const escaped = filepaths.map(fp => `'${escapeString(fp)}'`).join(', ');
+        await this.chunksTable.delete(`filepath IN (${escaped})`);
+        const countAfter = await this.chunksTable.countRows();
+        return countBefore - countAfter;
+    }
+    /**
+     * Get all chunks for a specific file.
+     */
+    async getChunksByFilepath(filepath) {
+        this.ensureConnected();
+        const results = await this.chunksTable.query()
+            .where(`filepath = '${escapeString(filepath)}'`)
+            .toArray();
+        return results.map(row => rowToChunk(row));
+    }
+    /**
+     * Get all unique filepaths in the database.
+     */
+    async getAllFilepaths() {
+        this.ensureConnected();
+        // Query all rows but only need filepath column
+        const results = await this.chunksTable.query()
+            .select(['filepath'])
+            .toArray();
+        const filepaths = new Set();
+        for (const row of results) {
+            filepaths.add(row.filepath);
+        }
+        return filepaths;
+    }
+    /**
+     * Count total number of chunks.
+     */
+    async countChunks() {
+        this.ensureConnected();
+        return this.chunksTable.countRows();
+    }
+    // ============================================================
+    // Cache Operations
+    // ============================================================
+    /**
+     * Get cached embeddings for a list of content hashes.
+     * @returns Map from content hash to vector
+     */
+    async getCachedEmbeddings(hashes) {
+        this.ensureConnected();
+        if (hashes.length === 0)
+            return new Map();
+        // Build IN clause
+        const escaped = hashes.map(h => `'${escapeString(h)}'`).join(', ');
+        const results = await this.cacheTable.query()
+            .where(`content_hash IN (${escaped})`)
+            .toArray();
+        const cache = new Map();
+        for (const row of results) {
+            const typed = row;
+            // Ensure vector is a plain array (LanceDB may return typed arrays)
+            const vector = Array.isArray(typed.vector)
+                ? typed.vector
+                : Array.from(typed.vector);
+            cache.set(typed.content_hash, vector);
+        }
+        return cache;
+    }
+    /**
+     * Cache embeddings for future use.
+     */
+    async cacheEmbeddings(entries) {
+        this.ensureConnected();
+        if (entries.length === 0)
+            return;
+        const rows = entries.map(embeddingToRow);
+        // Use merge insert for upsert behavior
+        await this.cacheTable.mergeInsert('content_hash')
+            .whenMatchedUpdateAll()
+            .whenNotMatchedInsertAll()
+            .execute(rows);
+    }
+    /**
+     * Count total number of cached embeddings.
+     */
+    async countCachedEmbeddings() {
+        this.ensureConnected();
+        return this.cacheTable.countRows();
+    }
+    // ============================================================
+    // Maintenance Operations
+    // ============================================================
+    /**
+     * Clear all chunks but keep the embedding cache.
+     */
+    async clearAll() {
+        this.ensureConnected();
+        // Delete all rows from chunks table
+        // LanceDB doesn't have a truncate, so we delete all
+        const count = await this.chunksTable.countRows();
+        if (count > 0) {
+            // Delete with a condition that matches all rows
+            await this.chunksTable.delete('id IS NOT NULL');
+        }
+    }
+    /**
+     * Drop and recreate the chunks table.
+     * Use this for force reindex to avoid schema mismatch issues.
+     */
+    async resetChunksTable() {
+        this.ensureConnected();
+        // Drop existing table
+        await this.db.dropTable(TABLE_NAMES.CODE_CHUNKS);
+        // Don't pre-create - let it be created from first data insert
+        // This avoids Arrow schema mismatch issues
+        this.chunksTable = null;
+    }
+    /**
+     * Clear the embedding cache.
+     */
+    async clearCache() {
+        this.ensureConnected();
+        const count = await this.cacheTable.countRows();
+        if (count > 0) {
+            await this.cacheTable.delete('content_hash IS NOT NULL');
+        }
+    }
+    /**
+     * Get the chunks table for direct querying (e.g., search).
+     */
+    getChunksTable() {
+        this.ensureConnected();
+        return this.chunksTable;
+    }
+}
+/**
+ * Escape a string for use in SQL-like LanceDB filter expressions.
+ * Escapes single quotes by doubling them.
+ */
+function escapeString(s) {
+    return s.replace(/'/g, "''");
+}

package/dist/rag/storage/lancedb-native.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Force bundler to include LanceDB native module.
+ * This file exists to make Bun's bundler statically analyze and include
+ * the LanceDB native binding, which is normally loaded dynamically.
+ */
+import '@lancedb/lancedb-darwin-arm64';
+export * from '@lancedb/lancedb';

package/dist/rag/storage/lancedb-native.js

@@ -0,0 +1,10 @@
+/**
+ * Force bundler to include LanceDB native module.
+ * This file exists to make Bun's bundler statically analyze and include
+ * the LanceDB native binding, which is normally loaded dynamically.
+ */
+// Force bundler to include the darwin-arm64 native module
+// @ts-ignore - native module
+import '@lancedb/lancedb-darwin-arm64';
+// Re-export everything from lancedb
+export * from '@lancedb/lancedb';

package/dist/rag/storage/schema.d.ts

@@ -0,0 +1,23 @@
+import { Schema } from 'apache-arrow';
+/**
+ * Current schema version. Increment when schema changes require reindex.
+ */
+export declare const SCHEMA_VERSION = 2;
+/**
+ * Arrow schema for the code_chunks table.
+ *
+ * Stores indexed code chunks with their embeddings.
+ *
+ * Schema v2 adds:
+ * - signature: Function/method signature line
+ * - docstring: Extracted documentation
+ * - is_exported: Whether symbol is exported
+ * - decorator_names: Comma-separated decorator names
+ */
+export declare function createCodeChunksSchema(dimensions?: number): Schema;
+/**
+ * Arrow schema for the embedding_cache table.
+ *
+ * Content-addressed cache for embeddings to avoid recomputation.
+ */
+export declare function createEmbeddingCacheSchema(dimensions?: number): Schema;

package/dist/rag/storage/schema.js

@@ -0,0 +1,50 @@
+import { Field, FixedSizeList, Float32, Int32, Schema, Utf8, Bool, } from 'apache-arrow';
+import { DEFAULT_EMBEDDING_DIMENSIONS } from '../constants.js';
+/**
+ * Current schema version. Increment when schema changes require reindex.
+ */
+export const SCHEMA_VERSION = 2;
+/**
+ * Arrow schema for the code_chunks table.
+ *
+ * Stores indexed code chunks with their embeddings.
+ *
+ * Schema v2 adds:
+ * - signature: Function/method signature line
+ * - docstring: Extracted documentation
+ * - is_exported: Whether symbol is exported
+ * - decorator_names: Comma-separated decorator names
+ */
+export function createCodeChunksSchema(dimensions = DEFAULT_EMBEDDING_DIMENSIONS) {
+    return new Schema([
+        new Field('id', new Utf8(), false), // "{filepath}:{startLine}"
+        new Field('vector', new FixedSizeList(dimensions, new Field('item', new Float32(), false)), false),
+        new Field('text', new Utf8(), false),
+        new Field('content_hash', new Utf8(), false),
+        new Field('filepath', new Utf8(), false),
+        new Field('filename', new Utf8(), false),
+        new Field('extension', new Utf8(), false),
+        new Field('type', new Utf8(), false), // function/class/method/module
+        new Field('name', new Utf8(), false),
+        new Field('start_line', new Int32(), false),
+        new Field('end_line', new Int32(), false),
+        new Field('file_hash', new Utf8(), false),
+        // New in schema v2: deterministic AST-derived metadata
+        new Field('signature', new Utf8(), true), // Function/class signature line (nullable)
+        new Field('docstring', new Utf8(), true), // Extracted documentation (nullable)
+        new Field('is_exported', new Bool(), false), // Has export modifier
+        new Field('decorator_names', new Utf8(), true), // Comma-separated decorators (nullable)
+    ]);
+}
+/**
+ * Arrow schema for the embedding_cache table.
+ *
+ * Content-addressed cache for embeddings to avoid recomputation.
+ */
+export function createEmbeddingCacheSchema(dimensions = DEFAULT_EMBEDDING_DIMENSIONS) {
+    return new Schema([
+        new Field('content_hash', new Utf8(), false), // Primary key
+        new Field('vector', new FixedSizeList(dimensions, new Field('item', new Float32(), false)), false),
+        new Field('created_at', new Utf8(), false), // ISO timestamp
+    ]);
+}

package/dist/rag/storage/types.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Types of code chunks extracted by tree-sitter.
+ */
+export type ChunkType = 'function' | 'class' | 'method' | 'module';
+/**
+ * A code chunk stored in LanceDB.
+ * Represents a semantic unit of code (function, class, method, or module).
+ */
+export interface CodeChunk {
+    /** Unique ID: "{filepath}:{startLine}" */
+    id: string;
+    /** Embedding vector (768 dimensions for Jina code embeddings) */
+    vector: number[];
+    /** Source code content */
+    text: string;
+    /** SHA256 hash of the text content */
+    contentHash: string;
+    /** Relative file path from project root */
+    filepath: string;
+    /** Just the filename (e.g., "utils.py") */
+    filename: string;
+    /** File extension (e.g., ".py") */
+    extension: string;
+    /** Chunk type: function, class, method, or module */
+    type: ChunkType;
+    /** Symbol name (empty for module chunks) */
+    name: string;
+    /** Start line number (1-indexed) */
+    startLine: number;
+    /** End line number (1-indexed) */
+    endLine: number;
+    /** SHA256 hash of the entire source file */
+    fileHash: string;
+    /** Function/method signature line (null for module/class without params) */
+    signature: string | null;
+    /** Extracted documentation (JSDoc, docstring, etc.) */
+    docstring: string | null;
+    /** Whether symbol has export modifier (JS/TS) or is in __all__ (Python) */
+    isExported: boolean;
+    /** Comma-separated decorator/annotation names (null if none) */
+    decoratorNames: string | null;
+}
+/**
+ * Row format for LanceDB code_chunks table.
+ * Uses snake_case to match Arrow/LanceDB conventions.
+ */
+export interface CodeChunkRow {
+    id: string;
+    vector: number[];
+    text: string;
+    content_hash: string;
+    filepath: string;
+    filename: string;
+    extension: string;
+    type: string;
+    name: string;
+    start_line: number;
+    end_line: number;
+    file_hash: string;
+    signature: string | null;
+    docstring: string | null;
+    is_exported: boolean;
+    decorator_names: string | null;
+}
+/**
+ * A cached embedding stored in LanceDB.
+ * Content-addressed by the hash of the text.
+ */
+export interface CachedEmbedding {
+    /** SHA256 hash of the text content (primary key) */
+    contentHash: string;
+    /** Embedding vector */
+    vector: number[];
+    /** ISO timestamp when cached */
+    createdAt: string;
+}
+/**
+ * Row format for LanceDB embedding_cache table.
+ */
+export interface CachedEmbeddingRow {
+    content_hash: string;
+    vector: number[];
+    created_at: string;
+}
+/**
+ * Convert a CodeChunk to a LanceDB row format.
+ */
+export declare function chunkToRow(chunk: CodeChunk): CodeChunkRow;
+/**
+ * Convert a LanceDB row to a CodeChunk.
+ */
+export declare function rowToChunk(row: CodeChunkRow): CodeChunk;
+/**
+ * Convert a CachedEmbedding to a LanceDB row format.
+ */
+export declare function embeddingToRow(embedding: CachedEmbedding): CachedEmbeddingRow;
+/**
+ * Convert a LanceDB row to a CachedEmbedding.
+ */
+export declare function rowToEmbedding(row: CachedEmbeddingRow): CachedEmbedding;

package/dist/rag/storage/types.js

@@ -0,0 +1,68 @@
+/**
+ * Convert a CodeChunk to a LanceDB row format.
+ */
+export function chunkToRow(chunk) {
+    return {
+        id: chunk.id,
+        vector: chunk.vector,
+        text: chunk.text,
+        content_hash: chunk.contentHash,
+        filepath: chunk.filepath,
+        filename: chunk.filename,
+        extension: chunk.extension,
+        type: chunk.type,
+        name: chunk.name,
+        start_line: chunk.startLine,
+        end_line: chunk.endLine,
+        file_hash: chunk.fileHash,
+        // New in schema v2
+        signature: chunk.signature,
+        docstring: chunk.docstring,
+        is_exported: chunk.isExported,
+        decorator_names: chunk.decoratorNames,
+    };
+}
+/**
+ * Convert a LanceDB row to a CodeChunk.
+ */
+export function rowToChunk(row) {
+    return {
+        id: row.id,
+        vector: row.vector,
+        text: row.text,
+        contentHash: row.content_hash,
+        filepath: row.filepath,
+        filename: row.filename,
+        extension: row.extension,
+        type: row.type,
+        name: row.name,
+        startLine: row.start_line,
+        endLine: row.end_line,
+        fileHash: row.file_hash,
+        // New in schema v2
+        signature: row.signature,
+        docstring: row.docstring,
+        isExported: row.is_exported,
+        decoratorNames: row.decorator_names,
+    };
+}
+/**
+ * Convert a CachedEmbedding to a LanceDB row format.
+ */
+export function embeddingToRow(embedding) {
+    return {
+        content_hash: embedding.contentHash,
+        vector: embedding.vector,
+        created_at: embedding.createdAt,
+    };
+}
+/**
+ * Convert a LanceDB row to a CachedEmbedding.
+ */
+export function rowToEmbedding(row) {
+    return {
+        contentHash: row.content_hash,
+        vector: row.vector,
+        createdAt: row.created_at,
+    };
+}