trellis 2.0.13 → 2.1.2

package/src/embeddings/store.ts

@@ -1,313 +0,0 @@
-/**
- * Embedding Vector Store
- *
- * Persistent storage for embedding vectors using bun:sqlite.
- * Vectors are stored as Float32Array blobs; cosine similarity
- * is computed in JavaScript for cross-platform portability.
- *
- * @see TRL-18
- */
-
-import { Database } from 'bun:sqlite';
-import type {
-  ChunkMeta,
-  ChunkType,
-  EmbeddingRecord,
-  SearchOptions,
-  SearchResult,
-} from './types.js';
-
-// ---------------------------------------------------------------------------
-// Schema
-// ---------------------------------------------------------------------------
-
-const SCHEMA_SQL = `
-CREATE TABLE IF NOT EXISTS chunks (
-  id TEXT PRIMARY KEY,
-  entity_id TEXT NOT NULL,
-  content TEXT NOT NULL,
-  chunk_type TEXT NOT NULL,
-  file_path TEXT,
-  updated_at TEXT NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS vectors (
-  id TEXT PRIMARY KEY,
-  embedding BLOB NOT NULL,
-  FOREIGN KEY (id) REFERENCES chunks(id) ON DELETE CASCADE
-);
-
-CREATE INDEX IF NOT EXISTS idx_chunks_entity ON chunks(entity_id);
-CREATE INDEX IF NOT EXISTS idx_chunks_type ON chunks(chunk_type);
-CREATE INDEX IF NOT EXISTS idx_chunks_file ON chunks(file_path);
-`;
-
-// ---------------------------------------------------------------------------
-// Vector Store
-// ---------------------------------------------------------------------------
-
-export class VectorStore {
-  private db: Database;
-
-  constructor(dbPath: string) {
-    this.db = new Database(dbPath);
-    this.db.exec('PRAGMA journal_mode=WAL;');
-    this.db.exec('PRAGMA foreign_keys=ON;');
-    this.db.exec(SCHEMA_SQL);
-  }
-
-  /**
-   * Insert or update a chunk with its embedding vector.
-   */
-  upsert(record: EmbeddingRecord): void {
-    const insertChunk = this.db.prepare(`
-      INSERT OR REPLACE INTO chunks (id, entity_id, content, chunk_type, file_path, updated_at)
-      VALUES ($id, $entityId, $content, $chunkType, $filePath, $updatedAt)
-    `);
-    const insertVector = this.db.prepare(`
-      INSERT OR REPLACE INTO vectors (id, embedding)
-      VALUES ($id, $embedding)
-    `);
-
-    const embeddingBlob = Buffer.from(record.embedding.buffer);
-
-    this.db.transaction(() => {
-      insertChunk.run({
-        $id: record.id,
-        $entityId: record.entityId,
-        $content: record.content,
-        $chunkType: record.chunkType,
-        $filePath: record.filePath ?? null,
-        $updatedAt: record.updatedAt,
-      });
-      insertVector.run({
-        $id: record.id,
-        $embedding: embeddingBlob,
-      });
-    })();
-  }
-
-  /**
-   * Batch upsert multiple records.
-   */
-  upsertBatch(records: EmbeddingRecord[]): void {
-    if (records.length === 0) return;
-
-    const insertChunk = this.db.prepare(`
-      INSERT OR REPLACE INTO chunks (id, entity_id, content, chunk_type, file_path, updated_at)
-      VALUES ($id, $entityId, $content, $chunkType, $filePath, $updatedAt)
-    `);
-    const insertVector = this.db.prepare(`
-      INSERT OR REPLACE INTO vectors (id, embedding)
-      VALUES ($id, $embedding)
-    `);
-
-    this.db.transaction(() => {
-      for (const record of records) {
-        const embeddingBlob = Buffer.from(record.embedding.buffer);
-        insertChunk.run({
-          $id: record.id,
-          $entityId: record.entityId,
-          $content: record.content,
-          $chunkType: record.chunkType,
-          $filePath: record.filePath ?? null,
-          $updatedAt: record.updatedAt,
-        });
-        insertVector.run({
-          $id: record.id,
-          $embedding: embeddingBlob,
-        });
-      }
-    })();
-  }
-
-  /**
-   * Delete a chunk and its vector by ID.
-   */
-  delete(id: string): void {
-    this.db.prepare('DELETE FROM vectors WHERE id = ?').run(id);
-    this.db.prepare('DELETE FROM chunks WHERE id = ?').run(id);
-  }
-
-  /**
-   * Delete all chunks for an entity.
-   */
-  deleteByEntity(entityId: string): void {
-    const ids = this.db
-      .prepare('SELECT id FROM chunks WHERE entity_id = ?')
-      .all(entityId) as Array<{ id: string }>;
-
-    if (ids.length === 0) return;
-
-    this.db.transaction(() => {
-      for (const { id } of ids) {
-        this.db.prepare('DELETE FROM vectors WHERE id = ?').run(id);
-        this.db.prepare('DELETE FROM chunks WHERE id = ?').run(id);
-      }
-    })();
-  }
-
-  /**
-   * Delete all chunks associated with a file path.
-   */
-  deleteByFile(filePath: string): void {
-    const ids = this.db
-      .prepare('SELECT id FROM chunks WHERE file_path = ?')
-      .all(filePath) as Array<{ id: string }>;
-
-    if (ids.length === 0) return;
-
-    this.db.transaction(() => {
-      for (const { id } of ids) {
-        this.db.prepare('DELETE FROM vectors WHERE id = ?').run(id);
-        this.db.prepare('DELETE FROM chunks WHERE id = ?').run(id);
-      }
-    })();
-  }
-
-  /**
-   * Get a chunk by ID (without vector).
-   */
-  getChunk(id: string): ChunkMeta | null {
-    const row = this.db
-      .prepare('SELECT * FROM chunks WHERE id = ?')
-      .get(id) as any;
-    if (!row) return null;
-    return rowToChunkMeta(row);
-  }
-
-  /**
-   * Search for chunks similar to the query vector.
-   * Uses brute-force cosine similarity scan.
-   */
-  search(queryVector: Float32Array, opts: SearchOptions = {}): SearchResult[] {
-    const limit = opts.limit ?? 10;
-    const minScore = opts.minScore ?? 0.0;
-
-    // Build SQL filter
-    const conditions: string[] = [];
-    const params: Record<string, unknown> = {};
-
-    if (opts.types && opts.types.length > 0) {
-      const placeholders = opts.types.map((_, i) => `$type${i}`).join(', ');
-      conditions.push(`c.chunk_type IN (${placeholders})`);
-      opts.types.forEach((t, i) => {
-        params[`$type${i}`] = t;
-      });
-    }
-
-    if (opts.filePrefix) {
-      conditions.push('c.file_path LIKE $filePrefix');
-      params.$filePrefix = `${opts.filePrefix}%`;
-    }
-
-    const where =
-      conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
-
-    const sql = `
-      SELECT c.id, c.entity_id, c.content, c.chunk_type, c.file_path, c.updated_at,
-             v.embedding
-      FROM chunks c
-      JOIN vectors v ON c.id = v.id
-      ${where}
-    `;
-
-    const rows = this.db.prepare(sql).all(params) as any[];
-
-    // Compute cosine similarity for each row
-    const scored: SearchResult[] = [];
-    for (const row of rows) {
-      const storedVec = new Float32Array(
-        (row.embedding as Buffer).buffer,
-        (row.embedding as Buffer).byteOffset,
-        (row.embedding as Buffer).byteLength / 4,
-      );
-      const score = cosineSimilarity(queryVector, storedVec);
-      if (score >= minScore) {
-        scored.push({
-          chunk: rowToChunkMeta(row),
-          score,
-        });
-      }
-    }
-
-    // Sort by score descending and limit
-    scored.sort((a, b) => b.score - a.score);
-    return scored.slice(0, limit);
-  }
-
-  /**
-   * Get total count of chunks in the store.
-   */
-  count(): number {
-    const row = this.db
-      .prepare('SELECT COUNT(*) as cnt FROM chunks')
-      .get() as any;
-    return row?.cnt ?? 0;
-  }
-
-  /**
-   * Get count by chunk type.
-   */
-  countByType(): Record<string, number> {
-    const rows = this.db
-      .prepare(
-        'SELECT chunk_type, COUNT(*) as cnt FROM chunks GROUP BY chunk_type',
-      )
-      .all() as any[];
-    const result: Record<string, number> = {};
-    for (const row of rows) {
-      result[row.chunk_type] = row.cnt;
-    }
-    return result;
-  }
-
-  /**
-   * Clear all data from the store.
-   */
-  clear(): void {
-    this.db.exec('DELETE FROM vectors');
-    this.db.exec('DELETE FROM chunks');
-  }
-
-  /**
-   * Close the database connection.
-   */
-  close(): void {
-    this.db.close();
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Helpers
-// ---------------------------------------------------------------------------
-
-function rowToChunkMeta(row: any): ChunkMeta {
-  return {
-    id: row.id,
-    entityId: row.entity_id,
-    content: row.content,
-    chunkType: row.chunk_type as ChunkType,
-    filePath: row.file_path ?? undefined,
-    updatedAt: row.updated_at,
-  };
-}
-
-/**
- * Compute cosine similarity between two vectors.
- * Both vectors should already be normalized (output of mean pooling + normalize).
- * For normalized vectors, cosine similarity = dot product.
- */
-export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
-  if (a.length !== b.length) return 0;
-  let dot = 0;
-  let normA = 0;
-  let normB = 0;
-  for (let i = 0; i < a.length; i++) {
-    dot += a[i] * b[i];
-    normA += a[i] * a[i];
-    normB += b[i] * b[i];
-  }
-  const denom = Math.sqrt(normA) * Math.sqrt(normB);
-  return denom === 0 ? 0 : dot / denom;
-}

package/src/embeddings/types.ts

@@ -1,92 +0,0 @@
-/**
- * Embedding Types
- *
- * Types for the semantic embedding and vector search system.
- *
- * @see TRL-18
- */
-
-// ---------------------------------------------------------------------------
-// Chunk types — what gets embedded
-// ---------------------------------------------------------------------------
-
-export type ChunkType =
-  | 'issue_title'
-  | 'issue_desc'
-  | 'milestone_msg'
-  | 'decision_rationale'
-  | 'summary_md'
-  | 'code_entity'
-  | 'doc_comment'
-  | 'markdown';
-
-export interface ChunkMeta {
-  /** Unique chunk ID, e.g. "issue:TRL-5:title", "file:src/engine.ts:chunk:0" */
-  id: string;
-  /** EAV entity ID this chunk belongs to */
-  entityId: string;
-  /** Original text content */
-  content: string;
-  /** Chunk classification */
-  chunkType: ChunkType;
-  /** Source file path (nullable for non-file entities) */
-  filePath?: string;
-  /** When this chunk was last updated */
-  updatedAt: string;
-}
-
-// ---------------------------------------------------------------------------
-// Embedding record — chunk + vector
-// ---------------------------------------------------------------------------
-
-export interface EmbeddingRecord extends ChunkMeta {
-  /** 384-dimensional embedding vector */
-  embedding: Float32Array;
-}
-
-// ---------------------------------------------------------------------------
-// Search
-// ---------------------------------------------------------------------------
-
-export interface SearchResult {
-  /** Chunk metadata */
-  chunk: ChunkMeta;
-  /** Cosine similarity score (0..1) */
-  score: number;
-}
-
-export interface SearchOptions {
-  /** Max results to return (default: 10) */
-  limit?: number;
-  /** Filter by chunk type(s) */
-  types?: ChunkType[];
-  /** Filter by file path prefix */
-  filePrefix?: string;
-  /** Minimum similarity threshold (default: 0.0) */
-  minScore?: number;
-}
-
-// ---------------------------------------------------------------------------
-// Embedding model config
-// ---------------------------------------------------------------------------
-
-export interface EmbeddingModelConfig {
-  /** Model name for transformers.js (default: "Xenova/all-MiniLM-L6-v2") */
-  modelName: string;
-  /** Embedding dimension (default: 384) */
-  dimension: number;
-  /** Cache directory for model files */
-  cacheDir?: string;
-}
-
-export const DEFAULT_MODEL_CONFIG: EmbeddingModelConfig = {
-  modelName: 'Xenova/all-MiniLM-L6-v2',
-  dimension: 384,
-};
-
-/** New chunk types for generic graph entities */
-export type GraphChunkType =
-  | ChunkType
-  | 'entity_summary'
-  | 'entity_fact'
-  | 'entity_link';