trellis 2.0.13 → 2.1.2

package/src/embeddings/auto-embed.ts

@@ -1,248 +0,0 @@
-/**
- * Auto-Embedding Middleware
- *
- * Kernel middleware that automatically embeds entity facts and links
- * on graph mutations. Runs after successful ops to index new/changed
- * content into the vector store.
- *
- * @module trellis/embeddings
- */
-
-import type { KernelOp } from '../core/persist/backend.js';
-import type { KernelMiddleware, MiddlewareContext, OpMiddlewareNext } from '../core/kernel/middleware.js';
-import type { Fact, Link } from '../core/store/eav-store.js';
-import type { ChunkMeta, EmbeddingRecord } from './types.js';
-import type { Embedder } from './search.js';
-import { VectorStore } from './store.js';
-import { embed } from './model.js';
-
-// ---------------------------------------------------------------------------
-// Entity text builder — converts facts/links into embeddable text
-// ---------------------------------------------------------------------------
-
-function factsToText(facts: Fact[]): string {
-  return facts
-    .filter((f) => f.a !== 'createdAt' && f.a !== 'updatedAt')
-    .map((f) => `${f.a}: ${f.v}`)
-    .join('\n');
-}
-
-function linksToText(links: Link[]): string {
-  return links.map((l) => `${l.e1} —[${l.a}]→ ${l.e2}`).join('\n');
-}
-
-function entitySummaryText(entityId: string, facts: Fact[], links: Link[]): string {
-  const type = facts.find((f) => f.a === 'type')?.v ?? 'Entity';
-  const name = facts.find((f) => f.a === 'name' || f.a === 'title')?.v ?? entityId;
-  const parts = [`${type}: ${name} (${entityId})`];
-
-  const attrs = facts.filter((f) => !['type', 'name', 'title', 'createdAt', 'updatedAt'].includes(f.a));
-  if (attrs.length > 0) {
-    parts.push(attrs.map((f) => `  ${f.a} = ${f.v}`).join('\n'));
-  }
-
-  if (links.length > 0) {
-    parts.push('Relations:');
-    parts.push(links.map((l) => `  ${l.a} → ${l.e2}`).join('\n'));
-  }
-
-  return parts.join('\n');
-}
-
-// ---------------------------------------------------------------------------
-// Middleware factory
-// ---------------------------------------------------------------------------
-
-export interface AutoEmbedOptions {
-  /** Path to the vector store SQLite database. */
-  dbPath: string;
-  /** Custom embedder function (default: transformers.js embed). */
-  embedFn?: Embedder;
-  /** Whether to embed facts individually (default: false — only entity summaries). */
-  embedIndividualFacts?: boolean;
-}
-
-/**
- * Creates a kernel middleware that auto-embeds entities on mutation.
- *
- * On addFacts/addLinks: embeds entity summaries into the vector store.
- * On deleteFacts/deleteLinks: removes stale embeddings.
- */
-export function createAutoEmbedMiddleware(options: AutoEmbedOptions): KernelMiddleware & { close: () => void } {
-  const store = new VectorStore(options.dbPath);
-  const embedFn = options.embedFn ?? embed;
-  const embedIndividual = options.embedIndividualFacts ?? false;
-
-  return {
-    name: 'auto-embed',
-
-    handleOp: async (op: KernelOp, ctx: MiddlewareContext, next: OpMiddlewareNext) => {
-      // Let the op proceed first
-      await next(op, ctx);
-
-      // Then asynchronously embed (don't block the mutation)
-      try {
-        await _processOp(op, store, embedFn, embedIndividual);
-      } catch {
-        // Embedding failures are non-fatal
-      }
-    },
-
-    close: () => {
-      store.close();
-    },
-  };
-}
-
-async function _processOp(
-  op: KernelOp,
-  store: VectorStore,
-  embedFn: Embedder,
-  embedIndividual: boolean,
-): Promise<void> {
-  const now = new Date().toISOString();
-
-  // Collect affected entity IDs
-  const entityIds = new Set<string>();
-  if (op.facts) for (const f of op.facts) entityIds.add(f.e);
-  if (op.links) for (const l of op.links) { entityIds.add(l.e1); entityIds.add(l.e2); }
-  if (op.deleteFacts) for (const f of op.deleteFacts) entityIds.add(f.e);
-  if (op.deleteLinks) for (const l of op.deleteLinks) { entityIds.add(l.e1); entityIds.add(l.e2); }
-
-  // Handle deletions — remove old embeddings for deleted entities
-  if (op.deleteFacts || op.deleteLinks) {
-    for (const eid of entityIds) {
-      store.deleteByEntity(eid);
-    }
-  }
-
-  // Handle additions — embed entity summaries
-  if (op.facts && op.facts.length > 0) {
-    // Group facts by entity
-    const factsByEntity = new Map<string, Fact[]>();
-    for (const f of op.facts) {
-      const existing = factsByEntity.get(f.e) ?? [];
-      existing.push(f);
-      factsByEntity.set(f.e, existing);
-    }
-
-    const linksByEntity = new Map<string, Link[]>();
-    if (op.links) {
-      for (const l of op.links) {
-        const existing = linksByEntity.get(l.e1) ?? [];
-        existing.push(l);
-        linksByEntity.set(l.e1, existing);
-      }
-    }
-
-    const records: EmbeddingRecord[] = [];
-
-    for (const [eid, facts] of factsByEntity) {
-      const links = linksByEntity.get(eid) ?? [];
-
-      // Entity summary embedding
-      const summaryText = entitySummaryText(eid, facts, links);
-      if (summaryText.trim()) {
-        try {
-          const vector = await embedFn(summaryText);
-          records.push({
-            id: `entity:${eid}:summary`,
-            entityId: eid,
-            content: summaryText,
-            chunkType: 'summary_md' as any,
-            updatedAt: now,
-            embedding: vector,
-          });
-        } catch {}
-      }
-
-      // Individual fact embeddings (optional)
-      if (embedIndividual) {
-        for (const fact of facts) {
-          if (['type', 'createdAt', 'updatedAt'].includes(fact.a)) continue;
-          const text = `${fact.a}: ${fact.v}`;
-          try {
-            const vector = await embedFn(text);
-            records.push({
-              id: `entity:${eid}:fact:${fact.a}`,
-              entityId: eid,
-              content: text,
-              chunkType: 'doc_comment' as any,
-              updatedAt: now,
-              embedding: vector,
-            });
-          } catch {}
-        }
-      }
-    }
-
-    if (records.length > 0) {
-      store.upsertBatch(records);
-    }
-  }
-}
-
-// ---------------------------------------------------------------------------
-// RAG Context Builder
-// ---------------------------------------------------------------------------
-
-export interface RAGContext {
-  /** The original query. */
-  query: string;
-  /** Retrieved chunks ranked by relevance. */
-  chunks: Array<{
-    content: string;
-    entityId: string;
-    score: number;
-    chunkType: string;
-  }>;
-  /** Total token estimate (rough: 1 token ≈ 4 chars). */
-  estimatedTokens: number;
-}
-
-/**
- * Build a RAG context from a natural language query.
- * Searches the vector store and assembles ranked context chunks.
- */
-export async function buildRAGContext(
-  query: string,
-  vectorStore: VectorStore,
-  embedFn: Embedder = embed,
-  options?: {
-    maxChunks?: number;
-    maxTokens?: number;
-    minScore?: number;
-  },
-): Promise<RAGContext> {
-  const maxChunks = options?.maxChunks ?? 10;
-  const maxTokens = options?.maxTokens ?? 4000;
-  const minScore = options?.minScore ?? 0.1;
-
-  const queryVector = await embedFn(query);
-  const results = vectorStore.search(queryVector, {
-    limit: maxChunks * 2,
-    minScore,
-  });
-
-  const chunks: RAGContext['chunks'] = [];
-  let totalChars = 0;
-
-  for (const r of results) {
-    if (chunks.length >= maxChunks) break;
-    if (totalChars + r.chunk.content.length > maxTokens * 4) break;
-
-    chunks.push({
-      content: r.chunk.content,
-      entityId: r.chunk.entityId,
-      score: r.score,
-      chunkType: r.chunk.chunkType,
-    });
-    totalChars += r.chunk.content.length;
-  }
-
-  return {
-    query,
-    chunks,
-    estimatedTokens: Math.ceil(totalChars / 4),
-  };
-}

package/src/embeddings/chunker.ts

@@ -1,327 +0,0 @@
-/**
- * Text Chunker
- *
- * Strategies for splitting different content types into embeddable chunks.
- * Short text is embedded as-is; markdown is split by headings; code entities
- * are chunked by declaration; large text uses sliding window.
- *
- * @see TRL-19
- */
-
-import type { ChunkMeta, ChunkType } from './types.js';
-
-// ---------------------------------------------------------------------------
-// Chunk configuration
-// ---------------------------------------------------------------------------
-
-/** Max characters per chunk before splitting (roughly ~128 tokens) */
-const MAX_CHUNK_CHARS = 512;
-/** Overlap characters for sliding window */
-const OVERLAP_CHARS = 64;
-
-// ---------------------------------------------------------------------------
-// Public API
-// ---------------------------------------------------------------------------
-
-/**
- * Chunk an issue into embeddable pieces.
- */
-export function chunkIssue(issue: {
-  id: string;
-  title?: string;
-  description?: string;
-}): ChunkMeta[] {
-  const now = new Date().toISOString();
-  const chunks: ChunkMeta[] = [];
-
-  if (issue.title) {
-    chunks.push({
-      id: `issue:${issue.id}:title`,
-      entityId: `issue:${issue.id}`,
-      content: issue.title,
-      chunkType: 'issue_title',
-      updatedAt: now,
-    });
-  }
-
-  if (issue.description) {
-    chunks.push({
-      id: `issue:${issue.id}:desc`,
-      entityId: `issue:${issue.id}`,
-      content: issue.description,
-      chunkType: 'issue_desc',
-      updatedAt: now,
-    });
-  }
-
-  return chunks;
-}
-
-/**
- * Chunk a decision trace into embeddable pieces.
- * Combines tool name, rationale, context, and output summary.
- */
-export function chunkDecision(decision: {
-  id: string;
-  toolName: string;
-  rationale?: string;
-  context?: string;
-  outputSummary?: string;
-}): ChunkMeta[] {
-  const parts: string[] = [];
-  parts.push(`Decision ${decision.id}: ${decision.toolName}`);
-  if (decision.rationale) parts.push(`Rationale: ${decision.rationale}`);
-  if (decision.context) parts.push(`Context: ${decision.context}`);
-  if (decision.outputSummary) parts.push(`Output: ${decision.outputSummary}`);
-
-  const content = parts.join('\n');
-  if (!content.trim()) return [];
-
-  return [
-    {
-      id: `decision:${decision.id}:rationale`,
-      entityId: `decision:${decision.id}`,
-      content,
-      chunkType: 'decision_rationale',
-      updatedAt: new Date().toISOString(),
-    },
-  ];
-}
-
-/**
- * Chunk a milestone message.
- */
-export function chunkMilestone(milestone: {
-  id: string;
-  message?: string;
-}): ChunkMeta[] {
-  if (!milestone.message) return [];
-
-  return [
-    {
-      id: `milestone:${milestone.id}:msg`,
-      entityId: `milestone:${milestone.id}`,
-      content: milestone.message,
-      chunkType: 'milestone_msg',
-      updatedAt: new Date().toISOString(),
-    },
-  ];
-}
-
-/**
- * Chunk a markdown file by heading sections.
- * Each H1/H2/H3 section becomes a separate chunk.
- * Sections that exceed MAX_CHUNK_CHARS get sliding-window split.
- */
-export function chunkMarkdown(filePath: string, content: string): ChunkMeta[] {
-  if (!content.trim()) return [];
-
-  const entityId = `file:${filePath}`;
-  const now = new Date().toISOString();
-  const sections = splitByHeadings(content);
-  const chunks: ChunkMeta[] = [];
-
-  for (let i = 0; i < sections.length; i++) {
-    const section = sections[i];
-    if (!section.text.trim()) continue;
-
-    if (section.text.length <= MAX_CHUNK_CHARS) {
-      chunks.push({
-        id: `${entityId}:section:${i}`,
-        entityId,
-        content: section.text,
-        chunkType: 'markdown',
-        filePath,
-        updatedAt: now,
-      });
-    } else {
-      // Split long sections with sliding window
-      const windows = slidingWindow(section.text);
-      for (let w = 0; w < windows.length; w++) {
-        chunks.push({
-          id: `${entityId}:section:${i}:w${w}`,
-          entityId,
-          content: windows[w],
-          chunkType: 'markdown',
-          filePath,
-          updatedAt: now,
-        });
-      }
-    }
-  }
-
-  return chunks;
-}
-
-/**
- * Chunk code entities (functions, classes, interfaces) from a parsed file.
- * Each declaration's signature + doc-comment becomes a chunk.
- */
-export function chunkCodeEntities(
-  filePath: string,
-  declarations: Array<{
-    id: string;
-    name: string;
-    kind: string;
-    signature: string;
-    docComment?: string;
-  }>,
-): ChunkMeta[] {
-  const now = new Date().toISOString();
-  const chunks: ChunkMeta[] = [];
-
-  for (const decl of declarations) {
-    const parts: string[] = [];
-    if (decl.docComment) parts.push(decl.docComment);
-    parts.push(`${decl.kind} ${decl.name}`);
-    parts.push(decl.signature);
-
-    const content = parts.join('\n').slice(0, MAX_CHUNK_CHARS);
-
-    chunks.push({
-      id: `symbol:${filePath}#${decl.name}`,
-      entityId: `symbol:${filePath}#${decl.name}`,
-      content,
-      chunkType: 'code_entity',
-      filePath,
-      updatedAt: now,
-    });
-  }
-
-  return chunks;
-}
-
-/**
- * Chunk doc-comments extracted from source files.
- */
-export function chunkDocComments(
-  filePath: string,
-  comments: Array<{ line: number; text: string }>,
-): ChunkMeta[] {
-  if (comments.length === 0) return [];
-
-  const entityId = `file:${filePath}`;
-  const now = new Date().toISOString();
-  const chunks: ChunkMeta[] = [];
-
-  for (let i = 0; i < comments.length; i++) {
-    const comment = comments[i];
-    if (!comment.text.trim()) continue;
-
-    chunks.push({
-      id: `${entityId}:doc:${i}`,
-      entityId,
-      content: comment.text.slice(0, MAX_CHUNK_CHARS),
-      chunkType: 'doc_comment',
-      filePath,
-      updatedAt: now,
-    });
-  }
-
-  return chunks;
-}
-
-/**
- * Chunk a summary.md or similar short-to-medium text file.
- */
-export function chunkSummary(filePath: string, content: string): ChunkMeta[] {
-  if (!content.trim()) return [];
-
-  const entityId = `file:${filePath}`;
-  const now = new Date().toISOString();
-
-  if (content.length <= MAX_CHUNK_CHARS) {
-    return [
-      {
-        id: `${entityId}:summary`,
-        entityId,
-        content,
-        chunkType: 'summary_md',
-        filePath,
-        updatedAt: now,
-      },
-    ];
-  }
-
-  // Split by headings for longer summaries
-  return chunkMarkdown(filePath, content).map((c) => ({
-    ...c,
-    chunkType: 'summary_md' as ChunkType,
-  }));
-}
-
-/**
- * Auto-detect file type and chunk accordingly.
- */
-export function chunkFile(filePath: string, content: string): ChunkMeta[] {
-  if (!content.trim()) return [];
-
-  const ext = filePath.split('.').pop()?.toLowerCase() ?? '';
-
-  // summary.md files
-  if (filePath.endsWith('summary.md')) {
-    return chunkSummary(filePath, content);
-  }
-
-  // Markdown files
-  if (ext === 'md') {
-    return chunkMarkdown(filePath, content);
-  }
-
-  // Source files — we only chunk doc-comments (code entities are handled separately)
-  // Return empty — code entities are handled via chunkCodeEntities from parsed AST
-  return [];
-}
-
-// ---------------------------------------------------------------------------
-// Internal helpers
-// ---------------------------------------------------------------------------
-
-interface Section {
-  heading?: string;
-  text: string;
-}
-
-/**
- * Split markdown content by heading boundaries (# H1, ## H2, ### H3).
- */
-function splitByHeadings(content: string): Section[] {
-  const lines = content.split('\n');
-  const sections: Section[] = [];
-  let currentSection: Section = { text: '' };
-
-  for (const line of lines) {
-    if (/^#{1,3}\s/.test(line)) {
-      // New heading — start a new section
-      if (currentSection.text.trim()) {
-        sections.push(currentSection);
-      }
-      currentSection = { heading: line, text: line + '\n' };
-    } else {
-      currentSection.text += line + '\n';
-    }
-  }
-
-  if (currentSection.text.trim()) {
-    sections.push(currentSection);
-  }
-
-  return sections;
-}
-
-/**
- * Split long text into overlapping windows.
- */
-export function slidingWindow(text: string): string[] {
-  const windows: string[] = [];
-  let start = 0;
-
-  while (start < text.length) {
-    const end = Math.min(start + MAX_CHUNK_CHARS, text.length);
-    windows.push(text.slice(start, end));
-    if (end >= text.length) break;
-    start += MAX_CHUNK_CHARS - OVERLAP_CHARS;
-  }
-
-  return windows;
-}

package/src/embeddings/index.ts

@@ -1,48 +0,0 @@
-/**
- * Embeddings Module — Public API
- *
- * Provides semantic embedding and vector search for TrellisVCS entities.
- *
- * @see TRL-18
- */
-
-// Types
-export type {
-  ChunkType,
-  ChunkMeta,
-  EmbeddingRecord,
-  SearchResult,
-  SearchOptions,
-  EmbeddingModelConfig,
-} from './types.js';
-export { DEFAULT_MODEL_CONFIG } from './types.js';
-
-// Model
-export { embed, embedBatch, loadModel, resetModel } from './model.js';
-
-// Store
-export { VectorStore, cosineSimilarity } from './store.js';
-
-// Search
-export { EmbeddingManager } from './search.js';
-export type { SearchableEngine, Embedder } from './search.js';
-
-// Auto-embedding middleware + RAG
-export { createAutoEmbedMiddleware, buildRAGContext } from './auto-embed.js';
-export type { AutoEmbedOptions, RAGContext } from './auto-embed.js';
-
-// New graph chunk types
-export type { GraphChunkType } from './types.js';
-
-// Chunker
-export {
-  chunkIssue,
-  chunkMilestone,
-  chunkDecision,
-  chunkMarkdown,
-  chunkCodeEntities,
-  chunkDocComments,
-  chunkSummary,
-  chunkFile,
-  slidingWindow,
-} from './chunker.js';