trellis 2.0.7 → 2.0.10

package/src/embeddings/auto-embed.ts

@@ -0,0 +1,248 @@
+/**
+ * Auto-Embedding Middleware
+ *
+ * Kernel middleware that automatically embeds entity facts and links
+ * on graph mutations. Runs after successful ops to index new/changed
+ * content into the vector store.
+ *
+ * @module trellis/embeddings
+ */
+
+import type { KernelOp } from '../core/persist/backend.js';
+import type { KernelMiddleware, MiddlewareContext, OpMiddlewareNext } from '../core/kernel/middleware.js';
+import type { Fact, Link } from '../core/store/eav-store.js';
+import type { ChunkMeta, EmbeddingRecord } from './types.js';
+import type { Embedder } from './search.js';
+import { VectorStore } from './store.js';
+import { embed } from './model.js';
+
+// ---------------------------------------------------------------------------
+// Entity text builder — converts facts/links into embeddable text
+// ---------------------------------------------------------------------------
+
+function factsToText(facts: Fact[]): string {
+  return facts
+    .filter((f) => f.a !== 'createdAt' && f.a !== 'updatedAt')
+    .map((f) => `${f.a}: ${f.v}`)
+    .join('\n');
+}
+
+function linksToText(links: Link[]): string {
+  return links.map((l) => `${l.e1} —[${l.a}]→ ${l.e2}`).join('\n');
+}
+
+function entitySummaryText(entityId: string, facts: Fact[], links: Link[]): string {
+  const type = facts.find((f) => f.a === 'type')?.v ?? 'Entity';
+  const name = facts.find((f) => f.a === 'name' || f.a === 'title')?.v ?? entityId;
+  const parts = [`${type}: ${name} (${entityId})`];
+
+  const attrs = facts.filter((f) => !['type', 'name', 'title', 'createdAt', 'updatedAt'].includes(f.a));
+  if (attrs.length > 0) {
+    parts.push(attrs.map((f) => `  ${f.a} = ${f.v}`).join('\n'));
+  }
+
+  if (links.length > 0) {
+    parts.push('Relations:');
+    parts.push(links.map((l) => `  ${l.a} → ${l.e2}`).join('\n'));
+  }
+
+  return parts.join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// Middleware factory
+// ---------------------------------------------------------------------------
+
+export interface AutoEmbedOptions {
+  /** Path to the vector store SQLite database. */
+  dbPath: string;
+  /** Custom embedder function (default: transformers.js embed). */
+  embedFn?: Embedder;
+  /** Whether to embed facts individually (default: false — only entity summaries). */
+  embedIndividualFacts?: boolean;
+}
+
+/**
+ * Creates a kernel middleware that auto-embeds entities on mutation.
+ *
+ * On addFacts/addLinks: embeds entity summaries into the vector store.
+ * On deleteFacts/deleteLinks: removes stale embeddings.
+ */
+export function createAutoEmbedMiddleware(options: AutoEmbedOptions): KernelMiddleware & { close: () => void } {
+  const store = new VectorStore(options.dbPath);
+  const embedFn = options.embedFn ?? embed;
+  const embedIndividual = options.embedIndividualFacts ?? false;
+
+  return {
+    name: 'auto-embed',
+
+    handleOp: async (op: KernelOp, ctx: MiddlewareContext, next: OpMiddlewareNext) => {
+      // Let the op proceed first
+      await next(op, ctx);
+
+      // Then asynchronously embed (don't block the mutation)
+      try {
+        await _processOp(op, store, embedFn, embedIndividual);
+      } catch {
+        // Embedding failures are non-fatal
+      }
+    },
+
+    close: () => {
+      store.close();
+    },
+  };
+}
+
+async function _processOp(
+  op: KernelOp,
+  store: VectorStore,
+  embedFn: Embedder,
+  embedIndividual: boolean,
+): Promise<void> {
+  const now = new Date().toISOString();
+
+  // Collect affected entity IDs
+  const entityIds = new Set<string>();
+  if (op.facts) for (const f of op.facts) entityIds.add(f.e);
+  if (op.links) for (const l of op.links) { entityIds.add(l.e1); entityIds.add(l.e2); }
+  if (op.deleteFacts) for (const f of op.deleteFacts) entityIds.add(f.e);
+  if (op.deleteLinks) for (const l of op.deleteLinks) { entityIds.add(l.e1); entityIds.add(l.e2); }
+
+  // Handle deletions — remove old embeddings for deleted entities
+  if (op.deleteFacts || op.deleteLinks) {
+    for (const eid of entityIds) {
+      store.deleteByEntity(eid);
+    }
+  }
+
+  // Handle additions — embed entity summaries
+  if (op.facts && op.facts.length > 0) {
+    // Group facts by entity
+    const factsByEntity = new Map<string, Fact[]>();
+    for (const f of op.facts) {
+      const existing = factsByEntity.get(f.e) ?? [];
+      existing.push(f);
+      factsByEntity.set(f.e, existing);
+    }
+
+    const linksByEntity = new Map<string, Link[]>();
+    if (op.links) {
+      for (const l of op.links) {
+        const existing = linksByEntity.get(l.e1) ?? [];
+        existing.push(l);
+        linksByEntity.set(l.e1, existing);
+      }
+    }
+
+    const records: EmbeddingRecord[] = [];
+
+    for (const [eid, facts] of factsByEntity) {
+      const links = linksByEntity.get(eid) ?? [];
+
+      // Entity summary embedding
+      const summaryText = entitySummaryText(eid, facts, links);
+      if (summaryText.trim()) {
+        try {
+          const vector = await embedFn(summaryText);
+          records.push({
+            id: `entity:${eid}:summary`,
+            entityId: eid,
+            content: summaryText,
+            chunkType: 'summary_md' as any,
+            updatedAt: now,
+            embedding: vector,
+          });
+        } catch {}
+      }
+
+      // Individual fact embeddings (optional)
+      if (embedIndividual) {
+        for (const fact of facts) {
+          if (['type', 'createdAt', 'updatedAt'].includes(fact.a)) continue;
+          const text = `${fact.a}: ${fact.v}`;
+          try {
+            const vector = await embedFn(text);
+            records.push({
+              id: `entity:${eid}:fact:${fact.a}`,
+              entityId: eid,
+              content: text,
+              chunkType: 'doc_comment' as any,
+              updatedAt: now,
+              embedding: vector,
+            });
+          } catch {}
+        }
+      }
+    }
+
+    if (records.length > 0) {
+      store.upsertBatch(records);
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// RAG Context Builder
+// ---------------------------------------------------------------------------
+
+export interface RAGContext {
+  /** The original query. */
+  query: string;
+  /** Retrieved chunks ranked by relevance. */
+  chunks: Array<{
+    content: string;
+    entityId: string;
+    score: number;
+    chunkType: string;
+  }>;
+  /** Total token estimate (rough: 1 token ≈ 4 chars). */
+  estimatedTokens: number;
+}
+
+/**
+ * Build a RAG context from a natural language query.
+ * Searches the vector store and assembles ranked context chunks.
+ */
+export async function buildRAGContext(
+  query: string,
+  vectorStore: VectorStore,
+  embedFn: Embedder = embed,
+  options?: {
+    maxChunks?: number;
+    maxTokens?: number;
+    minScore?: number;
+  },
+): Promise<RAGContext> {
+  const maxChunks = options?.maxChunks ?? 10;
+  const maxTokens = options?.maxTokens ?? 4000;
+  const minScore = options?.minScore ?? 0.1;
+
+  const queryVector = await embedFn(query);
+  const results = vectorStore.search(queryVector, {
+    limit: maxChunks * 2,
+    minScore,
+  });
+
+  const chunks: RAGContext['chunks'] = [];
+  let totalChars = 0;
+
+  for (const r of results) {
+    if (chunks.length >= maxChunks) break;
+    if (totalChars + r.chunk.content.length > maxTokens * 4) break;
+
+    chunks.push({
+      content: r.chunk.content,
+      entityId: r.chunk.entityId,
+      score: r.score,
+      chunkType: r.chunk.chunkType,
+    });
+    totalChars += r.chunk.content.length;
+  }
+
+  return {
+    query,
+    chunks,
+    estimatedTokens: Math.ceil(totalChars / 4),
+  };
+}

package/src/embeddings/index.ts

@@ -27,6 +27,13 @@ export { VectorStore, cosineSimilarity } from './store.js';
 export { EmbeddingManager } from './search.js';
 export type { SearchableEngine, Embedder } from './search.js';
 
+// Auto-embedding middleware + RAG
+export { createAutoEmbedMiddleware, buildRAGContext } from './auto-embed.js';
+export type { AutoEmbedOptions, RAGContext } from './auto-embed.js';
+
+// New graph chunk types
+export type { GraphChunkType } from './types.js';
+
 // Chunker
 export {
   chunkIssue,

package/src/embeddings/model.ts

@@ -1,7 +1,8 @@
 /**
  * Embedding Model
  *
- * Lazy-loads @xenova/transformers with all-MiniLM-L6-v2 (384-dim).
+ * Lazy-loads @huggingface/transformers (v3+) with all-MiniLM-L6-v2 (384-dim).
+ * Falls back to @xenova/transformers (v2) if the new package is unavailable.
  * Model is loaded once on first use and cached for subsequent calls.
  *
  * @see TRL-18
@@ -16,6 +17,24 @@ import { EmbeddingModelConfig, DEFAULT_MODEL_CONFIG } from './types.js';
 let pipeline: any = null;
 let loadPromise: Promise<any> | null = null;
 
+/**
+ * Dynamically import the transformers library.
+ * Tries @huggingface/transformers first (v3+), falls back to @xenova/transformers (v2).
+ */
+async function importTransformers(): Promise<{ pipeline: any }> {
+  try {
+    return await import('@huggingface/transformers' as string);
+  } catch {
+    try {
+      return await import('@xenova/transformers' as string);
+    } catch {
+      throw new Error(
+        'No transformers library found. Install @huggingface/transformers (recommended) or @xenova/transformers.',
+      );
+    }
+  }
+}
+
 /**
  * Load the embedding model lazily. Returns the feature-extraction pipeline.
  * Subsequent calls return the cached pipeline.
@@ -27,9 +46,7 @@ export async function loadModel(
 
   if (!loadPromise) {
     loadPromise = (async () => {
-      const { pipeline: createPipeline } = await import(
-        '@xenova/transformers'
-      );
+      const { pipeline: createPipeline } = await importTransformers();
       const opts: Record<string, unknown> = {};
       if (config.cacheDir) {
         opts.cache_dir = config.cacheDir;

package/src/embeddings/types.ts

@@ -71,7 +71,7 @@ export interface SearchOptions {
 // ---------------------------------------------------------------------------
 
 export interface EmbeddingModelConfig {
-  /** Model name for @xenova/transformers (default: "Xenova/all-MiniLM-L6-v2") */
+  /** Model name for transformers.js (default: "Xenova/all-MiniLM-L6-v2") */
   modelName: string;
   /** Embedding dimension (default: 384) */
   dimension: number;
@@ -83,3 +83,10 @@ export const DEFAULT_MODEL_CONFIG: EmbeddingModelConfig = {
   modelName: 'Xenova/all-MiniLM-L6-v2',
   dimension: 384,
 };
+
+/** New chunk types for generic graph entities */
+export type GraphChunkType =
+  | ChunkType
+  | 'entity_summary'
+  | 'entity_fact'
+  | 'entity_link';

package/src/engine.ts

@@ -22,7 +22,7 @@ import { readFile } from 'fs/promises';
 import { join, dirname } from 'path';
 import { EAVStore } from './core/store/eav-store.js';
 import type { Fact, Link } from './core/store/eav-store.js';
-import { FileWatcher } from './watcher/fs-watcher.js';
+import { FileWatcher, type ScanProgress } from './watcher/fs-watcher.js';
 import { Ingestion } from './watcher/ingestion.js';
 import { decompose } from './vcs/decompose.js';
 import { createVcsOp, isVcsOpKind } from './vcs/ops.js';
@@ -235,6 +235,13 @@ interface PersistedConfig {
 // Engine
 // ---------------------------------------------------------------------------
 
+export interface InitProgress {
+  phase: 'discovering' | 'hashing' | 'recording' | 'done';
+  current: number;
+  total: number;
+  message: string;
+}
+
 export class TrellisVcsEngine {
   private config: TrellisVcsConfig;
   private store: EAVStore;
@@ -281,7 +288,9 @@ export class TrellisVcsEngine {
   /**
    * Initialize a new TrellisVCS repo. Creates .trellis/ directory and config.
    */
-  async initRepo(): Promise<{ opsCreated: number }> {
+  async initRepo(opts?: {
+    onProgress?: (progress: InitProgress) => void;
+  }): Promise<{ opsCreated: number }> {
     const trellisDir = join(this.config.rootPath, '.trellis');
     if (!existsSync(trellisDir)) {
       mkdirSync(trellisDir, { recursive: true });
@@ -322,9 +331,27 @@ export class TrellisVcsEngine {
       debounceMs: this.config.debounceMs,
       onEvent: () => {},
     });
-    const events = await scanner.scan();
+    const events = await scanner.scan({
+      onProgress: (progress: ScanProgress) => {
+        if (progress.phase === 'done') {
+          return;
+        }
+        opts?.onProgress?.({
+          phase: progress.phase,
+          current: progress.current,
+          total: progress.total,
+          message: progress.message,
+        });
+      },
+    });
 
     let opsCreated = 1; // branch op
+    opts?.onProgress?.({
+      phase: 'recording',
+      current: 0,
+      total: events.length,
+      message: `Recording ${events.length} initial file operations…`,
+    });
     for (const event of events) {
       // Store file content in blob store
       if (event.contentHash) {
@@ -346,9 +373,24 @@ export class TrellisVcsEngine {
       });
       this.applyOp(op);
       opsCreated++;
+      const recordedFiles = opsCreated - 1;
+      if (recordedFiles % 25 === 0 || recordedFiles === events.length) {
+        opts?.onProgress?.({
+          phase: 'recording',
+          current: recordedFiles,
+          total: events.length,
+          message: `Recorded ${recordedFiles}/${events.length} initial file ops`,
+        });
+      }
     }
 
     await this.flushAutoCheckpoint();
+    opts?.onProgress?.({
+      phase: 'done',
+      current: opsCreated,
+      total: opsCreated,
+      message: `Initialized repository with ${opsCreated} operations`,
+    });
     return { opsCreated };
   }
 

package/src/index.ts

@@ -18,3 +18,12 @@ export { TrellisVcsEngine } from './engine.js';
 export * from './vcs/index.js';
 export { FileWatcher } from './watcher/fs-watcher.js';
 export { Ingestion } from './watcher/ingestion.js';
+
+// Core kernel (generic graph CRUD, independent of VCS)
+export { TrellisKernel } from './core/kernel/trellis-kernel.js';
+export { SqliteKernelBackend } from './core/persist/sqlite-backend.js';
+export type {
+  KernelConfig,
+  MutateResult,
+  EntityRecord,
+} from './core/kernel/trellis-kernel.js';

package/src/sync/http-transport.ts

@@ -0,0 +1,144 @@
+/**
+ * HTTP Sync Transport
+ *
+ * Implements SyncTransport over HTTP for network-based peer sync.
+ * Uses a simple JSON REST protocol:
+ *   POST /sync/message — send a sync message
+ *   GET  /sync/peers   — list connected peers
+ *
+ * The server side is a lightweight Bun HTTP server.
+ * The client side uses fetch() for outbound messages.
+ *
+ * @module trellis/sync
+ */
+
+import type { SyncTransport, SyncMessage, PeerId } from './types.js';
+
+// ---------------------------------------------------------------------------
+// HTTP Transport (Client)
+// ---------------------------------------------------------------------------
+
+export class HttpSyncTransport implements SyncTransport {
+  private localPeerId: string;
+  private peerUrls: Map<string, string> = new Map();
+  private messageHandler: ((msg: SyncMessage) => void) | null = null;
+  private knownPeers: Map<string, PeerId> = new Map();
+
+  constructor(localPeerId: string) {
+    this.localPeerId = localPeerId;
+  }
+
+  /**
+   * Add a remote peer by URL (e.g. "http://192.168.1.10:4200").
+   */
+  addPeer(peerId: string, url: string, name?: string): void {
+    this.peerUrls.set(peerId, url);
+    this.knownPeers.set(peerId, {
+      id: peerId,
+      name: name ?? peerId,
+      lastSeen: new Date().toISOString(),
+    });
+  }
+
+  /**
+   * Remove a remote peer.
+   */
+  removePeer(peerId: string): void {
+    this.peerUrls.delete(peerId);
+    this.knownPeers.delete(peerId);
+  }
+
+  async send(peerId: string, message: SyncMessage): Promise<void> {
+    const url = this.peerUrls.get(peerId);
+    if (!url) throw new Error(`Unknown peer "${peerId}". Add it with addPeer() first.`);
+
+    const resp = await fetch(`${url}/sync/message`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(message),
+    });
+
+    if (!resp.ok) {
+      throw new Error(`Sync message to ${peerId} failed: ${resp.status} ${resp.statusText}`);
+    }
+
+    // Check if the response contains a reply message
+    const contentType = resp.headers.get('content-type');
+    if (contentType?.includes('application/json')) {
+      const reply = await resp.json();
+      if (reply && reply.type && this.messageHandler) {
+        this.messageHandler(reply as SyncMessage);
+      }
+    }
+  }
+
+  onMessage(handler: (message: SyncMessage) => void): void {
+    this.messageHandler = handler;
+  }
+
+  /**
+   * Receive a message (called by the HTTP server handler).
+   */
+  receiveMessage(message: SyncMessage): void {
+    // Track peer
+    this.knownPeers.set(message.peerId, {
+      id: message.peerId,
+      name: message.peerId,
+      lastSeen: new Date().toISOString(),
+    });
+
+    if (this.messageHandler) {
+      this.messageHandler(message);
+    }
+  }
+
+  peers(): PeerId[] {
+    return [...this.knownPeers.values()];
+  }
+
+  getLocalPeerId(): string {
+    return this.localPeerId;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// HTTP Sync Server (creates Bun.serve handler)
+// ---------------------------------------------------------------------------
+
+export interface HttpSyncServerConfig {
+  port: number;
+  transport: HttpSyncTransport;
+}
+
+/**
+ * Create a Bun-compatible HTTP request handler for sync messages.
+ * Can be used with Bun.serve() or as middleware.
+ */
+export function createSyncHandler(transport: HttpSyncTransport): (req: Request) => Response | null {
+  return (req: Request): Response | null => {
+    const url = new URL(req.url);
+
+    if (url.pathname === '/sync/message' && req.method === 'POST') {
+      // Handle async parsing synchronously for Bun
+      return new Response(
+        req.json().then((body: any) => {
+          transport.receiveMessage(body as SyncMessage);
+          return JSON.stringify({ ok: true });
+        }) as any,
+        { headers: { 'Content-Type': 'application/json' } },
+      );
+    }
+
+    if (url.pathname === '/sync/peers' && req.method === 'GET') {
+      return new Response(
+        JSON.stringify({
+          localPeerId: transport.getLocalPeerId(),
+          peers: transport.peers(),
+        }),
+        { headers: { 'Content-Type': 'application/json' } },
+      );
+    }
+
+    return null; // Not a sync route
+  };
+}

package/src/sync/index.ts

@@ -30,3 +30,14 @@ export type { ReconcileResult, ReconcileConflict } from './reconciler.js';
 export { SyncEngine } from './sync-engine.js';
 
 export { MemoryTransport } from './memory-transport.js';
+
+export { HttpSyncTransport, createSyncHandler } from './http-transport.js';
+
+export { WebSocketSyncTransport } from './ws-transport.js';
+
+export {
+  MultiRepoManager,
+  parseCrossRepoRef,
+  formatCrossRepoRef,
+} from './multi-repo.js';
+export type { LinkedRepo, CrossRepoRef } from './multi-repo.js';