trellis 2.0.8 → 2.0.13

package/src/embeddings/auto-embed.ts

@@ -0,0 +1,248 @@
+/**
+ * Auto-Embedding Middleware
+ *
+ * Kernel middleware that automatically embeds entity facts and links
+ * on graph mutations. Runs after successful ops to index new/changed
+ * content into the vector store.
+ *
+ * @module trellis/embeddings
+ */
+
+import type { KernelOp } from '../core/persist/backend.js';
+import type { KernelMiddleware, MiddlewareContext, OpMiddlewareNext } from '../core/kernel/middleware.js';
+import type { Fact, Link } from '../core/store/eav-store.js';
+import type { ChunkMeta, EmbeddingRecord } from './types.js';
+import type { Embedder } from './search.js';
+import { VectorStore } from './store.js';
+import { embed } from './model.js';
+
+// ---------------------------------------------------------------------------
+// Entity text builder — converts facts/links into embeddable text
+// ---------------------------------------------------------------------------
+
+function factsToText(facts: Fact[]): string {
+  return facts
+    .filter((f) => f.a !== 'createdAt' && f.a !== 'updatedAt')
+    .map((f) => `${f.a}: ${f.v}`)
+    .join('\n');
+}
+
+function linksToText(links: Link[]): string {
+  return links.map((l) => `${l.e1} —[${l.a}]→ ${l.e2}`).join('\n');
+}
+
+function entitySummaryText(entityId: string, facts: Fact[], links: Link[]): string {
+  const type = facts.find((f) => f.a === 'type')?.v ?? 'Entity';
+  const name = facts.find((f) => f.a === 'name' || f.a === 'title')?.v ?? entityId;
+  const parts = [`${type}: ${name} (${entityId})`];
+
+  const attrs = facts.filter((f) => !['type', 'name', 'title', 'createdAt', 'updatedAt'].includes(f.a));
+  if (attrs.length > 0) {
+    parts.push(attrs.map((f) => `  ${f.a} = ${f.v}`).join('\n'));
+  }
+
+  if (links.length > 0) {
+    parts.push('Relations:');
+    parts.push(links.map((l) => `  ${l.a} → ${l.e2}`).join('\n'));
+  }
+
+  return parts.join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// Middleware factory
+// ---------------------------------------------------------------------------
+
+export interface AutoEmbedOptions {
+  /** Path to the vector store SQLite database. */
+  dbPath: string;
+  /** Custom embedder function (default: transformers.js embed). */
+  embedFn?: Embedder;
+  /** Whether to embed facts individually (default: false — only entity summaries). */
+  embedIndividualFacts?: boolean;
+}
+
+/**
+ * Creates a kernel middleware that auto-embeds entities on mutation.
+ *
+ * On addFacts/addLinks: embeds entity summaries into the vector store.
+ * On deleteFacts/deleteLinks: removes stale embeddings.
+ */
+export function createAutoEmbedMiddleware(options: AutoEmbedOptions): KernelMiddleware & { close: () => void } {
+  const store = new VectorStore(options.dbPath);
+  const embedFn = options.embedFn ?? embed;
+  const embedIndividual = options.embedIndividualFacts ?? false;
+
+  return {
+    name: 'auto-embed',
+
+    handleOp: async (op: KernelOp, ctx: MiddlewareContext, next: OpMiddlewareNext) => {
+      // Let the op proceed first
+      await next(op, ctx);
+
+      // Then asynchronously embed (don't block the mutation)
+      try {
+        await _processOp(op, store, embedFn, embedIndividual);
+      } catch {
+        // Embedding failures are non-fatal
+      }
+    },
+
+    close: () => {
+      store.close();
+    },
+  };
+}
+
+async function _processOp(
+  op: KernelOp,
+  store: VectorStore,
+  embedFn: Embedder,
+  embedIndividual: boolean,
+): Promise<void> {
+  const now = new Date().toISOString();
+
+  // Collect affected entity IDs
+  const entityIds = new Set<string>();
+  if (op.facts) for (const f of op.facts) entityIds.add(f.e);
+  if (op.links) for (const l of op.links) { entityIds.add(l.e1); entityIds.add(l.e2); }
+  if (op.deleteFacts) for (const f of op.deleteFacts) entityIds.add(f.e);
+  if (op.deleteLinks) for (const l of op.deleteLinks) { entityIds.add(l.e1); entityIds.add(l.e2); }
+
+  // Handle deletions — remove old embeddings for deleted entities
+  if (op.deleteFacts || op.deleteLinks) {
+    for (const eid of entityIds) {
+      store.deleteByEntity(eid);
+    }
+  }
+
+  // Handle additions — embed entity summaries
+  if (op.facts && op.facts.length > 0) {
+    // Group facts by entity
+    const factsByEntity = new Map<string, Fact[]>();
+    for (const f of op.facts) {
+      const existing = factsByEntity.get(f.e) ?? [];
+      existing.push(f);
+      factsByEntity.set(f.e, existing);
+    }
+
+    const linksByEntity = new Map<string, Link[]>();
+    if (op.links) {
+      for (const l of op.links) {
+        const existing = linksByEntity.get(l.e1) ?? [];
+        existing.push(l);
+        linksByEntity.set(l.e1, existing);
+      }
+    }
+
+    const records: EmbeddingRecord[] = [];
+
+    for (const [eid, facts] of factsByEntity) {
+      const links = linksByEntity.get(eid) ?? [];
+
+      // Entity summary embedding
+      const summaryText = entitySummaryText(eid, facts, links);
+      if (summaryText.trim()) {
+        try {
+          const vector = await embedFn(summaryText);
+          records.push({
+            id: `entity:${eid}:summary`,
+            entityId: eid,
+            content: summaryText,
+            chunkType: 'summary_md' as any,
+            updatedAt: now,
+            embedding: vector,
+          });
+        } catch {}
+      }
+
+      // Individual fact embeddings (optional)
+      if (embedIndividual) {
+        for (const fact of facts) {
+          if (['type', 'createdAt', 'updatedAt'].includes(fact.a)) continue;
+          const text = `${fact.a}: ${fact.v}`;
+          try {
+            const vector = await embedFn(text);
+            records.push({
+              id: `entity:${eid}:fact:${fact.a}`,
+              entityId: eid,
+              content: text,
+              chunkType: 'doc_comment' as any,
+              updatedAt: now,
+              embedding: vector,
+            });
+          } catch {}
+        }
+      }
+    }
+
+    if (records.length > 0) {
+      store.upsertBatch(records);
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// RAG Context Builder
+// ---------------------------------------------------------------------------
+
+export interface RAGContext {
+  /** The original query. */
+  query: string;
+  /** Retrieved chunks ranked by relevance. */
+  chunks: Array<{
+    content: string;
+    entityId: string;
+    score: number;
+    chunkType: string;
+  }>;
+  /** Total token estimate (rough: 1 token ≈ 4 chars). */
+  estimatedTokens: number;
+}
+
+/**
+ * Build a RAG context from a natural language query.
+ * Searches the vector store and assembles ranked context chunks.
+ */
+export async function buildRAGContext(
+  query: string,
+  vectorStore: VectorStore,
+  embedFn: Embedder = embed,
+  options?: {
+    maxChunks?: number;
+    maxTokens?: number;
+    minScore?: number;
+  },
+): Promise<RAGContext> {
+  const maxChunks = options?.maxChunks ?? 10;
+  const maxTokens = options?.maxTokens ?? 4000;
+  const minScore = options?.minScore ?? 0.1;
+
+  const queryVector = await embedFn(query);
+  const results = vectorStore.search(queryVector, {
+    limit: maxChunks * 2,
+    minScore,
+  });
+
+  const chunks: RAGContext['chunks'] = [];
+  let totalChars = 0;
+
+  for (const r of results) {
+    if (chunks.length >= maxChunks) break;
+    if (totalChars + r.chunk.content.length > maxTokens * 4) break;
+
+    chunks.push({
+      content: r.chunk.content,
+      entityId: r.chunk.entityId,
+      score: r.score,
+      chunkType: r.chunk.chunkType,
+    });
+    totalChars += r.chunk.content.length;
+  }
+
+  return {
+    query,
+    chunks,
+    estimatedTokens: Math.ceil(totalChars / 4),
+  };
+}

package/src/embeddings/index.ts

@@ -27,6 +27,13 @@ export { VectorStore, cosineSimilarity } from './store.js';
 export { EmbeddingManager } from './search.js';
 export type { SearchableEngine, Embedder } from './search.js';
 
+// Auto-embedding middleware + RAG
+export { createAutoEmbedMiddleware, buildRAGContext } from './auto-embed.js';
+export type { AutoEmbedOptions, RAGContext } from './auto-embed.js';
+
+// New graph chunk types
+export type { GraphChunkType } from './types.js';
+
 // Chunker
 export {
   chunkIssue,

package/src/embeddings/model.ts

@@ -1,7 +1,8 @@
 /**
  * Embedding Model
  *
- * Lazy-loads @xenova/transformers with all-MiniLM-L6-v2 (384-dim).
+ * Lazy-loads @huggingface/transformers (v3+) with all-MiniLM-L6-v2 (384-dim).
+ * Falls back to @xenova/transformers (v2) if the new package is unavailable.
  * Model is loaded once on first use and cached for subsequent calls.
  *
  * @see TRL-18
@@ -16,6 +17,24 @@ import { EmbeddingModelConfig, DEFAULT_MODEL_CONFIG } from './types.js';
 let pipeline: any = null;
 let loadPromise: Promise<any> | null = null;
 
+/**
+ * Dynamically import the transformers library.
+ * Tries @huggingface/transformers first (v3+), falls back to @xenova/transformers (v2).
+ */
+async function importTransformers(): Promise<{ pipeline: any }> {
+  try {
+    return await import('@huggingface/transformers' as string);
+  } catch {
+    try {
+      return await import('@xenova/transformers' as string);
+    } catch {
+      throw new Error(
+        'No transformers library found. Install @huggingface/transformers (recommended) or @xenova/transformers.',
+      );
+    }
+  }
+}
+
 /**
  * Load the embedding model lazily. Returns the feature-extraction pipeline.
  * Subsequent calls return the cached pipeline.
@@ -27,9 +46,7 @@ export async function loadModel(
 
   if (!loadPromise) {
     loadPromise = (async () => {
-      const { pipeline: createPipeline } = await import(
-        '@xenova/transformers'
-      );
+      const { pipeline: createPipeline } = await importTransformers();
       const opts: Record<string, unknown> = {};
       if (config.cacheDir) {
         opts.cache_dir = config.cacheDir;

package/src/embeddings/types.ts

@@ -71,7 +71,7 @@ export interface SearchOptions {
 // ---------------------------------------------------------------------------
 
 export interface EmbeddingModelConfig {
-  /** Model name for @xenova/transformers (default: "Xenova/all-MiniLM-L6-v2") */
+  /** Model name for transformers.js (default: "Xenova/all-MiniLM-L6-v2") */
   modelName: string;
   /** Embedding dimension (default: 384) */
   dimension: number;
@@ -83,3 +83,10 @@ export const DEFAULT_MODEL_CONFIG: EmbeddingModelConfig = {
   modelName: 'Xenova/all-MiniLM-L6-v2',
   dimension: 384,
 };
+
+/** New chunk types for generic graph entities */
+export type GraphChunkType =
+  | ChunkType
+  | 'entity_summary'
+  | 'entity_fact'
+  | 'entity_link';

package/src/index.ts

@@ -18,3 +18,12 @@ export { TrellisVcsEngine } from './engine.js';
 export * from './vcs/index.js';
 export { FileWatcher } from './watcher/fs-watcher.js';
 export { Ingestion } from './watcher/ingestion.js';
+
+// Core kernel (generic graph CRUD, independent of VCS)
+export { TrellisKernel } from './core/kernel/trellis-kernel.js';
+export { SqliteKernelBackend } from './core/persist/sqlite-backend.js';
+export type {
+  KernelConfig,
+  MutateResult,
+  EntityRecord,
+} from './core/kernel/trellis-kernel.js';

package/src/sync/http-transport.ts

@@ -0,0 +1,144 @@
+/**
+ * HTTP Sync Transport
+ *
+ * Implements SyncTransport over HTTP for network-based peer sync.
+ * Uses a simple JSON REST protocol:
+ *   POST /sync/message — send a sync message
+ *   GET  /sync/peers   — list connected peers
+ *
+ * The server side is a lightweight Bun HTTP server.
+ * The client side uses fetch() for outbound messages.
+ *
+ * @module trellis/sync
+ */
+
+import type { SyncTransport, SyncMessage, PeerId } from './types.js';
+
+// ---------------------------------------------------------------------------
+// HTTP Transport (Client)
+// ---------------------------------------------------------------------------
+
+export class HttpSyncTransport implements SyncTransport {
+  private localPeerId: string;
+  private peerUrls: Map<string, string> = new Map();
+  private messageHandler: ((msg: SyncMessage) => void) | null = null;
+  private knownPeers: Map<string, PeerId> = new Map();
+
+  constructor(localPeerId: string) {
+    this.localPeerId = localPeerId;
+  }
+
+  /**
+   * Add a remote peer by URL (e.g. "http://192.168.1.10:4200").
+   */
+  addPeer(peerId: string, url: string, name?: string): void {
+    this.peerUrls.set(peerId, url);
+    this.knownPeers.set(peerId, {
+      id: peerId,
+      name: name ?? peerId,
+      lastSeen: new Date().toISOString(),
+    });
+  }
+
+  /**
+   * Remove a remote peer.
+   */
+  removePeer(peerId: string): void {
+    this.peerUrls.delete(peerId);
+    this.knownPeers.delete(peerId);
+  }
+
+  async send(peerId: string, message: SyncMessage): Promise<void> {
+    const url = this.peerUrls.get(peerId);
+    if (!url) throw new Error(`Unknown peer "${peerId}". Add it with addPeer() first.`);
+
+    const resp = await fetch(`${url}/sync/message`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(message),
+    });
+
+    if (!resp.ok) {
+      throw new Error(`Sync message to ${peerId} failed: ${resp.status} ${resp.statusText}`);
+    }
+
+    // Check if the response contains a reply message
+    const contentType = resp.headers.get('content-type');
+    if (contentType?.includes('application/json')) {
+      const reply = await resp.json();
+      if (reply && reply.type && this.messageHandler) {
+        this.messageHandler(reply as SyncMessage);
+      }
+    }
+  }
+
+  onMessage(handler: (message: SyncMessage) => void): void {
+    this.messageHandler = handler;
+  }
+
+  /**
+   * Receive a message (called by the HTTP server handler).
+   */
+  receiveMessage(message: SyncMessage): void {
+    // Track peer
+    this.knownPeers.set(message.peerId, {
+      id: message.peerId,
+      name: message.peerId,
+      lastSeen: new Date().toISOString(),
+    });
+
+    if (this.messageHandler) {
+      this.messageHandler(message);
+    }
+  }
+
+  peers(): PeerId[] {
+    return [...this.knownPeers.values()];
+  }
+
+  getLocalPeerId(): string {
+    return this.localPeerId;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// HTTP Sync Server (creates Bun.serve handler)
+// ---------------------------------------------------------------------------
+
+export interface HttpSyncServerConfig {
+  port: number;
+  transport: HttpSyncTransport;
+}
+
+/**
+ * Create a Bun-compatible HTTP request handler for sync messages.
+ * Can be used with Bun.serve() or as middleware.
+ */
+export function createSyncHandler(transport: HttpSyncTransport): (req: Request) => Response | null {
+  return (req: Request): Response | null => {
+    const url = new URL(req.url);
+
+    if (url.pathname === '/sync/message' && req.method === 'POST') {
+      // Handle async parsing synchronously for Bun
+      return new Response(
+        req.json().then((body: any) => {
+          transport.receiveMessage(body as SyncMessage);
+          return JSON.stringify({ ok: true });
+        }) as any,
+        { headers: { 'Content-Type': 'application/json' } },
+      );
+    }
+
+    if (url.pathname === '/sync/peers' && req.method === 'GET') {
+      return new Response(
+        JSON.stringify({
+          localPeerId: transport.getLocalPeerId(),
+          peers: transport.peers(),
+        }),
+        { headers: { 'Content-Type': 'application/json' } },
+      );
+    }
+
+    return null; // Not a sync route
+  };
+}

package/src/sync/index.ts

@@ -30,3 +30,14 @@ export type { ReconcileResult, ReconcileConflict } from './reconciler.js';
 export { SyncEngine } from './sync-engine.js';
 
 export { MemoryTransport } from './memory-transport.js';
+
+export { HttpSyncTransport, createSyncHandler } from './http-transport.js';
+
+export { WebSocketSyncTransport } from './ws-transport.js';
+
+export {
+  MultiRepoManager,
+  parseCrossRepoRef,
+  formatCrossRepoRef,
+} from './multi-repo.js';
+export type { LinkedRepo, CrossRepoRef } from './multi-repo.js';

package/src/sync/multi-repo.ts

@@ -0,0 +1,200 @@
+/**
+ * Multi-Repo Linking — Cross-repo entity references.
+ *
+ * Enables entities in one Trellis repo to reference entities in another.
+ * Linked repos are registered with a local alias and remote path/URL.
+ * Cross-repo references use the format: `@alias:entityId`
+ *
+ * @module trellis/sync
+ */
+
+import type { TrellisKernel } from '../core/kernel/trellis-kernel.js';
+import type { Fact, Link } from '../core/store/eav-store.js';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface LinkedRepo {
+  /** Local alias for the remote repo (e.g. "backend", "shared-lib"). */
+  alias: string;
+  /** Path or URL to the remote repo. */
+  location: string;
+  /** Optional human-readable description. */
+  description?: string;
+  /** When this link was established. */
+  linkedAt: string;
+  /** Last sync timestamp. */
+  lastSyncedAt?: string;
+}
+
+export interface CrossRepoRef {
+  /** The repo alias. */
+  repoAlias: string;
+  /** The entity ID in the remote repo. */
+  entityId: string;
+}
+
+// ---------------------------------------------------------------------------
+// Multi-Repo Manager
+// ---------------------------------------------------------------------------
+
+export class MultiRepoManager {
+  private kernel: TrellisKernel;
+
+  constructor(kernel: TrellisKernel) {
+    this.kernel = kernel;
+  }
+
+  /**
+   * Link a remote repository.
+   */
+  async linkRepo(alias: string, location: string, description?: string): Promise<void> {
+    const id = `repo:${alias}`;
+    const existing = this.kernel.getEntity(id);
+    if (existing) {
+      throw new Error(`Repo alias "${alias}" is already linked to "${existing.facts.find(f => f.a === 'location')?.v}".`);
+    }
+
+    await this.kernel.createEntity(id, 'LinkedRepo', {
+      alias,
+      location,
+      ...(description ? { description } : {}),
+      linkedAt: new Date().toISOString(),
+    });
+  }
+
+  /**
+   * Unlink a remote repository.
+   */
+  async unlinkRepo(alias: string): Promise<void> {
+    const id = `repo:${alias}`;
+    await this.kernel.deleteEntity(id);
+  }
+
+  /**
+   * List all linked repos.
+   */
+  listLinkedRepos(): LinkedRepo[] {
+    const entities = this.kernel.listEntities('LinkedRepo');
+    return entities.map((e) => {
+      const get = (a: string) => e.facts.find((f) => f.a === a)?.v;
+      return {
+        alias: String(get('alias') ?? ''),
+        location: String(get('location') ?? ''),
+        description: get('description') as string | undefined,
+        linkedAt: String(get('linkedAt') ?? ''),
+        lastSyncedAt: get('lastSyncedAt') as string | undefined,
+      };
+    });
+  }
+
+  /**
+   * Get a linked repo by alias.
+   */
+  getLinkedRepo(alias: string): LinkedRepo | null {
+    const entity = this.kernel.getEntity(`repo:${alias}`);
+    if (!entity) return null;
+    const get = (a: string) => entity.facts.find((f) => f.a === a)?.v;
+    return {
+      alias: String(get('alias') ?? ''),
+      location: String(get('location') ?? ''),
+      description: get('description') as string | undefined,
+      linkedAt: String(get('linkedAt') ?? ''),
+      lastSyncedAt: get('lastSyncedAt') as string | undefined,
+    };
+  }
+
+  /**
+   * Create a cross-repo link: entity in this repo → entity in remote repo.
+   */
+  async addCrossRepoLink(
+    sourceEntityId: string,
+    attribute: string,
+    targetRepoAlias: string,
+    targetEntityId: string,
+  ): Promise<void> {
+    const repo = this.getLinkedRepo(targetRepoAlias);
+    if (!repo) {
+      throw new Error(`Repo alias "${targetRepoAlias}" is not linked. Use linkRepo() first.`);
+    }
+
+    const crossRef = `@${targetRepoAlias}:${targetEntityId}`;
+    await this.kernel.addLink(sourceEntityId, attribute, crossRef);
+  }
+
+  /**
+   * Remove a cross-repo link.
+   */
+  async removeCrossRepoLink(
+    sourceEntityId: string,
+    attribute: string,
+    targetRepoAlias: string,
+    targetEntityId: string,
+  ): Promise<void> {
+    const crossRef = `@${targetRepoAlias}:${targetEntityId}`;
+    await this.kernel.removeLink(sourceEntityId, attribute, crossRef);
+  }
+
+  /**
+   * Find all cross-repo references from a given entity.
+   */
+  getCrossRepoLinks(entityId: string): Array<{ attribute: string; ref: CrossRepoRef }> {
+    const store = this.kernel.getStore();
+    const links = store.getLinksByEntity(entityId);
+    const results: Array<{ attribute: string; ref: CrossRepoRef }> = [];
+
+    for (const link of links) {
+      if (link.e1 !== entityId) continue;
+      const parsed = parseCrossRepoRef(link.e2);
+      if (parsed) {
+        results.push({ attribute: link.a, ref: parsed });
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * Find all cross-repo references pointing to a specific remote entity.
+   */
+  findReferencesTo(repoAlias: string, entityId: string): Link[] {
+    const crossRef = `@${repoAlias}:${entityId}`;
+    const store = this.kernel.getStore();
+    return store.getAllLinks().filter((l) => l.e2 === crossRef);
+  }
+
+  /**
+   * Update the lastSyncedAt timestamp for a linked repo.
+   */
+  async markSynced(alias: string): Promise<void> {
+    await this.kernel.updateEntity(`repo:${alias}`, {
+      lastSyncedAt: new Date().toISOString(),
+    });
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse a cross-repo reference string.
+ * Format: `@alias:entityId`
+ */
+export function parseCrossRepoRef(ref: string): CrossRepoRef | null {
+  if (!ref.startsWith('@')) return null;
+  const colonIdx = ref.indexOf(':', 1);
+  if (colonIdx === -1) return null;
+  return {
+    repoAlias: ref.slice(1, colonIdx),
+    entityId: ref.slice(colonIdx + 1),
+  };
+}
+
+/**
+ * Format a cross-repo reference string.
+ */
+export function formatCrossRepoRef(repoAlias: string, entityId: string): string {
+  return `@${repoAlias}:${entityId}`;
+}