stellavault 0.1.0

package/packages/core/src/plugins/webhooks.ts

@@ -0,0 +1,110 @@
+// Webhook/Event System (F-A17)
+// Emit events on index, decay, gap detection → deliver via HTTP webhook
+
+import type { PluginEvent } from './index.js';
+
+export interface WebhookConfig {
+  url: string;
+  events: PluginEvent[];
+  secret?: string; // HMAC-SHA256 signing key
+  retries?: number;
+}
+
+export interface WebhookDelivery {
+  id: string;
+  event: PluginEvent;
+  url: string;
+  status: 'pending' | 'success' | 'failed';
+  attempts: number;
+  lastAttempt?: string;
+  response?: { status: number; body: string };
+}
+
+export class WebhookManager {
+  private configs: WebhookConfig[] = [];
+  private deliveries: WebhookDelivery[] = [];
+
+  register(config: WebhookConfig): void {
+    // MED: webhook URL 검증
+    try {
+      const parsed = new URL(config.url);
+      if (!['http:', 'https:'].includes(parsed.protocol)) throw new Error('Invalid protocol');
+      const host = parsed.hostname.toLowerCase();
+      if (host === 'localhost' || host === '127.0.0.1' || host.startsWith('192.168.') || host.startsWith('10.')) {
+        throw new Error('Internal URLs not allowed for webhooks');
+      }
+    } catch (e) { throw new Error(`Invalid webhook URL: ${e instanceof Error ? e.message : e}`); }
+    this.configs.push(config);
+  }
+
+  unregister(url: string): void {
+    this.configs = this.configs.filter(c => c.url !== url);
+  }
+
+  async emit(event: PluginEvent, data: unknown): Promise<WebhookDelivery[]> {
+    const matching = this.configs.filter(c => c.events.includes(event));
+    const results: WebhookDelivery[] = [];
+
+    for (const config of matching) {
+      const delivery = await this.deliver(config, event, data);
+      results.push(delivery);
+      this.deliveries.push(delivery);
+    }
+
+    // Keep last 100 deliveries
+    if (this.deliveries.length > 100) {
+      this.deliveries = this.deliveries.slice(-100);
+    }
+
+    return results;
+  }
+
+  private async deliver(config: WebhookConfig, event: PluginEvent, data: unknown): Promise<WebhookDelivery> {
+    const id = `wh-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    const maxRetries = config.retries ?? 3;
+    const delivery: WebhookDelivery = { id, event, url: config.url, status: 'pending', attempts: 0 };
+
+    const payload = JSON.stringify({ event, data, timestamp: new Date().toISOString(), source: 'stellavault' });
+
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      delivery.attempts = attempt + 1;
+      delivery.lastAttempt = new Date().toISOString();
+
+      try {
+        const headers: Record<string, string> = { 'Content-Type': 'application/json', 'User-Agent': 'stellavault-webhook/1.0' };
+
+        if (config.secret) {
+          const { createHmac } = await import('node:crypto');
+          const sig = createHmac('sha256', config.secret).update(payload).digest('hex');
+          headers['X-Stellavault-Signature'] = `sha256=${sig}`;
+        }
+
+        const res = await fetch(config.url, { method: 'POST', headers, body: payload, signal: AbortSignal.timeout(10000) });
+        delivery.response = { status: res.status, body: (await res.text()).slice(0, 500) };
+
+        if (res.ok) {
+          delivery.status = 'success';
+          return delivery;
+        }
+      } catch (err) {
+        delivery.response = { status: 0, body: err instanceof Error ? err.message : String(err) };
+      }
+
+      // Exponential backoff
+      if (attempt < maxRetries) {
+        await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt)));
+      }
+    }
+
+    delivery.status = 'failed';
+    return delivery;
+  }
+
+  getRecentDeliveries(limit = 20): WebhookDelivery[] {
+    return this.deliveries.slice(-limit).reverse();
+  }
+
+  listWebhooks(): WebhookConfig[] {
+    return [...this.configs];
+  }
+}

package/packages/core/src/search/bm25.ts

@@ -0,0 +1,16 @@
+// Design Ref: §6.2 — BM25 키워드 검색 (FTS5)
+
+import type { VectorStore } from '../store/types.js';
+import type { ScoredChunk } from '../types/chunk.js';
+
+export async function searchBm25(
+  store: VectorStore,
+  query: string,
+  limit: number,
+): Promise<ScoredChunk[]> {
+  // FTS5 쿼리 전처리: 특수문자 제거, 공백으로 OR 검색
+  const sanitized = query.replace(/[^\w\s가-힣]/g, ' ').trim();
+  if (!sanitized) return [];
+  const ftsQuery = sanitized.split(/\s+/).join(' OR ');
+  return store.searchKeyword(ftsQuery, limit);
+}

package/packages/core/src/search/index.ts

@@ -0,0 +1,83 @@
+// Design Ref: §4.2 — createSearchEngine
+
+import type { Embedder } from '../indexer/embedder.js';
+import type { VectorStore } from '../store/types.js';
+import type { SearchResult, SearchOptions } from '../types/search.js';
+import { searchBm25 } from './bm25.js';
+import { searchSemantic } from './semantic.js';
+import { rrfFusion } from './rrf.js';
+
+export { rrfFusion } from './rrf.js';
+
+export interface SearchEngine {
+  search(options: SearchOptions): Promise<SearchResult[]>;
+}
+
+export function createSearchEngine(deps: {
+  store: VectorStore;
+  embedder: Embedder;
+  rrfK?: number;
+}): SearchEngine {
+  const { store, embedder, rrfK = 60 } = deps;
+  const FETCH_LIMIT = 30; // 각 검색에서 가져올 후보 수
+
+  return {
+    async search(options: SearchOptions): Promise<SearchResult[]> {
+      const { query, limit = 10, threshold = 0.0, tags } = options;
+
+      // 병렬로 BM25 + Semantic 검색
+      const [bm25Results, semanticResults] = await Promise.all([
+        searchBm25(store, query, FETCH_LIMIT),
+        searchSemantic(store, embedder, query, FETCH_LIMIT),
+      ]);
+
+      // RRF Fusion
+      const fused = rrfFusion(semanticResults, bm25Results, rrfK, limit * 2);
+
+      // 청크+문서 조회 + 필터링
+      const results: SearchResult[] = [];
+      for (const scored of fused) {
+        if (scored.score < threshold) continue;
+
+        const chunk = await store.getChunk(scored.chunkId);
+        if (!chunk) continue;
+
+        const document = await store.getDocument(chunk.documentId);
+        if (!document) continue;
+
+        // 태그 필터
+        if (tags && tags.length > 0) {
+          const docTags = new Set(document.tags);
+          if (!tags.some(t => docTags.has(t))) continue;
+        }
+
+        results.push({
+          chunk,
+          document,
+          score: scored.score,
+          highlights: extractHighlights(chunk.content, query),
+        });
+
+        if (results.length >= limit) break;
+      }
+
+      return results;
+    },
+  };
+}
+
+function extractHighlights(content: string, query: string): string[] {
+  const words = query.toLowerCase().split(/\s+/).filter(w => w.length > 1);
+  const lines = content.split('\n');
+  const highlights: string[] = [];
+
+  for (const line of lines) {
+    const lower = line.toLowerCase();
+    if (words.some(w => lower.includes(w))) {
+      highlights.push(line.trim());
+      if (highlights.length >= 3) break;
+    }
+  }
+
+  return highlights;
+}

package/packages/core/src/search/rrf.ts

@@ -0,0 +1,31 @@
+// Design Ref: §6.2 — Reciprocal Rank Fusion (k=60)
+
+import type { ScoredChunk } from '../types/chunk.js';
+
+/**
+ * RRF: 두 랭킹 리스트를 통합하여 최종 점수를 산출합니다.
+ * score(d) = Σ 1/(k + rank_i) for each ranker i
+ */
+export function rrfFusion(
+  listA: ScoredChunk[],
+  listB: ScoredChunk[],
+  k: number = 60,
+  limit: number = 10,
+): ScoredChunk[] {
+  const scores = new Map<string, number>();
+
+  for (let i = 0; i < listA.length; i++) {
+    const id = listA[i].chunkId;
+    scores.set(id, (scores.get(id) ?? 0) + 1 / (k + i + 1));
+  }
+
+  for (let i = 0; i < listB.length; i++) {
+    const id = listB[i].chunkId;
+    scores.set(id, (scores.get(id) ?? 0) + 1 / (k + i + 1));
+  }
+
+  return [...scores.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, limit)
+    .map(([chunkId, score]) => ({ chunkId, score }));
+}

package/packages/core/src/search/semantic.ts

@@ -0,0 +1,15 @@
+// Design Ref: §6.2 — 시맨틱 벡터 검색
+
+import type { Embedder } from '../indexer/embedder.js';
+import type { VectorStore } from '../store/types.js';
+import type { ScoredChunk } from '../types/chunk.js';
+
+export async function searchSemantic(
+  store: VectorStore,
+  embedder: Embedder,
+  query: string,
+  limit: number,
+): Promise<ScoredChunk[]> {
+  const embedding = await embedder.embed(query);
+  return store.searchSemantic(embedding, limit);
+}

package/packages/core/src/store/index.ts

@@ -0,0 +1,2 @@
+export type { VectorStore } from './types.js';
+export { createSqliteVecStore, upsertDocument } from './sqlite-vec.js';

package/packages/core/src/store/sqlite-vec.ts

@@ -0,0 +1,290 @@
+// Design Ref: §3.3 — SQLite-vec 스키마
+// Design Ref: §3.2 — VectorStore 인터페이스 구현
+
+import Database from 'better-sqlite3';
+import * as sqliteVec from 'sqlite-vec';
+import { mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import type { VectorStore } from './types.js';
+import type { Chunk, ScoredChunk, Document, TopicInfo, StoreStats } from '../types/index.js';
+
+export function createSqliteVecStore(dbPath: string, dimensions: number = 384): VectorStore {
+  let db: Database.Database;
+
+  return {
+    async initialize() {
+      mkdirSync(dirname(dbPath), { recursive: true });
+      db = new Database(dbPath);
+      sqliteVec.load(db);
+      db.pragma('journal_mode = WAL');
+      db.pragma('foreign_keys = ON');
+      createTables(db, dimensions);
+    },
+
+    async upsertDocument(doc: Document) {
+      db.prepare(`
+        INSERT OR REPLACE INTO documents (id, file_path, title, content, frontmatter, tags, last_modified, content_hash, indexed_at, source, type)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `).run(
+        doc.id, doc.filePath, doc.title, doc.content,
+        JSON.stringify(doc.frontmatter), JSON.stringify(doc.tags),
+        doc.lastModified, doc.contentHash, new Date().toISOString(),
+        doc.source ?? 'local', doc.type ?? 'note'
+      );
+    },
+
+    async upsertChunks(chunks: Chunk[]) {
+      if (chunks.length === 0) return;
+      const docId = chunks[0].documentId;
+
+      const tx = db.transaction(() => {
+        // 기존 청크 삭제 (문서 단위 교체)
+        // 새로 삽입할 chunk ID들도 미리 삭제 (다른 문서에서 온 중복 방지)
+        const newChunkIds = chunks.map(c => c.id);
+        db.prepare('DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)').run(docId);
+        db.prepare('DELETE FROM chunks WHERE document_id = ?').run(docId);
+        for (const cid of newChunkIds) {
+          db.prepare('DELETE FROM chunk_embeddings WHERE chunk_id = ?').run(cid);
+          db.prepare('DELETE FROM chunks WHERE id = ?').run(cid);
+        }
+
+        const insertChunk = db.prepare(`
+          INSERT INTO chunks (id, document_id, content, heading, start_line, end_line, token_count)
+          VALUES (?, ?, ?, ?, ?, ?, ?)
+        `);
+        const insertEmbedding = db.prepare(`
+          INSERT INTO chunk_embeddings (chunk_id, embedding)
+          VALUES (?, ?)
+        `);
+
+        for (const chunk of chunks) {
+          insertChunk.run(
+            chunk.id, chunk.documentId, chunk.content,
+            chunk.heading, chunk.startLine, chunk.endLine, chunk.tokenCount
+          );
+          if (chunk.embedding) {
+            insertEmbedding.run(chunk.id, float32Buffer(chunk.embedding));
+          }
+        }
+      });
+      tx();
+    },
+
+    async deleteByDocumentId(documentId: string) {
+      const tx = db.transaction(() => {
+        db.prepare('DELETE FROM chunk_embeddings WHERE chunk_id IN (SELECT id FROM chunks WHERE document_id = ?)').run(documentId);
+        db.prepare('DELETE FROM chunks WHERE document_id = ?').run(documentId);
+        db.prepare('DELETE FROM documents WHERE id = ?').run(documentId);
+      });
+      tx();
+    },
+
+    async searchSemantic(embedding: number[], limit: number): Promise<ScoredChunk[]> {
+      const rows = db.prepare(`
+        SELECT chunk_id, distance
+        FROM chunk_embeddings
+        WHERE embedding MATCH ?
+        ORDER BY distance
+        LIMIT ?
+      `).all(float32Buffer(embedding), limit) as Array<{ chunk_id: string; distance: number }>;
+
+      return rows.map(r => ({
+        chunkId: r.chunk_id,
+        score: 1 / (1 + r.distance),  // distance → similarity score
+      }));
+    },
+
+    async searchKeyword(query: string, limit: number): Promise<ScoredChunk[]> {
+      const rows = db.prepare(`
+        SELECT c.id as chunk_id, rank
+        FROM chunks_fts f
+        JOIN chunks c ON c.rowid = f.rowid
+        WHERE chunks_fts MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      `).all(query, limit) as Array<{ chunk_id: string; rank: number }>;
+
+      return rows.map(r => ({
+        chunkId: r.chunk_id,
+        score: -r.rank,  // FTS5 rank is negative (lower = better)
+      }));
+    },
+
+    async getDocument(documentId: string): Promise<Document | null> {
+      const row = db.prepare('SELECT * FROM documents WHERE id = ?').get(documentId) as any;
+      if (!row) return null;
+      return rowToDocument(row);
+    },
+
+    async getChunk(chunkId: string): Promise<Chunk | null> {
+      const row = db.prepare('SELECT * FROM chunks WHERE id = ?').get(chunkId) as any;
+      if (!row) return null;
+      return rowToChunk(row);
+    },
+
+    async getAllDocuments(): Promise<Document[]> {
+      const rows = db.prepare('SELECT * FROM documents ORDER BY last_modified DESC').all() as any[];
+      return rows.map(rowToDocument);
+    },
+
+    async getTopics(): Promise<TopicInfo[]> {
+      const rows = db.prepare(`
+        SELECT je.value as tag, COUNT(DISTINCT d.id) as count
+        FROM documents d, json_each(d.tags) je
+        GROUP BY je.value
+        ORDER BY count DESC
+      `).all() as Array<{ tag: string; count: number }>;
+
+      return rows.map(r => ({
+        topic: r.tag,
+        count: r.count,
+        recentDocuments: [],
+      }));
+    },
+
+    async getStats(): Promise<StoreStats> {
+      const docCount = (db.prepare('SELECT COUNT(*) as c FROM documents').get() as any).c;
+      const chunkCount = (db.prepare('SELECT COUNT(*) as c FROM chunks').get() as any).c;
+      const lastRow = db.prepare('SELECT indexed_at FROM documents ORDER BY indexed_at DESC LIMIT 1').get() as any;
+      return {
+        documentCount: docCount,
+        chunkCount: chunkCount,
+        dbSizeBytes: 0, // 나중에 파일 크기 조회
+        lastIndexed: lastRow?.indexed_at ?? null,
+      };
+    },
+
+    async getDocumentEmbeddings(): Promise<Map<string, number[]>> {
+      // 각 문서의 첫 청크 임베딩을 문서 대표 벡터로 사용
+      const rows = db.prepare(`
+        SELECT c.document_id, ce.embedding
+        FROM chunks c
+        JOIN chunk_embeddings ce ON ce.chunk_id = c.id
+        WHERE c.id IN (
+          SELECT MIN(id) FROM chunks GROUP BY document_id
+        )
+      `).all() as Array<{ document_id: string; embedding: Buffer }>;
+
+      const result = new Map<string, number[]>();
+      for (const row of rows) {
+        result.set(row.document_id, bufferToFloat32(row.embedding));
+      }
+      return result;
+    },
+
+    async close() {
+      db.close();
+    },
+
+    getDb() {
+      return db;
+    },
+  };
+}
+
+function bufferToFloat32(buf: Buffer): number[] {
+  const float32 = new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
+  return Array.from(float32);
+}
+
+function createTables(db: Database.Database, dimensions: number = 384) {
+  // 기존 DB 마이그레이션: source/type 컬럼 추가
+  try {
+    db.exec(`ALTER TABLE documents ADD COLUMN source TEXT DEFAULT 'local'`);
+  } catch { /* 이미 존재 */ }
+  try {
+    db.exec(`ALTER TABLE documents ADD COLUMN type TEXT DEFAULT 'note'`);
+  } catch { /* 이미 존재 */ }
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS documents (
+      id TEXT PRIMARY KEY,
+      file_path TEXT NOT NULL UNIQUE,
+      title TEXT NOT NULL,
+      content TEXT NOT NULL,
+      frontmatter TEXT,
+      tags TEXT DEFAULT '[]',
+      last_modified TEXT NOT NULL,
+      content_hash TEXT NOT NULL,
+      indexed_at TEXT NOT NULL,
+      source TEXT DEFAULT 'local',
+      type TEXT DEFAULT 'note'
+    );
+
+    CREATE TABLE IF NOT EXISTS chunks (
+      id TEXT PRIMARY KEY,
+      document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+      content TEXT NOT NULL,
+      heading TEXT,
+      start_line INTEGER,
+      end_line INTEGER,
+      token_count INTEGER
+    );
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS chunk_embeddings USING vec0(
+      chunk_id TEXT PRIMARY KEY,
+      embedding FLOAT[${dimensions}]
+    );
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+      content,
+      heading,
+      content='chunks',
+      content_rowid='rowid'
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id);
+    CREATE INDEX IF NOT EXISTS idx_documents_content_hash ON documents(content_hash);
+  `);
+
+  // FTS5 트리거: chunks INSERT/DELETE 시 자동 동기화
+  db.exec(`
+    CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+      INSERT INTO chunks_fts(rowid, content, heading) VALUES (new.rowid, new.content, new.heading);
+    END;
+    CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+      INSERT INTO chunks_fts(chunks_fts, rowid, content, heading) VALUES('delete', old.rowid, old.content, old.heading);
+    END;
+  `);
+}
+
+// Document 헬퍼: upsertDocument는 indexer에서 호출
+export function upsertDocument(db: Database.Database, doc: Document) {
+  db.prepare(`
+    INSERT OR REPLACE INTO documents (id, file_path, title, content, frontmatter, tags, last_modified, content_hash, indexed_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(
+    doc.id, doc.filePath, doc.title, doc.content,
+    JSON.stringify(doc.frontmatter), JSON.stringify(doc.tags),
+    doc.lastModified, doc.contentHash, new Date().toISOString()
+  );
+}
+
+function rowToDocument(row: any): Document {
+  return {
+    id: row.id,
+    filePath: row.file_path,
+    title: row.title,
+    content: row.content,
+    frontmatter: JSON.parse(row.frontmatter || '{}'),
+    tags: JSON.parse(row.tags || '[]'),
+    lastModified: row.last_modified,
+    contentHash: row.content_hash,
+  };
+}
+
+function rowToChunk(row: any): Chunk {
+  return {
+    id: row.id,
+    documentId: row.document_id,
+    content: row.content,
+    heading: row.heading ?? '',
+    startLine: row.start_line,
+    endLine: row.end_line,
+    tokenCount: row.token_count,
+  };
+}
+
+function float32Buffer(arr: number[]): Buffer {
+  return Buffer.from(new Float32Array(arr).buffer);
+}

package/packages/core/src/store/types.ts

@@ -0,0 +1,22 @@
+// Design Ref: §3.2 — 교체 가능 설계 (VectorStore 인터페이스)
+
+import type { Chunk, ScoredChunk, Document, TopicInfo, StoreStats } from '../types/index.js';
+
+export interface VectorStore {
+  initialize(): Promise<void>;
+  upsertDocument(doc: Document): Promise<void>;
+  upsertChunks(chunks: Chunk[]): Promise<void>;
+  deleteByDocumentId(documentId: string): Promise<void>;
+  searchSemantic(embedding: number[], limit: number): Promise<ScoredChunk[]>;
+  searchKeyword(query: string, limit: number): Promise<ScoredChunk[]>;
+  getDocument(documentId: string): Promise<Document | null>;
+  getChunk(chunkId: string): Promise<Chunk | null>;
+  getAllDocuments(): Promise<Document[]>;
+  getTopics(): Promise<TopicInfo[]>;
+  getStats(): Promise<StoreStats>;
+  /** 각 문서의 첫 청크 임베딩 반환 (graph용) */
+  getDocumentEmbeddings(): Promise<Map<string, number[]>>;
+  close(): Promise<void>;
+  /** 내부 DB 인스턴스 접근 (Intelligence Layer용) */
+  getDb(): unknown;
+}