stellavault 0.1.0

package/packages/core/src/indexer/index.ts

@@ -0,0 +1,113 @@
+// Design Ref: §6.3 — Incremental Indexing + Pipeline
+
+import type { Embedder } from './embedder.js';
+import type { VectorStore } from '../store/types.js';
+import type { Document } from '../types/document.js';
+import type { Chunk } from '../types/chunk.js';
+import { scanVault } from './scanner.js';
+import { chunkDocument, type ChunkOptions } from './chunker.js';
+import { withRetry, errors } from '../utils/retry.js';
+
+export { type Embedder } from './embedder.js';
+export { createLocalEmbedder } from './local-embedder.js';
+export { scanVault } from './scanner.js';
+export { chunkDocument, estimateTokens } from './chunker.js';
+export { createWatcher } from './watcher.js';
+
+export interface IndexerOptions {
+  store: VectorStore;
+  embedder: Embedder;
+  chunkOptions?: Partial<ChunkOptions>;
+  onProgress?: (current: number, total: number, doc: Document) => void;
+}
+
+export interface IndexResult {
+  indexed: number;
+  skipped: number;
+  deleted: number;
+  failed: number;
+  totalChunks: number;
+  elapsedMs: number;
+}
+
+/**
+ * vault를 스캔하여 변경된 문서만 벡터화하는 증분 인덱서
+ */
+export async function indexVault(
+  vaultPath: string,
+  options: IndexerOptions,
+): Promise<IndexResult> {
+  const start = Date.now();
+  const { store, embedder, chunkOptions, onProgress } = options;
+
+  // 1. 스캔
+  const { documents } = scanVault(vaultPath);
+
+  // 2. 기존 인덱스 상태 조회
+  const existingDocs = await store.getAllDocuments();
+  const existingMap = new Map(existingDocs.map(d => [d.id, d.contentHash]));
+
+  let indexed = 0;
+  let skipped = 0;
+  let failed = 0;
+  let totalChunks = 0;
+
+  // 3. 증분 처리 (에러 복구 포함)
+  const scannedIds = new Set<string>();
+  for (let i = 0; i < documents.length; i++) {
+    const doc = documents[i];
+    scannedIds.add(doc.id);
+    onProgress?.(i + 1, documents.length, doc);
+
+    // content_hash 비교 → 변경 없으면 SKIP
+    if (existingMap.get(doc.id) === doc.contentHash) {
+      skipped++;
+      continue;
+    }
+
+    try {
+      // 청킹
+      const chunks = chunkDocument(doc.id, doc.content, chunkOptions);
+
+      // 임베딩 (retry with backoff)
+      const texts = chunks.map(c => c.content);
+      const embeddings = await withRetry(
+        () => embedder.embedBatch(texts),
+        { maxRetries: 2, baseDelayMs: 1000 },
+      );
+      const chunksWithEmbeddings: Chunk[] = chunks.map((c, j) => ({
+        ...c,
+        embedding: embeddings[j],
+      }));
+
+      // 저장 (document → chunks)
+      await store.upsertDocument(doc);
+      await store.upsertChunks(chunksWithEmbeddings);
+
+      indexed++;
+      totalChunks += chunks.length;
+    } catch (err) {
+      // Graceful degradation: skip failed file, continue with rest
+      failed++;
+      console.error(errors.indexingFailed(doc.filePath, err).format());
+    }
+  }
+
+  // 4. 삭제된 파일 처리
+  let deleted = 0;
+  for (const [existingId] of existingMap) {
+    if (!scannedIds.has(existingId)) {
+      await store.deleteByDocumentId(existingId);
+      deleted++;
+    }
+  }
+
+  return {
+    indexed,
+    skipped,
+    deleted,
+    failed,
+    totalChunks,
+    elapsedMs: Date.now() - start,
+  };
+}

package/packages/core/src/indexer/local-embedder.ts

@@ -0,0 +1,35 @@
+// Design Ref: §3.2 — Embedder 로컬 구현 (nomic-embed-text via @xenova/transformers)
+
+import type { Embedder } from './embedder.js';
+
+export function createLocalEmbedder(modelName: string = 'nomic-embed-text-v1.5'): Embedder {
+  let pipeline: any;
+  // all-MiniLM-L6-v2: 384, nomic-embed-text: 768
+  let dims = modelName.includes('MiniLM') ? 384 : 768;
+
+  return {
+    async initialize() {
+      const { pipeline: createPipeline } = await import('@xenova/transformers');
+      pipeline = await createPipeline('feature-extraction', `Xenova/${modelName}`, {
+        quantized: true,
+      });
+    },
+
+    async embed(text: string): Promise<number[]> {
+      const output = await pipeline(text, { pooling: 'mean', normalize: true });
+      return Array.from(output.data as Float32Array).slice(0, dims);
+    },
+
+    async embedBatch(texts: string[]): Promise<number[][]> {
+      const results: number[][] = [];
+      // 순차 처리 (메모리 절약)
+      for (const text of texts) {
+        results.push(await this.embed(text));
+      }
+      return results;
+    },
+
+    get dimensions() { return dims; },
+    get modelName() { return modelName; },
+  };
+}

package/packages/core/src/indexer/scanner.ts

@@ -0,0 +1,142 @@
+// Design Ref: §6 — Indexer (scanner: glob + frontmatter 파싱)
+
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join, relative, extname } from 'node:path';
+import { createHash } from 'node:crypto';
+import matter from 'gray-matter';
+import type { Document } from '../types/document.js';
+
+export interface ScanResult {
+  documents: Document[];
+  scannedFiles: number;
+  skippedFiles: number;
+}
+
+/**
+ * vault 디렉토리에서 모든 .md 파일을 스캔하여 Document 목록 반환
+ */
+export function scanVault(vaultPath: string): ScanResult {
+  const documents: Document[] = [];
+  let skippedFiles = 0;
+
+  const mdFiles = findMdFiles(vaultPath);
+
+  for (const filePath of mdFiles) {
+    try {
+      const doc = parseDocument(vaultPath, filePath);
+      documents.push(doc);
+    } catch {
+      skippedFiles++;
+    }
+  }
+
+  return { documents, scannedFiles: mdFiles.length, skippedFiles };
+}
+
+function findMdFiles(dir: string, files: string[] = []): string[] {
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (entry.name.startsWith('.') || entry.name === 'node_modules' || entry.name === 'zh-CN') continue;
+      findMdFiles(fullPath, files);
+    } else if (extname(entry.name) === '.md') {
+      files.push(fullPath);
+    }
+  }
+  return files;
+}
+
+function parseDocument(vaultPath: string, filePath: string): Document {
+  const raw = readFileSync(filePath, 'utf-8');
+  const stat = statSync(filePath);
+  const { data: frontmatter, content } = matter(raw);
+
+  const relativePath = relative(vaultPath, filePath).replace(/\\/g, '/');
+  const id = createHash('sha256').update(relativePath).digest('hex').slice(0, 16);
+  const contentHash = createHash('sha256').update(raw).digest('hex').slice(0, 16);
+
+  const title = (frontmatter.title as string)
+    ?? extractFirstHeading(content)
+    ?? relativePath.replace(/\.md$/, '');
+
+  const tags = extractTags(frontmatter, content);
+
+  // source/type 자동 추출 (원본 파일 수정 없이 DB에만 저장)
+  const source = inferSource(frontmatter, relativePath);
+  const type = inferType(frontmatter, relativePath);
+
+  return {
+    id,
+    filePath: relativePath,
+    title,
+    content,
+    frontmatter,
+    tags,
+    lastModified: stat.mtime.toISOString(),
+    contentHash,
+    source,
+    type,
+  };
+}
+
+function inferSource(frontmatter: Record<string, unknown>, filePath: string): string {
+  // frontmatter에 명시된 경우
+  if (frontmatter.source && typeof frontmatter.source === 'string') {
+    if (frontmatter.source.startsWith('http')) return 'clip';
+    return frontmatter.source;
+  }
+  // 경로 기반 추론
+  if (filePath.includes('clips/') || filePath.includes('clip/')) return 'clip';
+  if (filePath.includes('PDCA') || filePath.includes('pdca')) return 'local';
+  if (frontmatter['x-i18n']) return 'notion'; // Notion 번역 문서
+  if (frontmatter.clipped) return 'clip';
+  return 'local';
+}
+
+function inferType(frontmatter: Record<string, unknown>, filePath: string): string {
+  // frontmatter에 명시된 경우
+  if (frontmatter.type && typeof frontmatter.type === 'string') return frontmatter.type;
+  // tags 기반
+  const tags = Array.isArray(frontmatter.tags) ? frontmatter.tags : [];
+  if (tags.includes('bridge') || tags.includes('auto-generated')) return 'bridge';
+  if (tags.includes('clip') || tags.includes('youtube')) return 'clip';
+  if (tags.includes('decision')) return 'decision';
+  // 경로 기반
+  if (filePath.includes('clips/')) return 'clip';
+  if (filePath.includes('Decisions/') || filePath.includes('decisions/')) return 'decision';
+  if (filePath.includes('Sessions/') || filePath.includes('sessions/')) return 'session';
+  if (filePath.includes('Research/')) return 'research';
+  if (filePath.includes('Lessons/')) return 'lesson';
+  if (filePath.includes('Templates/')) return 'template';
+  return 'note';
+}
+
+function extractFirstHeading(content: string): string | null {
+  const match = content.match(/^#\s+(.+)$/m);
+  return match ? match[1].trim() : null;
+}
+
+function extractTags(frontmatter: Record<string, unknown>, content: string): string[] {
+  const tags = new Set<string>();
+
+  // frontmatter tags
+  const fmTags = frontmatter.tags;
+  if (Array.isArray(fmTags)) {
+    fmTags.forEach(t => tags.add(String(t)));
+  } else if (typeof fmTags === 'string') {
+    fmTags.split(',').map(t => t.trim()).filter(Boolean).forEach(t => tags.add(t));
+  }
+
+  // inline #tags (CSS 컬러코드, 순수 숫자, heading # 제외)
+  const inlineTags = content.match(/(?:^|\s)#([a-zA-Z가-힣][a-zA-Z가-힣\w-]*)/g);
+  if (inlineTags) {
+    for (const raw of inlineTags) {
+      const tag = raw.trim().slice(1);
+      // CSS hex 컬러 (#fff, #6c5ce7 등) 제외
+      if (/^[0-9a-fA-F]{3,8}$/.test(tag)) continue;
+      tags.add(tag);
+    }
+  }
+
+  return [...tags];
+}

package/packages/core/src/indexer/watcher.ts

@@ -0,0 +1,62 @@
+// Design Ref: §6.3 — 파일 감시 + 증분 인덱싱 (debounce 5s)
+
+import { watch, type FSWatcher } from 'chokidar';
+import { extname } from 'node:path';
+import type { Embedder } from './embedder.js';
+import type { VectorStore } from '../store/types.js';
+import { indexVault } from './index.js';
+import type { ChunkOptions } from './chunker.js';
+
+export interface WatcherOptions {
+  vaultPath: string;
+  store: VectorStore;
+  embedder: Embedder;
+  chunkOptions?: Partial<ChunkOptions>;
+  debounceMs?: number;
+  onReindex?: (result: { indexed: number; skipped: number }) => void;
+}
+
+export function createWatcher(options: WatcherOptions): { start(): void; stop(): void } {
+  const { vaultPath, store, embedder, chunkOptions, debounceMs = 5000, onReindex } = options;
+  let watcher: FSWatcher | null = null;
+  let debounceTimer: ReturnType<typeof setTimeout> | null = null;
+  let reindexing = false;
+
+  async function triggerReindex() {
+    if (reindexing) return;
+    reindexing = true;
+    try {
+      const result = await indexVault(vaultPath, { store, embedder, chunkOptions });
+      onReindex?.({ indexed: result.indexed, skipped: result.skipped });
+    } finally {
+      reindexing = false;
+    }
+  }
+
+  function scheduleReindex() {
+    if (debounceTimer) clearTimeout(debounceTimer);
+    debounceTimer = setTimeout(() => triggerReindex(), debounceMs);
+  }
+
+  return {
+    start() {
+      watcher = watch(vaultPath, {
+        ignored: /(^|[\/\\])\.|node_modules/,
+        persistent: true,
+        ignoreInitial: true,
+      });
+
+      watcher.on('all', (event, path) => {
+        if (extname(path) !== '.md') return;
+        if (['add', 'change', 'unlink'].includes(event)) {
+          scheduleReindex();
+        }
+      });
+    },
+
+    stop() {
+      if (debounceTimer) clearTimeout(debounceTimer);
+      watcher?.close();
+    },
+  };
+}

package/packages/core/src/intelligence/contradiction-detector.ts

@@ -0,0 +1,134 @@
+// Contradiction Detector (F-A12)
+// Finds potentially contradicting statements across notes
+// Uses embedding similarity + negation pattern detection
+
+import type { VectorStore } from '../store/types.js';
+
+export interface ContradictionPair {
+  docA: { id: string; title: string; filePath: string; statement: string };
+  docB: { id: string; title: string; filePath: string; statement: string };
+  similarity: number;
+  confidence: number; // 0-1, how likely this is a real contradiction
+  type: 'negation' | 'value_conflict' | 'temporal' | 'semantic';
+}
+
+// Negation/opposition patterns
+const NEGATION_PAIRS = [
+  ['should', 'should not'], ['must', 'must not'], ['always', 'never'],
+  ['best', 'worst'], ['good', 'bad'], ['correct', 'incorrect'],
+  ['true', 'false'], ['increase', 'decrease'], ['enable', 'disable'],
+  ['recommended', 'not recommended'], ['use', 'avoid'],
+  ['prefer', 'avoid'], ['do', "don't"], ['is', "isn't"],
+  ['can', "can't"], ['will', "won't"], ['important', 'unimportant'],
+  ['필요', '불필요'], ['해야', '하면 안'], ['좋', '나쁜'], ['맞', '틀'],
+];
+
+function extractKeyStatements(content: string): string[] {
+  return content
+    .split(/[.\n!?]/)
+    .map(s => s.trim())
+    .filter(s => s.length > 20 && s.length < 200)
+    .filter(s => {
+      const lower = s.toLowerCase();
+      return /should|must|always|never|best|important|recommend|prefer|avoid|필요|해야|좋|나쁜/.test(lower);
+    })
+    .slice(0, 10); // max 10 statements per document
+}
+
+function detectNegationConflict(stmtA: string, stmtB: string): { isConflict: boolean; confidence: number; type: ContradictionPair['type'] } {
+  const a = stmtA.toLowerCase();
+  const b = stmtB.toLowerCase();
+
+  // Check negation pairs
+  for (const [pos, neg] of NEGATION_PAIRS) {
+    if ((a.includes(pos) && b.includes(neg)) || (a.includes(neg) && b.includes(pos))) {
+      // Check if they're talking about the same subject (share words)
+      const wordsA = new Set(a.split(/\s+/).filter(w => w.length > 3));
+      const wordsB = new Set(b.split(/\s+/).filter(w => w.length > 3));
+      const overlap = [...wordsA].filter(w => wordsB.has(w)).length;
+      const minSize = Math.min(wordsA.size, wordsB.size) || 1;
+      const subjectOverlap = overlap / minSize;
+
+      if (subjectOverlap > 0.2) {
+        return { isConflict: true, confidence: Math.min(0.5 + subjectOverlap * 0.5, 0.95), type: 'negation' };
+      }
+    }
+  }
+
+  // Check numeric value conflicts (e.g., "timeout should be 30s" vs "timeout should be 5s")
+  const numsA = a.match(/\d+/g);
+  const numsB = b.match(/\d+/g);
+  if (numsA && numsB) {
+    const wordsA = new Set(a.replace(/\d+/g, '').split(/\s+/).filter(w => w.length > 3));
+    const wordsB = new Set(b.replace(/\d+/g, '').split(/\s+/).filter(w => w.length > 3));
+    const overlap = [...wordsA].filter(w => wordsB.has(w)).length;
+    if (overlap >= 2 && numsA[0] !== numsB[0]) {
+      return { isConflict: true, confidence: 0.6, type: 'value_conflict' };
+    }
+  }
+
+  return { isConflict: false, confidence: 0, type: 'semantic' };
+}
+
+export async function detectContradictions(
+  store: VectorStore,
+  limit = 20,
+): Promise<ContradictionPair[]> {
+  const docs = await store.getAllDocuments();
+  const embeddings = await store.getDocumentEmbeddings();
+
+  if (docs.length < 2) return [];
+
+  // Build document vectors + key statements
+  const docData = new Map<string, { vec: number[]; title: string; filePath: string; statements: string[] }>();
+  for (const doc of docs) {
+    const vec = embeddings.get(doc.id);
+    if (!vec) continue;
+    const statements = extractKeyStatements(doc.content);
+    if (statements.length === 0) continue;
+    docData.set(doc.id, { vec: Array.from(vec), title: doc.title, filePath: doc.filePath, statements });
+  }
+
+  const ids = [...docData.keys()];
+  const results: ContradictionPair[] = [];
+
+  // Compare documents with moderate similarity (same topic, possibly conflicting)
+  for (let i = 0; i < ids.length && results.length < limit * 3; i++) {
+    for (let j = i + 1; j < ids.length && results.length < limit * 3; j++) {
+      const a = docData.get(ids[i])!;
+      const b = docData.get(ids[j])!;
+      const sim = cosineSim(a.vec, b.vec);
+
+      // Sweet spot: similar enough to be same topic, not identical
+      if (sim < 0.3 || sim > 0.9) continue;
+
+      // Compare statements
+      for (const stmtA of a.statements) {
+        for (const stmtB of b.statements) {
+          const { isConflict, confidence, type } = detectNegationConflict(stmtA, stmtB);
+          if (isConflict && confidence >= 0.5) {
+            results.push({
+              docA: { id: ids[i], title: a.title, filePath: a.filePath, statement: stmtA },
+              docB: { id: ids[j], title: b.title, filePath: b.filePath, statement: stmtB },
+              similarity: Math.round(sim * 1000) / 1000,
+              confidence: Math.round(confidence * 100) / 100,
+              type,
+            });
+          }
+        }
+      }
+    }
+  }
+
+  return results
+    .sort((a, b) => b.confidence - a.confidence)
+    .slice(0, limit);
+}
+
+function cosineSim(a: number[], b: number[]): number {
+  if (a.length !== b.length) return 0;
+  let dot = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
+  const d = Math.sqrt(na) * Math.sqrt(nb);
+  return d === 0 ? 0 : dot / d;
+}