stellavault 0.2.0 → 0.2.1

package/packages/core/src/indexer/index.ts

@@ -1,113 +0,0 @@
-// Design Ref: §6.3 — Incremental Indexing + Pipeline
-
-import type { Embedder } from './embedder.js';
-import type { VectorStore } from '../store/types.js';
-import type { Document } from '../types/document.js';
-import type { Chunk } from '../types/chunk.js';
-import { scanVault } from './scanner.js';
-import { chunkDocument, type ChunkOptions } from './chunker.js';
-import { withRetry, errors } from '../utils/retry.js';
-
-export { type Embedder } from './embedder.js';
-export { createLocalEmbedder } from './local-embedder.js';
-export { scanVault } from './scanner.js';
-export { chunkDocument, estimateTokens } from './chunker.js';
-export { createWatcher } from './watcher.js';
-
-export interface IndexerOptions {
-  store: VectorStore;
-  embedder: Embedder;
-  chunkOptions?: Partial<ChunkOptions>;
-  onProgress?: (current: number, total: number, doc: Document) => void;
-}
-
-export interface IndexResult {
-  indexed: number;
-  skipped: number;
-  deleted: number;
-  failed: number;
-  totalChunks: number;
-  elapsedMs: number;
-}
-
-/**
- * vault를 스캔하여 변경된 문서만 벡터화하는 증분 인덱서
- */
-export async function indexVault(
-  vaultPath: string,
-  options: IndexerOptions,
-): Promise<IndexResult> {
-  const start = Date.now();
-  const { store, embedder, chunkOptions, onProgress } = options;
-
-  // 1. 스캔
-  const { documents } = scanVault(vaultPath);
-
-  // 2. 기존 인덱스 상태 조회
-  const existingDocs = await store.getAllDocuments();
-  const existingMap = new Map(existingDocs.map(d => [d.id, d.contentHash]));
-
-  let indexed = 0;
-  let skipped = 0;
-  let failed = 0;
-  let totalChunks = 0;
-
-  // 3. 증분 처리 (에러 복구 포함)
-  const scannedIds = new Set<string>();
-  for (let i = 0; i < documents.length; i++) {
-    const doc = documents[i];
-    scannedIds.add(doc.id);
-    onProgress?.(i + 1, documents.length, doc);
-
-    // content_hash 비교 → 변경 없으면 SKIP
-    if (existingMap.get(doc.id) === doc.contentHash) {
-      skipped++;
-      continue;
-    }
-
-    try {
-      // 청킹
-      const chunks = chunkDocument(doc.id, doc.content, chunkOptions);
-
-      // 임베딩 (retry with backoff)
-      const texts = chunks.map(c => c.content);
-      const embeddings = await withRetry(
-        () => embedder.embedBatch(texts),
-        { maxRetries: 2, baseDelayMs: 1000 },
-      );
-      const chunksWithEmbeddings: Chunk[] = chunks.map((c, j) => ({
-        ...c,
-        embedding: embeddings[j],
-      }));
-
-      // 저장 (document → chunks)
-      await store.upsertDocument(doc);
-      await store.upsertChunks(chunksWithEmbeddings);
-
-      indexed++;
-      totalChunks += chunks.length;
-    } catch (err) {
-      // Graceful degradation: skip failed file, continue with rest
-      failed++;
-      console.error(errors.indexingFailed(doc.filePath, err).format());
-    }
-  }
-
-  // 4. 삭제된 파일 처리
-  let deleted = 0;
-  for (const [existingId] of existingMap) {
-    if (!scannedIds.has(existingId)) {
-      await store.deleteByDocumentId(existingId);
-      deleted++;
-    }
-  }
-
-  return {
-    indexed,
-    skipped,
-    deleted,
-    failed,
-    totalChunks,
-    elapsedMs: Date.now() - start,
-  };
-}

package/packages/core/src/indexer/local-embedder.ts

@@ -1,35 +0,0 @@
-// Design Ref: §3.2 — Embedder 로컬 구현 (nomic-embed-text via @xenova/transformers)
-
-import type { Embedder } from './embedder.js';
-
-export function createLocalEmbedder(modelName: string = 'nomic-embed-text-v1.5'): Embedder {
-  let pipeline: any;
-  // all-MiniLM-L6-v2: 384, nomic-embed-text: 768
-  let dims = modelName.includes('MiniLM') ? 384 : 768;
-
-  return {
-    async initialize() {
-      const { pipeline: createPipeline } = await import('@xenova/transformers');
-      pipeline = await createPipeline('feature-extraction', `Xenova/${modelName}`, {
-        quantized: true,
-      });
-    },
-
-    async embed(text: string): Promise<number[]> {
-      const output = await pipeline(text, { pooling: 'mean', normalize: true });
-      return Array.from(output.data as Float32Array).slice(0, dims);
-    },
-
-    async embedBatch(texts: string[]): Promise<number[][]> {
-      const results: number[][] = [];
-      // 순차 처리 (메모리 절약)
-      for (const text of texts) {
-        results.push(await this.embed(text));
-      }
-      return results;
-    },
-
-    get dimensions() { return dims; },
-    get modelName() { return modelName; },
-  };
-}

package/packages/core/src/indexer/scanner.ts

@@ -1,142 +0,0 @@
-// Design Ref: §6 — Indexer (scanner: glob + frontmatter 파싱)
-
-import { readdirSync, readFileSync, statSync } from 'node:fs';
-import { join, relative, extname } from 'node:path';
-import { createHash } from 'node:crypto';
-import matter from 'gray-matter';
-import type { Document } from '../types/document.js';
-
-export interface ScanResult {
-  documents: Document[];
-  scannedFiles: number;
-  skippedFiles: number;
-}
-
-/**
- * vault 디렉토리에서 모든 .md 파일을 스캔하여 Document 목록 반환
- */
-export function scanVault(vaultPath: string): ScanResult {
-  const documents: Document[] = [];
-  let skippedFiles = 0;
-
-  const mdFiles = findMdFiles(vaultPath);
-
-  for (const filePath of mdFiles) {
-    try {
-      const doc = parseDocument(vaultPath, filePath);
-      documents.push(doc);
-    } catch {
-      skippedFiles++;
-    }
-  }
-
-  return { documents, scannedFiles: mdFiles.length, skippedFiles };
-}
-
-function findMdFiles(dir: string, files: string[] = []): string[] {
-  for (const entry of readdirSync(dir, { withFileTypes: true })) {
-    const fullPath = join(dir, entry.name);
-    if (entry.isDirectory()) {
-      if (entry.name.startsWith('.') || entry.name === 'node_modules' || entry.name === 'zh-CN') continue;
-      findMdFiles(fullPath, files);
-    } else if (extname(entry.name) === '.md') {
-      files.push(fullPath);
-    }
-  }
-  return files;
-}
-
-function parseDocument(vaultPath: string, filePath: string): Document {
-  const raw = readFileSync(filePath, 'utf-8');
-  const stat = statSync(filePath);
-  const { data: frontmatter, content } = matter(raw);
-
-  const relativePath = relative(vaultPath, filePath).replace(/\\/g, '/');
-  const id = createHash('sha256').update(relativePath).digest('hex').slice(0, 16);
-  const contentHash = createHash('sha256').update(raw).digest('hex').slice(0, 16);
-
-  const title = (frontmatter.title as string)
-    ?? extractFirstHeading(content)
-    ?? relativePath.replace(/\.md$/, '');
-
-  const tags = extractTags(frontmatter, content);
-
-  // source/type 자동 추출 (원본 파일 수정 없이 DB에만 저장)
-  const source = inferSource(frontmatter, relativePath);
-  const type = inferType(frontmatter, relativePath);
-
-  return {
-    id,
-    filePath: relativePath,
-    title,
-    content,
-    frontmatter,
-    tags,
-    lastModified: stat.mtime.toISOString(),
-    contentHash,
-    source,
-    type,
-  };
-}
-
-function inferSource(frontmatter: Record<string, unknown>, filePath: string): string {
-  // frontmatter에 명시된 경우
-  if (frontmatter.source && typeof frontmatter.source === 'string') {
-    if (frontmatter.source.startsWith('http')) return 'clip';
-    return frontmatter.source;
-  }
-  // 경로 기반 추론
-  if (filePath.includes('clips/') || filePath.includes('clip/')) return 'clip';
-  if (filePath.includes('PDCA') || filePath.includes('pdca')) return 'local';
-  if (frontmatter['x-i18n']) return 'notion'; // Notion 번역 문서
-  if (frontmatter.clipped) return 'clip';
-  return 'local';
-}
-
-function inferType(frontmatter: Record<string, unknown>, filePath: string): string {
-  // frontmatter에 명시된 경우
-  if (frontmatter.type && typeof frontmatter.type === 'string') return frontmatter.type;
-  // tags 기반
-  const tags = Array.isArray(frontmatter.tags) ? frontmatter.tags : [];
-  if (tags.includes('bridge') || tags.includes('auto-generated')) return 'bridge';
-  if (tags.includes('clip') || tags.includes('youtube')) return 'clip';
-  if (tags.includes('decision')) return 'decision';
-  // 경로 기반
-  if (filePath.includes('clips/')) return 'clip';
-  if (filePath.includes('Decisions/') || filePath.includes('decisions/')) return 'decision';
-  if (filePath.includes('Sessions/') || filePath.includes('sessions/')) return 'session';
-  if (filePath.includes('Research/')) return 'research';
-  if (filePath.includes('Lessons/')) return 'lesson';
-  if (filePath.includes('Templates/')) return 'template';
-  return 'note';
-}
-
-function extractFirstHeading(content: string): string | null {
-  const match = content.match(/^#\s+(.+)$/m);
-  return match ? match[1].trim() : null;
-}
-
-function extractTags(frontmatter: Record<string, unknown>, content: string): string[] {
-  const tags = new Set<string>();
-
-  // frontmatter tags
-  const fmTags = frontmatter.tags;
-  if (Array.isArray(fmTags)) {
-    fmTags.forEach(t => tags.add(String(t)));
-  } else if (typeof fmTags === 'string') {
-    fmTags.split(',').map(t => t.trim()).filter(Boolean).forEach(t => tags.add(t));
-  }
-
-  // inline #tags (CSS 컬러코드, 순수 숫자, heading # 제외)
-  const inlineTags = content.match(/(?:^|\s)#([a-zA-Z가-힣][a-zA-Z가-힣\w-]*)/g);
-  if (inlineTags) {
-    for (const raw of inlineTags) {
-      const tag = raw.trim().slice(1);
-      // CSS hex 컬러 (#fff, #6c5ce7 등) 제외
-      if (/^[0-9a-fA-F]{3,8}$/.test(tag)) continue;
-      tags.add(tag);
-    }
-  }
-
-  return [...tags];
-}

package/packages/core/src/indexer/watcher.ts

@@ -1,62 +0,0 @@
-// Design Ref: §6.3 — 파일 감시 + 증분 인덱싱 (debounce 5s)
-
-import { watch, type FSWatcher } from 'chokidar';
-import { extname } from 'node:path';
-import type { Embedder } from './embedder.js';
-import type { VectorStore } from '../store/types.js';
-import { indexVault } from './index.js';
-import type { ChunkOptions } from './chunker.js';
-
-export interface WatcherOptions {
-  vaultPath: string;
-  store: VectorStore;
-  embedder: Embedder;
-  chunkOptions?: Partial<ChunkOptions>;
-  debounceMs?: number;
-  onReindex?: (result: { indexed: number; skipped: number }) => void;
-}
-
-export function createWatcher(options: WatcherOptions): { start(): void; stop(): void } {
-  const { vaultPath, store, embedder, chunkOptions, debounceMs = 5000, onReindex } = options;
-  let watcher: FSWatcher | null = null;
-  let debounceTimer: ReturnType<typeof setTimeout> | null = null;
-  let reindexing = false;
-
-  async function triggerReindex() {
-    if (reindexing) return;
-    reindexing = true;
-    try {
-      const result = await indexVault(vaultPath, { store, embedder, chunkOptions });
-      onReindex?.({ indexed: result.indexed, skipped: result.skipped });
-    } finally {
-      reindexing = false;
-    }
-  }
-
-  function scheduleReindex() {
-    if (debounceTimer) clearTimeout(debounceTimer);
-    debounceTimer = setTimeout(() => triggerReindex(), debounceMs);
-  }
-
-  return {
-    start() {
-      watcher = watch(vaultPath, {
-        ignored: /(^|[\/\\])\.|node_modules/,
-        persistent: true,
-        ignoreInitial: true,
-      });
-
-      watcher.on('all', (event, path) => {
-        if (extname(path) !== '.md') return;
-        if (['add', 'change', 'unlink'].includes(event)) {
-          scheduleReindex();
-        }
-      });
-    },
-
-    stop() {
-      if (debounceTimer) clearTimeout(debounceTimer);
-      watcher?.close();
-    },
-  };
-}

package/packages/core/src/intelligence/contradiction-detector.ts

@@ -1,134 +0,0 @@
-// Contradiction Detector (F-A12)
-// Finds potentially contradicting statements across notes
-// Uses embedding similarity + negation pattern detection
-
-import type { VectorStore } from '../store/types.js';
-
-export interface ContradictionPair {
-  docA: { id: string; title: string; filePath: string; statement: string };
-  docB: { id: string; title: string; filePath: string; statement: string };
-  similarity: number;
-  confidence: number; // 0-1, how likely this is a real contradiction
-  type: 'negation' | 'value_conflict' | 'temporal' | 'semantic';
-}
-
-// Negation/opposition patterns
-const NEGATION_PAIRS = [
-  ['should', 'should not'], ['must', 'must not'], ['always', 'never'],
-  ['best', 'worst'], ['good', 'bad'], ['correct', 'incorrect'],
-  ['true', 'false'], ['increase', 'decrease'], ['enable', 'disable'],
-  ['recommended', 'not recommended'], ['use', 'avoid'],
-  ['prefer', 'avoid'], ['do', "don't"], ['is', "isn't"],
-  ['can', "can't"], ['will', "won't"], ['important', 'unimportant'],
-  ['필요', '불필요'], ['해야', '하면 안'], ['좋', '나쁜'], ['맞', '틀'],
-];
-
-function extractKeyStatements(content: string): string[] {
-  return content
-    .split(/[.\n!?]/)
-    .map(s => s.trim())
-    .filter(s => s.length > 20 && s.length < 200)
-    .filter(s => {
-      const lower = s.toLowerCase();
-      return /should|must|always|never|best|important|recommend|prefer|avoid|필요|해야|좋|나쁜/.test(lower);
-    })
-    .slice(0, 10); // max 10 statements per document
-}
-
-function detectNegationConflict(stmtA: string, stmtB: string): { isConflict: boolean; confidence: number; type: ContradictionPair['type'] } {
-  const a = stmtA.toLowerCase();
-  const b = stmtB.toLowerCase();
-
-  // Check negation pairs
-  for (const [pos, neg] of NEGATION_PAIRS) {
-    if ((a.includes(pos) && b.includes(neg)) || (a.includes(neg) && b.includes(pos))) {
-      // Check if they're talking about the same subject (share words)
-      const wordsA = new Set(a.split(/\s+/).filter(w => w.length > 3));
-      const wordsB = new Set(b.split(/\s+/).filter(w => w.length > 3));
-      const overlap = [...wordsA].filter(w => wordsB.has(w)).length;
-      const minSize = Math.min(wordsA.size, wordsB.size) || 1;
-      const subjectOverlap = overlap / minSize;
-
-      if (subjectOverlap > 0.2) {
-        return { isConflict: true, confidence: Math.min(0.5 + subjectOverlap * 0.5, 0.95), type: 'negation' };
-      }
-    }
-  }
-
-  // Check numeric value conflicts (e.g., "timeout should be 30s" vs "timeout should be 5s")
-  const numsA = a.match(/\d+/g);
-  const numsB = b.match(/\d+/g);
-  if (numsA && numsB) {
-    const wordsA = new Set(a.replace(/\d+/g, '').split(/\s+/).filter(w => w.length > 3));
-    const wordsB = new Set(b.replace(/\d+/g, '').split(/\s+/).filter(w => w.length > 3));
-    const overlap = [...wordsA].filter(w => wordsB.has(w)).length;
-    if (overlap >= 2 && numsA[0] !== numsB[0]) {
-      return { isConflict: true, confidence: 0.6, type: 'value_conflict' };
-    }
-  }
-
-  return { isConflict: false, confidence: 0, type: 'semantic' };
-}
-
-export async function detectContradictions(
-  store: VectorStore,
-  limit = 20,
-): Promise<ContradictionPair[]> {
-  const docs = await store.getAllDocuments();
-  const embeddings = await store.getDocumentEmbeddings();
-
-  if (docs.length < 2) return [];
-
-  // Build document vectors + key statements
-  const docData = new Map<string, { vec: number[]; title: string; filePath: string; statements: string[] }>();
-  for (const doc of docs) {
-    const vec = embeddings.get(doc.id);
-    if (!vec) continue;
-    const statements = extractKeyStatements(doc.content);
-    if (statements.length === 0) continue;
-    docData.set(doc.id, { vec: Array.from(vec), title: doc.title, filePath: doc.filePath, statements });
-  }
-
-  const ids = [...docData.keys()];
-  const results: ContradictionPair[] = [];
-
-  // Compare documents with moderate similarity (same topic, possibly conflicting)
-  for (let i = 0; i < ids.length && results.length < limit * 3; i++) {
-    for (let j = i + 1; j < ids.length && results.length < limit * 3; j++) {
-      const a = docData.get(ids[i])!;
-      const b = docData.get(ids[j])!;
-      const sim = cosineSim(a.vec, b.vec);
-
-      // Sweet spot: similar enough to be same topic, not identical
-      if (sim < 0.3 || sim > 0.9) continue;
-
-      // Compare statements
-      for (const stmtA of a.statements) {
-        for (const stmtB of b.statements) {
-          const { isConflict, confidence, type } = detectNegationConflict(stmtA, stmtB);
-          if (isConflict && confidence >= 0.5) {
-            results.push({
-              docA: { id: ids[i], title: a.title, filePath: a.filePath, statement: stmtA },
-              docB: { id: ids[j], title: b.title, filePath: b.filePath, statement: stmtB },
-              similarity: Math.round(sim * 1000) / 1000,
-              confidence: Math.round(confidence * 100) / 100,
-              type,
-            });
-          }
-        }
-      }
-    }
-  }
-
-  return results
-    .sort((a, b) => b.confidence - a.confidence)
-    .slice(0, limit);
-}
-
-function cosineSim(a: number[], b: number[]): number {
-  if (a.length !== b.length) return 0;
-  let dot = 0, na = 0, nb = 0;
-  for (let i = 0; i < a.length; i++) { dot += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
-  const d = Math.sqrt(na) * Math.sqrt(nb);
-  return d === 0 ? 0 : dot / d;
-}