sweet-search 0.0.1 → 2.3.0

package/core/embedding/embedding-telemetry.js

@@ -0,0 +1,219 @@
+/**
+ * Embedding Telemetry - Per-mode query telemetry for vocabulary prewarm pipeline.
+ * Extracted from embedding-cache.js for SOLID compliance (single responsibility).
+ */
+
+import fs from 'fs/promises';
+import { existsSync, mkdirSync, appendFileSync } from 'fs';
+import path from 'path';
+import { DB_PATHS } from '../infrastructure/config/index.js';
+
+// =============================================================================
+// PER-MODE QUERY TELEMETRY (Step 0 of Vocabulary Prewarm)
+// =============================================================================
+
+const TELEMETRY_PATH = path.join(path.dirname(DB_PATHS.vocabulary), 'query-telemetry.jsonl');
+const TELEMETRY_MAX_LINES = 10_000;
+const FLUSH_THRESHOLD = 50;
+
+export const telemetryStats = {
+  lexical:  { hits: 0, misses: 0, totalLatencyMs: 0, count: 0 },
+  semantic: { hits: 0, misses: 0, totalLatencyMs: 0, count: 0 },
+  hybrid:   { hits: 0, misses: 0, totalLatencyMs: 0, count: 0 },
+};
+
+// P1.2: Buffer entries in memory, flush periodically instead of per-query I/O
+const _telemetryBuffer = [];
+let _lineCountEstimate = -1; // -1 = unknown, needs initial read
+let _flushPromise = null;
+
+// Periodic flush interval (30s) and process exit hook to prevent tail loss.
+// The interval uses unref() so it doesn't keep the process alive.
+let _flushInterval = null;
+
+function _ensureShutdownHooks() {
+  if (_flushInterval) return; // already registered
+  _flushInterval = setInterval(() => {
+    if (_telemetryBuffer.length > 0) flushTelemetry().catch(() => {});
+  }, 30_000);
+  _flushInterval.unref(); // don't keep process alive for telemetry
+
+  // Synchronous best-effort flush on exit — beforeExit fires for empty event
+  // loop (normal exit), but NOT for SIGTERM/uncaughtException. We use both
+  // beforeExit (async-safe) and exit (sync-only, last resort).
+  process.on('beforeExit', _beforeExitFlush);
+  process.on('exit', _syncFlush);
+}
+
+async function _beforeExitFlush() {
+  try { await flushTelemetry(); } catch {}
+}
+
+function _syncFlush() {
+  // exit handler is synchronous — write what we can via writeFileSync
+  if (_telemetryBuffer.length === 0) return;
+  try {
+    mkdirSync(path.dirname(TELEMETRY_PATH), { recursive: true });
+    appendFileSync(TELEMETRY_PATH, _telemetryBuffer.splice(0).join('\n') + '\n');
+  } catch {}
+}
+
+/**
+ * Record a single query telemetry event.
+ * Updates in-memory stats and buffers the entry. Flushes to disk
+ * every FLUSH_THRESHOLD entries (default 50) to avoid per-query file I/O.
+ *
+ * @param {'lexical'|'semantic'|'hybrid'} mode - Search mode used
+ * @param {boolean} hit - Whether the cache was hit
+ * @param {number} latencyMs - Query latency in milliseconds
+ * @param {string} [query] - The query text (for promotion/demotion analysis)
+ * @param {string} [source] - Embedding source ('vocabulary', 'semantic-cache', 'api', 'local')
+ * @param {boolean} [lexicalHit] - For hybrid mode: whether lexical sub-path was a hit
+ * @param {boolean} [semanticHit] - For hybrid mode: whether semantic sub-path was a hit
+ */
+export async function recordQueryTelemetry(mode, hit, latencyMs, query, source, lexicalHit, semanticHit) {
+  const bucket = telemetryStats[mode];
+  if (!bucket) return;
+
+  bucket.count++;
+  bucket.totalLatencyMs += latencyMs;
+  if (hit) bucket.hits++; else bucket.misses++;
+
+  _telemetryBuffer.push(JSON.stringify({
+    mode,
+    hit,
+    latencyMs: Math.round(latencyMs * 100) / 100,
+    timestamp: new Date().toISOString(),
+    ...(query ? { query } : {}),
+    ...(source ? { source } : {}),
+    ...(lexicalHit != null ? { lexicalHit } : {}),
+    ...(semanticHit != null ? { semanticHit } : {}),
+  }));
+
+  _ensureShutdownHooks();
+
+  if (_telemetryBuffer.length >= FLUSH_THRESHOLD) {
+    await flushTelemetry();
+  }
+}
+
+/**
+ * Flush buffered telemetry entries to disk.
+ * Safe to call multiple times; coalesces concurrent flushes.
+ */
+export async function flushTelemetry() {
+  if (_telemetryBuffer.length === 0) return;
+  // Coalesce concurrent flush calls
+  if (_flushPromise) return _flushPromise;
+  _flushPromise = _flushTelemetryInner();
+  try { await _flushPromise; } finally { _flushPromise = null; }
+}
+
+async function _flushTelemetryInner() {
+  const batch = _telemetryBuffer.splice(0, _telemetryBuffer.length);
+  if (batch.length === 0) return;
+
+  try {
+    await fs.mkdir(path.dirname(TELEMETRY_PATH), { recursive: true });
+
+    // Lazy-init line count estimate (only once, not per query)
+    if (_lineCountEstimate < 0) {
+      if (existsSync(TELEMETRY_PATH)) {
+        const content = await fs.readFile(TELEMETRY_PATH, 'utf-8');
+        _lineCountEstimate = content.split('\n').filter(Boolean).length;
+      } else {
+        _lineCountEstimate = 0;
+      }
+    }
+
+    // Rotate if over limit
+    if (_lineCountEstimate + batch.length >= TELEMETRY_MAX_LINES) {
+      if (existsSync(TELEMETRY_PATH)) {
+        await fs.rename(TELEMETRY_PATH, TELEMETRY_PATH + '.bak');
+      }
+      _lineCountEstimate = 0;
+    }
+
+    await fs.appendFile(TELEMETRY_PATH, batch.join('\n') + '\n');
+    _lineCountEstimate += batch.length;
+  } catch (err) {
+    if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
+  }
+}
+
+/**
+ * Read the last N telemetry entries and compute per-mode statistics.
+ *
+ * @param {number} [lastN=100] - Number of recent entries to analyze
+ * @returns {Promise<{modes: Record<string, {hits: number, misses: number, hitRate: string, avgLatencyMs: number, count: number}>, total: number}>}
+ */
+export async function getTelemetryReport(lastN = 100) {
+  const modes = {
+    lexical:  { hits: 0, misses: 0, totalLatencyMs: 0, count: 0 },
+    semantic: { hits: 0, misses: 0, totalLatencyMs: 0, count: 0 },
+    hybrid:   { hits: 0, misses: 0, totalLatencyMs: 0, count: 0 },
+  };
+
+  try {
+    if (!existsSync(TELEMETRY_PATH)) return { modes: _formatModes(modes), total: 0 };
+
+    const content = await fs.readFile(TELEMETRY_PATH, 'utf-8');
+    const lines = content.split('\n').filter(Boolean);
+    const recent = lines.slice(-lastN);
+
+    for (const line of recent) {
+      try {
+        const entry = JSON.parse(line);
+        const bucket = modes[entry.mode];
+        if (!bucket) continue;
+        bucket.count++;
+        bucket.totalLatencyMs += entry.latencyMs || 0;
+        if (entry.hit) bucket.hits++; else bucket.misses++;
+      } catch (err) {
+        if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
+      }
+    }
+
+    return { modes: _formatModes(modes), total: recent.length };
+  } catch (err) {
+    if (process.env.DEBUG_CATCHES) process.stderr.write(`[non-fatal] ${err?.message || err}\n`);
+    return { modes: _formatModes(modes), total: 0 };
+  }
+}
+
+function _formatModes(modes) {
+  const result = {};
+  for (const [mode, stats] of Object.entries(modes)) {
+    const total = stats.hits + stats.misses;
+    result[mode] = {
+      hits: stats.hits,
+      misses: stats.misses,
+      hitRate: total > 0 ? `${(stats.hits / total * 100).toFixed(1)}%` : '0.0%',
+      avgLatencyMs: stats.count > 0 ? Math.round(stats.totalLatencyMs / stats.count * 100) / 100 : 0,
+      count: stats.count,
+    };
+  }
+  return result;
+}
+
+/**
+ * Reset in-memory telemetry stats and buffer (for testing).
+ * Also tears down shutdown hooks to prevent listener leaks in test suites.
+ */
+export function resetTelemetryStats() {
+  for (const bucket of Object.values(telemetryStats)) {
+    bucket.hits = 0;
+    bucket.misses = 0;
+    bucket.totalLatencyMs = 0;
+    bucket.count = 0;
+  }
+  _telemetryBuffer.length = 0;
+  _lineCountEstimate = -1;
+  // Tear down hooks to prevent listener leaks during tests
+  if (_flushInterval) {
+    clearInterval(_flushInterval);
+    _flushInterval = null;
+  }
+  process.removeListener('beforeExit', _beforeExitFlush);
+  process.removeListener('exit', _syncFlush);
+}

package/core/embedding/index.js

@@ -0,0 +1,40 @@
+/**
+ * Embedding Domain - Barrel export.
+ * Re-exports public API from all sub-modules.
+ *
+ * NOTE: expandVocabulary exists in both embedding-service.js and embedding-cache.js
+ * with different signatures. The service version (auto-supplies embedFn) is canonical.
+ * Telemetry symbols are re-exported by embedding-cache.js from embedding-telemetry.js,
+ * so we do NOT also export * from embedding-telemetry.js (ESM ambiguity).
+ */
+
+// Facade (has default export)
+export { default } from './embedding-service.js';
+export * from './embedding-service.js';
+
+// Remote (circuit breaker, rate limiters, API clients)
+export * from './embedding-remote.js';
+
+// Local model (ONNX inference)
+export * from './embedding-local-model.js';
+
+// Cache (LRU, vocabulary, semantic cache, deduplication)
+// NOTE: embedding-cache.js re-exports telemetry symbols from embedding-telemetry.js
+export * from './embedding-cache.js';
+
+// Resolve ESM ambiguity: expandVocabulary exists in both service and cache.
+// The service version (auto-supplies embedFn) is the public API.
+export { expandVocabulary } from './embedding-service.js';
+
+// CoreML cascade diagnostics — re-exported from the infrastructure
+// layer so consumers inside `core/embedding/` and out-of-domain callers
+// that import the embedding barrel can ask "what inference backend is
+// armed right now?" without reaching into infrastructure directly.
+// These are read-only views; the cascade itself is owned by
+// `core/infrastructure/coreml-cascade.js`.
+export {
+  isCoremlCascadeApplicable,
+  getCoremlCascadeState,
+  getCoremlCascadeReport,
+  getCoremlCascadeResolvedDirs,
+} from '../infrastructure/coreml-cascade.js';

package/core/graph/community-detector.js

@@ -0,0 +1,294 @@
+/**
+ * Community Detector Module
+ *
+ * Discovers logical communities of code entities using the Leiden algorithm
+ * on the entity/relationship graph stored in SQLite (code-graph.db).
+ *
+ * Algorithm core (Traag et al. 2019) lives in leiden-algorithm.js.
+ * This module handles DB I/O, graph construction, and the public API.
+ *
+ * Fallback: directory-based grouping when code-graph.db is absent or too large.
+ */
+
+import Database from 'better-sqlite3';
+import { createHash } from 'crypto';
+import { DB_PATHS } from '../infrastructure/config/index.js';
+import { applyReadPragmas } from '../infrastructure/db-utils.js';
+import { leidenCommunities, findConnectedComponents } from './leiden-algorithm.js';
+
+// Re-export for existing consumers
+export { leidenCommunities } from './leiden-algorithm.js';
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const ENTITY_CUTOFF = 50_000;
+const RELATIONSHIP_CUTOFF = 200_000;
+const DEFAULT_TIMEOUT_MS = 2000;
+
+// Relationship type weights for adjacency construction
+const REL_WEIGHTS = {
+  imports: 3,
+  extends: 3,
+  implements: 3,
+  calls: 1,
+  uses: 1,
+};
+
+// ---------------------------------------------------------------------------
+// Graph Hash
+// ---------------------------------------------------------------------------
+
+/**
+ * Compute sha256 hash of the code graph structure.
+ *
+ * @param {string} [dbPath] - Path to code-graph.db
+ * @returns {string} hex digest
+ */
+export function computeGraphHash(dbPath) {
+  const resolvedPath = dbPath || DB_PATHS.codeGraph;
+  const db = new Database(resolvedPath, { readonly: true, timeout: 5000 });
+  applyReadPragmas(db);
+
+  try {
+    const rows = db.prepare(`
+      SELECT source_id, target_id, type
+      FROM relationships
+      ORDER BY source_id, target_id, type
+    `).all();
+
+    const hash = createHash('sha256');
+    for (const row of rows) {
+      hash.update(`${row.source_id}\t${row.target_id}\t${row.type}\n`);
+    }
+
+    return hash.digest('hex');
+  } finally {
+    db.close();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Directory Fallback
+// ---------------------------------------------------------------------------
+
+/**
+ * Fallback: group entities by top-2 directory path segments.
+ *
+ * @param {Array<{id: number|string, name: string, file_path: string}>} entities
+ * @returns {Array<{id: number, entityIds: Array, fileIds: Array, entityCount: number}>}
+ */
+export function fallbackDirectoryGrouping(entities) {
+  const groups = new Map();
+
+  for (const ent of entities) {
+    const fp = ent.file_path || '';
+    const parts = fp.split('/').filter(Boolean);
+    const dirKey = parts.length >= 2 ? parts.slice(0, 2).join('/') : parts[0] || '(root)';
+
+    if (!groups.has(dirKey)) {
+      groups.set(dirKey, { entityIds: new Set(), fileIds: new Set() });
+    }
+    const g = groups.get(dirKey);
+    g.entityIds.add(ent.id);
+    if (fp) g.fileIds.add(fp);
+  }
+
+  let communityId = 0;
+  const communities = [];
+  for (const [, group] of groups) {
+    communities.push({
+      id: communityId++,
+      entityIds: [...group.entityIds],
+      fileIds: [...group.fileIds],
+      entityCount: group.entityIds.size,
+    });
+  }
+
+  return communities;
+}
+
+// ---------------------------------------------------------------------------
+// Main API
+// ---------------------------------------------------------------------------
+
+/**
+ * Detect communities from code-graph.db relationships.
+ *
+ * @param {string} [dbPath] - Path to code-graph.db
+ * @param {object} [options]
+ * @param {number} [options.resolution=1.0]
+ * @param {number} [options.maxIterations=10]
+ * @param {number} [options.timeoutMs=2000]
+ * @returns {{ communities: Array, graphHash: string, stale: boolean }}
+ */
+export function detectCommunities(dbPath, options = {}) {
+  const resolvedPath = dbPath || DB_PATHS.codeGraph;
+  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+
+  let db;
+  try {
+    db = new Database(resolvedPath, { readonly: true, timeout: 5000 });
+    applyReadPragmas(db);
+  } catch {
+    return { communities: [], graphHash: '', stale: false };
+  }
+
+  try {
+    const entities = db.prepare(`
+      SELECT id, name, type, file_path
+      FROM entities
+      WHERE stale_since IS NULL
+    `).all();
+
+    const relationships = db.prepare(`
+      SELECT source_id, target_id, type
+      FROM relationships
+    `).all();
+
+    // Hard cutoffs: fall back to directory grouping
+    if (entities.length > ENTITY_CUTOFF || relationships.length > RELATIONSHIP_CUTOFF) {
+      const graphHash = computeGraphHashFromRows(relationships);
+      return {
+        communities: fallbackDirectoryGrouping(entities),
+        graphHash,
+        stale: false,
+      };
+    }
+
+    // Build weighted adjacency (undirected)
+    const adjacency = buildWeightedAdjacency(entities, relationships);
+
+    // Run Leiden
+    const startTime = Date.now();
+    const { assignment, converged } = leidenCommunities(adjacency, {
+      resolution: options.resolution,
+      maxIterations: options.maxIterations,
+      timeoutMs,
+    });
+    const elapsed = Date.now() - startTime;
+    const stale = !converged && elapsed >= timeoutMs;
+
+    // Build entity lookup for file_path mapping
+    const entityMap = new Map();
+    for (const ent of entities) entityMap.set(ent.id, ent);
+
+    // Group by community
+    const commGroups = new Map();
+    for (const [nodeId, commId] of assignment) {
+      if (!commGroups.has(commId)) {
+        commGroups.set(commId, { entityIds: [], fileIds: new Set() });
+      }
+      const g = commGroups.get(commId);
+      g.entityIds.push(nodeId);
+      const ent = entityMap.get(nodeId);
+      if (ent?.file_path) g.fileIds.add(ent.file_path);
+    }
+
+    // Connectivity sanity check per community (BFS on induced subgraph)
+    const finalCommunities = [];
+    let nextId = 0;
+
+    for (const [, group] of commGroups) {
+      const memberSet = new Set(group.entityIds);
+      const subAdj = new Map();
+      for (const nodeId of group.entityIds) {
+        const neighbors = adjacency.get(nodeId);
+        if (!neighbors) continue;
+        const subNeighbors = new Map();
+        for (const [neighbor, w] of neighbors) {
+          if (memberSet.has(neighbor)) subNeighbors.set(neighbor, w);
+        }
+        if (subNeighbors.size > 0) subAdj.set(nodeId, subNeighbors);
+      }
+
+      const components = findConnectedComponents(group.entityIds, subAdj);
+      for (const comp of components) {
+        const fileIds = new Set();
+        const entityNames = [];
+        for (const nodeId of comp) {
+          const ent = entityMap.get(nodeId);
+          if (ent?.file_path) fileIds.add(ent.file_path);
+          if (ent?.name) entityNames.push(ent.name);
+        }
+        finalCommunities.push({
+          id: nextId++,
+          entityIds: comp,
+          entityNames,
+          fileIds: [...fileIds],
+          entityCount: comp.length,
+        });
+      }
+    }
+
+    const graphHash = computeGraphHashFromRows(relationships);
+    return { communities: finalCommunities, graphHash, stale };
+  } finally {
+    db.close();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Internal Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Build weighted undirected adjacency from entities and relationships.
+ * @param {Array} entities
+ * @param {Array} relationships
+ * @returns {Map<number, Map<number, number>>}
+ */
+function buildWeightedAdjacency(entities, relationships) {
+  const adjacency = new Map();
+  const entityIds = new Set();
+  for (const ent of entities) {
+    entityIds.add(ent.id);
+    adjacency.set(ent.id, new Map());
+  }
+
+  for (const rel of relationships) {
+    const src = rel.source_id;
+    const tgt = rel.target_id;
+    // Self-edges (src === tgt) are intentionally dropped: they have no
+    // effect on community structure (same-node edges don't influence
+    // modularity delta-Q), and Leiden's self-loop handling in local
+    // moving already skips them (leiden-algorithm.js line 172).
+    if (!src || !tgt || src === tgt) continue;
+    if (!entityIds.has(src) || !entityIds.has(tgt)) continue;
+
+    const w = REL_WEIGHTS[rel.type] || 1;
+
+    const srcNeighbors = adjacency.get(src);
+    const tgtNeighbors = adjacency.get(tgt);
+    srcNeighbors.set(tgt, (srcNeighbors.get(tgt) || 0) + w);
+    tgtNeighbors.set(src, (tgtNeighbors.get(src) || 0) + w);
+  }
+
+  return adjacency;
+}
+
+/**
+ * Compute graph hash from relationship rows (avoids re-querying DB).
+ * Sorts a copy to avoid mutating caller-owned arrays.
+ */
+function computeGraphHashFromRows(relationships) {
+  const sorted = [...relationships].sort((a, b) => {
+    if (a.source_id !== b.source_id) return a.source_id - b.source_id;
+    if (a.target_id !== b.target_id) return a.target_id - b.target_id;
+    return (a.type || '') < (b.type || '') ? -1 : (a.type || '') > (b.type || '') ? 1 : 0;
+  });
+
+  const hash = createHash('sha256');
+  for (const row of sorted) {
+    hash.update(`${row.source_id}\t${row.target_id}\t${row.type}\n`);
+  }
+  return hash.digest('hex');
+}
+
+export default {
+  detectCommunities,
+  computeGraphHash,
+  leidenCommunities,
+  fallbackDirectoryGrouping,
+};