skyloom 1.14.6 → 1.15.0

package/src/core/semantic.ts

@@ -1,211 +1,211 @@
-/**
- * Lightweight semantic retrieval — zero external dependencies.
- *
- * Why: The existing `recall_for_injection` uses SQL LIKE on tokens, which
- * misses semantically related facts that share few literal characters (e.g.
- * query "deploy" vs. stored fact "release_command"). This module provides
- * a character n-gram Jaccard similarity scorer that catches those cross-
- * lingual and synonym relationships without adding PyTorch / sentence-transformers.
- *
- * Design:
- * - Character n-grams (size 2-4) naturally handle CJK, mixed-language, and
- *   code identifiers better than word-level tokenization.
- * - Jaccard similarity on n-gram sets is fast (< 10 µs per pair) and
- *   well-correlated with human relevance judgments for short text.
- * - Zero dependencies beyond stdlib.
- */
-
-/**
- * Represents a candidate item for ranking
- */
-export interface Candidate {
-  key?: string;
-  value?: string | Record<string, unknown>;
-  [key: string]: unknown;
-}
-
-/**
- * Character n-gram semantic similarity scorer.
- *
- * Usage:
- * ```
- * const scorer = new SemanticScorer();
- * const score = scorer.similarity("deploy to prod", "release_command");
- * // score ≈ 0.15 (low but non-zero — catches partial overlap)
- *
- * const ranked = scorer.rank(
- *   "search query",
- *   [{value: "candidate A"}, {value: "candidate B"}]
- * );
- * // ranked → [[0.85, {value: "candidate A"}], [0.30, {value: "candidate B"}]]
- * ```
- */
-export class SemanticScorer {
-  private nRange: [number, number];
-  private cache: Map<string, Set<string>>;
-  private maxCacheSize = 512;
-
-  /**
-   * Initialize scorer with n-gram range.
-   * @param nRange - Tuple of [minSize, maxSize] for n-grams (default [2, 4])
-   */
-  constructor(nRange: [number, number] = [2, 4]) {
-    this.nRange = nRange;
-    this.cache = new Map();
-  }
-
-  /**
-   * Generate character n-gram fingerprint (cached).
-   * @param text - Input text to fingerprint
-   * @returns Set of n-grams for the text
-   */
-  private fingerprint(text: string): Set<string> {
-    // Return cached result if available
-    if (this.cache.has(text)) {
-      return this.cache.get(text)!;
-    }
-
-    const lowered = text.toLowerCase();
-    const ngrams = new Set<string>();
-
-    // Generate n-grams for each size in range
-    for (let n = this.nRange[0]; n <= this.nRange[1]; n++) {
-      if (lowered.length < n) {
-        continue;
-      }
-      for (let i = 0; i <= lowered.length - n; i++) {
-        ngrams.add(lowered.slice(i, i + n));
-      }
-    }
-
-    // Cache result if we haven't hit size limit
-    if (this.cache.size < this.maxCacheSize) {
-      this.cache.set(text, ngrams);
-    }
-
-    return ngrams;
-  }
-
-  /**
-   * Calculate Jaccard similarity between two texts using character n-grams.
-   * @param a - First text
-   * @param b - Second text
-   * @returns Similarity score between 0 and 1
-   */
-  similarity(a: string, b: string): number {
-    if (!a || !b) {
-      return 0.0;
-    }
-
-    const fpA = this.fingerprint(a);
-    const fpB = this.fingerprint(b);
-
-    if (fpA.size === 0 || fpB.size === 0) {
-      return 0.0;
-    }
-
-    // Calculate intersection
-    let intersection = 0;
-    for (const ngram of fpA) {
-      if (fpB.has(ngram)) {
-        intersection++;
-      }
-    }
-
-    // Calculate union
-    const union = fpA.size + fpB.size - intersection;
-
-    return union > 0 ? intersection / union : 0.0;
-  }
-
-  /**
-   * Rank candidates by semantic similarity to the query.
-   *
-   * Each candidate is a dict. The scorer reads `candidate[keyField]`
-   * for text to compare. Also scores the "key" field if present.
-   * Returns `[score, candidate]` tuples sorted descending, filtered by `minScore`.
-   *
-   * @param query - Query string to match against
-   * @param candidates - Array of candidate objects
-   * @param keyField - Field name to extract text from (default "value")
-   * @param topK - Maximum number of results to return (default 3)
-   * @param minScore - Minimum similarity threshold (default 0.02)
-   * @returns Array of [score, candidate] tuples, sorted by score descending
-   */
-  rank(
-    query: string,
-    candidates: Candidate[],
-    keyField: string = "value",
-    topK: number = 3,
-    minScore: number = 0.02
-  ): Array<[number, Candidate]> {
-    const scored: Array<[number, Candidate]> = [];
-
-    for (const candidate of candidates) {
-      let text = candidate[keyField];
-
-      // Skip if field is missing or empty
-      if (!text) {
-        continue;
-      }
-
-      // Convert objects to string representation
-      if (typeof text === "object") {
-        text = JSON.stringify(text);
-      } else {
-        text = String(text);
-      }
-
-      // Calculate similarity with the candidate's value field
-      let score = this.similarity(query, text as string);
-
-      // Also score the "key" field if present — often more discriminative
-      const keyText = candidate.key;
-      if (keyText) {
-        const keyScore = this.similarity(query, String(keyText));
-        score = Math.max(score, keyScore);
-      }
-
-      // Add to results if above threshold
-      if (score >= minScore) {
-        scored.push([score, candidate]);
-      }
-    }
-
-    // Sort descending by score
-    scored.sort((a, b) => b[0] - a[0]);
-
-    // Return top K results
-    return scored.slice(0, topK);
-  }
-
-  /**
-   * Clear the fingerprint cache.
-   */
-  clearCache(): void {
-    this.cache.clear();
-  }
-
-  /**
-   * Get current cache size.
-   */
-  getCacheSize(): number {
-    return this.cache.size;
-  }
-}
-
-/**
- * Module-level singleton instance (lazy initialization).
- */
-let scorer: SemanticScorer | null = null;
-
-/**
- * Get or create the module-level semantic scorer singleton.
- * @returns SemanticScorer instance
- */
-export function getScorer(): SemanticScorer {
-  if (scorer === null) {
-    scorer = new SemanticScorer();
-  }
-  return scorer;
-}
+/**
+ * Lightweight semantic retrieval — zero external dependencies.
+ *
+ * Why: The existing `recall_for_injection` uses SQL LIKE on tokens, which
+ * misses semantically related facts that share few literal characters (e.g.
+ * query "deploy" vs. stored fact "release_command"). This module provides
+ * a character n-gram Jaccard similarity scorer that catches those cross-
+ * lingual and synonym relationships without adding PyTorch / sentence-transformers.
+ *
+ * Design:
+ * - Character n-grams (size 2-4) naturally handle CJK, mixed-language, and
+ *   code identifiers better than word-level tokenization.
+ * - Jaccard similarity on n-gram sets is fast (< 10 µs per pair) and
+ *   well-correlated with human relevance judgments for short text.
+ * - Zero dependencies beyond stdlib.
+ */
+
+/**
+ * Represents a candidate item for ranking
+ */
+export interface Candidate {
+  key?: string;
+  value?: string | Record<string, unknown>;
+  [key: string]: unknown;
+}
+
+/**
+ * Character n-gram semantic similarity scorer.
+ *
+ * Usage:
+ * ```
+ * const scorer = new SemanticScorer();
+ * const score = scorer.similarity("deploy to prod", "release_command");
+ * // score ≈ 0.15 (low but non-zero — catches partial overlap)
+ *
+ * const ranked = scorer.rank(
+ *   "search query",
+ *   [{value: "candidate A"}, {value: "candidate B"}]
+ * );
+ * // ranked → [[0.85, {value: "candidate A"}], [0.30, {value: "candidate B"}]]
+ * ```
+ */
+export class SemanticScorer {
+  private nRange: [number, number];
+  private cache: Map<string, Set<string>>;
+  private maxCacheSize = 512;
+
+  /**
+   * Initialize scorer with n-gram range.
+   * @param nRange - Tuple of [minSize, maxSize] for n-grams (default [2, 4])
+   */
+  constructor(nRange: [number, number] = [2, 4]) {
+    this.nRange = nRange;
+    this.cache = new Map();
+  }
+
+  /**
+   * Generate character n-gram fingerprint (cached).
+   * @param text - Input text to fingerprint
+   * @returns Set of n-grams for the text
+   */
+  private fingerprint(text: string): Set<string> {
+    // Return cached result if available
+    if (this.cache.has(text)) {
+      return this.cache.get(text)!;
+    }
+
+    const lowered = text.toLowerCase();
+    const ngrams = new Set<string>();
+
+    // Generate n-grams for each size in range
+    for (let n = this.nRange[0]; n <= this.nRange[1]; n++) {
+      if (lowered.length < n) {
+        continue;
+      }
+      for (let i = 0; i <= lowered.length - n; i++) {
+        ngrams.add(lowered.slice(i, i + n));
+      }
+    }
+
+    // Cache result if we haven't hit size limit
+    if (this.cache.size < this.maxCacheSize) {
+      this.cache.set(text, ngrams);
+    }
+
+    return ngrams;
+  }
+
+  /**
+   * Calculate Jaccard similarity between two texts using character n-grams.
+   * @param a - First text
+   * @param b - Second text
+   * @returns Similarity score between 0 and 1
+   */
+  similarity(a: string, b: string): number {
+    if (!a || !b) {
+      return 0.0;
+    }
+
+    const fpA = this.fingerprint(a);
+    const fpB = this.fingerprint(b);
+
+    if (fpA.size === 0 || fpB.size === 0) {
+      return 0.0;
+    }
+
+    // Calculate intersection
+    let intersection = 0;
+    for (const ngram of fpA) {
+      if (fpB.has(ngram)) {
+        intersection++;
+      }
+    }
+
+    // Calculate union
+    const union = fpA.size + fpB.size - intersection;
+
+    return union > 0 ? intersection / union : 0.0;
+  }
+
+  /**
+   * Rank candidates by semantic similarity to the query.
+   *
+   * Each candidate is a dict. The scorer reads `candidate[keyField]`
+   * for text to compare. Also scores the "key" field if present.
+   * Returns `[score, candidate]` tuples sorted descending, filtered by `minScore`.
+   *
+   * @param query - Query string to match against
+   * @param candidates - Array of candidate objects
+   * @param keyField - Field name to extract text from (default "value")
+   * @param topK - Maximum number of results to return (default 3)
+   * @param minScore - Minimum similarity threshold (default 0.02)
+   * @returns Array of [score, candidate] tuples, sorted by score descending
+   */
+  rank(
+    query: string,
+    candidates: Candidate[],
+    keyField: string = "value",
+    topK: number = 3,
+    minScore: number = 0.02
+  ): Array<[number, Candidate]> {
+    const scored: Array<[number, Candidate]> = [];
+
+    for (const candidate of candidates) {
+      let text = candidate[keyField];
+
+      // Skip if field is missing or empty
+      if (!text) {
+        continue;
+      }
+
+      // Convert objects to string representation
+      if (typeof text === "object") {
+        text = JSON.stringify(text);
+      } else {
+        text = String(text);
+      }
+
+      // Calculate similarity with the candidate's value field
+      let score = this.similarity(query, text as string);
+
+      // Also score the "key" field if present — often more discriminative
+      const keyText = candidate.key;
+      if (keyText) {
+        const keyScore = this.similarity(query, String(keyText));
+        score = Math.max(score, keyScore);
+      }
+
+      // Add to results if above threshold
+      if (score >= minScore) {
+        scored.push([score, candidate]);
+      }
+    }
+
+    // Sort descending by score
+    scored.sort((a, b) => b[0] - a[0]);
+
+    // Return top K results
+    return scored.slice(0, topK);
+  }
+
+  /**
+   * Clear the fingerprint cache.
+   */
+  clearCache(): void {
+    this.cache.clear();
+  }
+
+  /**
+   * Get current cache size.
+   */
+  getCacheSize(): number {
+    return this.cache.size;
+  }
+}
+
+/**
+ * Module-level singleton instance (lazy initialization).
+ */
+let scorer: SemanticScorer | null = null;
+
+/**
+ * Get or create the module-level semantic scorer singleton.
+ * @returns SemanticScorer instance
+ */
+export function getScorer(): SemanticScorer {
+  if (scorer === null) {
+    scorer = new SemanticScorer();
+  }
+  return scorer;
+}