principles-disciple 1.8.1 → 1.8.2

package/{dist/core/dictionary.js → src/core/dictionary.ts}

@@ -1,17 +1,35 @@
 import * as fs from 'fs';
 import * as path from 'path';
+
+export type RuleType = 'regex' | 'exact_match';
+
+export interface PainRule {
+    type: RuleType;
+    pattern?: string;      // For regex
+    phrases?: string[];    // For exact_match
+    severity: number;
+    hits: number;
+    status: 'active' | 'dormant' | 'archived';
+}
+
+export interface PainDictionaryData {
+    rules: Record<string, PainRule>;
+}
+
+
 export const PAIN_PROTOCOL_TOKENS = [
     '[EVOLUTION_ACK]',
     'HEARTBEAT_OK',
     'HEARTBEAT_CHECK',
-];
-export function shouldIgnorePainProtocolText(text) {
+] as const;
+
+export function shouldIgnorePainProtocolText(text: string): boolean {
     const normalized = text.trim();
-    if (!normalized)
-        return false;
+    if (!normalized) return false;
     return PAIN_PROTOCOL_TOKENS.some((token) => normalized === token || normalized.startsWith(`${token} `) || normalized.includes(token));
 }
-const DEFAULT_RULES = {
+
+const DEFAULT_RULES: Record<string, PainRule> = {
     'P_CONFUSION_ZH': {
         type: 'regex',
         pattern: '我(似乎|好像)(不确定|不太确定|不清楚|困惑)',
@@ -41,33 +59,33 @@ const DEFAULT_RULES = {
         status: 'active'
     }
 };
+
 export class PainDictionary {
-    stateDir;
-    data = { rules: {} };
-    filePath;
-    compiledRegex = new Map();
-    constructor(stateDir) {
-        this.stateDir = stateDir;
+    private data: PainDictionaryData = { rules: {} };
+    private filePath: string;
+    private compiledRegex: Map<string, RegExp> = new Map();
+
+    constructor(private stateDir: string) {
         this.filePath = path.join(stateDir, 'pain_dictionary.json');
     }
-    load() {
+
+    load(): void {
         if (fs.existsSync(this.filePath)) {
             try {
                 this.data = JSON.parse(fs.readFileSync(this.filePath, 'utf8'));
-            }
-            catch (e) {
+            } catch (e) {
                 console.error('[PD] Failed to parse pain_dictionary.json, using defaults.');
                 this.data = { rules: { ...DEFAULT_RULES } };
             }
-        }
-        else {
+        } else {
             this.data = { rules: { ...DEFAULT_RULES } };
             console.log(`[PD:Dictionary] Dictionary not found at ${this.filePath}, creating with default rules`);
             this.flush();
         }
         this.compile();
     }
-    compile() {
+
+    private compile(): void {
         this.compiledRegex.clear();
         for (const [id, rule] of Object.entries(this.data.rules)) {
             if (rule.type === 'regex' && rule.pattern) {
@@ -75,16 +93,19 @@ export class PainDictionary {
             }
         }
     }
-    getRule(id) {
+
+    getRule(id: string): PainRule | undefined {
         return this.data.rules[id];
     }
-    getAllRules() {
+
+    getAllRules(): Record<string, PainRule> {
         return this.data.rules;
     }
+
     /**
      * Adds a new rule or updates an existing one.
      */
-    addRule(id, rule) {
+    addRule(id: string, rule: Omit<PainRule, 'hits'>): void {
         this.data.rules[id] = {
             ...rule,
             hits: this.data.rules[id]?.hits || 0
@@ -94,28 +115,29 @@ export class PainDictionary {
             this.compiledRegex.set(id, new RegExp(rule.pattern, 'i'));
         }
     }
-    match(text) {
-        if (shouldIgnorePainProtocolText(text))
-            return undefined;
-        let bestMatch = undefined;
+
+    match(text: string): { ruleId: string; severity: number } | undefined {
+        if (shouldIgnorePainProtocolText(text)) return undefined;
+
+        let bestMatch: { ruleId: string; severity: number } | undefined = undefined;
+
         for (const [id, rule] of Object.entries(this.data.rules)) {
-            if (rule.status !== 'active')
-                continue;
+            if (rule.status !== 'active') continue;
+
             let matched = false;
             if (rule.type === 'regex') {
                 const re = this.compiledRegex.get(id);
                 if (re) {
                     re.lastIndex = 0;
-                    if (re.test(text))
-                        matched = true;
+                    if (re.test(text)) matched = true;
                 }
-            }
-            else if (rule.type === 'exact_match' && rule.phrases) {
+            } else if (rule.type === 'exact_match' && rule.phrases) {
                 const lowerText = text.toLowerCase();
                 if (rule.phrases.some(p => lowerText.includes(p.toLowerCase()))) {
                     matched = true;
                 }
             }
+
             if (matched) {
                 rule.hits++;
                 if (!bestMatch || rule.severity > bestMatch.severity) {
@@ -125,18 +147,19 @@ export class PainDictionary {
         }
         return bestMatch;
     }
-    flush() {
+
+    flush(): void {
         try {
             if (!fs.existsSync(this.stateDir)) {
                 fs.mkdirSync(this.stateDir, { recursive: true });
             }
             fs.writeFileSync(this.filePath, JSON.stringify(this.data, null, 2), 'utf8');
-        }
-        catch (e) {
+        } catch (e) {
             console.error('[PD] Failed to flush pain_dictionary.json:', e);
         }
     }
-    getStats() {
+
+    getStats(): { totalRules: number; totalHits: number } {
         let totalHits = 0;
         for (const rule of Object.values(this.data.rules)) {
             totalHits += rule.hits || 0;

package/src/core/empathy-keyword-matcher.ts

@@ -0,0 +1,327 @@
+/**
+ * Empathy Keyword Matcher
+ * 
+ * Fast keyword-based empathy detection that replaces the previous
+ * LLM subagent-per-turn approach.
+ * 
+ * Flow:
+ *   User message → keyword matching → weighted score → GFI penalty
+ * 
+ * The keyword store is periodically optimized by a subagent that analyzes
+ * recent conversations and updates keyword weights, discovers new terms,
+ * and removes false positives.
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import {
+  EmpathyKeywordStore,
+  EmpathyKeywordEntry,
+  EmpathyKeywordStats,
+  EmpathyMatchResult,
+  EmpathyKeywordConfig,
+  SeedKeywordEntry,
+  EMPATHY_SEED_KEYWORDS,
+  DEFAULT_EMPATHY_KEYWORD_CONFIG,
+  scoreToSeverity,
+} from './empathy-types.js';
+
+const KEYWORD_STORE_FILE = 'empathy_keywords.json';
+
+// =========================================================================
+// Store Management
+// =========================================================================
+
+/**
+ * Creates a default keyword store from seed keywords.
+ * Supports both Chinese and English keywords.
+ */
+export function createDefaultKeywordStore(language: 'zh' | 'en' = 'zh'): EmpathyKeywordStore {
+  const now = new Date().toISOString();
+  const terms: Record<string, EmpathyKeywordEntry> = {};
+
+  // Include all seed keywords (both zh and en)
+  for (const seed of EMPATHY_SEED_KEYWORDS) {
+    // For Chinese language, include all keywords
+    // For English language, include only English keywords
+    const isChinese = /[\u4e00-\u9fa5]/.test(seed.term);
+    if (language === 'zh' || !isChinese) {
+      terms[seed.term] = {
+        weight: seed.weight,
+        source: 'seed',
+        hitCount: 0,
+        falsePositiveRate: seed.initialFalsePositiveRate ?? 0.15, // Differentiated FPR (Finding #6)
+      };
+    }
+  }
+
+  const stats: EmpathyKeywordStats = {
+    totalHits: 0,
+    totalFalsePositives: 0,
+    optimizationCount: 0,
+  };
+
+  return {
+    version: 1,
+    lastUpdated: now,
+    lastOptimizedAt: now,
+    terms,
+    stats,
+  };
+}
+
+/**
+ * Loads the keyword store from disk, or creates a default one if not found.
+ * Respects the configured language setting.
+ */
+export function loadKeywordStore(stateDir: string, language?: 'zh' | 'en'): EmpathyKeywordStore {
+  const filePath = path.join(stateDir, KEYWORD_STORE_FILE);
+  
+  try {
+    if (!fs.existsSync(filePath)) {
+      const store = createDefaultKeywordStore(language);
+      saveKeywordStore(stateDir, store);
+      return store;
+    }
+
+    const raw = fs.readFileSync(filePath, 'utf8');
+    const parsed = JSON.parse(raw);
+    
+    // Validate structure
+    if (!parsed.terms || !parsed.stats || !parsed.version) {
+      console.warn('[PD:Empathy] Invalid keyword store format, creating default');
+      const store = createDefaultKeywordStore(language);
+      saveKeywordStore(stateDir, store);
+      return store;
+    }
+
+    return parsed as EmpathyKeywordStore;
+  } catch (e) {
+    console.warn(`[PD:Empathy] Failed to load keyword store: ${e}`);
+    const store = createDefaultKeywordStore(language);
+    saveKeywordStore(stateDir, store);
+    return store;
+  }
+}
+
+/**
+ * Saves the keyword store to disk.
+ */
+export function saveKeywordStore(stateDir: string, store: EmpathyKeywordStore): void {
+  const filePath = path.join(stateDir, KEYWORD_STORE_FILE);
+  const dir = path.dirname(filePath);
+  
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+
+  store.lastUpdated = new Date().toISOString();
+  fs.writeFileSync(filePath, JSON.stringify(store, null, 2), 'utf8');
+}
+
+// =========================================================================
+// Keyword Matching
+// =========================================================================
+
+/**
+ * Matches text against the keyword store and returns a structured result.
+ * 
+ * This is the core function that replaces the previous LLM-based empathy detection.
+ * It runs in < 1ms for typical keyword stores (50-200 terms).
+ */
+export function matchEmpathyKeywords(
+  text: string,
+  store: EmpathyKeywordStore,
+  config: EmpathyKeywordConfig = DEFAULT_EMPATHY_KEYWORD_CONFIG,
+): EmpathyMatchResult {
+  if (!text || typeof text !== 'string' || text.trim().length === 0) {
+    return {
+      matched: false,
+      score: 0,
+      matchedTerms: [],
+      severity: 'mild',
+      confidence: 0,
+    };
+  }
+
+  const lowerText = text.toLowerCase();
+  let totalScore = 0;
+  const matchedTerms: string[] = [];
+
+  for (const [term, entry] of Object.entries(store.terms)) {
+    if (lowerText.includes(term.toLowerCase())) {
+      // Weight adjusted by false positive rate
+      const adjustedWeight = entry.weight * (1 - entry.falsePositiveRate);
+      totalScore += adjustedWeight;
+      matchedTerms.push(term);
+
+      // Update hit stats
+      entry.hitCount++;
+      entry.lastHitAt = new Date().toISOString();
+    }
+  }
+
+  // Cap score at 1.0
+  const cappedScore = Math.min(1, totalScore);
+
+  // Only consider matched if score exceeds threshold
+  const isMatched = cappedScore >= config.matchThreshold && matchedTerms.length > 0;
+
+  // Limit matched terms for performance
+  const limitedTerms = matchedTerms.slice(0, config.maxTermsPerMessage);
+
+  // Calculate confidence based on:
+  // - Number of matched terms (more terms = higher confidence)
+  // - Score relative to threshold (higher score = higher confidence)
+  const termConfidence = Math.min(1, limitedTerms.length / 3);
+  const scoreConfidence = Math.min(1, cappedScore / 0.8);
+  const confidence = Math.max(termConfidence, scoreConfidence);
+
+  const result: EmpathyMatchResult = {
+    matched: isMatched,
+    score: cappedScore,
+    matchedTerms: limitedTerms,
+    severity: scoreToSeverity(cappedScore),
+    confidence,
+  };
+
+  // Update store stats
+  if (isMatched) {
+    store.stats.totalHits += limitedTerms.length;
+  }
+
+  return result;
+}
+
+// =========================================================================
+// Keyword Store Updates
+// =========================================================================
+
+/**
+ * Applies keyword updates from subagent optimization.
+ * 
+ * This is called when the empathy optimizer subagent completes its analysis
+ * and returns suggested updates to the keyword store.
+ */
+export function applyKeywordUpdates(
+  store: EmpathyKeywordStore,
+  updates: Record<string, {
+    action: 'add' | 'update' | 'remove';
+    weight?: number;
+    falsePositiveRate?: number;
+    examples?: string[];
+    reasoning?: string;
+  }>,
+): { added: number; updated: number; removed: number } {
+  let added = 0;
+  let updated = 0;
+  let removed = 0;
+  const now = new Date().toISOString();
+
+  for (const [term, update] of Object.entries(updates)) {
+    switch (update.action) {
+      case 'add':
+        if (!store.terms[term]) {
+          store.terms[term] = {
+            weight: update.weight ?? 0.5,
+            source: 'llm_discovered',
+            hitCount: 0,
+            falsePositiveRate: update.falsePositiveRate ?? 0.2,
+            examples: update.examples,
+            discoveredAt: now,
+          };
+          added++;
+        }
+        break;
+
+      case 'update':
+        if (store.terms[term]) {
+          if (update.weight !== undefined) {
+            store.terms[term].weight = update.weight;
+          }
+          if (update.falsePositiveRate !== undefined) {
+            store.terms[term].falsePositiveRate = update.falsePositiveRate;
+          }
+          if (update.examples) {
+            store.terms[term].examples = update.examples;
+          }
+          updated++;
+        }
+        break;
+
+      case 'remove':
+        if (store.terms[term]) {
+          delete store.terms[term];
+          removed++;
+        }
+        break;
+    }
+  }
+
+  store.lastOptimizedAt = now;
+  store.stats.optimizationCount++;
+
+  return { added, updated, removed };
+}
+
+// =========================================================================
+// Optimization Trigger
+// =========================================================================
+
+/**
+ * Checks if it's time to trigger subagent optimization.
+ * 
+ * Returns true if either:
+ * - The number of turns since last optimization exceeds the interval
+ * - The time since last optimization exceeds the interval
+ */
+export function shouldTriggerOptimization(
+  store: EmpathyKeywordStore,
+  turnsSinceLastOptimization: number,
+  config: EmpathyKeywordConfig = DEFAULT_EMPATHY_KEYWORD_CONFIG,
+): boolean {
+  const turnsExceeded = turnsSinceLastOptimization >= config.optimizationIntervalTurns;
+  
+  const lastOpt = new Date(store.lastOptimizedAt).getTime();
+  const now = Date.now();
+  const timeExceeded = (now - lastOpt) >= config.optimizationIntervalMs;
+
+  return turnsExceeded || timeExceeded;
+}
+
+// =========================================================================
+// Store Inspection
+// =========================================================================
+
+/**
+ * Returns a summary of the keyword store for debugging/monitoring.
+ */
+export function getKeywordStoreSummary(store: EmpathyKeywordStore): {
+  totalTerms: number;
+  seedTerms: number;
+  discoveredTerms: number;
+  topHitTerms: { term: string; hitCount: number; weight: number }[];
+  highFalsePositiveTerms: { term: string; falsePositiveRate: number }[];
+} {
+  const terms = Object.entries(store.terms);
+  const seedTerms = terms.filter(([, e]) => e.source === 'seed');
+  const discoveredTerms = terms.filter(([, e]) => e.source === 'llm_discovered');
+
+  const topHitTerms = terms
+    .map(([term, entry]) => ({ term, hitCount: entry.hitCount, weight: entry.weight }))
+    .sort((a, b) => b.hitCount - a.hitCount)
+    .slice(0, 10);
+
+  const highFalsePositiveTerms = terms
+    .filter(([, e]) => e.falsePositiveRate > 0.3)
+    .map(([term, entry]) => ({ term, falsePositiveRate: entry.falsePositiveRate }))
+    .sort((a, b) => b.falsePositiveRate - a.falsePositiveRate);
+
+  return {
+    totalTerms: terms.length,
+    seedTerms: seedTerms.length,
+    discoveredTerms: discoveredTerms.length,
+    topHitTerms,
+    highFalsePositiveTerms,
+  };
+}