verifiable-thinking-mcp 0.4.0

package/src/lib/think/consistency.ts

@@ -0,0 +1,347 @@
+/**
+ * Consistency Checker - O(n) lightweight contradiction detection
+ *
+ * Detects obvious contradictions across reasoning steps:
+ * - Variable reassignment (x=5 then x=10)
+ * - Logical conflicts (always vs never, all vs none)
+ * - Sign/direction flips (positive→negative, increasing→decreasing)
+ *
+ * Keeps complexity O(n) by single-pass extraction + map-based comparison.
+ */
+
+/** A detected contradiction between reasoning steps */
+export interface Contradiction {
+  /** Type of contradiction found */
+  type: "value_reassignment" | "logical_conflict" | "sign_flip" | "direction_reversal";
+  /** Human-readable description */
+  description: string;
+  /** The variable/concept involved */
+  subject: string;
+  /** Step where original claim was made */
+  original_step: number;
+  /** Value/state in original step */
+  original_value: string;
+  /** Step where contradiction occurred */
+  conflicting_step: number;
+  /** Conflicting value/state */
+  conflicting_value: string;
+  /** Confidence in detection (0-1) */
+  confidence: number;
+}
+
+/** Result of consistency check */
+export interface ConsistencyResult {
+  /** Whether any contradictions were found */
+  has_contradictions: boolean;
+  /** List of detected contradictions */
+  contradictions: Contradiction[];
+  /** Number of steps analyzed */
+  steps_analyzed: number;
+}
+
+// Patterns for extracting variable assignments
+// Matches: x = 5, x = 10, let x = 3, x := 7
+const ASSIGNMENT_PATTERN = /(?:let\s+)?([a-zA-Z_][a-zA-Z0-9_]*)\s*[:=]=?\s*(-?\d+(?:\.\d+)?)/g;
+
+// Patterns for logical absolutes
+const LOGICAL_ABSOLUTES: Record<string, string[]> = {
+  always: ["never", "sometimes", "rarely"],
+  never: ["always", "sometimes", "often"],
+  all: ["none", "some", "few"],
+  none: ["all", "some", "many"],
+  every: ["no", "some", "few"],
+  must: ["cannot", "might", "may not"],
+  cannot: ["must", "can", "might"],
+  true: ["false"],
+  false: ["true"],
+  impossible: ["possible", "certain", "likely"],
+  certain: ["impossible", "uncertain", "unlikely"],
+};
+
+// Patterns for sign/direction words
+const SIGN_WORDS: Record<string, string[]> = {
+  positive: ["negative", "zero"],
+  negative: ["positive", "zero"],
+  increasing: ["decreasing", "constant"],
+  decreasing: ["increasing", "constant"],
+  greater: ["less", "equal"],
+  less: ["greater", "equal"],
+  above: ["below", "at"],
+  below: ["above", "at"],
+};
+
+/** Internal tracking structure for a claim */
+interface ClaimRecord {
+  step: number;
+  value: string;
+  context: string; // surrounding words for confidence
+}
+
+/**
+ * Extract variable assignments from text
+ * O(n) where n = text length
+ */
+function extractAssignments(text: string, stepNum: number): Map<string, ClaimRecord> {
+  const result = new Map<string, ClaimRecord>();
+  let match: RegExpExecArray | null;
+
+  // Reset regex state
+  ASSIGNMENT_PATTERN.lastIndex = 0;
+
+  while ((match = ASSIGNMENT_PATTERN.exec(text)) !== null) {
+    const varName = match[1]?.toLowerCase();
+    const value = match[2];
+    if (!varName || !value) continue;
+    // Get surrounding context (20 chars each side)
+    const start = Math.max(0, match.index - 20);
+    const end = Math.min(text.length, match.index + match[0].length + 20);
+    const context = text.slice(start, end);
+
+    result.set(varName, { step: stepNum, value, context });
+  }
+
+  return result;
+}
+
+/**
+ * Extract logical absolute claims from text
+ * O(n) where n = text length
+ *
+ * Strategy: Extract the word itself, look for conflicts later
+ */
+function extractLogicalClaims(text: string, stepNum: number): Map<string, ClaimRecord> {
+  const result = new Map<string, ClaimRecord>();
+  const lowerText = text.toLowerCase();
+
+  for (const [word, _conflicts] of Object.entries(LOGICAL_ABSOLUTES)) {
+    // Word boundary match
+    const regex = new RegExp(`\\b${word}\\b`, "gi");
+    if (regex.test(lowerText)) {
+      // Get context around the word
+      const match = lowerText.indexOf(word);
+      const start = Math.max(0, match - 30);
+      const end = Math.min(text.length, match + word.length + 30);
+      const context = text.slice(start, end);
+
+      // Simple key: just the word type (always, never, etc)
+      // We'll check conflicts based on value, not key matching
+      const key = `logical:${word}`;
+      result.set(key, { step: stepNum, value: word, context });
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Extract sign/direction claims from text
+ * O(n) where n = text length
+ */
+function extractSignClaims(text: string, stepNum: number): Map<string, ClaimRecord> {
+  const result = new Map<string, ClaimRecord>();
+  const lowerText = text.toLowerCase();
+
+  for (const [word, _conflicts] of Object.entries(SIGN_WORDS)) {
+    const regex = new RegExp(`\\b${word}\\b`, "gi");
+    if (regex.test(lowerText)) {
+      const match = lowerText.indexOf(word);
+      const start = Math.max(0, match - 30);
+      const end = Math.min(text.length, match + word.length + 30);
+      const context = text.slice(start, end);
+
+      // Include nearby noun/subject for specificity
+      // But use the word itself as the key for conflict matching
+      const varMatch = context.match(/\b(the\s+)?([a-zA-Z_][a-zA-Z0-9_]*)\b/i);
+      const subject = varMatch ? varMatch[2] || "value" : "value";
+      const key = `sign:${word}`;
+      result.set(key, { step: stepNum, value: word, context: `${subject}: ${context}` });
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Check for contradictions between two claims
+ */
+function findContradiction(
+  key: string,
+  oldClaim: ClaimRecord,
+  newClaim: ClaimRecord,
+): Contradiction | null {
+  // Value reassignment check
+  if (key.match(/^[a-z_][a-z0-9_]*$/i)) {
+    // Simple variable name - check if values differ
+    if (oldClaim.value !== newClaim.value) {
+      return {
+        type: "value_reassignment",
+        description: `Variable '${key}' was ${oldClaim.value} in step ${oldClaim.step}, now ${newClaim.value}`,
+        subject: key,
+        original_step: oldClaim.step,
+        original_value: oldClaim.value,
+        conflicting_step: newClaim.step,
+        conflicting_value: newClaim.value,
+        confidence: 0.9,
+      };
+    }
+  }
+
+  // Logical conflict check
+  if (key.startsWith("logical:")) {
+    const conflicts = LOGICAL_ABSOLUTES[oldClaim.value];
+    if (conflicts?.includes(newClaim.value)) {
+      return {
+        type: "logical_conflict",
+        description: `Logical conflict: '${oldClaim.value}' in step ${oldClaim.step} vs '${newClaim.value}' in step ${newClaim.step}`,
+        subject: key.split(":")[2] || "claim",
+        original_step: oldClaim.step,
+        original_value: oldClaim.value,
+        conflicting_step: newClaim.step,
+        conflicting_value: newClaim.value,
+        confidence: 0.85,
+      };
+    }
+  }
+
+  // Sign flip check
+  if (key.startsWith("sign:")) {
+    const conflicts = SIGN_WORDS[oldClaim.value];
+    if (conflicts?.includes(newClaim.value)) {
+      const subject = key.split(":")[1] || "value";
+      return {
+        type: "sign_flip",
+        description: `Sign flip for '${subject}': ${oldClaim.value} → ${newClaim.value}`,
+        subject,
+        original_step: oldClaim.step,
+        original_value: oldClaim.value,
+        conflicting_step: newClaim.step,
+        conflicting_value: newClaim.value,
+        confidence: 0.8,
+      };
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Check consistency across reasoning steps
+ *
+ * O(n*m) where n = total text length, m = number of steps
+ * Practically O(n) since m is bounded by session limits
+ *
+ * @param steps - Array of thought texts with step numbers
+ * @returns Consistency check result
+ */
+export function checkConsistency(
+  steps: Array<{ step: number; thought: string }>,
+): ConsistencyResult {
+  if (steps.length < 2) {
+    return {
+      has_contradictions: false,
+      contradictions: [],
+      steps_analyzed: steps.length,
+    };
+  }
+
+  const contradictions: Contradiction[] = [];
+
+  // Track all claims across steps
+  const allAssignments = new Map<string, ClaimRecord>();
+  // For logical/sign, track by value (the word itself) to find conflicts
+  const allLogicalByValue = new Map<string, ClaimRecord>();
+  const allSignsByValue = new Map<string, ClaimRecord>();
+
+  for (const { step, thought } of steps) {
+    // Extract claims from this step
+    const assignments = extractAssignments(thought, step);
+    const logical = extractLogicalClaims(thought, step);
+    const signs = extractSignClaims(thought, step);
+
+    // Check for contradictions with previous steps - variable reassignment
+    for (const [key, claim] of assignments) {
+      const existing = allAssignments.get(key);
+      if (existing && existing.step !== step) {
+        const contradiction = findContradiction(key, existing, claim);
+        if (contradiction) {
+          contradictions.push(contradiction);
+        }
+      }
+      // Update or set the claim (latest wins for tracking)
+      allAssignments.set(key, claim);
+    }
+
+    // Logical conflicts: check if this step's words conflict with prior words
+    for (const [_key, claim] of logical) {
+      const conflicts = LOGICAL_ABSOLUTES[claim.value];
+      if (conflicts) {
+        // Check if any conflicting word was seen in a prior step
+        for (const conflictWord of conflicts) {
+          const existing = allLogicalByValue.get(conflictWord);
+          if (existing && existing.step !== step) {
+            contradictions.push({
+              type: "logical_conflict",
+              description: `Logical conflict: '${existing.value}' in step ${existing.step} vs '${claim.value}' in step ${step}`,
+              subject: "claim",
+              original_step: existing.step,
+              original_value: existing.value,
+              conflicting_step: step,
+              conflicting_value: claim.value,
+              confidence: 0.85,
+            });
+          }
+        }
+      }
+      allLogicalByValue.set(claim.value, claim);
+    }
+
+    // Sign conflicts: check if this step's words conflict with prior words
+    for (const [_key, claim] of signs) {
+      const conflicts = SIGN_WORDS[claim.value];
+      if (conflicts) {
+        for (const conflictWord of conflicts) {
+          const existing = allSignsByValue.get(conflictWord);
+          if (existing && existing.step !== step) {
+            // Extract subject from context
+            const subjectMatch = claim.context.match(/^([^:]+):/);
+            const subject = subjectMatch?.[1] ?? "value";
+            contradictions.push({
+              type: "sign_flip",
+              description: `Sign flip for '${subject}': ${existing.value} → ${claim.value}`,
+              subject,
+              original_step: existing.step,
+              original_value: existing.value,
+              conflicting_step: step,
+              conflicting_value: claim.value,
+              confidence: 0.8,
+            });
+          }
+        }
+      }
+      allSignsByValue.set(claim.value, claim);
+    }
+  }
+
+  return {
+    has_contradictions: contradictions.length > 0,
+    contradictions,
+    steps_analyzed: steps.length,
+  };
+}
+
+/**
+ * Quick check if a new step contradicts any previous steps
+ * More efficient for incremental checking
+ *
+ * @param newStep - The new step to check
+ * @param priorSteps - Previous steps to check against
+ * @returns Array of contradictions (empty if none)
+ */
+export function checkStepConsistency(
+  newStep: { step: number; thought: string },
+  priorSteps: Array<{ step: number; thought: string }>,
+): Contradiction[] {
+  const result = checkConsistency([...priorSteps, newStep]);
+  // Only return contradictions involving the new step
+  return result.contradictions.filter((c) => c.conflicting_step === newStep.step);
+}

package/src/lib/think/guidance.ts

@@ -0,0 +1,188 @@
+/**
+ * Guidance Engine - Proactive reasoning assistance
+ * Research-backed failure pattern detection and guidance generation
+ */
+
+import type { ThoughtRecord } from "../session.ts";
+import type { VerificationDomain } from "../verification.ts";
+
+// ============================================================================
+// FAILURE PATTERNS - Research-backed reasoning hazards
+// ============================================================================
+
+export interface FailurePattern {
+  trigger: RegExp;
+  risk: string;
+  guidance: string;
+  checkpoint: boolean;
+  minLength?: number;
+}
+
+export const FAILURE_PATTERNS: Record<string, FailurePattern> = {
+  premature_conclusion: {
+    // Only flag if "answer is/=:" appears in first 100 chars AND total response is long
+    // (short responses are expected for structured phase outputs)
+    trigger: /^.{0,100}answer\s*(is|=|:)/im,
+    risk: "Concluding without showing intermediate steps",
+    guidance: "Show your work before concluding",
+    checkpoint: true,
+    minLength: 200, // Don't flag short structured outputs - they're intentional
+  },
+  arithmetic_chain: {
+    trigger: /\d+\s*[+\-*/]\s*\d+\s*[+\-*/]\s*\d+/,
+    risk: "Arithmetic chain prone to carry-forward errors",
+    guidance: "Verify each arithmetic step independently",
+    checkpoint: true,
+    minLength: 20,
+  },
+  short_answer: {
+    // Only flag for long-form reasoning contexts, not structured phase outputs
+    trigger: /^.{0,80}$/,
+    risk: "Answer too brief - may lack reasoning",
+    guidance: "Show intermediate steps",
+    checkpoint: false,
+    minLength: 0,
+    // NOTE: This pattern is disabled when guidance=false (phase-based iteration)
+    // It's designed for unstructured LLM outputs, not intentionally terse phases
+  },
+  contradiction: {
+    trigger: /but\s+(also|then|wait)|however.*but|on\s+the\s+other\s+hand.*yet/i,
+    risk: "Potential contradiction in reasoning",
+    guidance: "Resolve conflicting statements before proceeding",
+    checkpoint: true,
+    minLength: 100,
+  },
+  overconfident_complex: {
+    trigger: /obviously|clearly|trivially|of\s+course/i,
+    risk: "Overconfidence may mask errors",
+    guidance: "Verify 'obvious' steps explicitly",
+    checkpoint: false,
+    minLength: 100,
+  },
+  unchecked_assumption: {
+    trigger: /let's\s+assume|assuming\s+that|suppose\s+that/i,
+    risk: "Assumption may not hold",
+    guidance: "Verify assumption is warranted by the problem",
+    checkpoint: false,
+    minLength: 100,
+  },
+};
+
+// Valid purpose categories
+export const VALID_PURPOSES = new Set([
+  "analysis",
+  "action",
+  "reflection",
+  "decision",
+  "summary",
+  "validation",
+  "exploration",
+  "hypothesis",
+  "correction",
+  "planning",
+]);
+
+// ============================================================================
+// GUIDANCE ENGINE - Proactive reasoning assistance
+// ============================================================================
+
+export interface ThoughtAnalysis {
+  patterns_detected: string[];
+  guidance: string[];
+  checkpoint_recommended: boolean;
+  suggested_next: string | null;
+  risk_level: "low" | "medium" | "high";
+}
+
+export function analyzeThought(
+  thought: string,
+  step: number,
+  priorThoughts: ThoughtRecord[],
+  domain: VerificationDomain,
+): ThoughtAnalysis {
+  const patterns_detected: string[] = [];
+  const guidance: string[] = [];
+  let checkpoint_recommended = false;
+  let risk_score = 0;
+  const thoughtLength = thought.length;
+
+  // Check against known failure patterns
+  for (const [name, pattern] of Object.entries(FAILURE_PATTERNS)) {
+    const minLen = pattern.minLength ?? 0;
+    if (thoughtLength >= minLen && pattern.trigger.test(thought)) {
+      patterns_detected.push(name);
+      guidance.push(pattern.guidance);
+      if (pattern.checkpoint) checkpoint_recommended = true;
+      risk_score++;
+    }
+  }
+
+  // Domain-specific guidance
+  if (step > 1) {
+    if (domain === "math" && /=/.test(thought) && priorThoughts.length > 0) {
+      if (guidance.length === 0) {
+        guidance.push("Verify equation transformation preserves equality");
+      }
+    } else if (domain === "code" && /loop|iterate|recursive/i.test(thought)) {
+      guidance.push("Verify termination condition exists");
+      checkpoint_recommended = true;
+      risk_score++;
+    }
+  }
+
+  // Confidence trajectory analysis
+  const confidences = priorThoughts
+    .map((t) => t.verification?.confidence)
+    .filter((c): c is number => c !== undefined);
+
+  if (confidences.length >= 2) {
+    const recent = confidences.slice(-2);
+    const prev = recent[0] ?? 0;
+    const curr = recent[1] ?? 0;
+    if (curr < prev - 0.2) {
+      guidance.push("Confidence dropping - consider revisiting assumptions");
+      risk_score++;
+    }
+  }
+
+  // Suggest next action
+  let suggested_next: string | null = null;
+  if (checkpoint_recommended) {
+    suggested_next = "Pause and verify current step before proceeding";
+  } else if (patterns_detected.includes("premature_conclusion")) {
+    suggested_next = "Review all constraints before finalizing";
+  } else if (step >= 3 && !priorThoughts.some((t) => t.verification?.passed)) {
+    suggested_next = "Consider verifying intermediate steps";
+  }
+
+  const risk_level = risk_score >= 3 ? "high" : risk_score >= 1 ? "medium" : "low";
+
+  return {
+    patterns_detected,
+    guidance: guidance.slice(0, 3),
+    checkpoint_recommended,
+    suggested_next,
+    risk_level,
+  };
+}
+
+// ============================================================================
+// DOMAIN DETECTION
+// ============================================================================
+
+export function detectDomain(thought: string): VerificationDomain {
+  if (/\d+\s*[+\-*/^=]\s*\d+|equation|solve|derivative|integral|sum\s+of/i.test(thought)) {
+    return "math";
+  }
+  if (
+    /function|class|return|const|let|var|def\s|import\s|async|await|=>|->|fn\s|impl\s/i.test(
+      thought,
+    )
+  ) {
+    return "code";
+  }
+  if (/if\s+.+\s+then|therefore|implies|hence|thus|conclude|premise|valid|invalid/i.test(thought)) {
+    return "logic";
+  }
+  return "general";
+}