code-finder 0.1.0__py3-none-any.whl

claude_context/query_metrics.py

@@ -0,0 +1,164 @@
+"""
+Query Interpretation Metrics
+
+Tracks the performance and effectiveness of LLM-based query interpretation
+to measure impact on documentation quality.
+"""
+
+import logging
+from dataclasses import dataclass, field
+from typing import Dict, Any, List
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class QueryMetrics:
+    """
+    Track query interpretation quality and success rates.
+    
+    Used to monitor whether enhanced LLM interpretation improves
+    documentation quality by finding more relevant code.
+    """
+    
+    # Basic counts
+    questions_asked: int = 0
+    llm_interpretations: int = 0
+    results_found_after_llm: int = 0
+    
+    # Quality metrics
+    total_confidence: float = 0.0
+    
+    # Domain breakdown (for analysis)
+    ai_ml_questions: int = 0
+    k8s_questions: int = 0
+    web_questions: int = 0
+    cli_questions: int = 0
+    general_questions: int = 0
+    
+    # Track interpretations for detailed analysis
+    interpretation_log: List[Dict] = field(default_factory=list)
+    
+    def log_question(self, question: str, used_llm: bool, results_count: int, confidence: float):
+        """
+        Log a question and its outcome.
+        
+        Args:
+            question: The question asked
+            used_llm: Whether LLM interpretation was used
+            results_count: Number of results found
+            confidence: Confidence score of the answer
+        """
+        self.questions_asked += 1
+        self.total_confidence += confidence
+        
+        if used_llm:
+            self.llm_interpretations += 1
+            if results_count > 0:
+                self.results_found_after_llm += 1
+        
+        # Classify by domain (simple keyword matching)
+        question_lower = question.lower()
+        
+        if any(term in question_lower for term in [
+            "quantization", "quantize", "model", "training", "inference", 
+            "pruning", "distillation", "neural", "transformer"
+        ]):
+            self.ai_ml_questions += 1
+            domain = "ai_ml"
+        elif any(term in question_lower for term in [
+            "kubernetes", "kubectl", "operator", "crd", "configmap",
+            "pod", "deployment", "namespace"
+        ]):
+            self.k8s_questions += 1
+            domain = "k8s"
+        elif any(term in question_lower for term in [
+            "route", "endpoint", "api", "request", "response",
+            "middleware", "handler"
+        ]):
+            self.web_questions += 1
+            domain = "web"
+        elif any(term in question_lower for term in [
+            "command", "cli", "flag", "option", "subcommand"
+        ]):
+            self.cli_questions += 1
+            domain = "cli"
+        else:
+            self.general_questions += 1
+            domain = "general"
+        
+        # Store for detailed analysis
+        self.interpretation_log.append({
+            "question": question[:100],  # Truncate for storage
+            "domain": domain,
+            "used_llm": used_llm,
+            "results_count": results_count,
+            "confidence": confidence
+        })
+        
+        # Keep log size manageable (last 100 questions)
+        if len(self.interpretation_log) > 100:
+            self.interpretation_log = self.interpretation_log[-100:]
+    
+    def get_success_rate(self) -> float:
+        """
+        Calculate success rate of LLM interpretations.
+        
+        Returns:
+            Percentage of LLM interpretations that found results (0.0-1.0)
+        """
+        if self.llm_interpretations == 0:
+            return 0.0
+        return self.results_found_after_llm / self.llm_interpretations
+    
+    def get_avg_confidence(self) -> float:
+        """Get average confidence across all questions."""
+        if self.questions_asked == 0:
+            return 0.0
+        return self.total_confidence / self.questions_asked
+    
+    def get_summary(self) -> Dict[str, Any]:
+        """
+        Get comprehensive metrics summary.
+        
+        Returns:
+            Dictionary with all metrics and analysis
+        """
+        return {
+            "total_questions": self.questions_asked,
+            "llm_interpretations": self.llm_interpretations,
+            "llm_success_rate": self.get_success_rate(),
+            "avg_confidence": self.get_avg_confidence(),
+            "domain_breakdown": {
+                "ai_ml": self.ai_ml_questions,
+                "k8s": self.k8s_questions,
+                "web": self.web_questions,
+                "cli": self.cli_questions,
+                "general": self.general_questions
+            },
+            "llm_usage_rate": self.llm_interpretations / self.questions_asked if self.questions_asked > 0 else 0.0
+        }
+    
+    def print_report(self):
+        """Print human-readable metrics report."""
+        summary = self.get_summary()
+        
+        print("\n" + "="*60)
+        print("Query Interpretation Metrics")
+        print("="*60)
+        
+        print(f"\nQuestions Asked: {summary['total_questions']}")
+        print(f"LLM Interpretations Used: {summary['llm_interpretations']} ({summary['llm_usage_rate']:.1%})")
+        print(f"LLM Success Rate: {summary['llm_success_rate']:.1%}")
+        print(f"Average Confidence: {summary['avg_confidence']:.1%}")
+        
+        print(f"\nDomain Breakdown:")
+        for domain, count in summary['domain_breakdown'].items():
+            if count > 0:
+                print(f"  {domain}: {count}")
+        
+        print("="*60 + "\n")
+
+
+
+