ctrlcode 0.1.0__py3-none-any.whl

ctrlcode/metrics/dashboard.py

@@ -0,0 +1,283 @@
+"""Metrics dashboard for historical learning system."""
+
+import logging
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Optional
+
+from ..storage.history_db import HistoryDB
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class MetricsSummary:
+    """Summary of historical learning metrics."""
+
+    # Session stats
+    total_sessions: int
+    sessions_with_oracle_reuse: int
+    oracle_reuse_rate: float
+
+    # Bug detection stats
+    total_bugs_in_db: int
+    sessions_with_bug_warnings: int
+    bug_detection_rate: float
+
+    # Test stats
+    total_tests: int
+    avg_test_pass_rate: float
+    avg_tests_per_session: float
+
+    # Quality stats
+    avg_oracle_quality: float
+    avg_session_quality: float
+
+    # Efficiency stats
+    estimated_token_savings: int
+    estimated_llm_call_savings: int
+    estimated_time_savings_seconds: float
+
+    # Time range
+    earliest_session: Optional[datetime]
+    latest_session: Optional[datetime]
+
+
+class MetricsDashboard:
+    """Dashboard for tracking and visualizing historical learning metrics.
+
+    Provides insights into:
+    - Oracle reuse rate (how often we avoid fresh derivations)
+    - Bug pattern detection (proactive warnings)
+    - Test deduplication effectiveness
+    - Token and time savings
+    - Quality trends over time
+    """
+
+    def __init__(self, history_db: HistoryDB):
+        """Initialize metrics dashboard.
+
+        Args:
+            history_db: History database to analyze
+        """
+        self.history_db = history_db
+
+    def get_summary(self) -> MetricsSummary:
+        """Get overall metrics summary.
+
+        Returns:
+            MetricsSummary with all key metrics
+        """
+        stats = self.history_db.get_stats()
+
+        # Calculate efficiency metrics
+        oracle_reuse_count = stats.get("oracle_reuse_count", 0)
+        total_sessions = stats.get("total_sessions", 0)
+
+        # Each reused oracle saves ~2500 tokens and ~25 seconds
+        TOKEN_SAVINGS_PER_REUSE = 2500
+        TIME_SAVINGS_PER_REUSE = 25.0
+
+        estimated_token_savings = oracle_reuse_count * TOKEN_SAVINGS_PER_REUSE
+        estimated_llm_call_savings = oracle_reuse_count
+        estimated_time_savings = oracle_reuse_count * TIME_SAVINGS_PER_REUSE
+
+        # Get time range (would need to query DB for actual timestamps)
+        # For now, using None - would be enhanced with actual queries
+        earliest_session = None
+        latest_session = None
+
+        # Calculate averages
+        avg_tests_per_session = (
+            stats.get("total_tests", 0) / total_sessions if total_sessions > 0 else 0
+        )
+
+        # Estimate sessions with bug warnings (would need actual data)
+        # For now, using 0 - would be enhanced with actual tracking
+        sessions_with_bug_warnings = 0
+        bug_detection_rate = 0.0
+
+        return MetricsSummary(
+            total_sessions=total_sessions,
+            sessions_with_oracle_reuse=oracle_reuse_count,
+            oracle_reuse_rate=stats.get("oracle_reuse_rate", 0.0),
+            total_bugs_in_db=stats.get("total_bugs", 0),
+            sessions_with_bug_warnings=sessions_with_bug_warnings,
+            bug_detection_rate=bug_detection_rate,
+            total_tests=stats.get("total_tests", 0),
+            avg_test_pass_rate=stats.get("test_pass_rate", 0.0),
+            avg_tests_per_session=avg_tests_per_session,
+            avg_oracle_quality=stats.get("avg_oracle_quality", 0.0),
+            avg_session_quality=0.0,  # Would calculate from session quality_scores
+            estimated_token_savings=estimated_token_savings,
+            estimated_llm_call_savings=estimated_llm_call_savings,
+            estimated_time_savings_seconds=estimated_time_savings,
+            earliest_session=earliest_session,
+            latest_session=latest_session,
+        )
+
+    def format_report(self, summary: Optional[MetricsSummary] = None) -> str:
+        """Format metrics as human-readable report.
+
+        Args:
+            summary: Optional pre-computed summary (will compute if not provided)
+
+        Returns:
+            Formatted report string
+        """
+        if summary is None:
+            summary = self.get_summary()
+
+        lines = ["=" * 60]
+        lines.append("HISTORICAL LEARNING METRICS DASHBOARD")
+        lines.append("=" * 60)
+        lines.append("")
+
+        # Session stats
+        lines.append("📊 SESSION STATISTICS")
+        lines.append(f"  Total fuzzing sessions: {summary.total_sessions}")
+        lines.append(
+            f"  Sessions with oracle reuse: {summary.sessions_with_oracle_reuse}"
+        )
+        lines.append(f"  Oracle reuse rate: {summary.oracle_reuse_rate:.1%}")
+        lines.append("")
+
+        # Bug detection
+        lines.append("🐛 BUG PATTERN DETECTION")
+        lines.append(f"  Bug patterns in database: {summary.total_bugs_in_db}")
+        lines.append(
+            f"  Sessions with bug warnings: {summary.sessions_with_bug_warnings}"
+        )
+        lines.append(f"  Bug detection rate: {summary.bug_detection_rate:.1%}")
+        lines.append("")
+
+        # Test stats
+        lines.append("🧪 TEST EXECUTION")
+        lines.append(f"  Total tests executed: {summary.total_tests}")
+        lines.append(f"  Average test pass rate: {summary.avg_test_pass_rate:.1%}")
+        lines.append(f"  Avg tests per session: {summary.avg_tests_per_session:.1f}")
+        lines.append("")
+
+        # Quality metrics
+        lines.append("⭐ QUALITY METRICS")
+        lines.append(f"  Average oracle quality: {summary.avg_oracle_quality:.2f}/1.0")
+        lines.append(
+            f"  Average session quality: {summary.avg_session_quality:.2f}/1.0"
+        )
+        lines.append("")
+
+        # Efficiency gains
+        lines.append("⚡ EFFICIENCY GAINS")
+        lines.append(f"  Token savings: {summary.estimated_token_savings:,} tokens")
+        lines.append(
+            f"  LLM calls avoided: {summary.estimated_llm_call_savings} derivations"
+        )
+        lines.append(
+            f"  Time saved: {summary.estimated_time_savings_seconds:.1f} seconds "
+            f"({summary.estimated_time_savings_seconds / 60:.1f} minutes)"
+        )
+        lines.append("")
+
+        # Performance indicators
+        lines.append("📈 KEY PERFORMANCE INDICATORS")
+        if summary.oracle_reuse_rate >= 0.3:
+            lines.append("  ✅ Oracle reuse rate: EXCELLENT (≥30%)")
+        elif summary.oracle_reuse_rate >= 0.2:
+            lines.append("  ✓  Oracle reuse rate: GOOD (≥20%)")
+        elif summary.oracle_reuse_rate >= 0.1:
+            lines.append("  ⚠  Oracle reuse rate: FAIR (≥10%)")
+        else:
+            lines.append(
+                "  ❌ Oracle reuse rate: LOW (<10%) - need more diverse training data"
+            )
+
+        if summary.total_sessions < 10:
+            lines.append(
+                f"  ℹ️  Limited data: Only {summary.total_sessions} sessions - "
+                "metrics will improve with more usage"
+            )
+
+        lines.append("")
+        lines.append("=" * 60)
+
+        return "\n".join(lines)
+
+    def get_trend_data(self, days: int = 30) -> dict:
+        """Get trend data for the last N days.
+
+        Args:
+            days: Number of days to include
+
+        Returns:
+            Dictionary with trend data by day
+        """
+        # This would query the database for time-series data
+        # For now, returning placeholder structure
+        return {
+            "oracle_reuse_by_day": [],
+            "sessions_by_day": [],
+            "quality_by_day": [],
+            "bugs_detected_by_day": [],
+        }
+
+    def calculate_roi(self) -> dict:
+        """Calculate return on investment for historical learning.
+
+        Returns:
+            Dictionary with ROI metrics
+        """
+        summary = self.get_summary()
+
+        # Costs (rough estimates)
+        STORAGE_COST_PER_SESSION = 0.001  # $0.001 per session stored
+        EMBEDDING_COST_PER_SESSION = 0.0  # Free (local SentenceTransformers)
+
+        total_storage_cost = summary.total_sessions * STORAGE_COST_PER_SESSION
+
+        # Benefits (rough estimates)
+        TOKEN_COST_PER_1K = 0.003  # $3 per million tokens = $0.003 per 1K
+        TIME_VALUE_PER_SECOND = 0.01  # Value user's time at ~$36/hour = $0.01/second
+
+        token_savings_value = (summary.estimated_token_savings / 1000) * TOKEN_COST_PER_1K
+        time_savings_value = summary.estimated_time_savings_seconds * TIME_VALUE_PER_SECOND
+
+        total_benefit = token_savings_value + time_savings_value
+        total_cost = total_storage_cost
+
+        roi = ((total_benefit - total_cost) / total_cost * 100) if total_cost > 0 else 0
+
+        return {
+            "total_cost_usd": total_cost,
+            "total_benefit_usd": total_benefit,
+            "net_savings_usd": total_benefit - total_cost,
+            "roi_percentage": roi,
+            "token_savings_value_usd": token_savings_value,
+            "time_savings_value_usd": time_savings_value,
+        }
+
+    def get_top_oracles(self, limit: int = 10) -> list[dict]:
+        """Get top-performing oracles by quality and reuse.
+
+        Args:
+            limit: Number of top oracles to return
+
+        Returns:
+            List of oracle info dicts
+        """
+        # This would query oracle_embeddings table ordered by quality_score
+        # and join with usage counts
+        # For now, returning empty list - would be implemented with actual queries
+        return []
+
+    def get_common_bug_patterns(self, limit: int = 10) -> list[dict]:
+        """Get most common bug patterns.
+
+        Args:
+            limit: Number of patterns to return
+
+        Returns:
+            List of bug pattern info dicts
+        """
+        # This would query bug_patterns table and group by similar patterns
+        # For now, returning empty list - would be implemented with actual queries
+        return []