gitflow-analytics 1.0.1__py3-none-any.whl → 1.3.6__py3-none-any.whl

gitflow_analytics/qualitative/utils/cost_tracker.py

@@ -0,0 +1,345 @@
+"""Cost tracking utilities for LLM usage monitoring."""
+
+import json
+import logging
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Optional
+
+
+@dataclass
+class LLMCall:
+    """Record of a single LLM API call."""
+
+    timestamp: datetime
+    model: str
+    input_tokens: int
+    output_tokens: int
+    processing_time_ms: float
+    estimated_cost: float
+    batch_size: int = 1
+    success: bool = True
+    error_message: Optional[str] = None
+
+
+class CostTracker:
+    """Track and manage LLM API usage costs.
+
+    This class provides cost monitoring, budgeting, and optimization
+    features to keep LLM usage within acceptable limits while
+    maintaining analysis quality.
+    """
+
+    # OpenRouter pricing (approximate, in USD per 1M tokens)
+    MODEL_PRICING = {
+        # Anthropic models
+        "anthropic/claude-3-haiku": {"input": 0.25, "output": 1.25},
+        "anthropic/claude-3-sonnet": {"input": 3.0, "output": 15.0},
+        "anthropic/claude-3-opus": {"input": 15.0, "output": 75.0},
+        # OpenAI models
+        "openai/gpt-3.5-turbo": {"input": 0.5, "output": 1.5},
+        "openai/gpt-4": {"input": 30.0, "output": 60.0},
+        "openai/gpt-4-turbo": {"input": 10.0, "output": 30.0},
+        # Free models (Llama)
+        "meta-llama/llama-3.1-8b-instruct:free": {"input": 0.0, "output": 0.0},
+        "meta-llama/llama-3.1-70b-instruct:free": {"input": 0.0, "output": 0.0},
+        # Other popular models
+        "google/gemini-pro": {"input": 0.5, "output": 1.5},
+        "mistralai/mixtral-8x7b-instruct": {"input": 0.27, "output": 0.27},
+    }
+
+    def __init__(self, cache_dir: Optional[Path] = None, daily_budget: float = 5.0):
+        """Initialize cost tracker.
+
+        Args:
+            cache_dir: Directory to store cost tracking data
+            daily_budget: Maximum daily spending in USD
+        """
+        self.daily_budget = daily_budget
+        self.cache_dir = cache_dir or Path(".qualitative_cache")
+        self.cache_dir.mkdir(exist_ok=True)
+
+        self.cost_file = self.cache_dir / "llm_costs.json"
+        self.calls: list[LLMCall] = []
+        self.logger = logging.getLogger(__name__)
+
+        # Load existing cost data
+        self._load_cost_data()
+
+    def record_call(
+        self,
+        model: str,
+        input_tokens: int,
+        output_tokens: int,
+        processing_time: float,
+        batch_size: int = 1,
+        success: bool = True,
+        error_message: Optional[str] = None,
+    ) -> float:
+        """Record an LLM API call and return estimated cost.
+
+        Args:
+            model: Model name used
+            input_tokens: Number of input tokens
+            output_tokens: Number of output tokens
+            processing_time: Processing time in seconds
+            batch_size: Number of commits processed in this call
+            success: Whether the call was successful
+            error_message: Error message if call failed
+
+        Returns:
+            Estimated cost in USD
+        """
+        estimated_cost = self._calculate_cost(model, input_tokens, output_tokens)
+
+        call = LLMCall(
+            timestamp=datetime.utcnow(),
+            model=model,
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+            processing_time_ms=processing_time * 1000,
+            estimated_cost=estimated_cost,
+            batch_size=batch_size,
+            success=success,
+            error_message=error_message,
+        )
+
+        self.calls.append(call)
+        self._save_cost_data()
+
+        # Log cost information
+        self.logger.info(
+            f"LLM call: {model} | tokens: {input_tokens}+{output_tokens} | "
+            f"cost: ${estimated_cost:.4f} | batch: {batch_size}"
+        )
+
+        return estimated_cost
+
+    def get_daily_spend(self, date: Optional[datetime] = None) -> float:
+        """Get total spending for a specific date.
+
+        Args:
+            date: Date to check (defaults to today)
+
+        Returns:
+            Total spending in USD for the date
+        """
+        if date is None:
+            date = datetime.utcnow()
+
+        start_of_day = date.replace(hour=0, minute=0, second=0, microsecond=0)
+        end_of_day = start_of_day + timedelta(days=1)
+
+        daily_spend = sum(
+            call.estimated_cost
+            for call in self.calls
+            if start_of_day <= call.timestamp < end_of_day and call.success
+        )
+
+        return daily_spend
+
+    def check_budget_remaining(self) -> float:
+        """Check remaining budget for today.
+
+        Returns:
+            Remaining budget in USD (negative if over budget)
+        """
+        daily_spend = self.get_daily_spend()
+        return self.daily_budget - daily_spend
+
+    def can_afford_call(self, model: str, estimated_tokens: int) -> bool:
+        """Check if we can afford an API call within budget.
+
+        Args:
+            model: Model to use
+            estimated_tokens: Estimated total tokens (input + output)
+
+        Returns:
+            True if call is within budget
+        """
+        estimated_cost = self._calculate_cost(model, estimated_tokens // 2, estimated_tokens // 2)
+        remaining_budget = self.check_budget_remaining()
+
+        return remaining_budget >= estimated_cost
+
+    def get_usage_stats(self, days: int = 7) -> dict[str, any]:
+        """Get usage statistics for the last N days.
+
+        Args:
+            days: Number of days to analyze
+
+        Returns:
+            Dictionary with usage statistics
+        """
+        cutoff_date = datetime.utcnow() - timedelta(days=days)
+        recent_calls = [call for call in self.calls if call.timestamp >= cutoff_date]
+
+        if not recent_calls:
+            return {
+                "total_calls": 0,
+                "total_cost": 0.0,
+                "total_tokens": 0,
+                "avg_cost_per_call": 0.0,
+                "model_usage": {},
+                "success_rate": 1.0,
+            }
+
+        successful_calls = [call for call in recent_calls if call.success]
+
+        # Calculate statistics
+        total_cost = sum(call.estimated_cost for call in successful_calls)
+        total_tokens = sum(call.input_tokens + call.output_tokens for call in recent_calls)
+
+        # Model usage breakdown
+        model_usage = {}
+        for call in recent_calls:
+            if call.model not in model_usage:
+                model_usage[call.model] = {"calls": 0, "cost": 0.0, "tokens": 0}
+            model_usage[call.model]["calls"] += 1
+            model_usage[call.model]["cost"] += call.estimated_cost
+            model_usage[call.model]["tokens"] += call.input_tokens + call.output_tokens
+
+        return {
+            "total_calls": len(recent_calls),
+            "successful_calls": len(successful_calls),
+            "total_cost": total_cost,
+            "total_tokens": total_tokens,
+            "avg_cost_per_call": total_cost / len(successful_calls) if successful_calls else 0.0,
+            "model_usage": model_usage,
+            "success_rate": len(successful_calls) / len(recent_calls) if recent_calls else 1.0,
+            "daily_average_cost": total_cost / days,
+        }
+
+    def suggest_cost_optimizations(self) -> list[str]:
+        """Suggest ways to optimize costs based on usage patterns.
+
+        Returns:
+            List of optimization suggestions
+        """
+        suggestions = []
+        stats = self.get_usage_stats(days=7)
+
+        if stats["total_calls"] == 0:
+            return suggestions
+
+        # Check if expensive models are overused
+        model_usage = stats["model_usage"]
+        total_cost = stats["total_cost"]
+
+        expensive_models = ["anthropic/claude-3-opus", "openai/gpt-4"]
+        expensive_usage = sum(
+            model_usage.get(model, {}).get("cost", 0) for model in expensive_models
+        )
+
+        if expensive_usage > total_cost * 0.3:
+            suggestions.append(
+                "Consider using cheaper models (Claude Haiku, GPT-3.5) for routine classification"
+            )
+
+        # Check for free model opportunities
+        free_usage = model_usage.get("meta-llama/llama-3.1-8b-instruct:free", {}).get("calls", 0)
+        if free_usage < stats["total_calls"] * 0.5:
+            suggestions.append(
+                "Increase usage of free Llama models for simple classification tasks"
+            )
+
+        # Check daily spend
+        if self.get_daily_spend() > self.daily_budget * 0.8:
+            suggestions.append(
+                "Approaching daily budget limit - consider increasing NLP confidence threshold"
+            )
+
+        # Check batch efficiency
+        avg_batch_size = sum(call.batch_size for call in self.calls[-50:]) / min(  # Last 50 calls
+            50, len(self.calls)
+        )
+
+        if avg_batch_size < 3:
+            suggestions.append("Increase batch size for LLM calls to improve cost efficiency")
+
+        return suggestions
+
+    def _calculate_cost(self, model: str, input_tokens: int, output_tokens: int) -> float:
+        """Calculate estimated cost for an API call.
+
+        Args:
+            model: Model name
+            input_tokens: Number of input tokens
+            output_tokens: Number of output tokens
+
+        Returns:
+            Estimated cost in USD
+        """
+        if model not in self.MODEL_PRICING:
+            # Default to moderate pricing for unknown models
+            input_price = 1.0
+            output_price = 3.0
+            self.logger.warning(f"Unknown model pricing for {model}, using default rates")
+        else:
+            pricing = self.MODEL_PRICING[model]
+            input_price = pricing["input"]
+            output_price = pricing["output"]
+
+        # Calculate cost (pricing is per 1M tokens)
+        input_cost = (input_tokens / 1_000_000) * input_price
+        output_cost = (output_tokens / 1_000_000) * output_price
+
+        return input_cost + output_cost
+
+    def _load_cost_data(self) -> None:
+        """Load cost tracking data from file."""
+        if not self.cost_file.exists():
+            return
+
+        try:
+            with open(self.cost_file) as f:
+                data = json.load(f)
+
+            self.calls = []
+            for call_data in data.get("calls", []):
+                call = LLMCall(
+                    timestamp=datetime.fromisoformat(call_data["timestamp"]),
+                    model=call_data["model"],
+                    input_tokens=call_data["input_tokens"],
+                    output_tokens=call_data["output_tokens"],
+                    processing_time_ms=call_data["processing_time_ms"],
+                    estimated_cost=call_data["estimated_cost"],
+                    batch_size=call_data.get("batch_size", 1),
+                    success=call_data.get("success", True),
+                    error_message=call_data.get("error_message"),
+                )
+                self.calls.append(call)
+
+        except Exception as e:
+            self.logger.error(f"Failed to load cost data: {e}")
+            self.calls = []
+
+    def _save_cost_data(self) -> None:
+        """Save cost tracking data to file."""
+        try:
+            # Keep only last 1000 calls to prevent file from growing too large
+            recent_calls = self.calls[-1000:]
+
+            data = {
+                "calls": [
+                    {
+                        "timestamp": call.timestamp.isoformat(),
+                        "model": call.model,
+                        "input_tokens": call.input_tokens,
+                        "output_tokens": call.output_tokens,
+                        "processing_time_ms": call.processing_time_ms,
+                        "estimated_cost": call.estimated_cost,
+                        "batch_size": call.batch_size,
+                        "success": call.success,
+                        "error_message": call.error_message,
+                    }
+                    for call in recent_calls
+                ]
+            }
+
+            with open(self.cost_file, "w") as f:
+                json.dump(data, f, indent=2)
+
+        except Exception as e:
+            self.logger.error(f"Failed to save cost data: {e}")

gitflow_analytics/qualitative/utils/metrics.py

@@ -0,0 +1,361 @@
+"""Performance and accuracy metrics for qualitative analysis."""
+
+import logging
+import statistics
+from collections import defaultdict, deque
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+
+
+@dataclass
+class ProcessingMetrics:
+    """Metrics for a single processing operation."""
+
+    operation: str
+    processing_time_ms: float
+    items_processed: int
+    confidence_score: float
+    method_used: str  # 'nlp' or 'llm'
+    timestamp: datetime
+
+    @property
+    def items_per_second(self) -> float:
+        """Calculate processing rate."""
+        if self.processing_time_ms <= 0:
+            return 0.0
+        return (self.items_processed * 1000) / self.processing_time_ms
+
+
+class PerformanceMetrics:
+    """Track and analyze performance metrics for qualitative analysis.
+
+    This class provides comprehensive performance monitoring including
+    processing times, accuracy metrics, cost tracking, and system health
+    indicators for the qualitative analysis pipeline.
+    """
+
+    def __init__(self, max_history: int = 10000):
+        """Initialize performance metrics tracker.
+
+        Args:
+            max_history: Maximum number of metrics to keep in memory
+        """
+        self.max_history = max_history
+        self.logger = logging.getLogger(__name__)
+
+        # Processing metrics
+        self.processing_metrics: deque[ProcessingMetrics] = deque(maxlen=max_history)
+
+        # Method usage tracking
+        self.method_usage = defaultdict(int)
+        self.method_performance = defaultdict(list)
+
+        # Confidence tracking
+        self.confidence_history = deque(maxlen=max_history)
+
+        # Error tracking
+        self.error_counts = defaultdict(int)
+        self.error_history = deque(maxlen=1000)
+
+        # Cache performance
+        self.cache_hits = 0
+        self.cache_misses = 0
+
+        # Quality metrics
+        self.classification_accuracy = deque(maxlen=1000)
+
+    def record_processing(
+        self,
+        operation: str,
+        processing_time_ms: float,
+        items_processed: int,
+        confidence_score: float,
+        method_used: str,
+    ) -> None:
+        """Record a processing operation.
+
+        Args:
+            operation: Type of operation (e.g., 'classification', 'analysis')
+            processing_time_ms: Processing time in milliseconds
+            items_processed: Number of items processed
+            confidence_score: Average confidence score
+            method_used: Method used ('nlp' or 'llm')
+        """
+        metric = ProcessingMetrics(
+            operation=operation,
+            processing_time_ms=processing_time_ms,
+            items_processed=items_processed,
+            confidence_score=confidence_score,
+            method_used=method_used,
+            timestamp=datetime.utcnow(),
+        )
+
+        self.processing_metrics.append(metric)
+        self.method_usage[method_used] += items_processed
+        self.method_performance[method_used].append(processing_time_ms / items_processed)
+        self.confidence_history.append(confidence_score)
+
+    def record_cache_hit(self) -> None:
+        """Record a cache hit."""
+        self.cache_hits += 1
+
+    def record_cache_miss(self) -> None:
+        """Record a cache miss."""
+        self.cache_misses += 1
+
+    def record_error(self, error_type: str, error_message: str) -> None:
+        """Record an error occurrence.
+
+        Args:
+            error_type: Type of error
+            error_message: Error message
+        """
+        self.error_counts[error_type] += 1
+        self.error_history.append(
+            {"type": error_type, "message": error_message, "timestamp": datetime.utcnow()}
+        )
+
+    def record_classification_accuracy(self, accuracy: float) -> None:
+        """Record classification accuracy measurement.
+
+        Args:
+            accuracy: Accuracy score (0.0 to 1.0)
+        """
+        self.classification_accuracy.append(accuracy)
+
+    def get_processing_stats(self, hours: int = 24) -> dict[str, any]:
+        """Get processing statistics for the last N hours.
+
+        Args:
+            hours: Number of hours to analyze
+
+        Returns:
+            Dictionary with processing statistics
+        """
+        cutoff_time = datetime.utcnow() - timedelta(hours=hours)
+        recent_metrics = [m for m in self.processing_metrics if m.timestamp >= cutoff_time]
+
+        if not recent_metrics:
+            return {
+                "total_operations": 0,
+                "total_items_processed": 0,
+                "avg_processing_time_ms": 0.0,
+                "avg_items_per_second": 0.0,
+                "avg_confidence": 0.0,
+                "method_breakdown": {},
+                "cache_hit_rate": 0.0,
+            }
+
+        # Calculate statistics
+        total_items = sum(m.items_processed for m in recent_metrics)
+        total_time = sum(m.processing_time_ms for m in recent_metrics)
+
+        avg_processing_time = total_time / len(recent_metrics)
+        avg_items_per_second = statistics.mean([m.items_per_second for m in recent_metrics])
+        avg_confidence = statistics.mean([m.confidence_score for m in recent_metrics])
+
+        # Method breakdown
+        method_breakdown = {}
+        for method in ["nlp", "llm"]:
+            method_metrics = [m for m in recent_metrics if m.method_used == method]
+            if method_metrics:
+                method_items = sum(m.items_processed for m in method_metrics)
+                method_breakdown[method] = {
+                    "items_processed": method_items,
+                    "percentage": (method_items / total_items) * 100 if total_items > 0 else 0,
+                    "avg_confidence": statistics.mean([m.confidence_score for m in method_metrics]),
+                    "avg_processing_time_ms": statistics.mean(
+                        [m.processing_time_ms for m in method_metrics]
+                    ),
+                }
+
+        # Cache hit rate
+        total_cache_requests = self.cache_hits + self.cache_misses
+        cache_hit_rate = (
+            (self.cache_hits / total_cache_requests) if total_cache_requests > 0 else 0.0
+        )
+
+        return {
+            "total_operations": len(recent_metrics),
+            "total_items_processed": total_items,
+            "avg_processing_time_ms": avg_processing_time,
+            "avg_items_per_second": avg_items_per_second,
+            "avg_confidence": avg_confidence,
+            "method_breakdown": method_breakdown,
+            "cache_hit_rate": cache_hit_rate,
+            "time_period_hours": hours,
+        }
+
+    def get_quality_metrics(self) -> dict[str, any]:
+        """Get quality and accuracy metrics.
+
+        Returns:
+            Dictionary with quality metrics
+        """
+        if not self.confidence_history:
+            return {
+                "avg_confidence": 0.0,
+                "confidence_distribution": {},
+                "classification_accuracy": 0.0,
+                "quality_trend": "stable",
+            }
+
+        # Confidence statistics
+        confidences = list(self.confidence_history)
+        avg_confidence = statistics.mean(confidences)
+
+        # Confidence distribution
+        confidence_buckets = {
+            "high (>0.8)": sum(1 for c in confidences if c > 0.8),
+            "medium (0.6-0.8)": sum(1 for c in confidences if 0.6 <= c <= 0.8),
+            "low (<0.6)": sum(1 for c in confidences if c < 0.6),
+        }
+
+        # Quality trend (comparing recent vs. older metrics)
+        if len(confidences) >= 100:
+            recent_confidence = statistics.mean(confidences[-50:])
+            older_confidence = statistics.mean(confidences[-100:-50])
+
+            if recent_confidence > older_confidence + 0.05:
+                quality_trend = "improving"
+            elif recent_confidence < older_confidence - 0.05:
+                quality_trend = "declining"
+            else:
+                quality_trend = "stable"
+        else:
+            quality_trend = "insufficient_data"
+
+        # Classification accuracy
+        avg_accuracy = (
+            statistics.mean(self.classification_accuracy) if self.classification_accuracy else 0.0
+        )
+
+        return {
+            "avg_confidence": avg_confidence,
+            "confidence_distribution": confidence_buckets,
+            "classification_accuracy": avg_accuracy,
+            "quality_trend": quality_trend,
+            "total_samples": len(confidences),
+        }
+
+    def get_error_analysis(self) -> dict[str, any]:
+        """Get error analysis and system health metrics.
+
+        Returns:
+            Dictionary with error analysis
+        """
+        # Recent errors (last 24 hours)
+        cutoff_time = datetime.utcnow() - timedelta(hours=24)
+        recent_errors = [error for error in self.error_history if error["timestamp"] >= cutoff_time]
+
+        # Error type breakdown
+        error_type_counts = defaultdict(int)
+        for error in recent_errors:
+            error_type_counts[error["type"]] += 1
+
+        # Total operations for error rate calculation
+        total_operations = len([m for m in self.processing_metrics if m.timestamp >= cutoff_time])
+
+        error_rate = len(recent_errors) / total_operations if total_operations > 0 else 0.0
+
+        return {
+            "total_errors_24h": len(recent_errors),
+            "error_rate": error_rate,
+            "error_types": dict(error_type_counts),
+            "most_common_error": (
+                max(error_type_counts.keys(), key=error_type_counts.get)
+                if error_type_counts
+                else None
+            ),
+            "system_health": (
+                "healthy" if error_rate < 0.01 else "degraded" if error_rate < 0.05 else "unhealthy"
+            ),
+        }
+
+    def get_performance_alerts(self) -> list[str]:
+        """Get performance alerts and recommendations.
+
+        Returns:
+            List of alert messages
+        """
+        alerts = []
+
+        # Check recent performance
+        stats = self.get_processing_stats(hours=1)
+        quality = self.get_quality_metrics()
+        errors = self.get_error_analysis()
+
+        # Processing speed alerts
+        if stats["avg_items_per_second"] < 50:  # Less than 50 items/second
+            alerts.append("Processing speed below target (< 50 items/second)")
+
+        # Confidence alerts
+        if quality["avg_confidence"] < 0.6:
+            alerts.append("Average confidence below threshold (< 0.6)")
+
+        # Method balance alerts
+        if "llm" in stats["method_breakdown"]:
+            llm_percentage = stats["method_breakdown"]["llm"]["percentage"]
+            if llm_percentage > 20:  # More than 20% using LLM
+                alerts.append(
+                    f"High LLM usage ({llm_percentage:.1f}%) - consider tuning NLP thresholds"
+                )
+
+        # Error rate alerts
+        if errors["error_rate"] > 0.05:  # More than 5% error rate
+            alerts.append(f"High error rate ({errors['error_rate']:.1%})")
+
+        # Cache performance alerts
+        if stats["cache_hit_rate"] < 0.3:  # Less than 30% cache hit rate
+            alerts.append("Low cache hit rate - pattern learning may be ineffective")
+
+        # Quality trend alerts
+        if quality["quality_trend"] == "declining":
+            alerts.append("Quality trend declining - review recent changes")
+
+        return alerts
+
+    def get_optimization_suggestions(self) -> list[str]:
+        """Get optimization suggestions based on metrics.
+
+        Returns:
+            List of optimization suggestions
+        """
+        suggestions = []
+
+        stats = self.get_processing_stats(hours=24)
+        quality = self.get_quality_metrics()
+
+        # Performance optimizations
+        if stats["avg_items_per_second"] < 100:
+            suggestions.append("Consider increasing batch size or enabling parallel processing")
+
+        # Method optimization
+        method_breakdown = stats["method_breakdown"]
+        if "llm" in method_breakdown and method_breakdown["llm"]["percentage"] > 15:
+            suggestions.append(
+                "High LLM usage - consider lowering confidence threshold or improving NLP patterns"
+            )
+
+        if "nlp" in method_breakdown and method_breakdown["nlp"]["avg_confidence"] < 0.7:
+            suggestions.append("NLP confidence low - consider updating classification patterns")
+
+        # Quality optimizations
+        if quality["avg_confidence"] < 0.7:
+            suggestions.append(
+                "Overall confidence low - review classification accuracy and update models"
+            )
+
+        confidence_dist = quality["confidence_distribution"]
+        if confidence_dist.get("low (<0.6)", 0) > confidence_dist.get("high (>0.8)", 0):
+            suggestions.append(
+                "Many low-confidence predictions - consider additional training data"
+            )
+
+        # Cache optimizations
+        if stats["cache_hit_rate"] < 0.5:
+            suggestions.append(
+                "Low cache hit rate - increase cache size or improve pattern matching"
+            )
+
+        return suggestions