gitflow-analytics 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

gitflow_analytics/qualitative/utils/cost_tracker.py

@@ -0,0 +1,343 @@
+"""Cost tracking utilities for LLM usage monitoring."""
+
+import logging
+from datetime import datetime, timedelta
+from typing import Dict, List, Optional
+from dataclasses import dataclass
+from pathlib import Path
+import json
+
+
+@dataclass
+class LLMCall:
+    """Record of a single LLM API call."""
+    
+    timestamp: datetime
+    model: str
+    input_tokens: int
+    output_tokens: int
+    processing_time_ms: float
+    estimated_cost: float
+    batch_size: int = 1
+    success: bool = True
+    error_message: Optional[str] = None
+
+
+class CostTracker:
+    """Track and manage LLM API usage costs.
+    
+    This class provides cost monitoring, budgeting, and optimization
+    features to keep LLM usage within acceptable limits while
+    maintaining analysis quality.
+    """
+    
+    # OpenRouter pricing (approximate, in USD per 1M tokens)
+    MODEL_PRICING = {
+        # Anthropic models
+        'anthropic/claude-3-haiku': {'input': 0.25, 'output': 1.25},
+        'anthropic/claude-3-sonnet': {'input': 3.0, 'output': 15.0},
+        'anthropic/claude-3-opus': {'input': 15.0, 'output': 75.0},
+        
+        # OpenAI models
+        'openai/gpt-3.5-turbo': {'input': 0.5, 'output': 1.5},
+        'openai/gpt-4': {'input': 30.0, 'output': 60.0},
+        'openai/gpt-4-turbo': {'input': 10.0, 'output': 30.0},
+        
+        # Free models (Llama)
+        'meta-llama/llama-3.1-8b-instruct:free': {'input': 0.0, 'output': 0.0},
+        'meta-llama/llama-3.1-70b-instruct:free': {'input': 0.0, 'output': 0.0},
+        
+        # Other popular models
+        'google/gemini-pro': {'input': 0.5, 'output': 1.5},
+        'mistralai/mixtral-8x7b-instruct': {'input': 0.27, 'output': 0.27},
+    }
+    
+    def __init__(self, cache_dir: Optional[Path] = None, daily_budget: float = 5.0):
+        """Initialize cost tracker.
+        
+        Args:
+            cache_dir: Directory to store cost tracking data
+            daily_budget: Maximum daily spending in USD
+        """
+        self.daily_budget = daily_budget
+        self.cache_dir = cache_dir or Path(".qualitative_cache")
+        self.cache_dir.mkdir(exist_ok=True)
+        
+        self.cost_file = self.cache_dir / "llm_costs.json"
+        self.calls: List[LLMCall] = []
+        self.logger = logging.getLogger(__name__)
+        
+        # Load existing cost data
+        self._load_cost_data()
+        
+    def record_call(self, model: str, input_tokens: int, output_tokens: int,
+                   processing_time: float, batch_size: int = 1,
+                   success: bool = True, error_message: Optional[str] = None) -> float:
+        """Record an LLM API call and return estimated cost.
+        
+        Args:
+            model: Model name used
+            input_tokens: Number of input tokens
+            output_tokens: Number of output tokens  
+            processing_time: Processing time in seconds
+            batch_size: Number of commits processed in this call
+            success: Whether the call was successful
+            error_message: Error message if call failed
+            
+        Returns:
+            Estimated cost in USD
+        """
+        estimated_cost = self._calculate_cost(model, input_tokens, output_tokens)
+        
+        call = LLMCall(
+            timestamp=datetime.utcnow(),
+            model=model,
+            input_tokens=input_tokens,
+            output_tokens=output_tokens,
+            processing_time_ms=processing_time * 1000,
+            estimated_cost=estimated_cost,
+            batch_size=batch_size,
+            success=success,
+            error_message=error_message
+        )
+        
+        self.calls.append(call)
+        self._save_cost_data()
+        
+        # Log cost information
+        self.logger.info(
+            f"LLM call: {model} | tokens: {input_tokens}+{output_tokens} | "
+            f"cost: ${estimated_cost:.4f} | batch: {batch_size}"
+        )
+        
+        return estimated_cost
+        
+    def get_daily_spend(self, date: Optional[datetime] = None) -> float:
+        """Get total spending for a specific date.
+        
+        Args:
+            date: Date to check (defaults to today)
+            
+        Returns:
+            Total spending in USD for the date
+        """
+        if date is None:
+            date = datetime.utcnow()
+            
+        start_of_day = date.replace(hour=0, minute=0, second=0, microsecond=0)
+        end_of_day = start_of_day + timedelta(days=1)
+        
+        daily_spend = sum(
+            call.estimated_cost for call in self.calls
+            if start_of_day <= call.timestamp < end_of_day and call.success
+        )
+        
+        return daily_spend
+        
+    def check_budget_remaining(self) -> float:
+        """Check remaining budget for today.
+        
+        Returns:
+            Remaining budget in USD (negative if over budget)
+        """
+        daily_spend = self.get_daily_spend()
+        return self.daily_budget - daily_spend
+        
+    def can_afford_call(self, model: str, estimated_tokens: int) -> bool:
+        """Check if we can afford an API call within budget.
+        
+        Args:
+            model: Model to use
+            estimated_tokens: Estimated total tokens (input + output)
+            
+        Returns:
+            True if call is within budget
+        """
+        estimated_cost = self._calculate_cost(model, estimated_tokens // 2, estimated_tokens // 2)
+        remaining_budget = self.check_budget_remaining()
+        
+        return remaining_budget >= estimated_cost
+        
+    def get_usage_stats(self, days: int = 7) -> Dict[str, any]:
+        """Get usage statistics for the last N days.
+        
+        Args:
+            days: Number of days to analyze
+            
+        Returns:
+            Dictionary with usage statistics
+        """
+        cutoff_date = datetime.utcnow() - timedelta(days=days)
+        recent_calls = [call for call in self.calls if call.timestamp >= cutoff_date]
+        
+        if not recent_calls:
+            return {
+                'total_calls': 0,
+                'total_cost': 0.0,
+                'total_tokens': 0,
+                'avg_cost_per_call': 0.0,
+                'model_usage': {},
+                'success_rate': 1.0
+            }
+        
+        successful_calls = [call for call in recent_calls if call.success]
+        
+        # Calculate statistics
+        total_cost = sum(call.estimated_cost for call in successful_calls)
+        total_tokens = sum(call.input_tokens + call.output_tokens for call in recent_calls)
+        
+        # Model usage breakdown
+        model_usage = {}
+        for call in recent_calls:
+            if call.model not in model_usage:
+                model_usage[call.model] = {'calls': 0, 'cost': 0.0, 'tokens': 0}
+            model_usage[call.model]['calls'] += 1
+            model_usage[call.model]['cost'] += call.estimated_cost
+            model_usage[call.model]['tokens'] += call.input_tokens + call.output_tokens
+            
+        return {
+            'total_calls': len(recent_calls),
+            'successful_calls': len(successful_calls),
+            'total_cost': total_cost,
+            'total_tokens': total_tokens,
+            'avg_cost_per_call': total_cost / len(successful_calls) if successful_calls else 0.0,
+            'model_usage': model_usage,
+            'success_rate': len(successful_calls) / len(recent_calls) if recent_calls else 1.0,
+            'daily_average_cost': total_cost / days,
+        }
+        
+    def suggest_cost_optimizations(self) -> List[str]:
+        """Suggest ways to optimize costs based on usage patterns.
+        
+        Returns:
+            List of optimization suggestions
+        """
+        suggestions = []
+        stats = self.get_usage_stats(days=7)
+        
+        if stats['total_calls'] == 0:
+            return suggestions
+            
+        # Check if expensive models are overused
+        model_usage = stats['model_usage']
+        total_cost = stats['total_cost']
+        
+        expensive_models = ['anthropic/claude-3-opus', 'openai/gpt-4']
+        expensive_usage = sum(
+            model_usage.get(model, {}).get('cost', 0) 
+            for model in expensive_models
+        )
+        
+        if expensive_usage > total_cost * 0.3:
+            suggestions.append(
+                "Consider using cheaper models (Claude Haiku, GPT-3.5) for routine classification"
+            )
+            
+        # Check for free model opportunities
+        free_usage = model_usage.get('meta-llama/llama-3.1-8b-instruct:free', {}).get('calls', 0)
+        if free_usage < stats['total_calls'] * 0.5:
+            suggestions.append(
+                "Increase usage of free Llama models for simple classification tasks"
+            )
+            
+        # Check daily spend
+        if self.get_daily_spend() > self.daily_budget * 0.8:
+            suggestions.append(
+                "Approaching daily budget limit - consider increasing NLP confidence threshold"
+            )
+            
+        # Check batch efficiency
+        avg_batch_size = sum(
+            call.batch_size for call in self.calls[-50:]  # Last 50 calls
+        ) / min(50, len(self.calls))
+        
+        if avg_batch_size < 3:
+            suggestions.append(
+                "Increase batch size for LLM calls to improve cost efficiency"
+            )
+            
+        return suggestions
+        
+    def _calculate_cost(self, model: str, input_tokens: int, output_tokens: int) -> float:
+        """Calculate estimated cost for an API call.
+        
+        Args:
+            model: Model name
+            input_tokens: Number of input tokens
+            output_tokens: Number of output tokens
+            
+        Returns:
+            Estimated cost in USD
+        """
+        if model not in self.MODEL_PRICING:
+            # Default to moderate pricing for unknown models
+            input_price = 1.0
+            output_price = 3.0
+            self.logger.warning(f"Unknown model pricing for {model}, using default rates")
+        else:
+            pricing = self.MODEL_PRICING[model]
+            input_price = pricing['input']
+            output_price = pricing['output']
+            
+        # Calculate cost (pricing is per 1M tokens)
+        input_cost = (input_tokens / 1_000_000) * input_price
+        output_cost = (output_tokens / 1_000_000) * output_price
+        
+        return input_cost + output_cost
+        
+    def _load_cost_data(self) -> None:
+        """Load cost tracking data from file."""
+        if not self.cost_file.exists():
+            return
+            
+        try:
+            with open(self.cost_file, 'r') as f:
+                data = json.load(f)
+                
+            self.calls = []
+            for call_data in data.get('calls', []):
+                call = LLMCall(
+                    timestamp=datetime.fromisoformat(call_data['timestamp']),
+                    model=call_data['model'],
+                    input_tokens=call_data['input_tokens'],
+                    output_tokens=call_data['output_tokens'],
+                    processing_time_ms=call_data['processing_time_ms'],
+                    estimated_cost=call_data['estimated_cost'],
+                    batch_size=call_data.get('batch_size', 1),
+                    success=call_data.get('success', True),
+                    error_message=call_data.get('error_message')
+                )
+                self.calls.append(call)
+                
+        except Exception as e:
+            self.logger.error(f"Failed to load cost data: {e}")
+            self.calls = []
+            
+    def _save_cost_data(self) -> None:
+        """Save cost tracking data to file."""
+        try:
+            # Keep only last 1000 calls to prevent file from growing too large
+            recent_calls = self.calls[-1000:]
+            
+            data = {
+                'calls': [
+                    {
+                        'timestamp': call.timestamp.isoformat(),
+                        'model': call.model,
+                        'input_tokens': call.input_tokens,
+                        'output_tokens': call.output_tokens,
+                        'processing_time_ms': call.processing_time_ms,
+                        'estimated_cost': call.estimated_cost,
+                        'batch_size': call.batch_size,
+                        'success': call.success,
+                        'error_message': call.error_message
+                    }
+                    for call in recent_calls
+                ]
+            }
+            
+            with open(self.cost_file, 'w') as f:
+                json.dump(data, f, indent=2)
+                
+        except Exception as e:
+            self.logger.error(f"Failed to save cost data: {e}")

gitflow_analytics/qualitative/utils/metrics.py

@@ -0,0 +1,347 @@
+"""Performance and accuracy metrics for qualitative analysis."""
+
+import logging
+import time
+from collections import defaultdict, deque
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Dict, List, Optional, Tuple
+import statistics
+
+
+@dataclass 
+class ProcessingMetrics:
+    """Metrics for a single processing operation."""
+    
+    operation: str
+    processing_time_ms: float
+    items_processed: int
+    confidence_score: float
+    method_used: str  # 'nlp' or 'llm'
+    timestamp: datetime
+    
+    @property
+    def items_per_second(self) -> float:
+        """Calculate processing rate."""
+        if self.processing_time_ms <= 0:
+            return 0.0
+        return (self.items_processed * 1000) / self.processing_time_ms
+
+
+class PerformanceMetrics:
+    """Track and analyze performance metrics for qualitative analysis.
+    
+    This class provides comprehensive performance monitoring including
+    processing times, accuracy metrics, cost tracking, and system health
+    indicators for the qualitative analysis pipeline.
+    """
+    
+    def __init__(self, max_history: int = 10000):
+        """Initialize performance metrics tracker.
+        
+        Args:
+            max_history: Maximum number of metrics to keep in memory
+        """
+        self.max_history = max_history
+        self.logger = logging.getLogger(__name__)
+        
+        # Processing metrics
+        self.processing_metrics: deque[ProcessingMetrics] = deque(maxlen=max_history)
+        
+        # Method usage tracking
+        self.method_usage = defaultdict(int)
+        self.method_performance = defaultdict(list)
+        
+        # Confidence tracking
+        self.confidence_history = deque(maxlen=max_history)
+        
+        # Error tracking
+        self.error_counts = defaultdict(int)
+        self.error_history = deque(maxlen=1000)
+        
+        # Cache performance
+        self.cache_hits = 0
+        self.cache_misses = 0
+        
+        # Quality metrics
+        self.classification_accuracy = deque(maxlen=1000)
+        
+    def record_processing(self, operation: str, processing_time_ms: float,
+                         items_processed: int, confidence_score: float,
+                         method_used: str) -> None:
+        """Record a processing operation.
+        
+        Args:
+            operation: Type of operation (e.g., 'classification', 'analysis')
+            processing_time_ms: Processing time in milliseconds
+            items_processed: Number of items processed
+            confidence_score: Average confidence score
+            method_used: Method used ('nlp' or 'llm')
+        """
+        metric = ProcessingMetrics(
+            operation=operation,
+            processing_time_ms=processing_time_ms,
+            items_processed=items_processed,
+            confidence_score=confidence_score,
+            method_used=method_used,
+            timestamp=datetime.utcnow()
+        )
+        
+        self.processing_metrics.append(metric)
+        self.method_usage[method_used] += items_processed
+        self.method_performance[method_used].append(processing_time_ms / items_processed)
+        self.confidence_history.append(confidence_score)
+        
+    def record_cache_hit(self) -> None:
+        """Record a cache hit."""
+        self.cache_hits += 1
+        
+    def record_cache_miss(self) -> None:
+        """Record a cache miss."""
+        self.cache_misses += 1
+        
+    def record_error(self, error_type: str, error_message: str) -> None:
+        """Record an error occurrence.
+        
+        Args:
+            error_type: Type of error
+            error_message: Error message
+        """
+        self.error_counts[error_type] += 1
+        self.error_history.append({
+            'type': error_type,
+            'message': error_message,
+            'timestamp': datetime.utcnow()
+        })
+        
+    def record_classification_accuracy(self, accuracy: float) -> None:
+        """Record classification accuracy measurement.
+        
+        Args:
+            accuracy: Accuracy score (0.0 to 1.0)
+        """
+        self.classification_accuracy.append(accuracy)
+        
+    def get_processing_stats(self, hours: int = 24) -> Dict[str, any]:
+        """Get processing statistics for the last N hours.
+        
+        Args:
+            hours: Number of hours to analyze
+            
+        Returns:
+            Dictionary with processing statistics
+        """
+        cutoff_time = datetime.utcnow() - timedelta(hours=hours)
+        recent_metrics = [
+            m for m in self.processing_metrics 
+            if m.timestamp >= cutoff_time
+        ]
+        
+        if not recent_metrics:
+            return {
+                'total_operations': 0,
+                'total_items_processed': 0,
+                'avg_processing_time_ms': 0.0,
+                'avg_items_per_second': 0.0,
+                'avg_confidence': 0.0,
+                'method_breakdown': {},
+                'cache_hit_rate': 0.0
+            }
+            
+        # Calculate statistics
+        total_items = sum(m.items_processed for m in recent_metrics)
+        total_time = sum(m.processing_time_ms for m in recent_metrics)
+        
+        avg_processing_time = total_time / len(recent_metrics)
+        avg_items_per_second = statistics.mean([m.items_per_second for m in recent_metrics])
+        avg_confidence = statistics.mean([m.confidence_score for m in recent_metrics])
+        
+        # Method breakdown
+        method_breakdown = {}
+        for method in ['nlp', 'llm']:
+            method_metrics = [m for m in recent_metrics if m.method_used == method]
+            if method_metrics:
+                method_items = sum(m.items_processed for m in method_metrics)
+                method_breakdown[method] = {
+                    'items_processed': method_items,
+                    'percentage': (method_items / total_items) * 100 if total_items > 0 else 0,
+                    'avg_confidence': statistics.mean([m.confidence_score for m in method_metrics]),
+                    'avg_processing_time_ms': statistics.mean([m.processing_time_ms for m in method_metrics])
+                }
+        
+        # Cache hit rate
+        total_cache_requests = self.cache_hits + self.cache_misses
+        cache_hit_rate = (self.cache_hits / total_cache_requests) if total_cache_requests > 0 else 0.0
+        
+        return {
+            'total_operations': len(recent_metrics),
+            'total_items_processed': total_items,
+            'avg_processing_time_ms': avg_processing_time,
+            'avg_items_per_second': avg_items_per_second,
+            'avg_confidence': avg_confidence,
+            'method_breakdown': method_breakdown,
+            'cache_hit_rate': cache_hit_rate,
+            'time_period_hours': hours
+        }
+        
+    def get_quality_metrics(self) -> Dict[str, any]:
+        """Get quality and accuracy metrics.
+        
+        Returns:
+            Dictionary with quality metrics
+        """
+        if not self.confidence_history:
+            return {
+                'avg_confidence': 0.0,
+                'confidence_distribution': {},
+                'classification_accuracy': 0.0,
+                'quality_trend': 'stable'
+            }
+            
+        # Confidence statistics
+        confidences = list(self.confidence_history)
+        avg_confidence = statistics.mean(confidences)
+        
+        # Confidence distribution
+        confidence_buckets = {
+            'high (>0.8)': sum(1 for c in confidences if c > 0.8),
+            'medium (0.6-0.8)': sum(1 for c in confidences if 0.6 <= c <= 0.8),
+            'low (<0.6)': sum(1 for c in confidences if c < 0.6)
+        }
+        
+        # Quality trend (comparing recent vs. older metrics)
+        if len(confidences) >= 100:
+            recent_confidence = statistics.mean(confidences[-50:])
+            older_confidence = statistics.mean(confidences[-100:-50])
+            
+            if recent_confidence > older_confidence + 0.05:
+                quality_trend = 'improving'
+            elif recent_confidence < older_confidence - 0.05:
+                quality_trend = 'declining'
+            else:
+                quality_trend = 'stable'
+        else:
+            quality_trend = 'insufficient_data'
+            
+        # Classification accuracy
+        avg_accuracy = statistics.mean(self.classification_accuracy) if self.classification_accuracy else 0.0
+        
+        return {
+            'avg_confidence': avg_confidence,
+            'confidence_distribution': confidence_buckets,
+            'classification_accuracy': avg_accuracy,
+            'quality_trend': quality_trend,
+            'total_samples': len(confidences)
+        }
+        
+    def get_error_analysis(self) -> Dict[str, any]:
+        """Get error analysis and system health metrics.
+        
+        Returns:
+            Dictionary with error analysis
+        """
+        # Recent errors (last 24 hours)
+        cutoff_time = datetime.utcnow() - timedelta(hours=24)
+        recent_errors = [
+            error for error in self.error_history
+            if error['timestamp'] >= cutoff_time
+        ]
+        
+        # Error type breakdown
+        error_type_counts = defaultdict(int)
+        for error in recent_errors:
+            error_type_counts[error['type']] += 1
+            
+        # Total operations for error rate calculation
+        total_operations = len([
+            m for m in self.processing_metrics
+            if m.timestamp >= cutoff_time
+        ])
+        
+        error_rate = len(recent_errors) / total_operations if total_operations > 0 else 0.0
+        
+        return {
+            'total_errors_24h': len(recent_errors),
+            'error_rate': error_rate,
+            'error_types': dict(error_type_counts),
+            'most_common_error': max(error_type_counts.keys(), key=error_type_counts.get) if error_type_counts else None,
+            'system_health': 'healthy' if error_rate < 0.01 else 'degraded' if error_rate < 0.05 else 'unhealthy'
+        }
+        
+    def get_performance_alerts(self) -> List[str]:
+        """Get performance alerts and recommendations.
+        
+        Returns:
+            List of alert messages
+        """
+        alerts = []
+        
+        # Check recent performance
+        stats = self.get_processing_stats(hours=1)
+        quality = self.get_quality_metrics()
+        errors = self.get_error_analysis()
+        
+        # Processing speed alerts
+        if stats['avg_items_per_second'] < 50:  # Less than 50 items/second
+            alerts.append("Processing speed below target (< 50 items/second)")
+            
+        # Confidence alerts
+        if quality['avg_confidence'] < 0.6:
+            alerts.append("Average confidence below threshold (< 0.6)")
+            
+        # Method balance alerts
+        if 'llm' in stats['method_breakdown']:
+            llm_percentage = stats['method_breakdown']['llm']['percentage']
+            if llm_percentage > 20:  # More than 20% using LLM
+                alerts.append(f"High LLM usage ({llm_percentage:.1f}%) - consider tuning NLP thresholds")
+                
+        # Error rate alerts
+        if errors['error_rate'] > 0.05:  # More than 5% error rate
+            alerts.append(f"High error rate ({errors['error_rate']:.1%})")
+            
+        # Cache performance alerts
+        if stats['cache_hit_rate'] < 0.3:  # Less than 30% cache hit rate
+            alerts.append("Low cache hit rate - pattern learning may be ineffective")
+            
+        # Quality trend alerts
+        if quality['quality_trend'] == 'declining':
+            alerts.append("Quality trend declining - review recent changes")
+            
+        return alerts
+        
+    def get_optimization_suggestions(self) -> List[str]:
+        """Get optimization suggestions based on metrics.
+        
+        Returns:
+            List of optimization suggestions
+        """
+        suggestions = []
+        
+        stats = self.get_processing_stats(hours=24)
+        quality = self.get_quality_metrics()
+        
+        # Performance optimizations
+        if stats['avg_items_per_second'] < 100:
+            suggestions.append("Consider increasing batch size or enabling parallel processing")
+            
+        # Method optimization
+        method_breakdown = stats['method_breakdown']
+        if 'llm' in method_breakdown and method_breakdown['llm']['percentage'] > 15:
+            suggestions.append("High LLM usage - consider lowering confidence threshold or improving NLP patterns")
+            
+        if 'nlp' in method_breakdown and method_breakdown['nlp']['avg_confidence'] < 0.7:
+            suggestions.append("NLP confidence low - consider updating classification patterns")
+            
+        # Quality optimizations
+        if quality['avg_confidence'] < 0.7:
+            suggestions.append("Overall confidence low - review classification accuracy and update models")
+            
+        confidence_dist = quality['confidence_distribution']
+        if confidence_dist.get('low (<0.6)', 0) > confidence_dist.get('high (>0.8)', 0):
+            suggestions.append("Many low-confidence predictions - consider additional training data")
+            
+        # Cache optimizations
+        if stats['cache_hit_rate'] < 0.5:
+            suggestions.append("Low cache hit rate - increase cache size or improve pattern matching")
+            
+        return suggestions