gitflow-analytics 1.0.0__py3-none-any.whl → 1.0.3__py3-none-any.whl

gitflow_analytics/qualitative/models/schemas.py

@@ -0,0 +1,272 @@
+"""Data models and configuration schemas for qualitative analysis."""
+
+import time
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+
+@dataclass
+class QualitativeCommitData:
+    """Enhanced commit data with qualitative analysis results.
+    
+    This class extends basic commit information with semantic analysis results
+    including change type, business domain, risk assessment, and processing metadata.
+    """
+    
+    # Existing commit data from GitFlow Analytics
+    hash: str
+    message: str
+    author_name: str
+    author_email: str
+    timestamp: datetime
+    files_changed: List[str]
+    insertions: int
+    deletions: int
+    
+    # New qualitative analysis fields
+    change_type: str  # feature|bugfix|refactor|docs|test|chore|security|hotfix|config
+    change_type_confidence: float  # 0.0-1.0
+    business_domain: str  # frontend|backend|database|infrastructure|mobile|devops|unknown
+    domain_confidence: float  # 0.0-1.0
+    risk_level: str  # low|medium|high|critical
+    risk_factors: List[str]  # List of identified risk factors
+    intent_signals: Dict[str, Any]  # Intent analysis results
+    collaboration_patterns: Dict[str, Any]  # Team interaction patterns
+    technical_context: Dict[str, Any]  # Technical context information
+    
+    # Processing metadata
+    processing_method: str  # 'nlp' or 'llm'
+    processing_time_ms: float
+    confidence_score: float  # Overall confidence in analysis
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return {
+            'hash': self.hash,
+            'message': self.message,
+            'author_name': self.author_name,
+            'author_email': self.author_email,
+            'timestamp': self.timestamp.isoformat(),
+            'files_changed': self.files_changed,
+            'insertions': self.insertions,
+            'deletions': self.deletions,
+            'change_type': self.change_type,
+            'change_type_confidence': self.change_type_confidence,
+            'business_domain': self.business_domain,
+            'domain_confidence': self.domain_confidence,
+            'risk_level': self.risk_level,
+            'risk_factors': self.risk_factors,
+            'intent_signals': self.intent_signals,
+            'collaboration_patterns': self.collaboration_patterns,
+            'technical_context': self.technical_context,
+            'processing_method': self.processing_method,
+            'processing_time_ms': self.processing_time_ms,
+            'confidence_score': self.confidence_score,
+        }
+
+
+@dataclass
+class ChangeTypeConfig:
+    """Configuration for change type classification."""
+    
+    min_confidence: float = 0.7
+    semantic_weight: float = 0.6  # Weight for semantic features
+    file_pattern_weight: float = 0.4  # Weight for file pattern signals
+    enable_custom_patterns: bool = True
+    custom_patterns: Dict[str, Dict[str, List[str]]] = field(default_factory=dict)
+
+
+@dataclass
+class IntentConfig:
+    """Configuration for intent analysis."""
+    
+    urgency_keywords: Dict[str, List[str]] = field(default_factory=lambda: {
+        'critical': ['critical', 'urgent', 'hotfix', 'emergency', 'immediate'],
+        'important': ['important', 'priority', 'asap', 'needed'],
+        'routine': ['routine', 'regular', 'normal', 'standard']
+    })
+    confidence_threshold: float = 0.6
+    sentiment_analysis: bool = True
+
+
+@dataclass 
+class DomainConfig:
+    """Configuration for domain classification."""
+    
+    file_patterns: Dict[str, List[str]] = field(default_factory=lambda: {
+        'frontend': ['*.js', '*.jsx', '*.ts', '*.tsx', '*.vue', '*.html', '*.css', '*.scss'],
+        'backend': ['*.py', '*.java', '*.go', '*.rb', '*.php', '*.cs', '*.cpp'],
+        'database': ['*.sql', 'migrations/*', 'schema/*', '**/models/**'],
+        'infrastructure': ['Dockerfile', '*.yaml', '*.yml', 'terraform/*', '*.tf'],
+        'mobile': ['*.swift', '*.kt', '*.java', 'android/*', 'ios/*'],
+        'devops': ['*.yml', '*.yaml', 'ci/*', '.github/*', 'docker/*']
+    })
+    keyword_patterns: Dict[str, List[str]] = field(default_factory=lambda: {
+        'frontend': ['ui', 'component', 'styling', 'interface', 'layout'],
+        'backend': ['api', 'endpoint', 'service', 'server', 'logic'],
+        'database': ['query', 'schema', 'migration', 'data', 'model'],
+        'infrastructure': ['deploy', 'config', 'environment', 'setup'],
+        'mobile': ['android', 'ios', 'mobile', 'app'],
+        'devops': ['build', 'pipeline', 'deploy', 'ci', 'docker']
+    })
+    min_confidence: float = 0.6
+
+
+@dataclass
+class RiskConfig:
+    """Configuration for risk analysis."""
+    
+    high_risk_patterns: List[str] = field(default_factory=lambda: [
+        # Security-related patterns
+        'password', 'secret', 'key', 'token', 'auth', 'security',
+        # Critical system patterns  
+        'production', 'prod', 'critical', 'emergency',
+        # Infrastructure patterns
+        'database', 'migration', 'schema', 'deploy',
+        # Large change patterns
+        'refactor', 'rewrite', 'restructure'
+    ])
+    medium_risk_patterns: List[str] = field(default_factory=lambda: [
+        'config', 'configuration', 'settings', 'environment',
+        'api', 'endpoint', 'service', 'integration'
+    ])
+    file_risk_patterns: Dict[str, str] = field(default_factory=lambda: {
+        # High risk file patterns
+        '**/*prod*': 'high',
+        '**/migrations/**': 'high', 
+        '**/schema/**': 'high',
+        'Dockerfile': 'medium',
+        '*.yml': 'medium',
+        '*.yaml': 'medium',
+        '**/*config*': 'medium'
+    })
+    size_thresholds: Dict[str, int] = field(default_factory=lambda: {
+        'large_commit_files': 20,  # Files changed
+        'large_commit_lines': 500,  # Lines changed
+        'massive_commit_lines': 2000  # Very large changes
+    })
+
+
+@dataclass
+class NLPConfig:
+    """Configuration for NLP processing engine."""
+    
+    spacy_model: str = "en_core_web_sm"
+    spacy_batch_size: int = 1000
+    fast_mode: bool = True  # Disable parser/NER for speed
+    
+    # Component configurations
+    change_type_config: ChangeTypeConfig = field(default_factory=ChangeTypeConfig)
+    intent_config: IntentConfig = field(default_factory=IntentConfig)
+    domain_config: DomainConfig = field(default_factory=DomainConfig)
+    risk_config: RiskConfig = field(default_factory=RiskConfig)
+    
+    # Performance settings
+    enable_parallel_processing: bool = True
+    max_workers: int = 4
+
+
+@dataclass
+class LLMConfig:
+    """Configuration for LLM fallback processing via OpenRouter."""
+    
+    # OpenRouter API settings
+    openrouter_api_key: str = "${OPENROUTER_API_KEY}"
+    base_url: str = "https://openrouter.ai/api/v1"
+    
+    # Model selection strategy
+    primary_model: str = "anthropic/claude-3-haiku"  # Fast, cheap classification
+    fallback_model: str = "meta-llama/llama-3.1-8b-instruct:free"  # Free fallback
+    complex_model: str = "anthropic/claude-3-sonnet"  # For complex cases
+    
+    # Model routing thresholds
+    complexity_threshold: float = 0.5  # Route complex cases to better model
+    cost_threshold_per_1k: float = 0.01  # Max cost per 1k commits
+    
+    # Processing settings
+    max_tokens: int = 1000
+    temperature: float = 0.1
+    
+    # Batching settings  
+    max_group_size: int = 10  # Process up to 10 commits per batch
+    similarity_threshold: float = 0.8  # Group similar commits together
+    
+    # Rate limiting
+    requests_per_minute: int = 200  # Higher limit with OpenRouter
+    max_retries: int = 3
+    
+    # Cost control
+    max_daily_cost: float = 5.0  # Max daily spend in USD
+    enable_cost_tracking: bool = True
+
+
+@dataclass
+class CacheConfig:
+    """Configuration for qualitative analysis caching."""
+    
+    cache_dir: str = ".qualitative_cache"
+    semantic_cache_size: int = 10000  # Max cached patterns
+    pattern_cache_ttl_hours: int = 168  # 1 week
+    
+    # Learning settings
+    enable_pattern_learning: bool = True
+    learning_threshold: int = 10  # Min examples to learn pattern
+    confidence_boost_factor: float = 0.1  # Boost for learned patterns
+    
+    # Cache optimization
+    enable_compression: bool = True
+    max_cache_size_mb: int = 100
+
+
+@dataclass
+class QualitativeConfig:
+    """Main configuration for qualitative analysis system.
+    
+    This configuration orchestrates the entire qualitative analysis pipeline,
+    balancing performance, accuracy, and cost through intelligent NLP and 
+    strategic LLM usage.
+    """
+    
+    # Processing settings
+    enabled: bool = True
+    batch_size: int = 1000  # Commits processed per batch
+    max_llm_fallback_pct: float = 0.15  # Max 15% of commits use LLM
+    confidence_threshold: float = 0.7  # Min confidence for NLP results
+    
+    # Component configurations
+    nlp_config: NLPConfig = field(default_factory=NLPConfig)
+    llm_config: LLMConfig = field(default_factory=LLMConfig)
+    cache_config: CacheConfig = field(default_factory=CacheConfig)
+    
+    # Performance monitoring
+    enable_performance_tracking: bool = True
+    target_processing_time_ms: float = 2.0  # Target per-commit processing time
+    
+    # Quality settings
+    min_overall_confidence: float = 0.6  # Min confidence for any result
+    enable_quality_feedback: bool = True  # Learn from corrections
+    
+    def validate(self) -> List[str]:
+        """Validate configuration and return any warnings.
+        
+        Returns:
+            List of validation warning messages.
+        """
+        warnings = []
+        
+        if self.max_llm_fallback_pct > 0.3:
+            warnings.append("LLM fallback percentage > 30% may result in high costs")
+            
+        if self.confidence_threshold > 0.9:
+            warnings.append("Very high confidence threshold may route too many commits to LLM")
+            
+        if self.batch_size > 5000:
+            warnings.append("Large batch size may cause memory issues")
+            
+        # Validate LLM config if API key is set
+        if self.llm_config.openrouter_api_key and self.llm_config.openrouter_api_key != "${OPENROUTER_API_KEY}":
+            if self.llm_config.max_daily_cost < 1.0:
+                warnings.append("Very low daily cost limit may restrict LLM usage")
+        
+        return warnings

gitflow_analytics/qualitative/utils/__init__.py

@@ -0,0 +1,13 @@
+"""Utility functions for qualitative analysis."""
+
+from .text_processing import TextProcessor
+from .batch_processor import BatchProcessor  
+from .metrics import PerformanceMetrics
+from .cost_tracker import CostTracker
+
+__all__ = [
+    "TextProcessor",
+    "BatchProcessor",
+    "PerformanceMetrics",
+    "CostTracker",
+]

gitflow_analytics/qualitative/utils/batch_processor.py

@@ -0,0 +1,326 @@
+"""Batch processing utilities for efficient commit analysis."""
+
+import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any, Callable, Dict, Iterator, List, Optional, TypeVar
+from threading import Lock
+import time
+
+T = TypeVar('T')
+R = TypeVar('R')
+
+
+class BatchProcessor:
+    """Efficient batch processing for commit analysis.
+    
+    This class provides utilities for processing large numbers of commits
+    in batches with parallel execution, progress tracking, and error handling.
+    """
+    
+    def __init__(self, batch_size: int = 1000, max_workers: int = 4):
+        """Initialize batch processor.
+        
+        Args:
+            batch_size: Number of items to process per batch
+            max_workers: Maximum number of worker threads
+        """
+        self.batch_size = batch_size
+        self.max_workers = max_workers
+        self.logger = logging.getLogger(__name__)
+        self._stats_lock = Lock()
+        self._processing_stats = {
+            'total_processed': 0,
+            'total_errors': 0,
+            'batch_times': [],
+            'start_time': None
+        }
+        
+    def create_batches(self, items: List[T], batch_size: Optional[int] = None) -> Iterator[List[T]]:
+        """Split items into batches for processing.
+        
+        Args:
+            items: List of items to batch
+            batch_size: Override default batch size
+            
+        Yields:
+            Batches of items
+        """
+        batch_size = batch_size or self.batch_size
+        
+        for i in range(0, len(items), batch_size):
+            yield items[i:i + batch_size]
+            
+    def process_batches(self, items: List[T], processor_func: Callable[[List[T]], List[R]], 
+                       parallel: bool = True) -> List[R]:
+        """Process items in batches with optional parallelization.
+        
+        Args:
+            items: Items to process
+            processor_func: Function that processes a batch and returns results
+            parallel: Whether to use parallel processing
+            
+        Returns:
+            List of all processing results
+        """
+        if not items:
+            return []
+            
+        self._reset_stats()
+        self._processing_stats['start_time'] = time.time()
+        
+        batches = list(self.create_batches(items))
+        self.logger.info(f"Processing {len(items)} items in {len(batches)} batches")
+        
+        all_results = []
+        
+        if parallel and len(batches) > 1:
+            all_results = self._process_parallel(batches, processor_func)
+        else:
+            all_results = self._process_sequential(batches, processor_func)
+            
+        self._log_final_stats(len(items))
+        return all_results
+        
+    def process_with_callback(self, items: List[T], processor_func: Callable[[List[T]], List[R]],
+                            progress_callback: Optional[Callable[[int, int], None]] = None) -> List[R]:
+        """Process batches with progress callback.
+        
+        Args:
+            items: Items to process
+            processor_func: Function that processes a batch
+            progress_callback: Callback for progress updates (processed, total)
+            
+        Returns:
+            List of all processing results
+        """
+        if not items:
+            return []
+            
+        self._reset_stats()
+        batches = list(self.create_batches(items))
+        all_results = []
+        processed_count = 0
+        
+        for i, batch in enumerate(batches):
+            batch_start = time.time()
+            
+            try:
+                batch_results = processor_func(batch)
+                all_results.extend(batch_results)
+                processed_count += len(batch)
+                
+                with self._stats_lock:
+                    self._processing_stats['total_processed'] += len(batch)
+                    self._processing_stats['batch_times'].append(time.time() - batch_start)
+                    
+            except Exception as e:
+                self.logger.error(f"Error processing batch {i}: {e}")
+                with self._stats_lock:
+                    self._processing_stats['total_errors'] += len(batch)
+                    
+            # Call progress callback if provided
+            if progress_callback:
+                progress_callback(processed_count, len(items))
+                
+        return all_results
+        
+    def _process_parallel(self, batches: List[List[T]], 
+                         processor_func: Callable[[List[T]], List[R]]) -> List[R]:
+        """Process batches in parallel using ThreadPoolExecutor.
+        
+        Args:
+            batches: List of batches to process
+            processor_func: Function to process each batch
+            
+        Returns:
+            Combined results from all batches
+        """
+        all_results = []
+        
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            # Submit all batches
+            future_to_batch = {
+                executor.submit(self._process_batch_with_timing, batch, processor_func): i
+                for i, batch in enumerate(batches)
+            }
+            
+            # Collect results as they complete
+            for future in as_completed(future_to_batch):
+                batch_idx = future_to_batch[future]
+                
+                try:
+                    batch_results, batch_time = future.result()
+                    all_results.extend(batch_results)
+                    
+                    with self._stats_lock:
+                        self._processing_stats['total_processed'] += len(batches[batch_idx])
+                        self._processing_stats['batch_times'].append(batch_time)
+                        
+                except Exception as e:
+                    self.logger.error(f"Error processing batch {batch_idx}: {e}")
+                    with self._stats_lock:
+                        self._processing_stats['total_errors'] += len(batches[batch_idx])
+                        
+        return all_results
+        
+    def _process_sequential(self, batches: List[List[T]], 
+                           processor_func: Callable[[List[T]], List[R]]) -> List[R]:
+        """Process batches sequentially.
+        
+        Args:
+            batches: List of batches to process  
+            processor_func: Function to process each batch
+            
+        Returns:
+            Combined results from all batches
+        """
+        all_results = []
+        
+        for i, batch in enumerate(batches):
+            try:
+                batch_results, batch_time = self._process_batch_with_timing(batch, processor_func)
+                all_results.extend(batch_results)
+                
+                self._processing_stats['total_processed'] += len(batch)
+                self._processing_stats['batch_times'].append(batch_time)
+                
+            except Exception as e:
+                self.logger.error(f"Error processing batch {i}: {e}")
+                self._processing_stats['total_errors'] += len(batch)
+                
+        return all_results
+        
+    def _process_batch_with_timing(self, batch: List[T], 
+                                  processor_func: Callable[[List[T]], List[R]]) -> tuple[List[R], float]:
+        """Process a single batch with timing.
+        
+        Args:
+            batch: Batch to process
+            processor_func: Processing function
+            
+        Returns:
+            Tuple of (results, processing_time_seconds)
+        """
+        start_time = time.time()
+        results = processor_func(batch)
+        processing_time = time.time() - start_time
+        
+        return results, processing_time
+        
+    def _reset_stats(self) -> None:
+        """Reset processing statistics."""
+        with self._stats_lock:
+            self._processing_stats = {
+                'total_processed': 0,
+                'total_errors': 0,
+                'batch_times': [],
+                'start_time': time.time()
+            }
+            
+    def _log_final_stats(self, total_items: int) -> None:
+        """Log final processing statistics.
+        
+        Args:
+            total_items: Total number of items processed
+        """
+        with self._stats_lock:
+            stats = self._processing_stats.copy()
+            
+        if not stats['batch_times']:
+            return
+            
+        total_time = time.time() - stats['start_time']
+        avg_batch_time = sum(stats['batch_times']) / len(stats['batch_times'])
+        items_per_second = stats['total_processed'] / total_time if total_time > 0 else 0
+        
+        self.logger.info(
+            f"Batch processing complete: {stats['total_processed']}/{total_items} items processed "
+            f"in {total_time:.2f}s ({items_per_second:.1f} items/s), "
+            f"{stats['total_errors']} errors, avg batch time: {avg_batch_time:.2f}s"
+        )
+        
+    def get_processing_stats(self) -> Dict[str, Any]:
+        """Get current processing statistics.
+        
+        Returns:
+            Dictionary with processing statistics
+        """
+        with self._stats_lock:
+            stats = self._processing_stats.copy()
+            
+        if stats['start_time'] and stats['batch_times']:
+            elapsed_time = time.time() - stats['start_time']
+            avg_batch_time = sum(stats['batch_times']) / len(stats['batch_times'])
+            items_per_second = stats['total_processed'] / elapsed_time if elapsed_time > 0 else 0
+            
+            return {
+                'total_processed': stats['total_processed'],
+                'total_errors': stats['total_errors'],
+                'elapsed_time_seconds': elapsed_time,
+                'avg_batch_time_seconds': avg_batch_time,
+                'items_per_second': items_per_second,
+                'batches_completed': len(stats['batch_times']),
+                'error_rate': stats['total_errors'] / (stats['total_processed'] + stats['total_errors']) if (stats['total_processed'] + stats['total_errors']) > 0 else 0.0
+            }
+        else:
+            return {
+                'total_processed': 0,
+                'total_errors': 0,
+                'elapsed_time_seconds': 0,
+                'avg_batch_time_seconds': 0,
+                'items_per_second': 0,
+                'batches_completed': 0,
+                'error_rate': 0.0
+            }
+
+
+class ProgressTracker:
+    """Simple progress tracking for long-running operations."""
+    
+    def __init__(self, total: int, description: str = "Processing"):
+        """Initialize progress tracker.
+        
+        Args:
+            total: Total number of items to process
+            description: Description of the operation
+        """
+        self.total = total
+        self.description = description
+        self.processed = 0
+        self.start_time = time.time()
+        self.last_report = 0
+        self.logger = logging.getLogger(__name__)
+        
+    def update(self, count: int = 1) -> None:
+        """Update progress count.
+        
+        Args:
+            count: Number of items processed since last update
+        """
+        self.processed += count
+        
+        # Report progress every 10% or every 1000 items, whichever is less frequent
+        report_interval = max(self.total // 10, 1000)
+        
+        if self.processed - self.last_report >= report_interval or self.processed >= self.total:
+            self._report_progress()
+            self.last_report = self.processed
+            
+    def _report_progress(self) -> None:
+        """Report current progress."""
+        elapsed_time = time.time() - self.start_time
+        percentage = (self.processed / self.total) * 100 if self.total > 0 else 0
+        rate = self.processed / elapsed_time if elapsed_time > 0 else 0
+        
+        # Estimate time remaining
+        if rate > 0 and self.processed < self.total:
+            remaining_items = self.total - self.processed
+            eta_seconds = remaining_items / rate
+            eta_str = f", ETA: {eta_seconds:.0f}s"
+        else:
+            eta_str = ""
+            
+        self.logger.info(
+            f"{self.description}: {self.processed}/{self.total} ({percentage:.1f}%) "
+            f"at {rate:.1f} items/s{eta_str}"
+        )