gitflow-analytics 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

gitflow_analytics/qualitative/core/processor.py

@@ -0,0 +1,540 @@
+"""Main orchestrator for qualitative analysis of Git commits."""
+
+import logging
+import time
+from typing import Dict, List, Any, Tuple, Optional
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+from ...models.database import Database
+from ..models.schemas import QualitativeConfig, QualitativeCommitData
+from .nlp_engine import NLPEngine
+from .llm_fallback import LLMFallback
+from .pattern_cache import PatternCache
+from ..utils.batch_processor import BatchProcessor, ProgressTracker
+from ..utils.metrics import PerformanceMetrics
+
+
+class QualitativeProcessor:
+    """Main orchestrator for qualitative analysis of Git commits.
+    
+    This processor coordinates the entire qualitative analysis pipeline:
+    1. Pattern cache lookup for known commit patterns
+    2. Fast NLP processing for most commits
+    3. Strategic LLM fallback for uncertain cases
+    4. Pattern learning and cache updates
+    5. Performance monitoring and optimization
+    
+    The system is designed to process 10,000+ commits in under 60 seconds
+    while maintaining high accuracy and keeping LLM costs low.
+    """
+    
+    def __init__(self, config: QualitativeConfig, database: Database):
+        """Initialize qualitative processor.
+        
+        Args:
+            config: Configuration for qualitative analysis
+            database: Database instance for caching and storage
+        """
+        self.config = config
+        self.database = database
+        self.logger = logging.getLogger(__name__)
+        
+        # Initialize core components
+        self.nlp_engine = NLPEngine(config.nlp_config)
+        self.pattern_cache = PatternCache(config.cache_config, database)
+        
+        # Initialize LLM fallback if enabled
+        self.llm_fallback = None
+        if config.llm_config.openrouter_api_key:
+            try:
+                self.llm_fallback = LLMFallback(config.llm_config)
+                self.logger.info("LLM fallback system initialized")
+            except Exception as e:
+                self.logger.warning(f"LLM fallback initialization failed: {e}")
+        else:
+            self.logger.info("LLM fallback disabled (no API key configured)")
+            
+        # Initialize utilities
+        self.batch_processor = BatchProcessor(
+            batch_size=config.batch_size,
+            max_workers=config.nlp_config.max_workers
+        )
+        self.metrics = PerformanceMetrics()
+        
+        # Processing statistics
+        self.processing_stats = {
+            'total_processed': 0,
+            'cache_hits': 0,
+            'nlp_processed': 0,
+            'llm_processed': 0,
+            'processing_start_time': None,
+            'last_optimization': None
+        }
+        
+        self.logger.info("Qualitative processor initialized")
+        
+    def process_commits(self, commits: List[Dict[str, Any]], 
+                       show_progress: bool = True) -> List[QualitativeCommitData]:
+        """Process commits with qualitative analysis.
+        
+        Args:
+            commits: List of commit dictionaries from GitFlow Analytics
+            show_progress: Whether to show progress information
+            
+        Returns:
+            List of QualitativeCommitData with analysis results
+        """
+        if not commits:
+            return []
+            
+        if not self.config.enabled:
+            self.logger.info("Qualitative analysis disabled in configuration")
+            return self._create_disabled_results(commits)
+            
+        self.processing_stats['processing_start_time'] = time.time()
+        self.logger.info(f"Starting qualitative analysis of {len(commits)} commits")
+        
+        # Setup progress tracking
+        progress_tracker = ProgressTracker(
+            total=len(commits),
+            description="Qualitative Analysis"
+        ) if show_progress else None
+        
+        # Step 1: Check cache for known patterns
+        cached_results, uncached_commits = self._check_cache(commits, progress_tracker)
+        self.logger.info(f"Cache provided {len(cached_results)} results, processing {len(uncached_commits)} commits")
+        
+        # Step 2: Process uncached commits with NLP
+        nlp_results = []
+        if uncached_commits:
+            nlp_results = self._process_with_nlp(uncached_commits, progress_tracker)
+            
+        # Step 3: Identify uncertain cases for LLM processing
+        confident_results, uncertain_commits = self._separate_by_confidence(nlp_results)
+        self.logger.info(f"NLP confident: {len(confident_results)}, uncertain: {len(uncertain_commits)}")
+        
+        # Step 4: Process uncertain cases with LLM if available
+        llm_results = []
+        if uncertain_commits and self.llm_fallback:
+            llm_results = self._process_with_llm(uncertain_commits, progress_tracker)
+        else:
+            # If no LLM available, keep uncertain results with lower confidence
+            llm_results = [self._convert_to_uncertain_result(commit) for commit in uncertain_commits]
+            
+        # Step 5: Update cache with new high-confidence patterns
+        all_new_results = confident_results + llm_results
+        if self.config.cache_config.enable_pattern_learning:
+            self.pattern_cache.learn_from_results(all_new_results)
+            
+        # Step 6: Combine all results
+        all_results = cached_results + confident_results + llm_results
+        
+        # Update processing statistics
+        self._update_processing_stats(len(commits), len(cached_results), 
+                                    len(confident_results), len(llm_results))
+        
+        # Periodic cache optimization
+        if self._should_optimize_cache():
+            self._optimize_system()
+            
+        self.logger.info(f"Qualitative analysis completed in {time.time() - self.processing_stats['processing_start_time']:.2f}s")
+        return all_results
+        
+    def _check_cache(self, commits: List[Dict[str, Any]], 
+                    progress_tracker: Optional[ProgressTracker]) -> Tuple[List[QualitativeCommitData], List[Dict[str, Any]]]:
+        """Check pattern cache for known commit patterns.
+        
+        Args:
+            commits: List of commit dictionaries
+            progress_tracker: Optional progress tracker
+            
+        Returns:
+            Tuple of (cached_results, uncached_commits)
+        """
+        cached_results = []
+        uncached_commits = []
+        
+        for commit in commits:
+            cached_result = self.pattern_cache.lookup_pattern(
+                commit.get('message', ''),
+                commit.get('files_changed', [])
+            )
+            
+            if cached_result:
+                # Convert cached result to QualitativeCommitData
+                result = self._create_result_from_cache(commit, cached_result)
+                cached_results.append(result)
+                self.processing_stats['cache_hits'] += 1
+            else:
+                uncached_commits.append(commit)
+                
+            if progress_tracker:
+                progress_tracker.update(1)
+                
+        return cached_results, uncached_commits
+        
+    def _process_with_nlp(self, commits: List[Dict[str, Any]], 
+                         progress_tracker: Optional[ProgressTracker]) -> List[QualitativeCommitData]:
+        """Process commits using NLP engine.
+        
+        Args:
+            commits: List of commit dictionaries
+            progress_tracker: Optional progress tracker
+            
+        Returns:
+            List of QualitativeCommitData from NLP processing
+        """
+        if not commits:
+            return []
+            
+        def process_batch_with_progress(batch: List[Dict[str, Any]]) -> List[QualitativeCommitData]:
+            results = self.nlp_engine.process_batch(batch)
+            if progress_tracker:
+                progress_tracker.update(len(batch))
+            return results
+            
+        # Use batch processing for efficiency
+        if self.config.nlp_config.enable_parallel_processing and len(commits) > 1000:
+            all_results = self.batch_processor.process_batches(
+                commits, 
+                process_batch_with_progress,
+                parallel=True
+            )
+        else:
+            all_results = self.batch_processor.process_batches(
+                commits,
+                process_batch_with_progress, 
+                parallel=False
+            )
+            
+        self.processing_stats['nlp_processed'] += len(commits)
+        return all_results
+        
+    def _separate_by_confidence(self, results: List[QualitativeCommitData]) -> Tuple[List[QualitativeCommitData], List[Dict[str, Any]]]:
+        """Separate results by confidence threshold.
+        
+        Args:  
+            results: List of NLP analysis results
+            
+        Returns:
+            Tuple of (confident_results, uncertain_commit_dicts)
+        """
+        confident_results = []
+        uncertain_commits = []
+        
+        for result in results:
+            if result.confidence_score >= self.config.confidence_threshold:
+                confident_results.append(result)
+            else:
+                # Convert back to commit dict for LLM processing
+                commit_dict = {
+                    'hash': result.hash,
+                    'message': result.message,
+                    'author_name': result.author_name,
+                    'author_email': result.author_email,
+                    'timestamp': result.timestamp,
+                    'files_changed': result.files_changed,
+                    'insertions': result.insertions,
+                    'deletions': result.deletions
+                }
+                uncertain_commits.append(commit_dict)
+                
+        return confident_results, uncertain_commits
+        
+    def _process_with_llm(self, commits: List[Dict[str, Any]], 
+                         progress_tracker: Optional[ProgressTracker]) -> List[QualitativeCommitData]:
+        """Process uncertain commits with LLM fallback.
+        
+        Args:
+            commits: List of uncertain commit dictionaries
+            progress_tracker: Optional progress tracker
+            
+        Returns:
+            List of QualitativeCommitData from LLM processing
+        """
+        if not commits or not self.llm_fallback:
+            return []
+            
+        # Check LLM usage limits
+        max_llm_commits = int(len(commits) * self.config.max_llm_fallback_pct)
+        if len(commits) > max_llm_commits:
+            self.logger.warning(
+                f"LLM limit reached: processing {max_llm_commits} of {len(commits)} uncertain commits"
+            )
+            commits = commits[:max_llm_commits]
+            
+        # Group similar commits for batch processing
+        grouped_commits = self.llm_fallback.group_similar_commits(commits)
+        self.logger.debug(f"Grouped {len(commits)} commits into {len(grouped_commits)} groups for LLM processing")
+        
+        all_results = []
+        
+        for group in grouped_commits:
+            try:
+                group_results = self.llm_fallback.process_group(group)
+                all_results.extend(group_results)
+                
+                if progress_tracker:
+                    progress_tracker.update(len(group))
+                    
+            except Exception as e:
+                self.logger.error(f"LLM processing failed for group of {len(group)} commits: {e}")
+                # Create fallback results for this group
+                fallback_results = [self._convert_to_uncertain_result(commit) for commit in group]
+                all_results.extend(fallback_results)
+                
+                if progress_tracker:
+                    progress_tracker.update(len(group))
+                    
+        self.processing_stats['llm_processed'] += len(commits)
+        return all_results
+        
+    def _create_result_from_cache(self, commit: Dict[str, Any], 
+                                 cached_data: Dict[str, Any]) -> QualitativeCommitData:
+        """Create QualitativeCommitData from cached pattern.
+        
+        Args:
+            commit: Original commit dictionary
+            cached_data: Cached classification data
+            
+        Returns:
+            QualitativeCommitData object
+        """
+        return QualitativeCommitData(
+            # Copy commit fields
+            hash=commit.get('hash', ''),
+            message=commit.get('message', ''),
+            author_name=commit.get('author_name', ''),
+            author_email=commit.get('author_email', ''),
+            timestamp=commit.get('timestamp', time.time()),
+            files_changed=commit.get('files_changed', []),
+            insertions=commit.get('insertions', 0),
+            deletions=commit.get('deletions', 0),
+            
+            # Use cached classification data
+            change_type=cached_data.get('change_type', 'unknown'),
+            change_type_confidence=cached_data.get('change_type_confidence', 0.5),
+            business_domain=cached_data.get('business_domain', 'unknown'),
+            domain_confidence=cached_data.get('domain_confidence', 0.5),
+            risk_level=cached_data.get('risk_level', 'medium'),
+            risk_factors=cached_data.get('risk_factors', []),
+            intent_signals=cached_data.get('intent_signals', {}),
+            collaboration_patterns=cached_data.get('collaboration_patterns', {}),
+            technical_context={'processing_method': 'cached'},
+            
+            # Processing metadata
+            processing_method='cache',
+            processing_time_ms=0.5,  # Very fast for cached results
+            confidence_score=cached_data.get('confidence_score', 0.5)
+        )
+        
+    def _convert_to_uncertain_result(self, commit: Dict[str, Any]) -> QualitativeCommitData:
+        """Convert commit to uncertain result when LLM is unavailable.
+        
+        Args:
+            commit: Commit dictionary
+            
+        Returns:
+            QualitativeCommitData with uncertain classifications
+        """
+        return QualitativeCommitData(
+            # Copy commit fields
+            hash=commit.get('hash', ''),
+            message=commit.get('message', ''),
+            author_name=commit.get('author_name', ''),
+            author_email=commit.get('author_email', ''),
+            timestamp=commit.get('timestamp', time.time()),
+            files_changed=commit.get('files_changed', []),
+            insertions=commit.get('insertions', 0),
+            deletions=commit.get('deletions', 0),
+            
+            # Uncertain classifications
+            change_type='unknown',
+            change_type_confidence=0.3,
+            business_domain='unknown',
+            domain_confidence=0.3,
+            risk_level='medium',
+            risk_factors=['low_confidence_classification'],
+            intent_signals={'confidence': 0.3},
+            collaboration_patterns={},
+            technical_context={'processing_method': 'uncertain_fallback'},
+            
+            # Processing metadata
+            processing_method='nlp',
+            processing_time_ms=1.0,
+            confidence_score=0.3
+        )
+        
+    def _create_disabled_results(self, commits: List[Dict[str, Any]]) -> List[QualitativeCommitData]:
+        """Create disabled results when qualitative analysis is turned off.
+        
+        Args:
+            commits: List of commit dictionaries
+            
+        Returns:
+            List of QualitativeCommitData with disabled status
+        """
+        results = []
+        
+        for commit in commits:
+            result = QualitativeCommitData(
+                # Copy commit fields
+                hash=commit.get('hash', ''),
+                message=commit.get('message', ''),
+                author_name=commit.get('author_name', ''),
+                author_email=commit.get('author_email', ''),
+                timestamp=commit.get('timestamp', time.time()),
+                files_changed=commit.get('files_changed', []),
+                insertions=commit.get('insertions', 0),
+                deletions=commit.get('deletions', 0),
+                
+                # Disabled classifications
+                change_type='disabled',
+                change_type_confidence=0.0,
+                business_domain='disabled',
+                domain_confidence=0.0,
+                risk_level='unknown',
+                risk_factors=['qualitative_analysis_disabled'],
+                intent_signals={'disabled': True},
+                collaboration_patterns={},
+                technical_context={'processing_method': 'disabled'},
+                
+                # Processing metadata
+                processing_method='disabled',
+                processing_time_ms=0.0,
+                confidence_score=0.0
+            )
+            results.append(result)
+            
+        return results
+        
+    def _update_processing_stats(self, total_commits: int, cached: int, 
+                               nlp_processed: int, llm_processed: int) -> None:
+        """Update processing statistics.
+        
+        Args:
+            total_commits: Total number of commits processed
+            cached: Number of cache hits
+            nlp_processed: Number processed by NLP
+            llm_processed: Number processed by LLM
+        """
+        self.processing_stats['total_processed'] += total_commits
+        self.processing_stats['cache_hits'] += cached
+        self.processing_stats['nlp_processed'] += nlp_processed
+        self.processing_stats['llm_processed'] += llm_processed
+        
+        # Log processing breakdown
+        cache_pct = (cached / total_commits) * 100 if total_commits > 0 else 0
+        nlp_pct = (nlp_processed / total_commits) * 100 if total_commits > 0 else 0
+        llm_pct = (llm_processed / total_commits) * 100 if total_commits > 0 else 0
+        
+        self.logger.info(
+            f"Processing breakdown: {cache_pct:.1f}% cached, "
+            f"{nlp_pct:.1f}% NLP, {llm_pct:.1f}% LLM"
+        )
+        
+    def _should_optimize_cache(self) -> bool:
+        """Check if cache optimization should be performed.
+        
+        Returns:
+            True if optimization should be performed
+        """
+        # Optimize every 10,000 commits or every hour
+        if (self.processing_stats['total_processed'] % 10000 == 0 or
+            (self.processing_stats['last_optimization'] is None) or
+            (time.time() - self.processing_stats['last_optimization'] > 3600)):
+            return True
+        return False
+        
+    def _optimize_system(self) -> None:
+        """Perform system optimization."""
+        self.logger.info("Performing system optimization...")
+        
+        # Optimize pattern cache
+        self.pattern_cache.optimize_cache()
+        
+        # Update last optimization time
+        self.processing_stats['last_optimization'] = time.time()
+        
+        self.logger.info("System optimization completed")
+        
+    def get_processing_statistics(self) -> Dict[str, Any]:
+        """Get comprehensive processing statistics.
+        
+        Returns:
+            Dictionary with processing statistics
+        """
+        # Get component statistics
+        cache_stats = self.pattern_cache.get_cache_statistics()
+        nlp_stats = self.nlp_engine.get_performance_stats()
+        
+        # Calculate processing rates
+        total_time = time.time() - (self.processing_stats['processing_start_time'] or time.time())
+        commits_per_second = self.processing_stats['total_processed'] / total_time if total_time > 0 else 0
+        
+        # Calculate method percentages
+        total = self.processing_stats['total_processed']
+        method_percentages = {
+            'cache': (self.processing_stats['cache_hits'] / total * 100) if total > 0 else 0,
+            'nlp': (self.processing_stats['nlp_processed'] / total * 100) if total > 0 else 0,
+            'llm': (self.processing_stats['llm_processed'] / total * 100) if total > 0 else 0
+        }
+        
+        stats = {
+            'processing_summary': {
+                'total_commits_processed': self.processing_stats['total_processed'],
+                'commits_per_second': commits_per_second,
+                'total_processing_time_seconds': total_time,
+                'method_breakdown': method_percentages
+            },
+            'cache_statistics': cache_stats,
+            'nlp_statistics': nlp_stats,
+            'configuration': {
+                'enabled': self.config.enabled,
+                'confidence_threshold': self.config.confidence_threshold,
+                'max_llm_fallback_pct': self.config.max_llm_fallback_pct,
+                'batch_size': self.config.batch_size
+            }
+        }
+        
+        # Add LLM statistics if available
+        if self.llm_fallback:
+            stats['llm_statistics'] = {
+                'cost_tracking': self.llm_fallback.cost_tracker.get_usage_stats(),
+                'model_usage': 'available'
+            }
+        else:
+            stats['llm_statistics'] = {'model_usage': 'disabled'}
+            
+        return stats
+        
+    def validate_setup(self) -> Tuple[bool, List[str]]:
+        """Validate processor setup and dependencies.
+        
+        Returns:
+            Tuple of (is_valid, list_of_issues)
+        """
+        issues = []
+        
+        # Validate NLP engine
+        nlp_valid, nlp_issues = self.nlp_engine.validate_setup()
+        if not nlp_valid:
+            issues.extend([f"NLP: {issue}" for issue in nlp_issues])
+            
+        # Validate LLM fallback if configured
+        if self.config.llm_config.openrouter_api_key and self.llm_fallback is None:
+            issues.append("LLM: API key configured but fallback system failed to initialize")
+            
+        # Validate configuration
+        config_warnings = self.config.validate()
+        issues.extend([f"Config: {warning}" for warning in config_warnings])
+        
+        # Test database connection
+        try:
+            with self.database.get_session() as session:
+                session.execute("SELECT 1")
+        except Exception as e:
+            issues.append(f"Database: Connection failed - {e}")
+            
+        return len(issues) == 0, issues

gitflow_analytics/qualitative/models/__init__.py

@@ -0,0 +1,25 @@
+"""Data models and schemas for qualitative analysis."""
+
+from .schemas import (
+    QualitativeCommitData,
+    QualitativeConfig,
+    NLPConfig, 
+    LLMConfig,
+    CacheConfig,
+    ChangeTypeConfig,
+    IntentConfig,
+    DomainConfig,
+    RiskConfig,
+)
+
+__all__ = [
+    "QualitativeCommitData",
+    "QualitativeConfig",
+    "NLPConfig",
+    "LLMConfig", 
+    "CacheConfig",
+    "ChangeTypeConfig",
+    "IntentConfig",
+    "DomainConfig",
+    "RiskConfig",
+]