gitflow-analytics 1.0.1__py3-none-any.whl → 1.3.6__py3-none-any.whl

gitflow_analytics/qualitative/models/schemas.py

@@ -0,0 +1,306 @@
+"""Data models and configuration schemas for qualitative analysis."""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+@dataclass
+class QualitativeCommitData:
+    """Enhanced commit data with qualitative analysis results.
+
+    This class extends basic commit information with semantic analysis results
+    including change type, business domain, risk assessment, and processing metadata.
+    """
+
+    # Existing commit data from GitFlow Analytics
+    hash: str
+    message: str
+    author_name: str
+    author_email: str
+    timestamp: datetime
+    files_changed: list[str]
+    insertions: int
+    deletions: int
+
+    # New qualitative analysis fields
+    change_type: str  # feature|bugfix|refactor|docs|test|chore|security|hotfix|config
+    change_type_confidence: float  # 0.0-1.0
+    business_domain: str  # frontend|backend|database|infrastructure|mobile|devops|unknown
+    domain_confidence: float  # 0.0-1.0
+    risk_level: str  # low|medium|high|critical
+    risk_factors: list[str]  # List of identified risk factors
+    intent_signals: dict[str, Any]  # Intent analysis results
+    collaboration_patterns: dict[str, Any]  # Team interaction patterns
+    technical_context: dict[str, Any]  # Technical context information
+
+    # Processing metadata
+    processing_method: str  # 'nlp' or 'llm'
+    processing_time_ms: float
+    confidence_score: float  # Overall confidence in analysis
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return {
+            "hash": self.hash,
+            "message": self.message,
+            "author_name": self.author_name,
+            "author_email": self.author_email,
+            "timestamp": self.timestamp.isoformat(),
+            "files_changed": self.files_changed,
+            "insertions": self.insertions,
+            "deletions": self.deletions,
+            "change_type": self.change_type,
+            "change_type_confidence": self.change_type_confidence,
+            "business_domain": self.business_domain,
+            "domain_confidence": self.domain_confidence,
+            "risk_level": self.risk_level,
+            "risk_factors": self.risk_factors,
+            "intent_signals": self.intent_signals,
+            "collaboration_patterns": self.collaboration_patterns,
+            "technical_context": self.technical_context,
+            "processing_method": self.processing_method,
+            "processing_time_ms": self.processing_time_ms,
+            "confidence_score": self.confidence_score,
+        }
+
+
+@dataclass
+class ChangeTypeConfig:
+    """Configuration for change type classification."""
+
+    min_confidence: float = 0.7
+    semantic_weight: float = 0.6  # Weight for semantic features
+    file_pattern_weight: float = 0.4  # Weight for file pattern signals
+    enable_custom_patterns: bool = True
+    custom_patterns: dict[str, dict[str, list[str]]] = field(default_factory=dict)
+
+
+@dataclass
+class IntentConfig:
+    """Configuration for intent analysis."""
+
+    urgency_keywords: dict[str, list[str]] = field(
+        default_factory=lambda: {
+            "critical": ["critical", "urgent", "hotfix", "emergency", "immediate"],
+            "important": ["important", "priority", "asap", "needed"],
+            "routine": ["routine", "regular", "normal", "standard"],
+        }
+    )
+    confidence_threshold: float = 0.6
+    sentiment_analysis: bool = True
+
+
+@dataclass
+class DomainConfig:
+    """Configuration for domain classification."""
+
+    file_patterns: dict[str, list[str]] = field(
+        default_factory=lambda: {
+            "frontend": ["*.js", "*.jsx", "*.ts", "*.tsx", "*.vue", "*.html", "*.css", "*.scss"],
+            "backend": ["*.py", "*.java", "*.go", "*.rb", "*.php", "*.cs", "*.cpp"],
+            "database": ["*.sql", "migrations/*", "schema/*", "**/models/**"],
+            "infrastructure": ["Dockerfile", "*.yaml", "*.yml", "terraform/*", "*.tf"],
+            "mobile": ["*.swift", "*.kt", "*.java", "android/*", "ios/*"],
+            "devops": ["*.yml", "*.yaml", "ci/*", ".github/*", "docker/*"],
+        }
+    )
+    keyword_patterns: dict[str, list[str]] = field(
+        default_factory=lambda: {
+            "frontend": ["ui", "component", "styling", "interface", "layout"],
+            "backend": ["api", "endpoint", "service", "server", "logic"],
+            "database": ["query", "schema", "migration", "data", "model"],
+            "infrastructure": ["deploy", "config", "environment", "setup"],
+            "mobile": ["android", "ios", "mobile", "app"],
+            "devops": ["build", "pipeline", "deploy", "ci", "docker"],
+        }
+    )
+    min_confidence: float = 0.6
+
+
+@dataclass
+class RiskConfig:
+    """Configuration for risk analysis."""
+
+    high_risk_patterns: list[str] = field(
+        default_factory=lambda: [
+            # Security-related patterns
+            "password",
+            "secret",
+            "key",
+            "token",
+            "auth",
+            "security",
+            # Critical system patterns
+            "production",
+            "prod",
+            "critical",
+            "emergency",
+            # Infrastructure patterns
+            "database",
+            "migration",
+            "schema",
+            "deploy",
+            # Large change patterns
+            "refactor",
+            "rewrite",
+            "restructure",
+        ]
+    )
+    medium_risk_patterns: list[str] = field(
+        default_factory=lambda: [
+            "config",
+            "configuration",
+            "settings",
+            "environment",
+            "api",
+            "endpoint",
+            "service",
+            "integration",
+        ]
+    )
+    file_risk_patterns: dict[str, str] = field(
+        default_factory=lambda: {
+            # High risk file patterns
+            "**/*prod*": "high",
+            "**/migrations/**": "high",
+            "**/schema/**": "high",
+            "Dockerfile": "medium",
+            "*.yml": "medium",
+            "*.yaml": "medium",
+            "**/*config*": "medium",
+        }
+    )
+    size_thresholds: dict[str, int] = field(
+        default_factory=lambda: {
+            "large_commit_files": 20,  # Files changed
+            "large_commit_lines": 500,  # Lines changed
+            "massive_commit_lines": 2000,  # Very large changes
+        }
+    )
+
+
+@dataclass
+class NLPConfig:
+    """Configuration for NLP processing engine."""
+
+    spacy_model: str = "en_core_web_sm"
+    spacy_batch_size: int = 1000
+    fast_mode: bool = True  # Disable parser/NER for speed
+
+    # Component configurations
+    change_type_config: ChangeTypeConfig = field(default_factory=ChangeTypeConfig)
+    intent_config: IntentConfig = field(default_factory=IntentConfig)
+    domain_config: DomainConfig = field(default_factory=DomainConfig)
+    risk_config: RiskConfig = field(default_factory=RiskConfig)
+
+    # Performance settings
+    enable_parallel_processing: bool = True
+    max_workers: int = 4
+
+
+@dataclass
+class LLMConfig:
+    """Configuration for LLM fallback processing via OpenRouter."""
+
+    # OpenRouter API settings
+    openrouter_api_key: str = "${OPENROUTER_API_KEY}"
+    base_url: str = "https://openrouter.ai/api/v1"
+
+    # Model selection strategy
+    primary_model: str = "anthropic/claude-3-haiku"  # Fast, cheap classification
+    fallback_model: str = "meta-llama/llama-3.1-8b-instruct:free"  # Free fallback
+    complex_model: str = "anthropic/claude-3-sonnet"  # For complex cases
+
+    # Model routing thresholds
+    complexity_threshold: float = 0.5  # Route complex cases to better model
+    cost_threshold_per_1k: float = 0.01  # Max cost per 1k commits
+
+    # Processing settings
+    max_tokens: int = 1000
+    temperature: float = 0.1
+
+    # Batching settings
+    max_group_size: int = 10  # Process up to 10 commits per batch
+    similarity_threshold: float = 0.8  # Group similar commits together
+
+    # Rate limiting
+    requests_per_minute: int = 200  # Higher limit with OpenRouter
+    max_retries: int = 3
+
+    # Cost control
+    max_daily_cost: float = 5.0  # Max daily spend in USD
+    enable_cost_tracking: bool = True
+
+
+@dataclass
+class CacheConfig:
+    """Configuration for qualitative analysis caching."""
+
+    cache_dir: str = ".qualitative_cache"
+    semantic_cache_size: int = 10000  # Max cached patterns
+    pattern_cache_ttl_hours: int = 168  # 1 week
+
+    # Learning settings
+    enable_pattern_learning: bool = True
+    learning_threshold: int = 10  # Min examples to learn pattern
+    confidence_boost_factor: float = 0.1  # Boost for learned patterns
+
+    # Cache optimization
+    enable_compression: bool = True
+    max_cache_size_mb: int = 100
+
+
+@dataclass
+class QualitativeConfig:
+    """Main configuration for qualitative analysis system.
+
+    This configuration orchestrates the entire qualitative analysis pipeline,
+    balancing performance, accuracy, and cost through intelligent NLP and
+    strategic LLM usage.
+    """
+
+    # Processing settings
+    enabled: bool = True
+    batch_size: int = 1000  # Commits processed per batch
+    max_llm_fallback_pct: float = 0.15  # Max 15% of commits use LLM
+    confidence_threshold: float = 0.7  # Min confidence for NLP results
+
+    # Component configurations
+    nlp_config: NLPConfig = field(default_factory=NLPConfig)
+    llm_config: LLMConfig = field(default_factory=LLMConfig)
+    cache_config: CacheConfig = field(default_factory=CacheConfig)
+
+    # Performance monitoring
+    enable_performance_tracking: bool = True
+    target_processing_time_ms: float = 2.0  # Target per-commit processing time
+
+    # Quality settings
+    min_overall_confidence: float = 0.6  # Min confidence for any result
+    enable_quality_feedback: bool = True  # Learn from corrections
+
+    def validate(self) -> list[str]:
+        """Validate configuration and return any warnings.
+
+        Returns:
+            List of validation warning messages.
+        """
+        warnings = []
+
+        if self.max_llm_fallback_pct > 0.3:
+            warnings.append("LLM fallback percentage > 30% may result in high costs")
+
+        if self.confidence_threshold > 0.9:
+            warnings.append("Very high confidence threshold may route too many commits to LLM")
+
+        if self.batch_size > 5000:
+            warnings.append("Large batch size may cause memory issues")
+
+        # Validate LLM config if API key is set
+        if (
+            self.llm_config.openrouter_api_key
+            and self.llm_config.openrouter_api_key != "${OPENROUTER_API_KEY}"
+        ) and self.llm_config.max_daily_cost < 1.0:
+            warnings.append("Very low daily cost limit may restrict LLM usage")
+
+        return warnings

gitflow_analytics/qualitative/utils/__init__.py

@@ -0,0 +1,13 @@
+"""Utility functions for qualitative analysis."""
+
+from .batch_processor import BatchProcessor
+from .cost_tracker import CostTracker
+from .metrics import PerformanceMetrics
+from .text_processing import TextProcessor
+
+__all__ = [
+    "TextProcessor",
+    "BatchProcessor",
+    "PerformanceMetrics",
+    "CostTracker",
+]

gitflow_analytics/qualitative/utils/batch_processor.py

@@ -0,0 +1,339 @@
+"""Batch processing utilities for efficient commit analysis."""
+
+import logging
+import time
+from collections.abc import Iterator
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from threading import Lock
+from typing import Any, Callable, Optional, TypeVar
+
+T = TypeVar("T")
+R = TypeVar("R")
+
+
+class BatchProcessor:
+    """Efficient batch processing for commit analysis.
+
+    This class provides utilities for processing large numbers of commits
+    in batches with parallel execution, progress tracking, and error handling.
+    """
+
+    def __init__(self, batch_size: int = 1000, max_workers: int = 4):
+        """Initialize batch processor.
+
+        Args:
+            batch_size: Number of items to process per batch
+            max_workers: Maximum number of worker threads
+        """
+        self.batch_size = batch_size
+        self.max_workers = max_workers
+        self.logger = logging.getLogger(__name__)
+        self._stats_lock = Lock()
+        self._processing_stats = {
+            "total_processed": 0,
+            "total_errors": 0,
+            "batch_times": [],
+            "start_time": None,
+        }
+
+    def create_batches(self, items: list[T], batch_size: Optional[int] = None) -> Iterator[list[T]]:
+        """Split items into batches for processing.
+
+        Args:
+            items: List of items to batch
+            batch_size: Override default batch size
+
+        Yields:
+            Batches of items
+        """
+        batch_size = batch_size or self.batch_size
+
+        for i in range(0, len(items), batch_size):
+            yield items[i : i + batch_size]
+
+    def process_batches(
+        self, items: list[T], processor_func: Callable[[list[T]], list[R]], parallel: bool = True
+    ) -> list[R]:
+        """Process items in batches with optional parallelization.
+
+        Args:
+            items: Items to process
+            processor_func: Function that processes a batch and returns results
+            parallel: Whether to use parallel processing
+
+        Returns:
+            List of all processing results
+        """
+        if not items:
+            return []
+
+        self._reset_stats()
+        self._processing_stats["start_time"] = time.time()
+
+        batches = list(self.create_batches(items))
+        self.logger.info(f"Processing {len(items)} items in {len(batches)} batches")
+
+        all_results = []
+
+        if parallel and len(batches) > 1:
+            all_results = self._process_parallel(batches, processor_func)
+        else:
+            all_results = self._process_sequential(batches, processor_func)
+
+        self._log_final_stats(len(items))
+        return all_results
+
+    def process_with_callback(
+        self,
+        items: list[T],
+        processor_func: Callable[[list[T]], list[R]],
+        progress_callback: Optional[Callable[[int, int], None]] = None,
+    ) -> list[R]:
+        """Process batches with progress callback.
+
+        Args:
+            items: Items to process
+            processor_func: Function that processes a batch
+            progress_callback: Callback for progress updates (processed, total)
+
+        Returns:
+            List of all processing results
+        """
+        if not items:
+            return []
+
+        self._reset_stats()
+        batches = list(self.create_batches(items))
+        all_results = []
+        processed_count = 0
+
+        for i, batch in enumerate(batches):
+            batch_start = time.time()
+
+            try:
+                batch_results = processor_func(batch)
+                all_results.extend(batch_results)
+                processed_count += len(batch)
+
+                with self._stats_lock:
+                    self._processing_stats["total_processed"] += len(batch)
+                    self._processing_stats["batch_times"].append(time.time() - batch_start)
+
+            except Exception as e:
+                self.logger.error(f"Error processing batch {i}: {e}")
+                with self._stats_lock:
+                    self._processing_stats["total_errors"] += len(batch)
+
+            # Call progress callback if provided
+            if progress_callback:
+                progress_callback(processed_count, len(items))
+
+        return all_results
+
+    def _process_parallel(
+        self, batches: list[list[T]], processor_func: Callable[[list[T]], list[R]]
+    ) -> list[R]:
+        """Process batches in parallel using ThreadPoolExecutor.
+
+        Args:
+            batches: List of batches to process
+            processor_func: Function to process each batch
+
+        Returns:
+            Combined results from all batches
+        """
+        all_results = []
+
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            # Submit all batches
+            future_to_batch = {
+                executor.submit(self._process_batch_with_timing, batch, processor_func): i
+                for i, batch in enumerate(batches)
+            }
+
+            # Collect results as they complete
+            for future in as_completed(future_to_batch):
+                batch_idx = future_to_batch[future]
+
+                try:
+                    batch_results, batch_time = future.result()
+                    all_results.extend(batch_results)
+
+                    with self._stats_lock:
+                        self._processing_stats["total_processed"] += len(batches[batch_idx])
+                        self._processing_stats["batch_times"].append(batch_time)
+
+                except Exception as e:
+                    self.logger.error(f"Error processing batch {batch_idx}: {e}")
+                    with self._stats_lock:
+                        self._processing_stats["total_errors"] += len(batches[batch_idx])
+
+        return all_results
+
+    def _process_sequential(
+        self, batches: list[list[T]], processor_func: Callable[[list[T]], list[R]]
+    ) -> list[R]:
+        """Process batches sequentially.
+
+        Args:
+            batches: List of batches to process
+            processor_func: Function to process each batch
+
+        Returns:
+            Combined results from all batches
+        """
+        all_results = []
+
+        for i, batch in enumerate(batches):
+            try:
+                batch_results, batch_time = self._process_batch_with_timing(batch, processor_func)
+                all_results.extend(batch_results)
+
+                self._processing_stats["total_processed"] += len(batch)
+                self._processing_stats["batch_times"].append(batch_time)
+
+            except Exception as e:
+                self.logger.error(f"Error processing batch {i}: {e}")
+                self._processing_stats["total_errors"] += len(batch)
+
+        return all_results
+
+    def _process_batch_with_timing(
+        self, batch: list[T], processor_func: Callable[[list[T]], list[R]]
+    ) -> tuple[list[R], float]:
+        """Process a single batch with timing.
+
+        Args:
+            batch: Batch to process
+            processor_func: Processing function
+
+        Returns:
+            Tuple of (results, processing_time_seconds)
+        """
+        start_time = time.time()
+        results = processor_func(batch)
+        processing_time = time.time() - start_time
+
+        return results, processing_time
+
+    def _reset_stats(self) -> None:
+        """Reset processing statistics."""
+        with self._stats_lock:
+            self._processing_stats = {
+                "total_processed": 0,
+                "total_errors": 0,
+                "batch_times": [],
+                "start_time": time.time(),
+            }
+
+    def _log_final_stats(self, total_items: int) -> None:
+        """Log final processing statistics.
+
+        Args:
+            total_items: Total number of items processed
+        """
+        with self._stats_lock:
+            stats = self._processing_stats.copy()
+
+        if not stats["batch_times"]:
+            return
+
+        total_time = time.time() - stats["start_time"]
+        avg_batch_time = sum(stats["batch_times"]) / len(stats["batch_times"])
+        items_per_second = stats["total_processed"] / total_time if total_time > 0 else 0
+
+        self.logger.info(
+            f"Batch processing complete: {stats['total_processed']}/{total_items} items processed "
+            f"in {total_time:.2f}s ({items_per_second:.1f} items/s), "
+            f"{stats['total_errors']} errors, avg batch time: {avg_batch_time:.2f}s"
+        )
+
+    def get_processing_stats(self) -> dict[str, Any]:
+        """Get current processing statistics.
+
+        Returns:
+            Dictionary with processing statistics
+        """
+        with self._stats_lock:
+            stats = self._processing_stats.copy()
+
+        if stats["start_time"] and stats["batch_times"]:
+            elapsed_time = time.time() - stats["start_time"]
+            avg_batch_time = sum(stats["batch_times"]) / len(stats["batch_times"])
+            items_per_second = stats["total_processed"] / elapsed_time if elapsed_time > 0 else 0
+
+            return {
+                "total_processed": stats["total_processed"],
+                "total_errors": stats["total_errors"],
+                "elapsed_time_seconds": elapsed_time,
+                "avg_batch_time_seconds": avg_batch_time,
+                "items_per_second": items_per_second,
+                "batches_completed": len(stats["batch_times"]),
+                "error_rate": (
+                    stats["total_errors"] / (stats["total_processed"] + stats["total_errors"])
+                    if (stats["total_processed"] + stats["total_errors"]) > 0
+                    else 0.0
+                ),
+            }
+        else:
+            return {
+                "total_processed": 0,
+                "total_errors": 0,
+                "elapsed_time_seconds": 0,
+                "avg_batch_time_seconds": 0,
+                "items_per_second": 0,
+                "batches_completed": 0,
+                "error_rate": 0.0,
+            }
+
+
+class ProgressTracker:
+    """Simple progress tracking for long-running operations."""
+
+    def __init__(self, total: int, description: str = "Processing"):
+        """Initialize progress tracker.
+
+        Args:
+            total: Total number of items to process
+            description: Description of the operation
+        """
+        self.total = total
+        self.description = description
+        self.processed = 0
+        self.start_time = time.time()
+        self.last_report = 0
+        self.logger = logging.getLogger(__name__)
+
+    def update(self, count: int = 1) -> None:
+        """Update progress count.
+
+        Args:
+            count: Number of items processed since last update
+        """
+        self.processed += count
+
+        # Report progress every 10% or every 1000 items, whichever is less frequent
+        report_interval = max(self.total // 10, 1000)
+
+        if self.processed - self.last_report >= report_interval or self.processed >= self.total:
+            self._report_progress()
+            self.last_report = self.processed
+
+    def _report_progress(self) -> None:
+        """Report current progress."""
+        elapsed_time = time.time() - self.start_time
+        percentage = (self.processed / self.total) * 100 if self.total > 0 else 0
+        rate = self.processed / elapsed_time if elapsed_time > 0 else 0
+
+        # Estimate time remaining
+        if rate > 0 and self.processed < self.total:
+            remaining_items = self.total - self.processed
+            eta_seconds = remaining_items / rate
+            eta_str = f", ETA: {eta_seconds:.0f}s"
+        else:
+            eta_str = ""
+
+        self.logger.info(
+            f"{self.description}: {self.processed}/{self.total} ({percentage:.1f}%) "
+            f"at {rate:.1f} items/s{eta_str}"
+        )