gitflow-analytics 1.0.1__py3-none-any.whl → 1.3.6__py3-none-any.whl

gitflow_analytics/core/progress.py

@@ -0,0 +1,372 @@
+"""Centralized progress reporting service for GitFlow Analytics.
+
+This module provides a unified interface for progress reporting across the application,
+replacing scattered tqdm usage with a centralized, testable, and configurable service.
+
+WHY: Progress reporting was scattered across multiple modules (analyzer.py, data_fetcher.py,
+batch_classifier.py, etc.), violating DRY principles and making it difficult to maintain
+consistent progress UX. This service centralizes all progress management.
+
+DESIGN DECISIONS:
+- Context-based API: Each progress bar gets a context object for clean lifecycle management
+- Thread-safe: Uses threading locks to ensure safe concurrent access
+- Testable: Can be globally disabled for testing, with event capture capability
+- Nested support: Handles nested progress contexts with proper positioning
+- Consistent styling: All progress bars follow the same formatting rules
+
+USAGE:
+    from gitflow_analytics.core.progress import get_progress_service
+
+    progress = get_progress_service()
+    context = progress.create_progress(100, "Processing items")
+    for item in items:
+        # Process item
+        progress.update(context)
+    progress.complete(context)
+"""
+
+import os
+import sys
+import threading
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from tqdm import tqdm
+
+
+@dataclass
+class ProgressContext:
+    """Context object for a single progress operation.
+
+    Encapsulates all state for a progress bar, allowing clean lifecycle management
+    and preventing resource leaks.
+    """
+
+    progress_bar: Optional[Any]  # tqdm instance or None if disabled
+    description: str
+    total: int
+    unit: str
+    position: int
+    current: int = 0
+    is_nested: bool = False
+    parent_context: Optional["ProgressContext"] = None
+
+
+@dataclass
+class ProgressEvent:
+    """Event captured during progress operations for testing.
+
+    Allows tests to verify that progress operations occurred without
+    actually displaying progress bars.
+    """
+
+    event_type: str  # 'create', 'update', 'complete'
+    description: str
+    total: Optional[int] = None
+    increment: Optional[int] = None
+    current: Optional[int] = None
+
+
+class ProgressService:
+    """Centralized service for managing progress reporting.
+
+    This service provides a unified interface for creating and managing progress bars
+    throughout the application. It supports nested progress contexts, global disable
+    for testing, and event capture for verification.
+    """
+
+    def __init__(self):
+        """Initialize the progress service."""
+        self._enabled = True
+        self._lock = threading.Lock()
+        self._active_contexts: list[ProgressContext] = []
+        self._position_counter = 0
+        self._capture_events = False
+        self._captured_events: list[ProgressEvent] = []
+
+        # Check environment for testing mode
+        self._check_testing_environment()
+
+    def _check_testing_environment(self):
+        """Check if running in a testing environment and disable if needed.
+
+        WHY: Progress bars interfere with test output and can cause issues in CI/CD.
+        This automatically detects common testing scenarios and disables progress.
+        """
+        # Disable in pytest
+        if "pytest" in sys.modules:
+            self._enabled = False
+
+        # Disable if explicitly requested via environment
+        if os.environ.get("GITFLOW_DISABLE_PROGRESS", "").lower() in ("1", "true", "yes"):
+            self._enabled = False
+
+        # Disable if not in a TTY (e.g., CI/CD, piped output)
+        if not sys.stdout.isatty():
+            self._enabled = False
+
+    def create_progress(
+        self,
+        total: int,
+        description: str,
+        unit: str = "items",
+        nested: bool = False,
+        leave: bool = True,
+        position: Optional[int] = None,
+    ) -> ProgressContext:
+        """Create a new progress context.
+
+        Args:
+            total: Total number of items to process
+            description: Description shown next to the progress bar
+            unit: Unit label for items (e.g., "commits", "repos", "files")
+            nested: Whether this is a nested progress bar
+            leave: Whether to leave the progress bar on screen after completion
+            position: Explicit position for the progress bar (for nested contexts)
+
+        Returns:
+            ProgressContext object to use for updates
+
+        DESIGN: Returns a context object rather than the tqdm instance directly
+        to provide better lifecycle management and prevent resource leaks.
+        """
+        with self._lock:
+            # Capture event if needed
+            if self._capture_events:
+                self._captured_events.append(ProgressEvent("create", description, total=total))
+
+            # Determine position for nested progress bars
+            if position is None:
+                if nested:
+                    self._position_counter += 1
+                position = self._position_counter
+
+            # Create context
+            context = ProgressContext(
+                progress_bar=None,
+                description=description,
+                total=total,
+                unit=unit,
+                position=position,
+                is_nested=nested,
+            )
+
+            # Create actual progress bar if enabled
+            if self._enabled:
+                context.progress_bar = tqdm(
+                    total=total,
+                    desc=description,
+                    unit=unit,
+                    position=position,
+                    leave=leave,
+                    # Consistent styling
+                    bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}]",
+                    dynamic_ncols=True,
+                )
+
+            self._active_contexts.append(context)
+            return context
+
+    def update(
+        self, context: ProgressContext, increment: int = 1, description: Optional[str] = None
+    ):
+        """Update progress for a given context.
+
+        Args:
+            context: The progress context to update
+            increment: Number of items completed (default: 1)
+            description: Optional new description to set
+
+        WHY: Centralizes update logic and ensures consistent behavior across
+        all progress bars in the application.
+        """
+        with self._lock:
+            context.current += increment
+
+            # Capture event if needed
+            if self._capture_events:
+                self._captured_events.append(
+                    ProgressEvent(
+                        "update",
+                        description or context.description,
+                        increment=increment,
+                        current=context.current,
+                    )
+                )
+
+            # Update actual progress bar if it exists
+            if context.progress_bar:
+                context.progress_bar.update(increment)
+                if description:
+                    context.progress_bar.set_description(description)
+
+    def set_description(self, context: ProgressContext, description: str):
+        """Update the description of a progress context.
+
+        Args:
+            context: The progress context to update
+            description: New description to display
+        """
+        with self._lock:
+            context.description = description
+            if context.progress_bar:
+                context.progress_bar.set_description(description)
+
+    def complete(self, context: ProgressContext):
+        """Mark a progress context as complete and clean up resources.
+
+        Args:
+            context: The progress context to complete
+
+        IMPORTANT: Always call this method when done with a progress context
+        to ensure proper resource cleanup.
+        """
+        with self._lock:
+            # Capture event if needed
+            if self._capture_events:
+                self._captured_events.append(
+                    ProgressEvent("complete", context.description, current=context.current)
+                )
+
+            # Remove from active contexts BEFORE modifying progress_bar
+            # to avoid comparison issues with None
+            if context in self._active_contexts:
+                self._active_contexts.remove(context)
+
+            # Close actual progress bar if it exists
+            if context.progress_bar:
+                context.progress_bar.close()
+                context.progress_bar = None
+
+            # Reset position counter if no nested contexts remain
+            if context.is_nested and not any(c.is_nested for c in self._active_contexts):
+                self._position_counter = 0
+
+    @contextmanager
+    def progress(
+        self,
+        total: int,
+        description: str,
+        unit: str = "items",
+        nested: bool = False,
+        leave: bool = True,
+    ):
+        """Context manager for progress operations.
+
+        Args:
+            total: Total number of items to process
+            description: Description shown next to the progress bar
+            unit: Unit label for items
+            nested: Whether this is a nested progress bar
+            leave: Whether to leave the progress bar on screen
+
+        Yields:
+            ProgressContext object for updates
+
+        Example:
+            with progress.progress(100, "Processing") as ctx:
+                for item in items:
+                    process(item)
+                    progress.update(ctx)
+        """
+        context = self.create_progress(total, description, unit, nested, leave)
+        try:
+            yield context
+        finally:
+            self.complete(context)
+
+    def disable(self):
+        """Disable all progress reporting globally.
+
+        Useful for testing or quiet mode operation.
+        """
+        with self._lock:
+            self._enabled = False
+            # Close any active progress bars
+            for context in self._active_contexts[:]:
+                if context.progress_bar:
+                    context.progress_bar.close()
+                    context.progress_bar = None
+
+    def enable(self):
+        """Enable progress reporting globally."""
+        with self._lock:
+            self._enabled = True
+
+    def is_enabled(self) -> bool:
+        """Check if progress reporting is enabled."""
+        return self._enabled
+
+    def start_event_capture(self):
+        """Start capturing progress events for testing.
+
+        WHY: Allows tests to verify that progress operations occurred
+        without actually displaying progress bars.
+        """
+        with self._lock:
+            self._capture_events = True
+            self._captured_events = []
+
+    def stop_event_capture(self) -> list[ProgressEvent]:
+        """Stop capturing events and return captured events.
+
+        Returns:
+            List of ProgressEvent objects that were captured
+        """
+        with self._lock:
+            self._capture_events = False
+            events = self._captured_events[:]
+            self._captured_events = []
+            return events
+
+    def get_captured_events(self) -> list[ProgressEvent]:
+        """Get currently captured events without stopping capture.
+
+        Returns:
+            List of ProgressEvent objects captured so far
+        """
+        with self._lock:
+            return self._captured_events[:]
+
+    def clear_captured_events(self):
+        """Clear captured events without stopping capture."""
+        with self._lock:
+            self._captured_events = []
+
+
+# Global singleton instance
+_progress_service: Optional[ProgressService] = None
+_service_lock = threading.Lock()
+
+
+def get_progress_service() -> ProgressService:
+    """Get the global progress service instance.
+
+    Returns:
+        The singleton ProgressService instance
+
+    Thread-safe singleton pattern ensures only one progress service exists.
+    """
+    global _progress_service
+
+    if _progress_service is None:
+        with _service_lock:
+            if _progress_service is None:
+                _progress_service = ProgressService()
+
+    return _progress_service
+
+
+def reset_progress_service():
+    """Reset the global progress service instance.
+
+    WARNING: Only use this in tests or during application shutdown.
+    This will close all active progress bars and create a new service instance.
+    """
+    global _progress_service
+
+    with _service_lock:
+        if _progress_service:
+            _progress_service.disable()
+        _progress_service = None

gitflow_analytics/core/schema_version.py

@@ -0,0 +1,269 @@
+"""Schema versioning for tracking data structure changes."""
+
+import hashlib
+import json
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+from sqlalchemy import Column, DateTime, String, Text, create_engine
+from sqlalchemy.orm import declarative_base, sessionmaker
+
+Base = declarative_base()
+
+
+class SchemaVersion(Base):
+    """Track schema versions for incremental data processing."""
+
+    __tablename__ = "schema_versions"
+
+    component = Column(String, primary_key=True)  # e.g., 'qualitative', 'identity', 'core'
+    version_hash = Column(String, nullable=False)  # Hash of schema definition
+    schema_definition = Column(Text, nullable=False)  # JSON schema definition
+    created_at = Column(DateTime, default=datetime.utcnow)
+    last_processed_date = Column(
+        DateTime, nullable=True
+    )  # Last date we processed data with this schema
+
+
+class SchemaVersionManager:
+    """Manages schema versions and determines if incremental processing is possible."""
+
+    # Define current schema versions for each component
+    CURRENT_SCHEMAS = {
+        "qualitative": {
+            "version": "2.0",
+            "fields": [
+                "change_type",
+                "change_type_confidence",
+                "business_domain",
+                "domain_confidence",
+                "risk_level",
+                "risk_factors",
+                "intent_signals",
+                "collaboration_patterns",
+                "technical_context",
+                "processing_method",
+                "processing_time_ms",
+                "confidence_score",
+            ],
+            "config_fields": [
+                "nlp_config",
+                "llm_config",
+                "cache_config",
+                "confidence_threshold",
+                "max_llm_fallback_pct",
+            ],
+        },
+        "identity": {
+            "version": "1.3",
+            "fields": [
+                "canonical_id",
+                "primary_name",
+                "primary_email",
+                "manual_mappings",
+                "similarity_threshold",
+                "auto_analysis",
+                "display_names",
+                "preferred_name_field",
+            ],
+        },
+        "core": {
+            "version": "1.0",
+            "fields": [
+                "story_points",
+                "ticket_references",
+                "files_changed",
+                "insertions",
+                "deletions",
+                "complexity_delta",
+                "branch_mapping_rules",
+            ],
+        },
+        "github": {
+            "version": "1.0",
+            "fields": [
+                "pr_data",
+                "pr_metrics",
+                "issue_data",
+                "rate_limit_retries",
+                "backoff_factor",
+                "allowed_ticket_platforms",
+            ],
+        },
+        "jira": {
+            "version": "1.0",
+            "fields": ["story_point_fields", "project_keys", "base_url", "issue_data"],
+        },
+    }
+
+    def __init__(self, cache_dir: Path):
+        """Initialize schema version manager."""
+        self.cache_dir = cache_dir
+        self.db_path = cache_dir / "schema_versions.db"
+        self.engine = create_engine(f"sqlite:///{self.db_path}")
+        Base.metadata.create_all(self.engine)
+        self.session_factory = sessionmaker(bind=self.engine)
+
+    def get_schema_hash(self, component: str, config: Optional[dict[str, Any]] = None) -> str:
+        """Generate hash for a component's schema including configuration."""
+        if component not in self.CURRENT_SCHEMAS:
+            raise ValueError(f"Unknown component: {component}")
+
+        schema_def = self.CURRENT_SCHEMAS[component].copy()
+
+        # Include relevant configuration in the hash
+        if config and "config_fields" in schema_def:
+            relevant_config = {}
+            for field in schema_def["config_fields"]:
+                if field in config:
+                    relevant_config[field] = self._normalize_config_value(config[field])
+            schema_def["config"] = relevant_config
+
+        # Create deterministic hash
+        schema_json = json.dumps(schema_def, sort_keys=True)
+        return hashlib.sha256(schema_json.encode()).hexdigest()[:16]
+
+    def _normalize_config_value(self, value: Any) -> Any:
+        """Normalize config values for consistent hashing."""
+        if isinstance(value, dict):
+            return {k: self._normalize_config_value(v) for k, v in sorted(value.items())}
+        elif isinstance(value, list):
+            return sorted([self._normalize_config_value(v) for v in value])
+        elif isinstance(value, (int, float, str, bool, type(None))):
+            return value
+        else:
+            # Convert complex objects to string representation
+            return str(value)
+
+    def has_schema_changed(self, component: str, config: Optional[dict[str, Any]] = None) -> bool:
+        """Check if schema has changed since last processing."""
+        current_hash = self.get_schema_hash(component, config)
+
+        with self.session_factory() as session:
+            stored_version = session.query(SchemaVersion).filter_by(component=component).first()
+
+            if not stored_version:
+                return True  # No previous schema, consider changed
+
+            return stored_version.version_hash != current_hash
+
+    def update_schema_version(
+        self,
+        component: str,
+        config: Optional[dict[str, Any]] = None,
+        last_processed_date: Optional[datetime] = None,
+    ):
+        """Update stored schema version."""
+        current_hash = self.get_schema_hash(component, config)
+        schema_def = json.dumps(self.CURRENT_SCHEMAS[component], sort_keys=True)
+
+        # Ensure date is timezone-aware before storing
+        if last_processed_date and last_processed_date.tzinfo is None:
+            last_processed_date = last_processed_date.replace(tzinfo=timezone.utc)
+
+        with self.session_factory() as session:
+            stored_version = session.query(SchemaVersion).filter_by(component=component).first()
+
+            if stored_version:
+                stored_version.version_hash = current_hash
+                stored_version.schema_definition = schema_def
+                if last_processed_date:
+                    stored_version.last_processed_date = last_processed_date
+            else:
+                stored_version = SchemaVersion(
+                    component=component,
+                    version_hash=current_hash,
+                    schema_definition=schema_def,
+                    last_processed_date=last_processed_date,
+                )
+                session.add(stored_version)
+
+            session.commit()
+
+    def get_last_processed_date(self, component: str) -> Optional[datetime]:
+        """Get the last date data was processed for this component."""
+        with self.session_factory() as session:
+            stored_version = session.query(SchemaVersion).filter_by(component=component).first()
+            return stored_version.last_processed_date if stored_version else None
+
+    def should_process_date(
+        self, component: str, date: datetime, config: Optional[dict[str, Any]] = None
+    ) -> bool:
+        """Determine if we should process data for a given date."""
+        # Always process if schema has changed
+        if self.has_schema_changed(component, config):
+            return True
+
+        # Check if we've already processed this date
+        last_processed = self.get_last_processed_date(component)
+        if not last_processed:
+            return True
+
+        # Ensure both dates are timezone-aware for comparison
+        if date.tzinfo is None:
+            date = date.replace(tzinfo=timezone.utc)
+        if last_processed.tzinfo is None:
+            last_processed = last_processed.replace(tzinfo=timezone.utc)
+
+        # Process if date is after last processed date
+        return date > last_processed
+
+    def mark_date_processed(
+        self, component: str, date: datetime, config: Optional[dict[str, Any]] = None
+    ):
+        """Mark a date as processed for incremental tracking."""
+        # Ensure date is timezone-aware before storing
+        if date.tzinfo is None:
+            date = date.replace(tzinfo=timezone.utc)
+
+        with self.session_factory() as session:
+            stored_version = session.query(SchemaVersion).filter_by(component=component).first()
+
+            if stored_version:
+                # Update to the latest processed date
+                if not stored_version.last_processed_date:
+                    stored_version.last_processed_date = date
+                    session.commit()
+                else:
+                    # Ensure stored date is timezone-aware for comparison
+                    stored_date = stored_version.last_processed_date
+                    if stored_date.tzinfo is None:
+                        stored_date = stored_date.replace(tzinfo=timezone.utc)
+
+                    if date > stored_date:
+                        stored_version.last_processed_date = date
+                        session.commit()
+            else:
+                # Create new entry
+                self.update_schema_version(component, config, date)
+
+    def get_schema_info(self, component: str) -> dict[str, Any]:
+        """Get detailed schema information for debugging."""
+        with self.session_factory() as session:
+            stored_version = session.query(SchemaVersion).filter_by(component=component).first()
+
+            current_hash = self.get_schema_hash(component)
+
+            return {
+                "component": component,
+                "current_schema_hash": current_hash,
+                "stored_schema_hash": stored_version.version_hash if stored_version else None,
+                "schema_changed": self.has_schema_changed(component),
+                "last_processed": stored_version.last_processed_date if stored_version else None,
+                "created_at": stored_version.created_at if stored_version else None,
+            }
+
+    def reset_component(self, component: str):
+        """Reset schema version for a component (forces full reprocessing)."""
+        with self.session_factory() as session:
+            stored_version = session.query(SchemaVersion).filter_by(component=component).first()
+            if stored_version:
+                session.delete(stored_version)
+                session.commit()
+
+
+def create_schema_manager(cache_dir: Path) -> SchemaVersionManager:
+    """Factory function to create a schema version manager."""
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    return SchemaVersionManager(cache_dir)

gitflow_analytics/extractors/base.py

@@ -1,11 +1,12 @@
 """Base classes for pluggable extractors."""
+
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional
+from typing import Any, Optional
 
 
 class ExtractorBase(ABC):
     """Base class for all extractors."""
-    
+
     @abstractmethod
     def extract_from_text(self, text: str) -> Any:
         """Extract information from text."""
@@ -14,28 +15,29 @@ class ExtractorBase(ABC):
 
 class StoryPointExtractorBase(ExtractorBase):
     """Base class for story point extractors."""
-    
+
     @abstractmethod
     def extract_from_text(self, text: str) -> Optional[int]:
         """Extract story points from text."""
         pass
-    
+
     @abstractmethod
-    def extract_from_pr(self, pr_data: Dict[str, Any], 
-                       commit_messages: Optional[List[str]] = None) -> Optional[int]:
+    def extract_from_pr(
+        self, pr_data: dict[str, Any], commit_messages: Optional[list[str]] = None
+    ) -> Optional[int]:
         """Extract story points from pull request."""
         pass
 
 
 class TicketExtractorBase(ExtractorBase):
     """Base class for ticket extractors."""
-    
+
     @abstractmethod
-    def extract_from_text(self, text: str) -> List[Dict[str, str]]:
+    def extract_from_text(self, text: str) -> list[dict[str, str]]:
         """Extract ticket references from text."""
         pass
-    
+
     @abstractmethod
-    def extract_by_platform(self, text: str) -> Dict[str, List[str]]:
+    def extract_by_platform(self, text: str) -> dict[str, list[str]]:
         """Extract tickets grouped by platform."""
-        pass
+        pass