gitflow-analytics 1.0.3__py3-none-any.whl → 1.3.11__py3-none-any.whl

gitflow_analytics/config/schema.py

@@ -0,0 +1,444 @@
+"""Configuration schema definitions and defaults for GitFlow Analytics."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional
+
+if TYPE_CHECKING:
+    from ..qualitative.models.schemas import QualitativeConfig
+
+
+@dataclass
+class RepositoryConfig:
+    """Configuration for a single repository."""
+
+    name: str
+    path: Path
+    github_repo: Optional[str] = None
+    project_key: Optional[str] = None
+    branch: Optional[str] = None
+
+    def __post_init__(self) -> None:
+        self.path = Path(self.path).expanduser().resolve()
+        if not self.project_key:
+            self.project_key = self.name.upper().replace("-", "_")
+
+
+@dataclass
+class GitHubConfig:
+    """GitHub API configuration."""
+
+    token: Optional[str] = None
+    owner: Optional[str] = None
+    organization: Optional[str] = None
+    base_url: str = "https://api.github.com"
+    max_retries: int = 3
+    backoff_factor: int = 2
+
+    def get_repo_full_name(self, repo_name: str) -> str:
+        """Get full repository name including owner."""
+        if "/" in repo_name:
+            return repo_name
+        if self.owner:
+            return f"{self.owner}/{repo_name}"
+        raise ValueError(f"Repository {repo_name} needs owner specified")
+
+
+@dataclass
+class MLCategorization:
+    """ML-based commit categorization configuration."""
+
+    enabled: bool = True
+    min_confidence: float = 0.6
+    semantic_weight: float = 0.7
+    file_pattern_weight: float = 0.3
+    hybrid_threshold: float = 0.5  # Confidence threshold for using ML vs rule-based
+    cache_duration_days: int = 30
+    batch_size: int = 100
+    enable_caching: bool = True
+    spacy_model: str = "en_core_web_sm"  # Preferred spaCy model
+
+
+@dataclass
+class LLMClassificationConfig:
+    """LLM-based commit classification configuration.
+
+    This configuration enables Large Language Model-based commit classification
+    via OpenRouter API for more accurate and context-aware categorization.
+    """
+
+    # Enable/disable LLM classification
+    enabled: bool = False  # Disabled by default to avoid unexpected API costs
+
+    # OpenRouter API configuration
+    api_key: Optional[str] = None  # Set via environment variable or config
+    api_base_url: str = "https://openrouter.ai/api/v1"
+    model: str = "mistralai/mistral-7b-instruct"  # Fast, affordable model
+
+    # Alternative models for different use cases:
+    # - "meta-llama/llama-3-8b-instruct" (Higher accuracy, slightly more expensive)
+    # - "openai/gpt-3.5-turbo" (Good balance, more expensive)
+
+    # Classification parameters
+    confidence_threshold: float = 0.7  # Minimum confidence for LLM predictions
+    max_tokens: int = 50  # Keep responses short for cost optimization
+    temperature: float = 0.1  # Low temperature for consistent results
+    timeout_seconds: float = 30.0  # API request timeout
+
+    # Caching configuration (aggressive caching for cost optimization)
+    cache_duration_days: int = 90  # Long cache duration
+    enable_caching: bool = True
+
+    # Cost and rate limiting
+    max_daily_requests: int = 1000  # Daily API request limit
+
+    # Domain-specific terms for better classification accuracy
+    domain_terms: dict[str, list[str]] = field(
+        default_factory=lambda: {
+            "media": [
+                "video",
+                "audio",
+                "streaming",
+                "player",
+                "media",
+                "content",
+                "broadcast",
+                "live",
+                "recording",
+                "episode",
+                "program",
+                "tv",
+                "radio",
+                "podcast",
+                "channel",
+                "playlist",
+            ],
+            "localization": [
+                "translation",
+                "i18n",
+                "l10n",
+                "locale",
+                "language",
+                "spanish",
+                "french",
+                "german",
+                "italian",
+                "portuguese",
+                "multilingual",
+                "translate",
+                "localize",
+                "regional",
+            ],
+            "integration": [
+                "api",
+                "webhook",
+                "third-party",
+                "external",
+                "service",
+                "integration",
+                "sync",
+                "import",
+                "export",
+                "connector",
+                "oauth",
+                "auth",
+                "authentication",
+                "sso",
+            ],
+            "content": [
+                "copy",
+                "text",
+                "wording",
+                "messaging",
+                "editorial",
+                "article",
+                "blog",
+                "news",
+                "story",
+                "caption",
+                "title",
+                "headline",
+                "description",
+                "summary",
+                "metadata",
+            ],
+        }
+    )
+
+    # Fallback behavior when LLM is unavailable
+    fallback_to_rules: bool = True  # Fall back to rule-based classification
+    fallback_to_ml: bool = True  # Fall back to existing ML classification
+
+
+@dataclass
+class CommitClassificationConfig:
+    """Configuration for commit classification system.
+
+    This configuration controls the Random Forest-based commit classification
+    system that analyzes commits to categorize them into types like feature,
+    bugfix, refactor, docs, test, etc.
+    """
+
+    enabled: bool = True
+    confidence_threshold: float = 0.5  # Minimum confidence for reliable predictions
+    batch_size: int = 100  # Commits processed per batch
+    auto_retrain: bool = True  # Automatically check if model needs retraining
+    retrain_threshold_days: int = 30  # Days after which to suggest retraining
+
+    # Model hyperparameters
+    model: dict[str, Any] = field(
+        default_factory=lambda: {
+            "n_estimators": 100,  # Number of trees in random forest
+            "max_depth": 20,  # Maximum depth of trees
+            "min_samples_split": 5,  # Minimum samples to split a node
+            "min_samples_leaf": 2,  # Minimum samples at leaf node
+            "random_state": 42,  # For reproducible results
+            "n_jobs": -1,  # Use all available CPU cores
+        }
+    )
+
+    # Feature extraction settings
+    feature_extraction: dict[str, Any] = field(
+        default_factory=lambda: {
+            "enable_temporal_features": True,
+            "enable_author_features": True,
+            "enable_file_analysis": True,
+            "keyword_categories": [
+                "feature",
+                "bugfix",
+                "refactor",
+                "docs",
+                "test",
+                "config",
+                "security",
+                "performance",
+                "ui",
+                "api",
+                "database",
+                "deployment",
+            ],
+        }
+    )
+
+    # Training settings
+    training: dict[str, Any] = field(
+        default_factory=lambda: {
+            "validation_split": 0.2,  # Fraction for validation
+            "min_training_samples": 20,  # Minimum samples needed for training
+            "cross_validation_folds": 5,  # K-fold cross validation
+            "class_weight": "balanced",  # Handle class imbalance
+        }
+    )
+
+    # Supported classification categories
+    categories: dict[str, str] = field(
+        default_factory=lambda: {
+            "feature": "New functionality or capabilities",
+            "bugfix": "Bug fixes and error corrections",
+            "refactor": "Code restructuring and optimization",
+            "docs": "Documentation changes and updates",
+            "test": "Testing-related changes",
+            "config": "Configuration and settings changes",
+            "chore": "Maintenance and housekeeping tasks",
+            "security": "Security-related changes",
+            "hotfix": "Emergency production fixes",
+            "style": "Code style and formatting changes",
+            "build": "Build system and dependency changes",
+            "ci": "Continuous integration changes",
+            "revert": "Reverts of previous changes",
+            "merge": "Merge commits and integration",
+            "wip": "Work in progress commits",
+        }
+    )
+
+
+@dataclass
+class BranchAnalysisConfig:
+    """Configuration for branch analysis optimization.
+
+    This configuration controls how branches are analyzed to prevent performance
+    issues on large organizations with many repositories and branches.
+    """
+
+    # Branch analysis strategy
+    strategy: str = "smart"  # Options: "all", "smart", "main_only"
+
+    # Smart analysis parameters
+    max_branches_per_repo: int = 50  # Maximum branches to analyze per repository
+    active_days_threshold: int = 90  # Days to consider a branch "active"
+    include_main_branches: bool = True  # Always include main/master branches
+
+    # Branch name patterns to always include/exclude
+    always_include_patterns: list[str] = field(
+        default_factory=lambda: [
+            r"^(main|master|develop|dev)$",  # Main development branches
+            r"^release/.*",  # Release branches
+            r"^hotfix/.*",  # Hotfix branches
+        ]
+    )
+
+    always_exclude_patterns: list[str] = field(
+        default_factory=lambda: [
+            r"^dependabot/.*",  # Dependabot branches
+            r"^renovate/.*",  # Renovate branches
+            r".*-backup$",  # Backup branches
+            r".*-temp$",  # Temporary branches
+        ]
+    )
+
+    # Performance limits
+    enable_progress_logging: bool = True  # Log branch analysis progress
+    branch_commit_limit: int = 1000  # Max commits to analyze per branch
+
+
+@dataclass
+class AnalysisConfig:
+    """Analysis-specific configuration."""
+
+    story_point_patterns: list[str] = field(default_factory=list)
+    exclude_authors: list[str] = field(default_factory=list)
+    exclude_message_patterns: list[str] = field(default_factory=list)
+    exclude_paths: list[str] = field(default_factory=list)
+    similarity_threshold: float = 0.85
+    manual_identity_mappings: list[dict[str, Any]] = field(default_factory=list)
+    default_ticket_platform: Optional[str] = None
+    branch_mapping_rules: dict[str, list[str]] = field(default_factory=dict)
+    ticket_platforms: Optional[list[str]] = None
+    auto_identity_analysis: bool = True  # Enable automatic identity analysis by default
+    branch_patterns: Optional[list[str]] = (
+        None  # Branch patterns to analyze (e.g., ["*"] for all branches)
+    )
+    branch_analysis: BranchAnalysisConfig = field(default_factory=BranchAnalysisConfig)
+    ml_categorization: MLCategorization = field(default_factory=MLCategorization)
+    commit_classification: CommitClassificationConfig = field(
+        default_factory=CommitClassificationConfig
+    )
+    llm_classification: LLMClassificationConfig = field(default_factory=LLMClassificationConfig)
+
+
+@dataclass
+class OutputConfig:
+    """Output configuration."""
+
+    directory: Optional[Path] = None
+    formats: list[str] = field(default_factory=lambda: ["csv", "markdown"])
+    csv_delimiter: str = ","
+    csv_encoding: str = "utf-8"
+    anonymize_enabled: bool = False
+    anonymize_fields: list[str] = field(default_factory=list)
+    anonymize_method: str = "hash"
+
+
+@dataclass
+class CacheConfig:
+    """Cache configuration."""
+
+    directory: Path = Path(".gitflow-cache")
+    ttl_hours: int = 168
+    max_size_mb: int = 500
+
+
+@dataclass
+class JIRAConfig:
+    """JIRA configuration."""
+
+    access_user: str
+    access_token: str
+    base_url: Optional[str] = None
+
+
+@dataclass
+class JIRAIntegrationConfig:
+    """JIRA integration specific configuration."""
+
+    enabled: bool = True
+    fetch_story_points: bool = True
+    project_keys: list[str] = field(default_factory=list)
+    story_point_fields: list[str] = field(
+        default_factory=lambda: ["customfield_10016", "customfield_10021", "Story Points"]
+    )
+
+
+@dataclass
+class PMPlatformConfig:
+    """Base PM platform configuration."""
+
+    enabled: bool = True
+    platform_type: str = ""
+    config: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class PMIntegrationConfig:
+    """PM framework integration configuration."""
+
+    enabled: bool = False
+    primary_platform: Optional[str] = None
+    correlation: dict[str, Any] = field(default_factory=dict)
+    platforms: dict[str, PMPlatformConfig] = field(default_factory=dict)
+
+
+@dataclass
+class Config:
+    """Main configuration container."""
+
+    repositories: list[RepositoryConfig]
+    github: GitHubConfig
+    analysis: AnalysisConfig
+    output: OutputConfig
+    cache: CacheConfig
+    jira: Optional[JIRAConfig] = None
+    jira_integration: Optional[JIRAIntegrationConfig] = None
+    pm: Optional[Any] = None  # Modern PM framework config
+    pm_integration: Optional[PMIntegrationConfig] = None
+    qualitative: Optional["QualitativeConfig"] = None
+
+    def discover_organization_repositories(
+        self, clone_base_path: Optional[Path] = None
+    ) -> list[RepositoryConfig]:
+        """Discover repositories from GitHub organization.
+
+        Args:
+            clone_base_path: Base directory where repos should be cloned/found.
+                           If None, uses output directory.
+
+        Returns:
+            List of discovered repository configurations.
+        """
+        if not self.github.organization or not self.github.token:
+            return []
+
+        from github import Github
+
+        github_client = Github(self.github.token, base_url=self.github.base_url)
+
+        try:
+            org = github_client.get_organization(self.github.organization)
+            discovered_repos = []
+
+            base_path = clone_base_path or self.output.directory
+            if base_path is None:
+                raise ValueError("No base path available for repository cloning")
+
+            for repo in org.get_repos():
+                # Skip archived repositories
+                if repo.archived:
+                    continue
+
+                # Create repository configuration
+                repo_path = base_path / repo.name
+                repo_config = RepositoryConfig(
+                    name=repo.name,
+                    path=repo_path,
+                    github_repo=repo.full_name,
+                    project_key=repo.name.upper().replace("-", "_"),
+                    branch=repo.default_branch,
+                )
+                discovered_repos.append(repo_config)
+
+            return discovered_repos
+
+        except Exception as e:
+            raise ValueError(
+                f"Failed to discover repositories from organization {self.github.organization}: {e}"
+            ) from e

gitflow_analytics/config/validator.py

@@ -0,0 +1,154 @@
+"""Configuration validation logic for GitFlow Analytics."""
+
+from pathlib import Path
+
+from .schema import Config
+
+
+class ConfigValidator:
+    """Validates configuration settings."""
+
+    @staticmethod
+    def validate_config(config: Config) -> list[str]:
+        """Validate configuration and return list of warnings.
+
+        Args:
+            config: Configuration to validate
+
+        Returns:
+            List of warning messages
+        """
+        warnings = []
+
+        # Check repository paths exist
+        for repo in config.repositories:
+            if not repo.path.exists():
+                warnings.append(f"Repository path does not exist: {repo.path}")
+            elif not (repo.path / ".git").exists():
+                warnings.append(f"Path is not a git repository: {repo.path}")
+
+        # Check GitHub token if GitHub repos are specified
+        has_github_repos = any(r.github_repo for r in config.repositories)
+        if has_github_repos and not config.github.token:
+            warnings.append("GitHub repositories specified but no GitHub token provided")
+
+        # Check if owner is needed
+        for repo in config.repositories:
+            if repo.github_repo and "/" not in repo.github_repo and not config.github.owner:
+                warnings.append(f"Repository {repo.github_repo} needs owner specified")
+
+        # Check cache directory permissions
+        try:
+            config.cache.directory.mkdir(exist_ok=True, parents=True)
+        except PermissionError:
+            warnings.append(f"Cannot create cache directory: {config.cache.directory}")
+
+        return warnings
+
+    @staticmethod
+    def validate_analysis_config(analysis_config: dict, config_path: Path) -> None:
+        """Validate analysis configuration section.
+
+        Args:
+            analysis_config: Analysis configuration dictionary
+            config_path: Path to configuration file (for error messages)
+
+        Raises:
+            InvalidValueError: If configuration values are invalid
+        """
+        from .errors import InvalidValueError
+
+        # Validate similarity threshold
+        if "identity" in analysis_config:
+            threshold = analysis_config["identity"].get("similarity_threshold")
+            if threshold is not None and not (0.0 <= threshold <= 1.0):
+                raise InvalidValueError(
+                    "similarity_threshold",
+                    threshold,
+                    "must be between 0.0 and 1.0",
+                    config_path,
+                    valid_values=["0.0 to 1.0"],
+                )
+
+        # Validate ML categorization settings
+        if "ml_categorization" in analysis_config:
+            ml_config = analysis_config["ml_categorization"]
+
+            if "min_confidence" in ml_config:
+                conf = ml_config["min_confidence"]
+                if not (0.0 <= conf <= 1.0):
+                    raise InvalidValueError(
+                        "ml_categorization.min_confidence",
+                        conf,
+                        "must be between 0.0 and 1.0",
+                        config_path,
+                        valid_values=["0.0 to 1.0"],
+                    )
+
+            if "semantic_weight" in ml_config:
+                weight = ml_config["semantic_weight"]
+                if not (0.0 <= weight <= 1.0):
+                    raise InvalidValueError(
+                        "ml_categorization.semantic_weight",
+                        weight,
+                        "must be between 0.0 and 1.0",
+                        config_path,
+                        valid_values=["0.0 to 1.0"],
+                    )
+
+        # Validate branch analysis strategy
+        if "branch_analysis" in analysis_config:
+            branch_config = analysis_config["branch_analysis"]
+            if "strategy" in branch_config:
+                strategy = branch_config["strategy"]
+                valid_strategies = ["all", "smart", "main_only"]
+                if strategy not in valid_strategies:
+                    raise InvalidValueError(
+                        "branch_analysis.strategy",
+                        strategy,
+                        "invalid branch analysis strategy",
+                        config_path,
+                        valid_values=valid_strategies,
+                    )
+
+    @staticmethod
+    def validate_output_config(output_config: dict, config_path: Path) -> None:
+        """Validate output configuration section.
+
+        Args:
+            output_config: Output configuration dictionary
+            config_path: Path to configuration file (for error messages)
+
+        Raises:
+            InvalidValueError: If configuration values are invalid
+        """
+        from .errors import InvalidValueError
+
+        # Validate output formats
+        if "formats" in output_config:
+            formats = output_config["formats"]
+            valid_formats = ["csv", "markdown", "json"]
+            for fmt in formats:
+                if fmt not in valid_formats:
+                    raise InvalidValueError(
+                        "output.formats",
+                        fmt,
+                        "invalid output format",
+                        config_path,
+                        valid_values=valid_formats,
+                    )
+
+        # Validate anonymization method
+        if "anonymization" in output_config:
+            anon_config = output_config["anonymization"]
+            if "method" in anon_config:
+                method = anon_config["method"]
+                valid_methods = ["hash", "random", "sequential"]
+                if method not in valid_methods:
+                    raise InvalidValueError(
+                        "output.anonymization.method",
+                        method,
+                        "invalid anonymization method",
+                        config_path,
+                        valid_values=valid_methods,
+                    )