gitflow-analytics 1.0.0__py3-none-any.whl

gitflow_analytics/__init__.py

@@ -0,0 +1,22 @@
+"""GitFlow Analytics - Git repository productivity analysis tool."""
+
+from ._version import __version__, __version_info__
+
+__author__ = 'Bob Matyas'
+__email__ = 'bobmatnyc@gmail.com'
+
+from .core.analyzer import GitAnalyzer
+from .core.cache import GitAnalysisCache
+from .core.identity import DeveloperIdentityResolver
+from .extractors.story_points import StoryPointExtractor
+from .extractors.tickets import TicketExtractor
+from .reports.csv_writer import CSVReportGenerator
+
+__all__ = [
+    'GitAnalyzer',
+    'GitAnalysisCache',
+    'DeveloperIdentityResolver',
+    'StoryPointExtractor',
+    'TicketExtractor',
+    'CSVReportGenerator',
+]

gitflow_analytics/_version.py

@@ -0,0 +1,4 @@
+"""Version information for gitflow-analytics."""
+
+__version__ = "1.0.0"
+__version_info__ = tuple(int(x) for x in __version__.split("."))

gitflow_analytics/cli.py

@@ -0,0 +1,441 @@
+"""Command-line interface for GitFlow Analytics."""
+import click
+import yaml
+from pathlib import Path
+from datetime import datetime, timedelta
+from typing import Optional
+import sys
+import pandas as pd
+
+from .config import ConfigLoader, Config
+from .core.cache import GitAnalysisCache
+from .core.analyzer import GitAnalyzer
+from .core.identity import DeveloperIdentityResolver
+from .extractors.story_points import StoryPointExtractor
+from .extractors.tickets import TicketExtractor
+from .reports.csv_writer import CSVReportGenerator
+from .reports.analytics_writer import AnalyticsReportGenerator
+from .reports.narrative_writer import NarrativeReportGenerator
+from .metrics.dora import DORAMetricsCalculator
+from .integrations.orchestrator import IntegrationOrchestrator
+
+
+@click.group()
+@click.version_option(version='0.1.0', prog_name='GitFlow Analytics')
+def cli():
+    """GitFlow Analytics - Analyze Git repositories for productivity insights."""
+    pass
+
+
+@cli.command()
+@click.option('--config', '-c', 
+              type=click.Path(exists=True, path_type=Path),
+              required=True,
+              help='Path to YAML configuration file')
+@click.option('--weeks', '-w', 
+              type=int, 
+              default=12,
+              help='Number of weeks to analyze (default: 12)')
+@click.option('--output', '-o', 
+              type=click.Path(path_type=Path),
+              default=None,
+              help='Output directory for reports (overrides config file)')
+@click.option('--anonymize', 
+              is_flag=True,
+              help='Anonymize developer information in reports')
+@click.option('--no-cache', 
+              is_flag=True,
+              help='Disable caching (slower but always fresh)')
+@click.option('--validate-only', 
+              is_flag=True,
+              help='Validate configuration without running analysis')
+@click.option('--clear-cache', 
+              is_flag=True,
+              help='Clear cache before running analysis')
+def analyze(config: Path, weeks: int, output: Optional[Path], anonymize: bool, 
+           no_cache: bool, validate_only: bool, clear_cache: bool):
+    """Analyze Git repositories using configuration file."""
+    
+    try:
+        # Load configuration
+        click.echo(f"📋 Loading configuration from {config}...")
+        cfg = ConfigLoader.load(config)
+        
+        # Validate configuration
+        warnings = ConfigLoader.validate_config(cfg)
+        if warnings:
+            click.echo("⚠️  Configuration warnings:")
+            for warning in warnings:
+                click.echo(f"   - {warning}")
+        
+        if validate_only:
+            if not warnings:
+                click.echo("✅ Configuration is valid!")
+            else:
+                click.echo("❌ Configuration has issues that should be addressed.")
+            return
+        
+        # Use output directory from CLI or config
+        if output is None:
+            output = cfg.output.directory if cfg.output.directory else Path('./reports')
+        
+        # Setup output directory
+        output.mkdir(parents=True, exist_ok=True)
+        
+        # Initialize components
+        cache_dir = cfg.cache.directory
+        if clear_cache:
+            click.echo("🗑️  Clearing cache...")
+            import shutil
+            if cache_dir.exists():
+                shutil.rmtree(cache_dir)
+        
+        cache = GitAnalysisCache(
+            cache_dir, 
+            ttl_hours=0 if no_cache else cfg.cache.ttl_hours
+        )
+        
+        identity_resolver = DeveloperIdentityResolver(
+            cache_dir / 'identities.db',
+            similarity_threshold=cfg.analysis.similarity_threshold,
+            manual_mappings=cfg.analysis.manual_identity_mappings
+        )
+        
+        analyzer = GitAnalyzer(cache, branch_mapping_rules=cfg.analysis.branch_mapping_rules)
+        orchestrator = IntegrationOrchestrator(cfg, cache)
+        
+        # Analysis period
+        end_date = datetime.now()
+        start_date = end_date - timedelta(weeks=weeks)
+        
+        click.echo(f"\n🚀 Analyzing {len(cfg.repositories)} repositories...")
+        click.echo(f"   Period: {start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}")
+        
+        # Analyze repositories
+        all_commits = []
+        all_prs = []
+        all_enrichments = {}
+        
+        for repo_config in cfg.repositories:
+            click.echo(f"\n📁 Analyzing {repo_config.name}...")
+            
+            # Check if repo exists
+            if not repo_config.path.exists():
+                click.echo(f"   ❌ Repository path not found: {repo_config.path}")
+                continue
+            
+            # Analyze repository
+            try:
+                commits = analyzer.analyze_repository(
+                    repo_config.path,
+                    start_date,
+                    repo_config.branch
+                )
+                
+                # Add project key and resolve developer identities
+                for commit in commits:
+                    # Use configured project key or fall back to inferred project
+                    if repo_config.project_key and repo_config.project_key != 'UNKNOWN':
+                        commit['project_key'] = repo_config.project_key
+                    else:
+                        commit['project_key'] = commit.get('inferred_project', 'UNKNOWN')
+                    
+                    commit['canonical_id'] = identity_resolver.resolve_developer(
+                        commit['author_name'],
+                        commit['author_email']
+                    )
+                
+                all_commits.extend(commits)
+                click.echo(f"   ✅ Found {len(commits)} commits")
+                
+                # Enrich with integration data
+                enrichment = orchestrator.enrich_repository_data(
+                    repo_config, commits, start_date
+                )
+                all_enrichments[repo_config.name] = enrichment
+                
+                if enrichment['prs']:
+                    all_prs.extend(enrichment['prs'])
+                    click.echo(f"   ✅ Found {len(enrichment['prs'])} pull requests")
+                
+            except Exception as e:
+                click.echo(f"   ❌ Error: {e}")
+                continue
+        
+        if not all_commits:
+            click.echo("\n❌ No commits found in the specified period!")
+            return
+        
+        # Update developer statistics
+        click.echo("\n👥 Resolving developer identities...")
+        identity_resolver.update_commit_stats(all_commits)
+        developer_stats = identity_resolver.get_developer_stats()
+        click.echo(f"   ✅ Identified {len(developer_stats)} unique developers")
+        
+        # Analyze tickets
+        click.echo("\n🎫 Analyzing ticket references...")
+        ticket_extractor = TicketExtractor()
+        ticket_analysis = ticket_extractor.analyze_ticket_coverage(all_commits, all_prs)
+        
+        for platform, count in ticket_analysis['ticket_summary'].items():
+            click.echo(f"   - {platform.title()}: {count} unique tickets")
+        
+        # Generate reports
+        click.echo("\n📊 Generating reports...")
+        report_gen = CSVReportGenerator(anonymize=anonymize or cfg.output.anonymize_enabled)
+        analytics_gen = AnalyticsReportGenerator(anonymize=anonymize or cfg.output.anonymize_enabled)
+        
+        # Weekly metrics report
+        weekly_report = output / f'weekly_metrics_{datetime.now().strftime("%Y%m%d")}.csv'
+        report_gen.generate_weekly_report(
+            all_commits, 
+            developer_stats, 
+            weekly_report,
+            weeks
+        )
+        click.echo(f"   ✅ Weekly metrics: {weekly_report}")
+        
+        # Summary report
+        summary_report = output / f'summary_{datetime.now().strftime("%Y%m%d")}.csv'
+        report_gen.generate_summary_report(
+            all_commits,
+            all_prs,
+            developer_stats,
+            ticket_analysis,
+            summary_report
+        )
+        click.echo(f"   ✅ Summary stats: {summary_report}")
+        
+        # Developer report
+        developer_report = output / f'developers_{datetime.now().strftime("%Y%m%d")}.csv'
+        report_gen.generate_developer_report(
+            developer_stats,
+            developer_report
+        )
+        click.echo(f"   ✅ Developer stats: {developer_report}")
+        
+        # Activity distribution report
+        activity_report = output / f'activity_distribution_{datetime.now().strftime("%Y%m%d")}.csv'
+        analytics_gen.generate_activity_distribution_report(
+            all_commits,
+            developer_stats,
+            activity_report
+        )
+        click.echo(f"   ✅ Activity distribution: {activity_report}")
+        
+        # Developer focus report
+        focus_report = output / f'developer_focus_{datetime.now().strftime("%Y%m%d")}.csv'
+        analytics_gen.generate_developer_focus_report(
+            all_commits,
+            developer_stats,
+            focus_report,
+            weeks
+        )
+        click.echo(f"   ✅ Developer focus: {focus_report}")
+        
+        # Qualitative insights report
+        insights_report = output / f'qualitative_insights_{datetime.now().strftime("%Y%m%d")}.csv'
+        analytics_gen.generate_qualitative_insights_report(
+            all_commits,
+            developer_stats,
+            ticket_analysis,
+            insights_report
+        )
+        click.echo(f"   ✅ Qualitative insights: {insights_report}")
+        
+        # Calculate DORA metrics
+        dora_calculator = DORAMetricsCalculator()
+        dora_metrics = dora_calculator.calculate_dora_metrics(
+            all_commits, all_prs, start_date, end_date
+        )
+        
+        # Aggregate PR metrics
+        pr_metrics = {}
+        for enrichment in all_enrichments.values():
+            if enrichment.get('pr_metrics'):
+                # Combine metrics (simplified - in production would properly aggregate)
+                pr_metrics = enrichment['pr_metrics']
+                break
+        
+        # Generate narrative report if markdown format is enabled
+        if 'markdown' in cfg.output.formats:
+            narrative_gen = NarrativeReportGenerator()
+            
+            # Load activity distribution data
+            activity_df = pd.read_csv(activity_report)
+            activity_data = activity_df.to_dict('records')
+            
+            # Load focus data
+            focus_df = pd.read_csv(focus_report)
+            focus_data = focus_df.to_dict('records')
+            
+            # Load insights data
+            insights_df = pd.read_csv(insights_report)
+            insights_data = insights_df.to_dict('records')
+            
+            narrative_report = output / f'narrative_report_{datetime.now().strftime("%Y%m%d")}.md'
+            narrative_gen.generate_narrative_report(
+                all_commits,
+                all_prs,
+                developer_stats,
+                activity_data,
+                focus_data,
+                insights_data,
+                ticket_analysis,
+                pr_metrics,
+                narrative_report,
+                weeks
+            )
+            click.echo(f"   ✅ Narrative report: {narrative_report}")
+        
+        # Generate JSON export if enabled
+        if 'json' in cfg.output.formats:
+            json_report = output / f'gitflow_export_{datetime.now().strftime("%Y%m%d")}.json'
+            
+            project_metrics = {
+                'ticket_analysis': ticket_analysis,
+                'pr_metrics': pr_metrics,
+                'enrichments': all_enrichments
+            }
+            
+            orchestrator.export_to_json(
+                all_commits,
+                all_prs,
+                developer_stats,
+                project_metrics,
+                dora_metrics,
+                str(json_report)
+            )
+            click.echo(f"   ✅ JSON export: {json_report}")
+        
+        # Print summary
+        click.echo("\n📈 Analysis Summary:")
+        click.echo(f"   - Total commits: {len(all_commits)}")
+        click.echo(f"   - Total PRs: {len(all_prs)}")
+        click.echo(f"   - Active developers: {len(developer_stats)}")
+        click.echo(f"   - Ticket coverage: {ticket_analysis['commit_coverage_pct']:.1f}%")
+        
+        total_story_points = sum(c.get('story_points', 0) or 0 for c in all_commits)
+        click.echo(f"   - Total story points: {total_story_points}")
+        
+        if dora_metrics:
+            click.echo(f"\n🎯 DORA Metrics:")
+            click.echo(f"   - Deployment frequency: {dora_metrics['deployment_frequency']['category']}")
+            click.echo(f"   - Lead time: {dora_metrics['lead_time_hours']:.1f} hours")
+            click.echo(f"   - Change failure rate: {dora_metrics['change_failure_rate']:.1f}%")
+            click.echo(f"   - MTTR: {dora_metrics['mttr_hours']:.1f} hours")
+            click.echo(f"   - Performance level: {dora_metrics['performance_level']}")
+        
+        click.echo(f"\n✅ Analysis complete! Reports saved to {output}")
+        
+    except Exception as e:
+        click.echo(f"\n❌ Error: {e}", err=True)
+        if '--debug' in sys.argv:
+            raise
+        sys.exit(1)
+
+
+@cli.command()
+@click.option('--config', '-c',
+              type=click.Path(exists=True, path_type=Path),
+              required=True,
+              help='Path to YAML configuration file')
+def cache_stats(config: Path):
+    """Show cache statistics."""
+    try:
+        cfg = ConfigLoader.load(config)
+        cache = GitAnalysisCache(cfg.cache.directory)
+        
+        stats = cache.get_cache_stats()
+        
+        click.echo("📊 Cache Statistics:")
+        click.echo(f"   - Cached commits: {stats['cached_commits']}")
+        click.echo(f"   - Cached PRs: {stats['cached_prs']}")
+        click.echo(f"   - Cached issues: {stats['cached_issues']}")
+        click.echo(f"   - Stale entries: {stats['stale_commits']}")
+        
+        # Calculate cache size
+        import os
+        cache_size = 0
+        for root, dirs, files in os.walk(cfg.cache.directory):
+            for f in files:
+                cache_size += os.path.getsize(os.path.join(root, f))
+        
+        click.echo(f"   - Cache size: {cache_size / 1024 / 1024:.1f} MB")
+        
+    except Exception as e:
+        click.echo(f"❌ Error: {e}", err=True)
+        sys.exit(1)
+
+
+@cli.command()
+@click.option('--config', '-c',
+              type=click.Path(exists=True, path_type=Path),
+              required=True,
+              help='Path to YAML configuration file')
+@click.argument('dev1')
+@click.argument('dev2')
+def merge_identity(config: Path, dev1: str, dev2: str):
+    """Merge two developer identities."""
+    try:
+        cfg = ConfigLoader.load(config)
+        identity_resolver = DeveloperIdentityResolver(
+            cfg.cache.directory / 'identities.db'
+        )
+        
+        click.echo(f"🔄 Merging {dev2} into {dev1}...")
+        identity_resolver.merge_identities(dev1, dev2)
+        click.echo("✅ Identities merged successfully!")
+        
+    except Exception as e:
+        click.echo(f"❌ Error: {e}", err=True)
+        sys.exit(1)
+
+
+@cli.command()
+@click.option('--config', '-c',
+              type=click.Path(exists=True, path_type=Path),
+              required=True,
+              help='Path to YAML configuration file')
+def list_developers(config: Path):
+    """List all known developers."""
+    try:
+        cfg = ConfigLoader.load(config)
+        identity_resolver = DeveloperIdentityResolver(
+            cfg.cache.directory / 'identities.db'
+        )
+        
+        developers = identity_resolver.get_developer_stats()
+        
+        if not developers:
+            click.echo("No developers found. Run analysis first.")
+            return
+        
+        click.echo("👥 Known Developers:")
+        click.echo(f"{'Name':<30} {'Email':<40} {'Commits':<10} {'Points':<10} {'Aliases'}")
+        click.echo("-" * 100)
+        
+        for dev in developers[:20]:  # Show top 20
+            click.echo(
+                f"{dev['primary_name']:<30} "
+                f"{dev['primary_email']:<40} "
+                f"{dev['total_commits']:<10} "
+                f"{dev['total_story_points']:<10} "
+                f"{dev['alias_count']}"
+            )
+        
+        if len(developers) > 20:
+            click.echo(f"\n... and {len(developers) - 20} more developers")
+        
+    except Exception as e:
+        click.echo(f"❌ Error: {e}", err=True)
+        sys.exit(1)
+
+
+def main():
+    """Main entry point."""
+    cli()
+
+
+if __name__ == '__main__':
+    main()

gitflow_analytics/config.py

@@ -0,0 +1,215 @@
+"""Configuration management for GitFlow Analytics."""
+import os
+import yaml
+from pathlib import Path
+from typing import Dict, Any, Optional, List
+from dataclasses import dataclass, field
+
+@dataclass
+class RepositoryConfig:
+    """Configuration for a single repository."""
+    name: str
+    path: Path
+    github_repo: Optional[str] = None
+    project_key: Optional[str] = None
+    branch: Optional[str] = None
+    
+    def __post_init__(self):
+        self.path = Path(self.path).expanduser().resolve()
+        if not self.project_key:
+            self.project_key = self.name.upper().replace('-', '_')
+
+@dataclass
+class GitHubConfig:
+    """GitHub API configuration."""
+    token: Optional[str] = None
+    owner: Optional[str] = None
+    base_url: str = "https://api.github.com"
+    max_retries: int = 3
+    backoff_factor: int = 2
+    
+    def get_repo_full_name(self, repo_name: str) -> str:
+        """Get full repository name including owner."""
+        if '/' in repo_name:
+            return repo_name
+        if self.owner:
+            return f"{self.owner}/{repo_name}"
+        raise ValueError(f"Repository {repo_name} needs owner specified")
+
+@dataclass
+class AnalysisConfig:
+    """Analysis-specific configuration."""
+    story_point_patterns: List[str] = field(default_factory=list)
+    exclude_authors: List[str] = field(default_factory=list)
+    exclude_message_patterns: List[str] = field(default_factory=list)
+    similarity_threshold: float = 0.85
+    manual_identity_mappings: List[Dict[str, Any]] = field(default_factory=list)
+    default_ticket_platform: Optional[str] = None
+    branch_mapping_rules: Dict[str, List[str]] = field(default_factory=dict)
+
+@dataclass
+class OutputConfig:
+    """Output configuration."""
+    directory: Optional[Path] = None
+    formats: List[str] = field(default_factory=lambda: ["csv", "markdown"])
+    csv_delimiter: str = ","
+    csv_encoding: str = "utf-8"
+    anonymize_enabled: bool = False
+    anonymize_fields: List[str] = field(default_factory=list)
+    anonymize_method: str = "hash"
+
+@dataclass
+class CacheConfig:
+    """Cache configuration."""
+    directory: Path = Path(".gitflow-cache")
+    ttl_hours: int = 168
+    max_size_mb: int = 500
+
+@dataclass
+class Config:
+    """Main configuration container."""
+    repositories: List[RepositoryConfig]
+    github: GitHubConfig
+    analysis: AnalysisConfig
+    output: OutputConfig
+    cache: CacheConfig
+
+class ConfigLoader:
+    """Load and validate configuration from YAML files."""
+    
+    @staticmethod
+    def load(config_path: Path) -> Config:
+        """Load configuration from YAML file."""
+        with open(config_path, 'r') as f:
+            data = yaml.safe_load(f)
+        
+        # Validate version
+        version = data.get('version', '1.0')
+        if version not in ['1.0']:
+            raise ValueError(f"Unsupported config version: {version}")
+        
+        # Process GitHub config
+        github_data = data.get('github', {})
+        github_config = GitHubConfig(
+            token=ConfigLoader._resolve_env_var(github_data.get('token')),
+            owner=ConfigLoader._resolve_env_var(github_data.get('owner')),
+            base_url=github_data.get('base_url', 'https://api.github.com'),
+            max_retries=github_data.get('rate_limit', {}).get('max_retries', 3),
+            backoff_factor=github_data.get('rate_limit', {}).get('backoff_factor', 2)
+        )
+        
+        # Process repositories
+        repositories = []
+        for repo_data in data.get('repositories', []):
+            # Handle github_repo with owner fallback
+            github_repo = repo_data.get('github_repo')
+            if github_repo and github_config.owner and '/' not in github_repo:
+                github_repo = f"{github_config.owner}/{github_repo}"
+            
+            repo_config = RepositoryConfig(
+                name=repo_data['name'],
+                path=repo_data['path'],
+                github_repo=github_repo,
+                project_key=repo_data.get('project_key'),
+                branch=repo_data.get('branch')
+            )
+            repositories.append(repo_config)
+        
+        if not repositories:
+            raise ValueError("No repositories defined in configuration")
+        
+        # Process analysis settings
+        analysis_data = data.get('analysis', {})
+        analysis_config = AnalysisConfig(
+            story_point_patterns=analysis_data.get('story_point_patterns', [
+                r"(?:story\s*points?|sp|pts?)\s*[:=]\s*(\d+)",
+                r"\[(\d+)\s*(?:sp|pts?)\]",
+                r"#(\d+)sp"
+            ]),
+            exclude_authors=analysis_data.get('exclude', {}).get('authors', [
+                "dependabot[bot]",
+                "renovate[bot]"
+            ]),
+            exclude_message_patterns=analysis_data.get('exclude', {}).get('message_patterns', []),
+            similarity_threshold=analysis_data.get('identity', {}).get('similarity_threshold', 0.85),
+            manual_identity_mappings=analysis_data.get('identity', {}).get('manual_mappings', []),
+            default_ticket_platform=analysis_data.get('default_ticket_platform'),
+            branch_mapping_rules=analysis_data.get('branch_mapping_rules', {})
+        )
+        
+        # Process output settings
+        output_data = data.get('output', {})
+        output_dir = output_data.get('directory')
+        if output_dir:
+            output_dir = Path(output_dir).expanduser().resolve()
+        
+        output_config = OutputConfig(
+            directory=output_dir,
+            formats=output_data.get('formats', ['csv', 'markdown']),
+            csv_delimiter=output_data.get('csv', {}).get('delimiter', ','),
+            csv_encoding=output_data.get('csv', {}).get('encoding', 'utf-8'),
+            anonymize_enabled=output_data.get('anonymization', {}).get('enabled', False),
+            anonymize_fields=output_data.get('anonymization', {}).get('fields', []),
+            anonymize_method=output_data.get('anonymization', {}).get('method', 'hash')
+        )
+        
+        # Process cache settings
+        cache_data = data.get('cache', {})
+        cache_config = CacheConfig(
+            directory=Path(cache_data.get('directory', '.gitflow-cache')),
+            ttl_hours=cache_data.get('ttl_hours', 168),
+            max_size_mb=cache_data.get('max_size_mb', 500)
+        )
+        
+        return Config(
+            repositories=repositories,
+            github=github_config,
+            analysis=analysis_config,
+            output=output_config,
+            cache=cache_config
+        )
+    
+    @staticmethod
+    def _resolve_env_var(value: Optional[str]) -> Optional[str]:
+        """Resolve environment variable references."""
+        if not value:
+            return None
+            
+        if value.startswith('${') and value.endswith('}'):
+            env_var = value[2:-1]
+            resolved = os.environ.get(env_var)
+            if not resolved:
+                raise ValueError(f"Environment variable {env_var} not set")
+            return resolved
+        
+        return value
+    
+    @staticmethod
+    def validate_config(config: Config) -> List[str]:
+        """Validate configuration and return list of warnings."""
+        warnings = []
+        
+        # Check repository paths exist
+        for repo in config.repositories:
+            if not repo.path.exists():
+                warnings.append(f"Repository path does not exist: {repo.path}")
+            elif not (repo.path / '.git').exists():
+                warnings.append(f"Path is not a git repository: {repo.path}")
+        
+        # Check GitHub token if GitHub repos are specified
+        has_github_repos = any(r.github_repo for r in config.repositories)
+        if has_github_repos and not config.github.token:
+            warnings.append("GitHub repositories specified but no GitHub token provided")
+        
+        # Check if owner is needed
+        for repo in config.repositories:
+            if repo.github_repo and '/' not in repo.github_repo and not config.github.owner:
+                warnings.append(f"Repository {repo.github_repo} needs owner specified")
+        
+        # Check cache directory permissions
+        try:
+            config.cache.directory.mkdir(exist_ok=True, parents=True)
+        except PermissionError:
+            warnings.append(f"Cannot create cache directory: {config.cache.directory}")
+        
+        return warnings