gitflow-analytics 1.0.0__py3-none-any.whl

gitflow_analytics/core/analyzer.py

@@ -0,0 +1,195 @@
+"""Git repository analyzer with batch processing support."""
+import re
+from datetime import datetime
+from typing import List, Dict, Any, Optional, Tuple, Generator
+from pathlib import Path
+import git
+from git import Repo
+from tqdm import tqdm
+
+from .cache import GitAnalysisCache
+from ..extractors.story_points import StoryPointExtractor
+from ..extractors.tickets import TicketExtractor
+from .branch_mapper import BranchToProjectMapper
+
+
+class GitAnalyzer:
+    """Analyze Git repositories with caching and batch processing."""
+    
+    def __init__(self, cache: GitAnalysisCache, batch_size: int = 1000, 
+                 branch_mapping_rules: Optional[Dict[str, List[str]]] = None):
+        """Initialize analyzer with cache."""
+        self.cache = cache
+        self.batch_size = batch_size
+        self.story_point_extractor = StoryPointExtractor()
+        self.ticket_extractor = TicketExtractor()
+        self.branch_mapper = BranchToProjectMapper(branch_mapping_rules)
+    
+    def analyze_repository(self, repo_path: Path, since: datetime, 
+                         branch: Optional[str] = None) -> List[Dict[str, Any]]:
+        """Analyze a Git repository with batch processing."""
+        try:
+            repo = Repo(repo_path)
+        except Exception as e:
+            raise ValueError(f"Failed to open repository at {repo_path}: {e}")
+        
+        # Get commits to analyze
+        commits = self._get_commits(repo, since, branch)
+        total_commits = len(commits)
+        
+        if total_commits == 0:
+            return []
+        
+        analyzed_commits = []
+        
+        # Process in batches with progress bar
+        with tqdm(total=total_commits, desc=f"Analyzing {repo_path.name}") as pbar:
+            for batch in self._batch_commits(commits, self.batch_size):
+                batch_results = self._process_batch(repo, repo_path, batch)
+                analyzed_commits.extend(batch_results)
+                
+                # Cache the batch
+                self.cache.cache_commits_batch(str(repo_path), batch_results)
+                
+                pbar.update(len(batch))
+        
+        return analyzed_commits
+    
+    def _get_commits(self, repo: Repo, since: datetime, 
+                    branch: Optional[str] = None) -> List[git.Commit]:
+        """Get commits from repository."""
+        if branch:
+            try:
+                commits = list(repo.iter_commits(branch, since=since))
+            except git.GitCommandError:
+                # Branch doesn't exist
+                return []
+        else:
+            # Get commits from all branches
+            commits = []
+            for ref in repo.refs:
+                if ref.name.startswith('origin/'):
+                    continue  # Skip remote branches
+                try:
+                    branch_commits = list(repo.iter_commits(ref, since=since))
+                    commits.extend(branch_commits)
+                except git.GitCommandError:
+                    continue
+            
+            # Remove duplicates while preserving order
+            seen = set()
+            unique_commits = []
+            for commit in commits:
+                if commit.hexsha not in seen:
+                    seen.add(commit.hexsha)
+                    unique_commits.append(commit)
+            
+            commits = unique_commits
+        
+        # Sort by date
+        return sorted(commits, key=lambda c: c.committed_datetime)
+    
+    def _batch_commits(self, commits: List[git.Commit], 
+                      batch_size: int) -> Generator[List[git.Commit], None, None]:
+        """Yield batches of commits."""
+        for i in range(0, len(commits), batch_size):
+            yield commits[i:i + batch_size]
+    
+    def _process_batch(self, repo: Repo, repo_path: Path, 
+                      commits: List[git.Commit]) -> List[Dict[str, Any]]:
+        """Process a batch of commits."""
+        results = []
+        
+        for commit in commits:
+            # Check cache first
+            cached = self.cache.get_cached_commit(str(repo_path), commit.hexsha)
+            if cached:
+                results.append(cached)
+                continue
+            
+            # Analyze commit
+            commit_data = self._analyze_commit(repo, commit, repo_path)
+            results.append(commit_data)
+        
+        return results
+    
+    def _analyze_commit(self, repo: Repo, commit: git.Commit, repo_path: Path) -> Dict[str, Any]:
+        """Analyze a single commit."""
+        # Basic commit data
+        commit_data = {
+            'hash': commit.hexsha,
+            'author_name': commit.author.name,
+            'author_email': commit.author.email,
+            'message': commit.message,
+            'timestamp': commit.committed_datetime,
+            'is_merge': len(commit.parents) > 1
+        }
+        
+        # Get branch name
+        commit_data['branch'] = self._get_commit_branch(repo, commit)
+        
+        # Map branch to project
+        commit_data['inferred_project'] = self.branch_mapper.map_branch_to_project(
+            commit_data['branch'], repo_path
+        )
+        
+        # Calculate metrics
+        stats = commit.stats.total
+        commit_data['files_changed'] = stats.get('files', 0)
+        commit_data['insertions'] = stats.get('insertions', 0)
+        commit_data['deletions'] = stats.get('deletions', 0)
+        
+        # Extract story points
+        commit_data['story_points'] = self.story_point_extractor.extract_from_text(
+            commit.message
+        )
+        
+        # Extract ticket references
+        commit_data['ticket_references'] = self.ticket_extractor.extract_from_text(
+            commit.message
+        )
+        
+        # Calculate complexity delta
+        commit_data['complexity_delta'] = self._calculate_complexity_delta(commit)
+        
+        return commit_data
+    
+    def _get_commit_branch(self, repo: Repo, commit: git.Commit) -> str:
+        """Get the branch name for a commit."""
+        # This is a simplified approach - getting the first branch that contains the commit
+        for branch in repo.branches:
+            if commit in repo.iter_commits(branch):
+                return branch.name
+        return 'unknown'
+    
+    def _calculate_complexity_delta(self, commit: git.Commit) -> float:
+        """Calculate complexity change for a commit."""
+        total_delta = 0.0
+        
+        for diff in commit.diff(commit.parents[0] if commit.parents else None):
+            if not self._is_code_file(diff.b_path or diff.a_path or ''):
+                continue
+            
+            # Simple complexity estimation based on diff size
+            # In a real implementation, you'd parse the code and calculate cyclomatic complexity
+            if diff.new_file:
+                total_delta += diff.b_blob.size / 100 if diff.b_blob else 0
+            elif diff.deleted_file:
+                total_delta -= diff.a_blob.size / 100 if diff.a_blob else 0
+            else:
+                # Modified file - estimate based on change size
+                added = len(diff.diff.decode('utf-8', errors='ignore').split('\n+')) if diff.diff else 0
+                removed = len(diff.diff.decode('utf-8', errors='ignore').split('\n-')) if diff.diff else 0
+                total_delta += (added - removed) / 10
+        
+        return total_delta
+    
+    def _is_code_file(self, filepath: str) -> bool:
+        """Check if file is a code file."""
+        code_extensions = {
+            '.py', '.js', '.ts', '.java', '.cpp', '.c', '.h', '.hpp',
+            '.go', '.rs', '.rb', '.php', '.swift', '.kt', '.scala',
+            '.cs', '.vb', '.r', '.m', '.mm', '.f90', '.f95', '.lua'
+        }
+        
+        return any(filepath.endswith(ext) for ext in code_extensions)

gitflow_analytics/core/branch_mapper.py

@@ -0,0 +1,221 @@
+"""Map git branches to projects based on naming conventions."""
+import re
+from typing import Dict, List, Optional, Tuple
+from pathlib import Path
+
+
+class BranchToProjectMapper:
+    """Maps git branches to project keys based on conventions."""
+    
+    def __init__(self, mapping_rules: Optional[Dict[str, List[str]]] = None):
+        """
+        Initialize with custom mapping rules.
+        
+        Args:
+            mapping_rules: Dict mapping project keys to list of branch patterns
+                          e.g., {'FRONTEND': ['feature/fe-*', 'frontend/*']}
+        """
+        self.mapping_rules = mapping_rules or self._get_default_rules()
+        self.compiled_rules = self._compile_patterns()
+    
+    def _get_default_rules(self) -> Dict[str, List[str]]:
+        """Get default branch mapping rules."""
+        return {
+            'FRONTEND': [
+                r'^feature/fe[-/_]',
+                r'^feature/frontend[-/_]',
+                r'^frontend/',
+                r'^fe/',
+                r'[-/_]frontend[-/_]',
+                r'[-/_]fe[-/_]',
+                r'[-/_]ui[-/_]',
+                r'[-/_]web[-/_]'
+            ],
+            'BACKEND': [
+                r'^feature/be[-/_]',
+                r'^feature/backend[-/_]',
+                r'^backend/',
+                r'^be/',
+                r'^api/',
+                r'[-/_]backend[-/_]',
+                r'[-/_]be[-/_]',
+                r'[-/_]api[-/_]',
+                r'[-/_]server[-/_]'
+            ],
+            'SERVICE': [
+                r'^feature/service[-/_]',
+                r'^feature/svc[-/_]',
+                r'^service/',
+                r'^svc/',
+                r'[-/_]service[-/_]',
+                r'[-/_]svc[-/_]',
+                r'[-/_]microservice[-/_]'
+            ],
+            'MOBILE': [
+                r'^feature/mobile[-/_]',
+                r'^feature/app[-/_]',
+                r'^mobile/',
+                r'^app/',
+                r'^ios/',
+                r'^android/',
+                r'[-/_]mobile[-/_]',
+                r'[-/_]app[-/_]',
+                r'[-/_]ios[-/_]',
+                r'[-/_]android[-/_]'
+            ],
+            'DATA': [
+                r'^feature/data[-/_]',
+                r'^feature/etl[-/_]',
+                r'^data/',
+                r'^etl/',
+                r'^pipeline/',
+                r'[-/_]data[-/_]',
+                r'[-/_]etl[-/_]',
+                r'[-/_]pipeline[-/_]',
+                r'[-/_]analytics[-/_]'
+            ],
+            'INFRA': [
+                r'^feature/infra[-/_]',
+                r'^feature/devops[-/_]',
+                r'^infra/',
+                r'^devops/',
+                r'^ops/',
+                r'[-/_]infra[-/_]',
+                r'[-/_]devops[-/_]',
+                r'[-/_]ops[-/_]',
+                r'[-/_]deployment[-/_]'
+            ],
+            'SCRAPER': [
+                r'^feature/scraper[-/_]',
+                r'^feature/crawler[-/_]',
+                r'^scraper/',
+                r'^crawler/',
+                r'[-/_]scraper[-/_]',
+                r'[-/_]crawler[-/_]',
+                r'[-/_]scraping[-/_]'
+            ]
+        }
+    
+    def _compile_patterns(self) -> Dict[str, List[re.Pattern]]:
+        """Compile regex patterns for efficiency."""
+        compiled = {}
+        for project, patterns in self.mapping_rules.items():
+            compiled[project] = [re.compile(pattern, re.IGNORECASE) for pattern in patterns]
+        return compiled
+    
+    def map_branch_to_project(self, branch_name: str, repo_path: Optional[Path] = None) -> str:
+        """
+        Map a branch name to a project key.
+        
+        Args:
+            branch_name: Git branch name
+            repo_path: Optional repository path for context
+            
+        Returns:
+            Project key or 'UNKNOWN'
+        """
+        if not branch_name or branch_name in ['main', 'master', 'develop', 'development']:
+            # Try to infer from repo path if available
+            if repo_path:
+                return self._infer_from_repo_path(repo_path)
+            return 'UNKNOWN'
+        
+        # Check against compiled patterns
+        for project, patterns in self.compiled_rules.items():
+            for pattern in patterns:
+                if pattern.search(branch_name):
+                    return project
+        
+        # Try to extract from ticket references in branch name
+        ticket_project = self._extract_from_ticket(branch_name)
+        if ticket_project:
+            return ticket_project
+        
+        # Try to infer from repo path if available
+        if repo_path:
+            return self._infer_from_repo_path(repo_path)
+        
+        return 'UNKNOWN'
+    
+    def _extract_from_ticket(self, branch_name: str) -> Optional[str]:
+        """Extract project from ticket reference in branch name."""
+        # Common ticket patterns
+        ticket_patterns = [
+            r'([A-Z]{2,})-\d+',  # JIRA style: PROJ-123
+            r'#([A-Z]{2,})\d+',  # Hash prefix: #PROJ123
+            r'([A-Z]{2,})_\d+',  # Underscore: PROJ_123
+        ]
+        
+        for pattern in ticket_patterns:
+            match = re.search(pattern, branch_name, re.IGNORECASE)
+            if match:
+                prefix = match.group(1).upper()
+                # Map common prefixes to projects
+                prefix_map = {
+                    'FE': 'FRONTEND',
+                    'BE': 'BACKEND',
+                    'SVC': 'SERVICE',
+                    'MOB': 'MOBILE',
+                    'DATA': 'DATA',
+                    'ETL': 'DATA',
+                    'INFRA': 'INFRA',
+                    'OPS': 'INFRA',
+                    'SCRAPE': 'SCRAPER',
+                    'CRAWL': 'SCRAPER'
+                }
+                
+                if prefix in prefix_map:
+                    return prefix_map[prefix]
+                
+                # Check if prefix matches any project key
+                for project in self.mapping_rules.keys():
+                    if prefix == project or prefix in project:
+                        return project
+        
+        return None
+    
+    def _infer_from_repo_path(self, repo_path: Path) -> str:
+        """Infer project from repository path."""
+        repo_name = repo_path.name.lower()
+        
+        # Direct mapping
+        path_map = {
+            'frontend': 'FRONTEND',
+            'backend': 'BACKEND',
+            'service': 'SERVICE',
+            'service-ts': 'SERVICE_TS',
+            'services': 'SERVICES',
+            'mobile': 'MOBILE',
+            'ios': 'MOBILE',
+            'android': 'MOBILE',
+            'data': 'DATA',
+            'etl': 'DATA',
+            'infra': 'INFRA',
+            'infrastructure': 'INFRA',
+            'scraper': 'SCRAPER',
+            'crawler': 'SCRAPER',
+            'scrapers': 'SCRAPER'
+        }
+        
+        for key, project in path_map.items():
+            if key in repo_name:
+                return project
+        
+        # Check parent directory
+        if repo_path.parent.name.lower() in path_map:
+            return path_map[repo_path.parent.name.lower()]
+        
+        return 'UNKNOWN'
+    
+    def add_mapping_rule(self, project: str, patterns: List[str]):
+        """Add custom mapping rules for a project."""
+        if project not in self.mapping_rules:
+            self.mapping_rules[project] = []
+        
+        self.mapping_rules[project].extend(patterns)
+        
+        # Recompile patterns
+        self.compiled_rules[project] = [
+            re.compile(pattern, re.IGNORECASE) 
+            for pattern in self.mapping_rules[project]
+        ]

gitflow_analytics/core/cache.py

@@ -0,0 +1,275 @@
+"""Caching layer for Git analysis with SQLite backend."""
+import hashlib
+from datetime import datetime, timedelta
+from typing import List, Optional, Dict, Any
+from pathlib import Path
+from contextlib import contextmanager
+
+from sqlalchemy.orm import Session
+from sqlalchemy import and_
+
+from ..models.database import Database, CachedCommit, PullRequestCache, IssueCache
+
+
+class GitAnalysisCache:
+    """Cache for Git analysis results."""
+    
+    def __init__(self, cache_dir: Path, ttl_hours: int = 168):
+        """Initialize cache with SQLite backend."""
+        self.cache_dir = cache_dir
+        self.ttl_hours = ttl_hours
+        self.db = Database(cache_dir / 'gitflow_cache.db')
+    
+    @contextmanager
+    def get_session(self):
+        """Get database session context manager."""
+        session = self.db.get_session()
+        try:
+            yield session
+            session.commit()
+        except Exception:
+            session.rollback()
+            raise
+        finally:
+            session.close()
+    
+    def get_cached_commit(self, repo_path: str, commit_hash: str) -> Optional[Dict[str, Any]]:
+        """Retrieve cached commit data if not stale."""
+        with self.get_session() as session:
+            cached = session.query(CachedCommit).filter(
+                and_(
+                    CachedCommit.repo_path == repo_path,
+                    CachedCommit.commit_hash == commit_hash
+                )
+            ).first()
+            
+            if cached and not self._is_stale(cached.cached_at):
+                return self._commit_to_dict(cached)
+            
+            return None
+    
+    def cache_commit(self, repo_path: str, commit_data: Dict[str, Any]):
+        """Cache commit analysis results."""
+        with self.get_session() as session:
+            # Check if already exists
+            existing = session.query(CachedCommit).filter(
+                and_(
+                    CachedCommit.repo_path == repo_path,
+                    CachedCommit.commit_hash == commit_data['hash']
+                )
+            ).first()
+            
+            if existing:
+                # Update existing
+                for key, value in commit_data.items():
+                    if hasattr(existing, key):
+                        setattr(existing, key, value)
+                existing.cached_at = datetime.utcnow()
+            else:
+                # Create new
+                cached_commit = CachedCommit(
+                    repo_path=repo_path,
+                    commit_hash=commit_data['hash'],
+                    author_name=commit_data.get('author_name'),
+                    author_email=commit_data.get('author_email'),
+                    message=commit_data.get('message'),
+                    timestamp=commit_data.get('timestamp'),
+                    branch=commit_data.get('branch'),
+                    is_merge=commit_data.get('is_merge', False),
+                    files_changed=commit_data.get('files_changed', 0),
+                    insertions=commit_data.get('insertions', 0),
+                    deletions=commit_data.get('deletions', 0),
+                    complexity_delta=commit_data.get('complexity_delta', 0.0),
+                    story_points=commit_data.get('story_points'),
+                    ticket_references=commit_data.get('ticket_references', [])
+                )
+                session.add(cached_commit)
+    
+    def cache_commits_batch(self, repo_path: str, commits: List[Dict[str, Any]]):
+        """Cache multiple commits in a single transaction."""
+        with self.get_session() as session:
+            for commit_data in commits:
+                # Check if already exists
+                existing = session.query(CachedCommit).filter(
+                    and_(
+                        CachedCommit.repo_path == repo_path,
+                        CachedCommit.commit_hash == commit_data['hash']
+                    )
+                ).first()
+                
+                if existing:
+                    # Update existing
+                    for key, value in commit_data.items():
+                        if key != 'hash' and hasattr(existing, key):
+                            setattr(existing, key, value)
+                    existing.cached_at = datetime.utcnow()
+                else:
+                    # Create new
+                    cached_commit = CachedCommit(
+                        repo_path=repo_path,
+                        commit_hash=commit_data['hash'],
+                        author_name=commit_data.get('author_name'),
+                        author_email=commit_data.get('author_email'),
+                        message=commit_data.get('message'),
+                        timestamp=commit_data.get('timestamp'),
+                        branch=commit_data.get('branch'),
+                        is_merge=commit_data.get('is_merge', False),
+                        files_changed=commit_data.get('files_changed', 0),
+                        insertions=commit_data.get('insertions', 0),
+                        deletions=commit_data.get('deletions', 0),
+                        complexity_delta=commit_data.get('complexity_delta', 0.0),
+                        story_points=commit_data.get('story_points'),
+                        ticket_references=commit_data.get('ticket_references', [])
+                    )
+                    session.add(cached_commit)
+    
+    def get_cached_pr(self, repo_path: str, pr_number: int) -> Optional[Dict[str, Any]]:
+        """Retrieve cached pull request data."""
+        with self.get_session() as session:
+            cached = session.query(PullRequestCache).filter(
+                and_(
+                    PullRequestCache.repo_path == repo_path,
+                    PullRequestCache.pr_number == pr_number
+                )
+            ).first()
+            
+            if cached and not self._is_stale(cached.cached_at):
+                return self._pr_to_dict(cached)
+            
+            return None
+    
+    def cache_pr(self, repo_path: str, pr_data: Dict[str, Any]):
+        """Cache pull request data."""
+        with self.get_session() as session:
+            cached_pr = PullRequestCache(
+                repo_path=repo_path,
+                pr_number=pr_data['number'],
+                title=pr_data.get('title'),
+                description=pr_data.get('description'),
+                author=pr_data.get('author'),
+                created_at=pr_data.get('created_at'),
+                merged_at=pr_data.get('merged_at'),
+                story_points=pr_data.get('story_points'),
+                labels=pr_data.get('labels', []),
+                commit_hashes=pr_data.get('commit_hashes', [])
+            )
+            session.merge(cached_pr)
+    
+    def cache_issue(self, platform: str, issue_data: Dict[str, Any]):
+        """Cache issue data from various platforms."""
+        with self.get_session() as session:
+            cached_issue = IssueCache(
+                platform=platform,
+                issue_id=str(issue_data['id']),
+                project_key=issue_data['project_key'],
+                title=issue_data.get('title'),
+                description=issue_data.get('description'),
+                status=issue_data.get('status'),
+                assignee=issue_data.get('assignee'),
+                created_at=issue_data.get('created_at'),
+                updated_at=issue_data.get('updated_at'),
+                resolved_at=issue_data.get('resolved_at'),
+                story_points=issue_data.get('story_points'),
+                labels=issue_data.get('labels', []),
+                platform_data=issue_data.get('platform_data', {})
+            )
+            session.merge(cached_issue)
+    
+    def get_cached_issues(self, platform: str, project_key: str) -> List[Dict[str, Any]]:
+        """Get all cached issues for a platform and project."""
+        with self.get_session() as session:
+            issues = session.query(IssueCache).filter(
+                and_(
+                    IssueCache.platform == platform,
+                    IssueCache.project_key == project_key
+                )
+            ).all()
+            
+            return [self._issue_to_dict(issue) for issue in issues 
+                    if not self._is_stale(issue.cached_at)]
+    
+    def clear_stale_cache(self):
+        """Remove stale cache entries."""
+        cutoff_time = datetime.utcnow() - timedelta(hours=self.ttl_hours)
+        
+        with self.get_session() as session:
+            session.query(CachedCommit).filter(
+                CachedCommit.cached_at < cutoff_time
+            ).delete()
+            
+            session.query(PullRequestCache).filter(
+                PullRequestCache.cached_at < cutoff_time
+            ).delete()
+            
+            session.query(IssueCache).filter(
+                IssueCache.cached_at < cutoff_time
+            ).delete()
+    
+    def get_cache_stats(self) -> Dict[str, int]:
+        """Get cache statistics."""
+        with self.get_session() as session:
+            stats = {
+                'cached_commits': session.query(CachedCommit).count(),
+                'cached_prs': session.query(PullRequestCache).count(),
+                'cached_issues': session.query(IssueCache).count(),
+                'stale_commits': session.query(CachedCommit).filter(
+                    CachedCommit.cached_at < datetime.utcnow() - timedelta(hours=self.ttl_hours)
+                ).count()
+            }
+            return stats
+    
+    def _is_stale(self, cached_at: datetime) -> bool:
+        """Check if cache entry is stale."""
+        if self.ttl_hours == 0:  # No expiration
+            return False
+        return cached_at < datetime.utcnow() - timedelta(hours=self.ttl_hours)
+    
+    def _commit_to_dict(self, commit: CachedCommit) -> Dict[str, Any]:
+        """Convert CachedCommit to dictionary."""
+        return {
+            'hash': commit.commit_hash,
+            'author_name': commit.author_name,
+            'author_email': commit.author_email,
+            'message': commit.message,
+            'timestamp': commit.timestamp,
+            'branch': commit.branch,
+            'is_merge': commit.is_merge,
+            'files_changed': commit.files_changed,
+            'insertions': commit.insertions,
+            'deletions': commit.deletions,
+            'complexity_delta': commit.complexity_delta,
+            'story_points': commit.story_points,
+            'ticket_references': commit.ticket_references or []
+        }
+    
+    def _pr_to_dict(self, pr: PullRequestCache) -> Dict[str, Any]:
+        """Convert PullRequestCache to dictionary."""
+        return {
+            'number': pr.pr_number,
+            'title': pr.title,
+            'description': pr.description,
+            'author': pr.author,
+            'created_at': pr.created_at,
+            'merged_at': pr.merged_at,
+            'story_points': pr.story_points,
+            'labels': pr.labels or [],
+            'commit_hashes': pr.commit_hashes or []
+        }
+    
+    def _issue_to_dict(self, issue: IssueCache) -> Dict[str, Any]:
+        """Convert IssueCache to dictionary."""
+        return {
+            'platform': issue.platform,
+            'id': issue.issue_id,
+            'project_key': issue.project_key,
+            'title': issue.title,
+            'description': issue.description,
+            'status': issue.status,
+            'assignee': issue.assignee,
+            'created_at': issue.created_at,
+            'updated_at': issue.updated_at,
+            'resolved_at': issue.resolved_at,
+            'story_points': issue.story_points,
+            'labels': issue.labels or [],
+            'platform_data': issue.platform_data or {}
+        }