gitflow-analytics 1.0.0__py3-none-any.whl

gitflow_analytics/core/identity.py

@@ -0,0 +1,402 @@
+"""Developer identity resolution with persistence."""
+import uuid
+import difflib
+from datetime import datetime
+from typing import Dict, List, Optional, Set, Tuple, Any
+from collections import defaultdict
+from contextlib import contextmanager
+
+from sqlalchemy.orm import Session
+from sqlalchemy import and_, or_
+
+from ..models.database import Database, DeveloperIdentity, DeveloperAlias
+
+
+class DeveloperIdentityResolver:
+    """Resolve and normalize developer identities across repositories."""
+    
+    def __init__(self, db_path, similarity_threshold: float = 0.85, 
+                 manual_mappings: Optional[List[Dict[str, Any]]] = None):
+        """Initialize with database for persistence."""
+        self.similarity_threshold = similarity_threshold
+        self.db = Database(db_path)
+        self._cache = {}  # In-memory cache for performance
+        self._load_cache()
+        
+        # Store manual mappings to apply later
+        self.manual_mappings = manual_mappings
+    
+    @contextmanager
+    def get_session(self):
+        """Get database session context manager."""
+        session = self.db.get_session()
+        try:
+            yield session
+            session.commit()
+        except Exception:
+            session.rollback()
+            raise
+        finally:
+            session.close()
+    
+    def _load_cache(self):
+        """Load identities into memory cache."""
+        with self.get_session() as session:
+            # Load all identities
+            identities = session.query(DeveloperIdentity).all()
+            for identity in identities:
+                self._cache[identity.canonical_id] = {
+                    'primary_name': identity.primary_name,
+                    'primary_email': identity.primary_email,
+                    'github_username': identity.github_username
+                }
+            
+            # Load all aliases
+            aliases = session.query(DeveloperAlias).all()
+            for alias in aliases:
+                key = f"{alias.email.lower()}:{alias.name.lower()}"
+                self._cache[key] = alias.canonical_id
+    
+    def _apply_manual_mappings(self, manual_mappings: List[Dict[str, Any]]):
+        """Apply manual identity mappings from configuration."""
+        # Clear cache to ensure we get fresh data
+        self._cache.clear()
+        self._load_cache()
+        
+        with self.get_session() as session:
+            for mapping in manual_mappings:
+                canonical_email = mapping.get('canonical_email', '').lower().strip()
+                aliases = mapping.get('aliases', [])
+                
+                if not canonical_email or not aliases:
+                    continue
+                
+                # Find the canonical identity
+                canonical_identity = session.query(DeveloperIdentity).filter(
+                    DeveloperIdentity.primary_email == canonical_email
+                ).first()
+                
+                if not canonical_identity:
+                    # Skip if canonical identity doesn't exist yet
+                    print(f"Warning: Canonical identity not found for email: {canonical_email}")
+                    continue
+                
+                # Process each alias
+                for alias_email in aliases:
+                    alias_email = alias_email.lower().strip()
+                    
+                    # Check if alias identity exists as a primary identity
+                    alias_identity = session.query(DeveloperIdentity).filter(
+                        DeveloperIdentity.primary_email == alias_email
+                    ).first()
+                    
+                    if alias_identity:
+                        if alias_identity.canonical_id != canonical_identity.canonical_id:
+                            # Merge the identities - commit before merge to avoid locks
+                            session.commit()
+                            print(f"Merging identity: {alias_identity.primary_name} ({alias_email}) into {canonical_identity.primary_name} ({canonical_email})")
+                            self.merge_identities(canonical_identity.canonical_id, alias_identity.canonical_id)
+                            # Refresh session after merge
+                            session.expire_all()
+                    else:
+                        # Just add as an alias if not a primary identity
+                        existing_alias = session.query(DeveloperAlias).filter(
+                            and_(
+                                DeveloperAlias.email == alias_email,
+                                DeveloperAlias.canonical_id == canonical_identity.canonical_id
+                            )
+                        ).first()
+                        
+                        if not existing_alias:
+                            # Get the name from any existing alias with this email
+                            name_for_alias = None
+                            any_alias = session.query(DeveloperAlias).filter(
+                                DeveloperAlias.email == alias_email
+                            ).first()
+                            if any_alias:
+                                name_for_alias = any_alias.name
+                            else:
+                                name_for_alias = canonical_identity.primary_name
+                            
+                            new_alias = DeveloperAlias(
+                                canonical_id=canonical_identity.canonical_id,
+                                name=name_for_alias,
+                                email=alias_email
+                            )
+                            session.add(new_alias)
+                            print(f"Added alias: {alias_email} for {canonical_identity.primary_name}")
+        
+        # Reload cache after all mappings
+        self._cache.clear()
+        self._load_cache()
+    
+    def resolve_developer(self, name: str, email: str, 
+                         github_username: Optional[str] = None) -> str:
+        """Resolve developer identity and return canonical ID."""
+        # Normalize inputs
+        name = name.strip()
+        email = email.lower().strip()
+        
+        # Check cache first
+        cache_key = f"{email}:{name.lower()}"
+        if cache_key in self._cache:
+            canonical_id = self._cache[cache_key]
+            # Update stats
+            self._update_developer_stats(canonical_id)
+            return canonical_id
+        
+        # Check exact email match in database
+        with self.get_session() as session:
+            # Check aliases
+            alias = session.query(DeveloperAlias).filter(
+                DeveloperAlias.email == email
+            ).first()
+            
+            if alias:
+                self._cache[cache_key] = alias.canonical_id
+                self._update_developer_stats(alias.canonical_id)
+                return alias.canonical_id
+            
+            # Check primary identities
+            identity = session.query(DeveloperIdentity).filter(
+                DeveloperIdentity.primary_email == email
+            ).first()
+            
+            if identity:
+                # Add as alias if name is different
+                if identity.primary_name.lower() != name.lower():
+                    self._add_alias(identity.canonical_id, name, email)
+                self._cache[cache_key] = identity.canonical_id
+                return identity.canonical_id
+        
+        # Find similar developer
+        best_match = self._find_best_match(name, email)
+        
+        if best_match and best_match[1] >= self.similarity_threshold:
+            canonical_id = best_match[0]
+            self._add_alias(canonical_id, name, email)
+            self._cache[cache_key] = canonical_id
+            return canonical_id
+        
+        # Create new identity
+        canonical_id = self._create_identity(name, email, github_username)
+        self._cache[cache_key] = canonical_id
+        return canonical_id
+    
+    def _find_best_match(self, name: str, email: str) -> Optional[Tuple[str, float]]:
+        """Find the best matching existing developer."""
+        best_score = 0.0
+        best_canonical_id = None
+        
+        name_lower = name.lower().strip()
+        email_domain = email.split('@')[1] if '@' in email else ''
+        
+        with self.get_session() as session:
+            # Get all identities for comparison
+            identities = session.query(DeveloperIdentity).all()
+            
+            for identity in identities:
+                score = 0.0
+                
+                # Name similarity (40% weight)
+                name_sim = difflib.SequenceMatcher(
+                    None, name_lower, identity.primary_name.lower()
+                ).ratio()
+                score += name_sim * 0.4
+                
+                # Email domain similarity (30% weight)
+                identity_domain = (identity.primary_email.split('@')[1] 
+                                 if '@' in identity.primary_email else '')
+                if email_domain and email_domain == identity_domain:
+                    score += 0.3
+                
+                # Check aliases (30% weight)
+                aliases = session.query(DeveloperAlias).filter(
+                    DeveloperAlias.canonical_id == identity.canonical_id
+                ).all()
+                
+                best_alias_score = 0.0
+                for alias in aliases:
+                    alias_name_sim = difflib.SequenceMatcher(
+                        None, name_lower, alias.name.lower()
+                    ).ratio()
+                    
+                    # Bonus for same email domain in aliases
+                    alias_domain = alias.email.split('@')[1] if '@' in alias.email else ''
+                    domain_bonus = 0.2 if alias_domain == email_domain else 0.0
+                    
+                    alias_score = alias_name_sim + domain_bonus
+                    best_alias_score = max(best_alias_score, alias_score)
+                
+                score += min(best_alias_score * 0.3, 0.3)
+                
+                if score > best_score:
+                    best_score = score
+                    best_canonical_id = identity.canonical_id
+        
+        return (best_canonical_id, best_score) if best_canonical_id else None
+    
+    def _create_identity(self, name: str, email: str, 
+                        github_username: Optional[str] = None) -> str:
+        """Create new developer identity."""
+        canonical_id = str(uuid.uuid4())
+        
+        with self.get_session() as session:
+            identity = DeveloperIdentity(
+                canonical_id=canonical_id,
+                primary_name=name,
+                primary_email=email,
+                github_username=github_username,
+                total_commits=0,
+                total_story_points=0
+            )
+            session.add(identity)
+        
+        # Update cache
+        self._cache[canonical_id] = {
+            'primary_name': name,
+            'primary_email': email,
+            'github_username': github_username
+        }
+        
+        return canonical_id
+    
+    def _add_alias(self, canonical_id: str, name: str, email: str):
+        """Add alias for existing developer."""
+        with self.get_session() as session:
+            # Check if alias already exists
+            existing = session.query(DeveloperAlias).filter(
+                and_(
+                    DeveloperAlias.canonical_id == canonical_id,
+                    DeveloperAlias.email == email.lower()
+                )
+            ).first()
+            
+            if not existing:
+                alias = DeveloperAlias(
+                    canonical_id=canonical_id,
+                    name=name,
+                    email=email.lower()
+                )
+                session.add(alias)
+    
+    def _update_developer_stats(self, canonical_id: str):
+        """Update developer statistics."""
+        with self.get_session() as session:
+            identity = session.query(DeveloperIdentity).filter(
+                DeveloperIdentity.canonical_id == canonical_id
+            ).first()
+            
+            if identity:
+                identity.last_seen = datetime.utcnow()
+    
+    def merge_identities(self, canonical_id1: str, canonical_id2: str):
+        """Merge two developer identities."""
+        # First, add the alias outside of the main merge transaction
+        with self.get_session() as session:
+            identity2 = session.query(DeveloperIdentity).filter(
+                DeveloperIdentity.canonical_id == canonical_id2
+            ).first()
+            if identity2:
+                identity2_name = identity2.primary_name
+                identity2_email = identity2.primary_email
+        
+        # Add identity2's primary as alias to identity1 first
+        self._add_alias(canonical_id1, identity2_name, identity2_email)
+        
+        # Now do the merge in a separate transaction
+        with self.get_session() as session:
+            # Get both identities fresh
+            identity1 = session.query(DeveloperIdentity).filter(
+                DeveloperIdentity.canonical_id == canonical_id1
+            ).first()
+            identity2 = session.query(DeveloperIdentity).filter(
+                DeveloperIdentity.canonical_id == canonical_id2
+            ).first()
+            
+            if not identity1 or not identity2:
+                raise ValueError("One or both identities not found")
+            
+            # Keep identity1, merge identity2 into it
+            identity1.total_commits += identity2.total_commits
+            identity1.total_story_points += identity2.total_story_points
+            identity1.first_seen = min(identity1.first_seen, identity2.first_seen)
+            identity1.last_seen = max(identity1.last_seen, identity2.last_seen)
+            
+            # Move all aliases from identity2 to identity1
+            aliases = session.query(DeveloperAlias).filter(
+                DeveloperAlias.canonical_id == canonical_id2
+            ).all()
+            
+            for alias in aliases:
+                alias.canonical_id = canonical_id1
+            
+            # Delete identity2
+            session.delete(identity2)
+        
+        # Clear cache to force reload
+        self._cache.clear()
+        self._load_cache()
+    
+    def get_developer_stats(self) -> List[Dict[str, Any]]:
+        """Get statistics for all developers."""
+        stats = []
+        
+        with self.get_session() as session:
+            identities = session.query(DeveloperIdentity).all()
+            
+            for identity in identities:
+                # Count aliases
+                alias_count = session.query(DeveloperAlias).filter(
+                    DeveloperAlias.canonical_id == identity.canonical_id
+                ).count()
+                
+                stats.append({
+                    'canonical_id': identity.canonical_id,
+                    'primary_name': identity.primary_name,
+                    'primary_email': identity.primary_email,
+                    'github_username': identity.github_username,
+                    'total_commits': identity.total_commits,
+                    'total_story_points': identity.total_story_points,
+                    'alias_count': alias_count,
+                    'first_seen': identity.first_seen,
+                    'last_seen': identity.last_seen
+                })
+        
+        # Sort by total commits
+        return sorted(stats, key=lambda x: x['total_commits'], reverse=True)
+    
+    def update_commit_stats(self, commits: List[Dict[str, Any]]):
+        """Update developer statistics based on commits."""
+        # Aggregate stats by canonical ID
+        stats_by_dev = defaultdict(lambda: {'commits': 0, 'story_points': 0})
+        
+        for commit in commits:
+            canonical_id = self.resolve_developer(
+                commit['author_name'],
+                commit['author_email']
+            )
+            
+            stats_by_dev[canonical_id]['commits'] += 1
+            stats_by_dev[canonical_id]['story_points'] += commit.get('story_points', 0) or 0
+        
+        # Update database
+        with self.get_session() as session:
+            for canonical_id, stats in stats_by_dev.items():
+                identity = session.query(DeveloperIdentity).filter(
+                    DeveloperIdentity.canonical_id == canonical_id
+                ).first()
+                
+                if identity:
+                    identity.total_commits += stats['commits']
+                    identity.total_story_points += stats['story_points']
+                    identity.last_seen = datetime.utcnow()
+        
+        # Apply manual mappings after all identities are created
+        if self.manual_mappings:
+            self.apply_manual_mappings()
+    
+    def apply_manual_mappings(self):
+        """Apply manual mappings - can be called explicitly after identities are created."""
+        if self.manual_mappings:
+            self._apply_manual_mappings(self.manual_mappings)

gitflow_analytics/extractors/base.py

@@ -0,0 +1,41 @@
+"""Base classes for pluggable extractors."""
+from abc import ABC, abstractmethod
+from typing import Any, Optional, List, Dict
+
+
+class ExtractorBase(ABC):
+    """Base class for all extractors."""
+    
+    @abstractmethod
+    def extract_from_text(self, text: str) -> Any:
+        """Extract information from text."""
+        pass
+
+
+class StoryPointExtractorBase(ExtractorBase):
+    """Base class for story point extractors."""
+    
+    @abstractmethod
+    def extract_from_text(self, text: str) -> Optional[int]:
+        """Extract story points from text."""
+        pass
+    
+    @abstractmethod
+    def extract_from_pr(self, pr_data: Dict[str, Any], 
+                       commit_messages: Optional[List[str]] = None) -> Optional[int]:
+        """Extract story points from pull request."""
+        pass
+
+
+class TicketExtractorBase(ExtractorBase):
+    """Base class for ticket extractors."""
+    
+    @abstractmethod
+    def extract_from_text(self, text: str) -> List[Dict[str, str]]:
+        """Extract ticket references from text."""
+        pass
+    
+    @abstractmethod
+    def extract_by_platform(self, text: str) -> Dict[str, List[str]]:
+        """Extract tickets grouped by platform."""
+        pass

gitflow_analytics/extractors/story_points.py

@@ -0,0 +1,128 @@
+"""Story point extraction from commits and pull requests."""
+import re
+from typing import Optional, List, Dict, Any
+
+
+class StoryPointExtractor:
+    """Extract story points from text using configurable patterns."""
+    
+    def __init__(self, patterns: Optional[List[str]] = None):
+        """Initialize with extraction patterns."""
+        if patterns is None:
+            patterns = [
+                r'(?:story\s*points?|sp|pts?)\s*[:=]\s*(\d+)',  # SP: 5, story points = 3
+                r'\[(\d+)\s*(?:sp|pts?)\]',                     # [3sp], [5 pts]
+                r'#(\d+)sp',                                     # #3sp
+                r'estimate:\s*(\d+)',                            # estimate: 5
+                r'\bSP(\d+)\b',                                  # SP5, SP13
+                r'points?:\s*(\d+)',                             # points: 8
+            ]
+        
+        self.patterns = [re.compile(pattern, re.IGNORECASE) for pattern in patterns]
+    
+    def extract_from_text(self, text: str) -> Optional[int]:
+        """Extract story points from text."""
+        if not text:
+            return None
+        
+        for pattern in self.patterns:
+            match = pattern.search(text)
+            if match:
+                try:
+                    points = int(match.group(1))
+                    # Sanity check - story points should be reasonable
+                    if 0 < points <= 100:
+                        return points
+                except (ValueError, IndexError):
+                    continue
+        
+        return None
+    
+    def extract_from_pr(self, pr_data: Dict[str, Any], 
+                       commit_messages: Optional[List[str]] = None) -> Optional[int]:
+        """Extract story points from PR with fallback to commits."""
+        # Try PR description first (most authoritative)
+        points = self.extract_from_text(pr_data.get('description', ''))
+        if points:
+            return points
+        
+        # Try PR title
+        points = self.extract_from_text(pr_data.get('title', ''))
+        if points:
+            return points
+        
+        # Try PR body (if different from description)
+        if 'body' in pr_data:
+            points = self.extract_from_text(pr_data['body'])
+            if points:
+                return points
+        
+        # Fallback to commit messages
+        if commit_messages:
+            commit_points = []
+            for message in commit_messages:
+                points = self.extract_from_text(message)
+                if points:
+                    commit_points.append(points)
+            
+            if commit_points:
+                # Use the most common value or max if no consensus
+                from collections import Counter
+                point_counts = Counter(commit_points)
+                most_common = point_counts.most_common(1)
+                if most_common:
+                    return most_common[0][0]
+        
+        return None
+    
+    def aggregate_story_points(self, prs: List[Dict[str, Any]], 
+                             commits: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """Aggregate story points from PRs and commits."""
+        # Map commits to PRs
+        pr_by_commit = {}
+        for pr in prs:
+            for commit_hash in pr.get('commit_hashes', []):
+                pr_by_commit[commit_hash] = pr
+        
+        # Track which commits are associated with PRs
+        pr_commits = set(pr_by_commit.keys())
+        
+        # Aggregate results
+        results = {
+            'total_story_points': 0,
+            'pr_story_points': 0,
+            'commit_story_points': 0,
+            'orphan_commits': [],  # Commits without PRs
+            'unestimated_prs': []  # PRs without story points
+        }
+        
+        # Process PRs
+        for pr in prs:
+            pr_points = pr.get('story_points', 0)
+            if pr_points:
+                results['pr_story_points'] += pr_points
+                results['total_story_points'] += pr_points
+            else:
+                results['unestimated_prs'].append({
+                    'number': pr['number'],
+                    'title': pr['title']
+                })
+        
+        # Process commits not in PRs
+        for commit in commits:
+            if commit['hash'] not in pr_commits:
+                commit_points = commit.get('story_points', 0)
+                if commit_points:
+                    results['commit_story_points'] += commit_points
+                    results['total_story_points'] += commit_points
+                
+                # Track significant orphan commits
+                if commit['files_changed'] > 5 or commit['insertions'] > 100:
+                    results['orphan_commits'].append({
+                        'hash': commit['hash'][:7],
+                        'message': commit['message'].split('\n')[0][:80],
+                        'story_points': commit_points,
+                        'files_changed': commit['files_changed']
+                    })
+        
+        return results