gitflow-analytics 1.0.0__py3-none-any.whl → 1.0.3__py3-none-any.whl

gitflow_analytics/qualitative/utils/text_processing.py

@@ -0,0 +1,243 @@
+"""Text processing utilities for qualitative analysis."""
+
+import hashlib
+import re
+from typing import Dict, List, Set
+
+
+class TextProcessor:
+    """Utility class for text preprocessing and feature extraction.
+    
+    This class provides common text processing operations needed across
+    the qualitative analysis pipeline, including normalization, feature
+    extraction, and similarity calculations.
+    """
+    
+    def __init__(self) -> None:
+        """Initialize text processor with common patterns."""
+        # Common patterns for normalization
+        self.url_pattern = re.compile(r'https?://[^\s]+')
+        self.email_pattern = re.compile(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b')
+        self.hash_pattern = re.compile(r'\b[a-f0-9]{7,40}\b')  # Git hashes
+        self.ticket_pattern = re.compile(r'\b(?:JIRA|TICKET|ISSUE|BUG|TASK)-?\d+\b', re.IGNORECASE)
+        
+        # Stop words for feature extraction
+        self.stop_words: Set[str] = {
+            'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 
+            'of', 'with', 'by', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+            'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+            'should', 'may', 'might', 'can', 'this', 'that', 'these', 'those'
+        }
+        
+    def normalize_message(self, message: str) -> str:
+        """Normalize commit message for consistent processing.
+        
+        This method standardizes commit messages by removing URLs, emails,
+        hashes, and other variable content that doesn't contribute to 
+        semantic classification.
+        
+        Args:
+            message: Raw commit message
+            
+        Returns:
+            Normalized message suitable for classification
+        """
+        if not message:
+            return ""
+            
+        # Convert to lowercase for consistency
+        normalized = message.lower().strip()
+        
+        # Remove URLs, emails, and hashes
+        normalized = self.url_pattern.sub('[URL]', normalized)
+        normalized = self.email_pattern.sub('[EMAIL]', normalized)
+        normalized = self.hash_pattern.sub('[HASH]', normalized)
+        
+        # Normalize ticket references
+        normalized = self.ticket_pattern.sub('[TICKET]', normalized)
+        
+        # Remove extra whitespace
+        normalized = re.sub(r'\s+', ' ', normalized)
+        
+        return normalized.strip()
+        
+    def extract_keywords(self, text: str, min_length: int = 3) -> List[str]:
+        """Extract meaningful keywords from text.
+        
+        Extracts keywords by removing stop words, punctuation, and short words
+        that are unlikely to be semantically meaningful.
+        
+        Args:
+            text: Input text to extract keywords from
+            min_length: Minimum length for keywords
+            
+        Returns:
+            List of extracted keywords
+        """
+        if not text:
+            return []
+            
+        # Split into words and clean
+        words = re.findall(r'\b[a-zA-Z]+\b', text.lower())
+        
+        # Filter stop words and short words
+        keywords = [
+            word for word in words 
+            if word not in self.stop_words and len(word) >= min_length
+        ]
+        
+        return keywords
+        
+    def create_semantic_fingerprint(self, message: str, files: List[str]) -> str:
+        """Create a semantic fingerprint for similarity matching.
+        
+        Creates a hash-based fingerprint that captures the semantic essence
+        of a commit for pattern matching and caching.
+        
+        Args:
+            message: Commit message
+            files: List of changed files
+            
+        Returns:
+            Hex-encoded fingerprint string
+        """
+        # Normalize message for consistent fingerprinting
+        normalized_msg = self.normalize_message(message)
+        keywords = self.extract_keywords(normalized_msg)
+        
+        # Extract file patterns (extensions, directories)
+        file_patterns = []
+        for file_path in files[:10]:  # Limit to prevent huge fingerprints
+            # Get file extension
+            if '.' in file_path:
+                ext = file_path.split('.')[-1].lower()
+                file_patterns.append(f"ext:{ext}")
+                
+            # Get directory patterns  
+            parts = file_path.split('/')
+            if len(parts) > 1:
+                # First directory
+                file_patterns.append(f"dir:{parts[0]}")
+                # Last directory before file
+                if len(parts) > 2:
+                    file_patterns.append(f"dir:{parts[-2]}")
+        
+        # Combine keywords and file patterns
+        semantic_elements = sorted(keywords[:10]) + sorted(set(file_patterns))
+        
+        # Create fingerprint
+        fingerprint_text = '|'.join(semantic_elements)
+        return hashlib.md5(fingerprint_text.encode()).hexdigest()
+        
+    def calculate_message_similarity(self, msg1: str, msg2: str) -> float:
+        """Calculate semantic similarity between two commit messages.
+        
+        Uses keyword overlap to estimate semantic similarity between
+        commit messages for grouping similar commits.
+        
+        Args:
+            msg1: First commit message
+            msg2: Second commit message
+            
+        Returns:
+            Similarity score between 0.0 and 1.0
+        """
+        if not msg1 or not msg2:
+            return 0.0
+            
+        # Extract keywords from both messages
+        keywords1 = set(self.extract_keywords(self.normalize_message(msg1)))
+        keywords2 = set(self.extract_keywords(self.normalize_message(msg2)))
+        
+        if not keywords1 or not keywords2:
+            return 0.0
+            
+        # Calculate Jaccard similarity
+        intersection = len(keywords1.intersection(keywords2))
+        union = len(keywords1.union(keywords2))
+        
+        return intersection / union if union > 0 else 0.0
+        
+    def extract_file_patterns(self, files: List[str]) -> Dict[str, int]:
+        """Extract file patterns for domain classification.
+        
+        Analyzes file paths to extract patterns useful for determining
+        the technical domain of changes.
+        
+        Args:
+            files: List of file paths
+            
+        Returns:
+            Dictionary mapping pattern types to counts
+        """
+        patterns = {
+            'extensions': {},
+            'directories': {},
+            'special_files': {},
+        }
+        
+        for file_path in files:
+            # File extensions
+            if '.' in file_path:
+                ext = file_path.split('.')[-1].lower()
+                patterns['extensions'][ext] = patterns['extensions'].get(ext, 0) + 1
+                
+            # Directory patterns
+            parts = file_path.split('/')
+            for part in parts[:-1]:  # Exclude filename
+                if part:  # Skip empty parts
+                    patterns['directories'][part] = patterns['directories'].get(part, 0) + 1
+                    
+            # Special files
+            filename = parts[-1].lower()
+            special_files = [
+                'dockerfile', 'makefile', 'readme', 'license', 'changelog',
+                'package.json', 'requirements.txt', 'setup.py', 'pom.xml'
+            ]
+            for special in special_files:
+                if special in filename:
+                    patterns['special_files'][special] = patterns['special_files'].get(special, 0) + 1
+                    
+        return patterns
+        
+    def calculate_commit_complexity(self, message: str, files: List[str], 
+                                  insertions: int, deletions: int) -> Dict[str, float]:
+        """Calculate various complexity metrics for a commit.
+        
+        Estimates the complexity of a commit based on message content,
+        file changes, and line changes to help with risk assessment.
+        
+        Args:
+            message: Commit message
+            files: List of changed files
+            insertions: Number of lines inserted
+            deletions: Number of lines deleted
+            
+        Returns:
+            Dictionary of complexity metrics
+        """
+        metrics = {}
+        
+        # Message complexity (length, keywords)
+        metrics['message_length'] = len(message)
+        keywords = self.extract_keywords(message)
+        metrics['keyword_count'] = len(keywords)
+        metrics['message_complexity'] = min(1.0, len(keywords) / 10.0)
+        
+        # File complexity
+        metrics['files_changed'] = len(files)
+        metrics['file_complexity'] = min(1.0, len(files) / 20.0)
+        
+        # Line change complexity
+        total_changes = insertions + deletions
+        metrics['total_changes'] = total_changes
+        metrics['change_complexity'] = min(1.0, total_changes / 500.0)
+        
+        # Overall complexity score (0.0 to 1.0)
+        metrics['overall_complexity'] = (
+            metrics['message_complexity'] * 0.2 +
+            metrics['file_complexity'] * 0.3 +
+            metrics['change_complexity'] * 0.5
+        )
+        
+        return metrics

gitflow_analytics/reports/analytics_writer.py

@@ -1,6 +1,6 @@
 """Advanced analytics report generation with percentage and qualitative metrics."""
 import csv
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import List, Dict, Any, Tuple
 from collections import defaultdict
@@ -26,7 +26,11 @@ class AnalyticsReportGenerator:
         
         # Calculate totals
         total_commits = len(commits)
-        total_lines = sum(c['insertions'] + c['deletions'] for c in commits)
+        total_lines = sum(
+            c.get('filtered_insertions', c.get('insertions', 0)) + 
+            c.get('filtered_deletions', c.get('deletions', 0)) 
+            for c in commits
+        )
         total_files = sum(c['files_changed'] for c in commits)
         
         # Group by developer and project
@@ -39,8 +43,11 @@ class AnalyticsReportGenerator:
             project = commit.get('project_key', 'UNKNOWN')
             
             dev_project_activity[dev_id][project]['commits'] += 1
-            dev_project_activity[dev_id][project]['lines'] += commit['insertions'] + commit['deletions']
-            dev_project_activity[dev_id][project]['files'] += commit['files_changed']
+            dev_project_activity[dev_id][project]['lines'] += (
+                commit.get('filtered_insertions', commit.get('insertions', 0)) + 
+                commit.get('filtered_deletions', commit.get('deletions', 0))
+            )
+            dev_project_activity[dev_id][project]['files'] += commit.get('filtered_files_changed', commit.get('files_changed', 0))
             dev_project_activity[dev_id][project]['story_points'] += commit.get('story_points', 0) or 0
         
         # Build report data
@@ -120,8 +127,8 @@ class AnalyticsReportGenerator:
                                       output_path: Path,
                                       weeks: int = 12) -> Path:
         """Generate developer focus analysis showing concentration patterns and activity across all projects."""
-        # Calculate week boundaries
-        end_date = datetime.now()
+        # Calculate week boundaries (timezone-aware to match commit timestamps)
+        end_date = datetime.now(timezone.utc)
         start_date = end_date - timedelta(weeks=weeks)
         
         # Build developer lookup
@@ -162,7 +169,10 @@ class AnalyticsReportGenerator:
                 projects[project_key] += 1
                 
                 # Lines changed per project
-                lines_changed = commit['insertions'] + commit['deletions']
+                lines_changed = (
+                    commit.get('filtered_insertions', commit.get('insertions', 0)) + 
+                    commit.get('filtered_deletions', commit.get('deletions', 0))
+                )
                 project_lines[project_key] += lines_changed
                 
                 # Weekly distribution
@@ -432,8 +442,15 @@ class AnalyticsReportGenerator:
     
     def _get_week_start(self, date: datetime) -> datetime:
         """Get Monday of the week for a given date."""
+        # Ensure consistent timezone handling - keep timezone info
         if hasattr(date, 'tzinfo') and date.tzinfo is not None:
-            date = date.replace(tzinfo=None)
+            # Keep timezone-aware but ensure it's UTC
+            if date.tzinfo != timezone.utc:
+                date = date.astimezone(timezone.utc)
+        else:
+            # Convert naive datetime to UTC timezone-aware
+            date = date.replace(tzinfo=timezone.utc)
+        
         days_since_monday = date.weekday()
         monday = date - timedelta(days=days_since_monday)
         return monday.replace(hour=0, minute=0, second=0, microsecond=0)

gitflow_analytics/reports/csv_writer.py

@@ -1,8 +1,8 @@
 """CSV report generation for GitFlow Analytics."""
 import csv
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
-from typing import List, Dict, Any, Optional
+from typing import List, Dict, Any, Optional, Tuple
 from collections import defaultdict
 import pandas as pd
 
@@ -13,7 +13,7 @@ class CSVReportGenerator:
     def __init__(self, anonymize: bool = False):
         """Initialize report generator."""
         self.anonymize = anonymize
-        self._anonymization_map = {}
+        self._anonymization_map: Dict[str, str] = {}
         self._anonymous_counter = 0
     
     def generate_weekly_report(self, commits: List[Dict[str, Any]], 
@@ -21,12 +21,12 @@ class CSVReportGenerator:
                              output_path: Path,
                              weeks: int = 12) -> Path:
         """Generate weekly metrics CSV report."""
-        # Calculate week boundaries
-        end_date = datetime.now()
+        # Calculate week boundaries (timezone-aware to match commit timestamps)
+        end_date = datetime.now(timezone.utc)
         start_date = end_date - timedelta(weeks=weeks)
         
         # Group commits by week and developer
-        weekly_data = self._aggregate_weekly_data(commits, start_date, end_date)
+        weekly_data: Dict[Tuple[datetime, str, str], Dict[str, Any]] = self._aggregate_weekly_data(commits, start_date, end_date)
         
         # Create developer lookup
         dev_lookup = {dev['canonical_id']: dev for dev in developer_stats}
@@ -90,7 +90,12 @@ class CSVReportGenerator:
         # Overall statistics
         total_commits = len(commits)
         total_story_points = sum(c.get('story_points', 0) or 0 for c in commits)
-        total_lines = sum(c['insertions'] + c['deletions'] for c in commits)
+        # Use filtered stats if available, otherwise fall back to raw stats
+        total_lines = sum(
+            c.get('filtered_insertions', c.get('insertions', 0)) + 
+            c.get('filtered_deletions', c.get('deletions', 0)) 
+            for c in commits
+        )
         
         summary_data.append({
             'metric': 'Total Commits',
@@ -193,9 +198,9 @@ class CSVReportGenerator:
     
     def _aggregate_weekly_data(self, commits: List[Dict[str, Any]], 
                              start_date: datetime, 
-                             end_date: datetime) -> Dict[tuple, Dict[str, Any]]:
+                             end_date: datetime) -> Dict[Tuple[datetime, str, str], Dict[str, Any]]:
         """Aggregate commit data by week."""
-        weekly_data = defaultdict(lambda: {
+        weekly_data: defaultdict[Tuple[datetime, str, str], Dict[str, Any]] = defaultdict(lambda: {
             'commits': 0,
             'story_points': 0,
             'lines_added': 0,
@@ -209,10 +214,14 @@ class CSVReportGenerator:
         
         for commit in commits:
             timestamp = commit['timestamp']
-            # Handle both timezone-aware and naive datetimes
+            # Ensure consistent timezone handling
             if hasattr(timestamp, 'tzinfo') and timestamp.tzinfo is not None:
-                # Convert timezone-aware to naive (UTC)
-                timestamp = timestamp.replace(tzinfo=None)
+                # Keep timezone-aware but ensure it's UTC
+                if timestamp.tzinfo != timezone.utc:
+                    timestamp = timestamp.astimezone(timezone.utc)
+            else:
+                # Convert naive datetime to UTC timezone-aware
+                timestamp = timestamp.replace(tzinfo=timezone.utc)
             
             if timestamp < start_date or timestamp > end_date:
                 continue
@@ -229,43 +238,53 @@ class CSVReportGenerator:
             key = (week_start, canonical_id, project_key)
             
             # Aggregate metrics
-            weekly_data[key]['commits'] += 1
-            weekly_data[key]['story_points'] += commit.get('story_points', 0) or 0
-            weekly_data[key]['lines_added'] += commit.get('insertions', 0)
-            weekly_data[key]['lines_removed'] += commit.get('deletions', 0)
-            weekly_data[key]['files_changed'] += commit.get('files_changed', 0)
-            weekly_data[key]['complexity_delta'] += commit.get('complexity_delta', 0.0)
+            data = weekly_data[key]
+            data['commits'] += 1
+            data['story_points'] += commit.get('story_points', 0) or 0
+            
+            # Use filtered stats if available, otherwise fall back to raw stats
+            data['lines_added'] += commit.get('filtered_insertions', commit.get('insertions', 0)) or 0
+            data['lines_removed'] += commit.get('filtered_deletions', commit.get('deletions', 0)) or 0
+            data['files_changed'] += commit.get('filtered_files_changed', commit.get('files_changed', 0)) or 0
+            
+            data['complexity_delta'] += commit.get('complexity_delta', 0.0) or 0.0
             
             # Track tickets
-            if commit.get('ticket_references'):
-                weekly_data[key]['commits_with_tickets'] += 1
-                for ticket in commit['ticket_references']:
+            ticket_refs = commit.get('ticket_references', [])
+            if ticket_refs:
+                data['commits_with_tickets'] += 1
+                tickets_set = data['tickets']
+                for ticket in ticket_refs:
                     if isinstance(ticket, dict):
-                        weekly_data[key]['tickets'].add(ticket.get('full_id', ''))
+                        tickets_set.add(ticket.get('full_id', ''))
                     else:
-                        weekly_data[key]['tickets'].add(str(ticket))
+                        tickets_set.add(str(ticket))
             
             # Track PRs (if available)
-            if commit.get('pr_number'):
-                weekly_data[key]['prs'].add(commit['pr_number'])
+            pr_number = commit.get('pr_number')
+            if pr_number:
+                prs_set = data['prs']
+                prs_set.add(pr_number)
         
         # Calculate derived metrics
-        result = {}
+        result: Dict[Tuple[datetime, str, str], Dict[str, Any]] = {}
         for key, metrics in weekly_data.items():
-            commits = metrics['commits']
-            if commits > 0:
+            commits_count = metrics['commits']
+            if commits_count > 0:
                 metrics['ticket_coverage_pct'] = (
-                    metrics['commits_with_tickets'] / commits * 100
+                    metrics['commits_with_tickets'] / commits_count * 100
                 )
                 metrics['avg_commit_size'] = (
-                    (metrics['lines_added'] + metrics['lines_removed']) / commits
+                    (metrics['lines_added'] + metrics['lines_removed']) / commits_count
                 )
             else:
                 metrics['ticket_coverage_pct'] = 0
                 metrics['avg_commit_size'] = 0
             
-            metrics['unique_tickets'] = len(metrics['tickets'])
-            metrics['prs_involved'] = len(metrics['prs'])
+            tickets_set = metrics['tickets']
+            prs_set = metrics['prs']
+            metrics['unique_tickets'] = len(tickets_set)
+            metrics['prs_involved'] = len(prs_set)
             
             # Remove sets before returning
             del metrics['tickets']
@@ -278,6 +297,15 @@ class CSVReportGenerator:
     
     def _get_week_start(self, date: datetime) -> datetime:
         """Get Monday of the week for a given date."""
+        # Ensure consistent timezone handling - keep timezone info
+        if hasattr(date, 'tzinfo') and date.tzinfo is not None:
+            # Keep timezone-aware but ensure it's UTC
+            if date.tzinfo != timezone.utc:
+                date = date.astimezone(timezone.utc)
+        else:
+            # Convert naive datetime to UTC timezone-aware
+            date = date.replace(tzinfo=timezone.utc)
+        
         days_since_monday = date.weekday()
         monday = date - timedelta(days=days_since_monday)
         return monday.replace(hour=0, minute=0, second=0, microsecond=0)

gitflow_analytics/reports/narrative_writer.py

@@ -1,14 +1,14 @@
 """Narrative report generation in Markdown format."""
 from datetime import datetime
 from pathlib import Path
-from typing import List, Dict, Any
+from typing import List, Dict, Any, Set
 from io import StringIO
 
 
 class NarrativeReportGenerator:
     """Generate human-readable narrative reports in Markdown."""
     
-    def __init__(self):
+    def __init__(self) -> None:
         """Initialize narrative report generator."""
         self.templates = {
             'high_performer': "{name} led development with {commits} commits ({pct}% of total activity)",
@@ -74,11 +74,15 @@ class NarrativeReportGenerator:
     
     def _write_executive_summary(self, report: StringIO, commits: List[Dict[str, Any]], 
                                developer_stats: List[Dict[str, Any]], 
-                               ticket_analysis: Dict[str, Any]):
+                               ticket_analysis: Dict[str, Any]) -> None:
         """Write executive summary section."""
         total_commits = len(commits)
         total_developers = len(developer_stats)
-        total_lines = sum(c['insertions'] + c['deletions'] for c in commits)
+        total_lines = sum(
+            c.get('filtered_insertions', c.get('insertions', 0)) + 
+            c.get('filtered_deletions', c.get('deletions', 0)) 
+            for c in commits
+        )
         
         report.write(f"- **Total Commits**: {total_commits:,}\n")
         report.write(f"- **Active Developers**: {total_developers}\n")
@@ -96,7 +100,7 @@ class NarrativeReportGenerator:
             report.write(f"({top_dev['total_commits']} commits)\n")
     
     def _write_team_composition(self, report: StringIO, developer_stats: List[Dict[str, Any]], 
-                              focus_data: List[Dict[str, Any]]):
+                              focus_data: List[Dict[str, Any]]) -> None:
         """Write team composition analysis."""
         report.write("### Developer Profiles\n\n")
         
@@ -121,10 +125,10 @@ class NarrativeReportGenerator:
             report.write("\n")
     
     def _write_project_activity(self, report: StringIO, activity_dist: List[Dict[str, Any]], 
-                              commits: List[Dict[str, Any]]):
+                              commits: List[Dict[str, Any]]) -> None:
         """Write project activity breakdown."""
         # Aggregate by project
-        project_totals = {}
+        project_totals: Dict[str, Dict[str, Any]] = {}
         for row in activity_dist:
             project = row['project']
             if project not in project_totals:
@@ -133,9 +137,11 @@ class NarrativeReportGenerator:
                     'lines': 0,
                     'developers': set()
                 }
-            project_totals[project]['commits'] += row['commits']
-            project_totals[project]['lines'] += row['lines_changed']
-            project_totals[project]['developers'].add(row['developer'])
+            data = project_totals[project]
+            data['commits'] += row['commits']
+            data['lines'] += row['lines_changed']
+            developers_set: Set[str] = data['developers']
+            developers_set.add(row['developer'])
         
         # Sort by commits
         sorted_projects = sorted(project_totals.items(), 
@@ -150,12 +156,12 @@ class NarrativeReportGenerator:
             report.write(f"- Active Developers: {len(data['developers'])}\n\n")
     
     def _write_development_patterns(self, report: StringIO, insights: List[Dict[str, Any]], 
-                                  focus_data: List[Dict[str, Any]]):
+                                  focus_data: List[Dict[str, Any]]) -> None:
         """Write development patterns analysis."""
         report.write("### Key Patterns Identified\n\n")
         
         # Group insights by category
-        by_category = {}
+        by_category: Dict[str, List[Dict[str, Any]]] = {}
         for insight in insights:
             category = insight['category']
             if category not in by_category:
@@ -182,7 +188,7 @@ class NarrativeReportGenerator:
                 report.write("suggests high context switching\n")
     
     def _write_pr_analysis(self, report: StringIO, pr_metrics: Dict[str, Any], 
-                         prs: List[Dict[str, Any]]):
+                         prs: List[Dict[str, Any]]) -> None:
         """Write pull request analysis."""
         report.write(f"- **Total PRs Merged**: {pr_metrics['total_prs']}\n")
         report.write(f"- **Average PR Size**: {pr_metrics['avg_pr_size']:.0f} lines\n")
@@ -194,7 +200,7 @@ class NarrativeReportGenerator:
             avg_comments = pr_metrics['total_review_comments'] / pr_metrics['total_prs']
             report.write(f"- **Average Comments per PR**: {avg_comments:.1f}\n")
     
-    def _write_ticket_tracking(self, report: StringIO, ticket_analysis: Dict[str, Any]):
+    def _write_ticket_tracking(self, report: StringIO, ticket_analysis: Dict[str, Any]) -> None:
         """Write ticket tracking analysis."""
         report.write("### Platform Usage\n\n")
         
@@ -217,7 +223,7 @@ class NarrativeReportGenerator:
                 report.write(f"({commit['files_changed']} files)\n")
     
     def _write_recommendations(self, report: StringIO, insights: List[Dict[str, Any]], 
-                             ticket_analysis: Dict[str, Any], focus_data: List[Dict[str, Any]]):
+                             ticket_analysis: Dict[str, Any], focus_data: List[Dict[str, Any]]) -> None:
         """Write recommendations based on analysis."""
         recommendations = []
         

gitflow_analytics/tui/__init__.py

@@ -0,0 +1,5 @@
+"""Terminal User Interface for GitFlow Analytics."""
+
+from .app import GitFlowAnalyticsApp
+
+__all__ = ["GitFlowAnalyticsApp"]