gitflow-analytics 1.3.6__py3-none-any.whl → 3.3.0__py3-none-any.whl

gitflow_analytics/qualitative/core/llm_fallback.py

@@ -113,6 +113,11 @@ class LLMFallback:
         # Batch processing cache
         self.batch_cache = {}
 
+        # Circuit breaker state
+        self.consecutive_failures = 0
+        self.circuit_breaker_open = False
+        self.max_consecutive_failures = 3  # Open circuit after 3 failures
+
         # Token encoder for cost estimation
         try:
             self.encoding = tiktoken.get_encoding("cl100k_base")  # GPT-4 encoding
@@ -142,6 +147,7 @@ class LLMFallback:
         return openai.OpenAI(
             base_url=self.config.base_url,
             api_key=api_key,
+            timeout=30.0,  # 30 second timeout to prevent hanging
             default_headers={
                 "HTTP-Referer": "https://github.com/bobmatnyc/gitflow-analytics",
                 "X-Title": "GitFlow Analytics - Qualitative Analysis",
@@ -213,6 +219,14 @@ class LLMFallback:
         if not commits:
             return []
 
+        # Check circuit breaker state
+        if self.circuit_breaker_open:
+            self.logger.warning(
+                f"Circuit breaker open ({self.consecutive_failures} consecutive failures), "
+                "using fallback classification"
+            )
+            return self._create_fallback_results(commits)
+
         start_time = time.time()
 
         # Check cache first
@@ -263,6 +277,15 @@ class LLMFallback:
             if results:
                 self.batch_cache[cache_key] = self._create_template_from_results(results)
 
+            # Reset circuit breaker on success
+            if self.consecutive_failures > 0:
+                self.logger.info(
+                    f"API call succeeded, resetting circuit breaker "
+                    f"(was {self.consecutive_failures} failures)"
+                )
+                self.consecutive_failures = 0
+                self.circuit_breaker_open = False
+
             # Update processing time in results
             for result in results:
                 result.processing_time_ms = (processing_time * 1000) / len(results)
@@ -272,6 +295,15 @@ class LLMFallback:
         except Exception as e:
             self.logger.error(f"OpenRouter processing failed: {e}")
 
+            # Increment failure counter and check circuit breaker
+            self.consecutive_failures += 1
+            if self.consecutive_failures >= self.max_consecutive_failures:
+                self.circuit_breaker_open = True
+                self.logger.error(
+                    f"Circuit breaker opened after {self.consecutive_failures} consecutive failures. "
+                    "All future LLM calls will use fallback classification until manual reset."
+                )
+
             # Record failed call
             self.cost_tracker.record_call(
                 model=selected_model,
@@ -283,8 +315,8 @@ class LLMFallback:
                 error_message=str(e),
             )
 
-            # Try fallback model if primary failed
-            if selected_model != self.config.fallback_model:
+            # Try fallback model if primary failed AND circuit breaker not open
+            if selected_model != self.config.fallback_model and not self.circuit_breaker_open:
                 return self._retry_with_fallback_model(commits, prompt)
             else:
                 return self._create_fallback_results(commits)

gitflow_analytics/reports/narrative_writer.py

@@ -7,6 +7,7 @@ from pathlib import Path
 from typing import Any
 
 from ..metrics.activity_scoring import ActivityScorer
+from ..core.progress import get_progress_service
 
 # Get logger for this module
 logger = logging.getLogger(__name__)
@@ -174,110 +175,153 @@ class NarrativeReportGenerator:
         # Store analysis period for use in weekly trends calculation
         self._analysis_start_date = analysis_start_date
         self._analysis_end_date = analysis_end_date
-        
+
         logger.debug(f"DEBUG NARRATIVE: Starting report generation with exclude_authors: {exclude_authors}")
         logger.debug(f"DEBUG NARRATIVE: Analysis period: {analysis_start_date} to {analysis_end_date}")
         logger.debug(f"DEBUG NARRATIVE: Input data sizes - commits: {len(commits)}, developer_stats: {len(developer_stats)}, "
                     f"activity_dist: {len(activity_dist)}, focus_data: {len(focus_data)}")
-        
+
         # Sample some developer_stats to see their structure
         if developer_stats:
             for i, dev in enumerate(developer_stats[:3]):
                 logger.debug(f"DEBUG NARRATIVE: Sample developer_stats[{i}]: canonical_id='{dev.get('canonical_id', '')}', "
                            f"primary_name='{dev.get('primary_name', '')}', name='{dev.get('name', '')}', "
                            f"primary_email='{dev.get('primary_email', '')}'")
-        
+
         # Filter out excluded authors in Phase 2 using canonical_id
         if exclude_authors:
             logger.debug(f"DEBUG NARRATIVE: Applying exclusion filter with {len(exclude_authors)} excluded authors")
-            
+
             original_commits = len(commits)
             commits = self._filter_excluded_authors(commits, exclude_authors)
             filtered_commits = original_commits - len(commits)
-            
+
             # Filter other data structures too
             logger.debug(f"DEBUG NARRATIVE: Filtering developer_stats (original: {len(developer_stats)})")
-            developer_stats = self._filter_excluded_authors(developer_stats, exclude_authors)  
+            developer_stats = self._filter_excluded_authors(developer_stats, exclude_authors)
             logger.debug(f"DEBUG NARRATIVE: After filtering developer_stats: {len(developer_stats)}")
-            
+
             activity_dist = self._filter_excluded_authors(activity_dist, exclude_authors)
             focus_data = self._filter_excluded_authors(focus_data, exclude_authors)
-            
+
             if filtered_commits > 0:
                 logger.info(f"Filtered out {filtered_commits} commits from {len(exclude_authors)} excluded authors in narrative report")
-            
+
             # Log remaining developers after filtering
             if developer_stats:
                 remaining_devs = [dev.get('primary_name', dev.get('name', 'Unknown')) for dev in developer_stats]
                 logger.debug(f"DEBUG NARRATIVE: Remaining developers after filtering: {remaining_devs}")
         else:
             logger.debug("DEBUG NARRATIVE: No exclusion filter applied")
-        
-        report = StringIO()
-
-        # Header
-        report.write("# GitFlow Analytics Report\n\n")
 
-        # Log datetime formatting
-        now = datetime.now()
-        logger.debug(
-            f"Formatting current datetime for report header: {now} (tzinfo: {getattr(now, 'tzinfo', 'N/A')})"
-        )
-        formatted_time = now.strftime("%Y-%m-%d %H:%M:%S")
-        logger.debug(f"  Formatted time: {formatted_time}")
-
-        report.write(f"**Generated**: {formatted_time}\n")
-        report.write(f"**Analysis Period**: Last {weeks} weeks\n\n")
-
-        # Executive Summary
-        report.write("## Executive Summary\n\n")
-        self._write_executive_summary(report, commits, developer_stats, ticket_analysis, prs, branch_health_metrics, pm_data)
-
-        # Add ChatGPT qualitative insights if available
-        if chatgpt_summary:
-            report.write("\n## Qualitative Analysis\n\n")
-            report.write(chatgpt_summary)
-            report.write("\n")
-
-        # Team Composition
-        report.write("\n## Team Composition\n\n")
-        self._write_team_composition(report, developer_stats, focus_data, commits, prs, ticket_analysis, weeks)
-
-        # Project Activity
-        report.write("\n## Project Activity\n\n")
-        self._write_project_activity(report, activity_dist, commits, branch_health_metrics, ticket_analysis, weeks)
-
-
-        # Development Patterns
-        report.write("\n## Development Patterns\n\n")
-        self._write_development_patterns(report, insights, focus_data)
-
-        # Commit Classification Analysis (if ML analysis is available)
-        if ticket_analysis.get("ml_analysis", {}).get("enabled", False):
-            report.write("\n## Commit Classification Analysis\n\n")
-            self._write_commit_classification_analysis(report, ticket_analysis)
-
-        # Pull Request Analysis (if available)
-        if pr_metrics and pr_metrics.get("total_prs", 0) > 0:
-            report.write("\n## Pull Request Analysis\n\n")
-            self._write_pr_analysis(report, pr_metrics, prs)
-
-        # Ticket Tracking
-        report.write("\n## Issue Tracking\n\n")
-        self._write_ticket_tracking(report, ticket_analysis, developer_stats)
-
-        # PM Platform Insights
-        if pm_data and "metrics" in pm_data:
-            report.write("\n## PM Platform Integration\n\n")
-            self._write_pm_insights(report, pm_data)
-
-        # Recommendations
-        report.write("\n## Recommendations\n\n")
-        self._write_recommendations(report, insights, ticket_analysis, focus_data)
-
-        # Write to file
-        with open(output_path, "w") as f:
-            f.write(report.getvalue())
+        # Initialize progress tracking for narrative report generation
+        progress_service = get_progress_service()
+
+        # Count all sections to be generated (including conditional ones)
+        sections = []
+        sections.append(("Executive Summary", True))
+        sections.append(("Qualitative Analysis", bool(chatgpt_summary)))
+        sections.append(("Team Composition", True))
+        sections.append(("Project Activity", True))
+        sections.append(("Development Patterns", True))
+        sections.append(("Commit Classification Analysis", ticket_analysis.get("ml_analysis", {}).get("enabled", False)))
+        sections.append(("Pull Request Analysis", bool(pr_metrics and pr_metrics.get("total_prs", 0) > 0)))
+        sections.append(("Issue Tracking", True))
+        sections.append(("PM Platform Integration", bool(pm_data and "metrics" in pm_data)))
+        sections.append(("Recommendations", True))
+
+        # Filter to only included sections
+        active_sections = [name for name, include in sections if include]
+        total_sections = len(active_sections)
+
+        logger.debug(f"Generating narrative report with {total_sections} sections: {', '.join(active_sections)}")
+
+        # Create progress context for narrative report generation
+        with progress_service.progress(total_sections, "Generating narrative report sections", unit="sections") as progress_ctx:
+            report = StringIO()
+
+            # Header
+            report.write("# GitFlow Analytics Report\n\n")
+
+            # Log datetime formatting
+            now = datetime.now()
+            logger.debug(
+                f"Formatting current datetime for report header: {now} (tzinfo: {getattr(now, 'tzinfo', 'N/A')})"
+            )
+            formatted_time = now.strftime("%Y-%m-%d %H:%M:%S")
+            logger.debug(f"  Formatted time: {formatted_time}")
+
+            report.write(f"**Generated**: {formatted_time}\n")
+            report.write(f"**Analysis Period**: Last {weeks} weeks\n\n")
+
+            # Executive Summary
+            progress_service.set_description(progress_ctx, "Generating Executive Summary")
+            report.write("## Executive Summary\n\n")
+            self._write_executive_summary(report, commits, developer_stats, ticket_analysis, prs, branch_health_metrics, pm_data)
+            progress_service.update(progress_ctx)
+
+            # Add ChatGPT qualitative insights if available
+            if chatgpt_summary:
+                progress_service.set_description(progress_ctx, "Generating Qualitative Analysis")
+                report.write("\n## Qualitative Analysis\n\n")
+                report.write(chatgpt_summary)
+                report.write("\n")
+                progress_service.update(progress_ctx)
+
+            # Team Composition
+            progress_service.set_description(progress_ctx, "Generating Team Composition")
+            report.write("\n## Team Composition\n\n")
+            self._write_team_composition(report, developer_stats, focus_data, commits, prs, ticket_analysis, weeks)
+            progress_service.update(progress_ctx)
+
+            # Project Activity
+            progress_service.set_description(progress_ctx, "Generating Project Activity")
+            report.write("\n## Project Activity\n\n")
+            self._write_project_activity(report, activity_dist, commits, branch_health_metrics, ticket_analysis, weeks)
+            progress_service.update(progress_ctx)
+
+            # Development Patterns
+            progress_service.set_description(progress_ctx, "Generating Development Patterns")
+            report.write("\n## Development Patterns\n\n")
+            self._write_development_patterns(report, insights, focus_data)
+            progress_service.update(progress_ctx)
+
+            # Commit Classification Analysis (if ML analysis is available)
+            if ticket_analysis.get("ml_analysis", {}).get("enabled", False):
+                progress_service.set_description(progress_ctx, "Generating Commit Classification Analysis")
+                report.write("\n## Commit Classification Analysis\n\n")
+                self._write_commit_classification_analysis(report, ticket_analysis)
+                progress_service.update(progress_ctx)
+
+            # Pull Request Analysis (if available)
+            if pr_metrics and pr_metrics.get("total_prs", 0) > 0:
+                progress_service.set_description(progress_ctx, "Generating Pull Request Analysis")
+                report.write("\n## Pull Request Analysis\n\n")
+                self._write_pr_analysis(report, pr_metrics, prs)
+                progress_service.update(progress_ctx)
+
+            # Issue Tracking (includes Enhanced Untracked Analysis)
+            progress_service.set_description(progress_ctx, "Generating Issue Tracking")
+            report.write("\n## Issue Tracking\n\n")
+            self._write_ticket_tracking(report, ticket_analysis, developer_stats)
+            progress_service.update(progress_ctx)
+
+            # PM Platform Insights
+            if pm_data and "metrics" in pm_data:
+                progress_service.set_description(progress_ctx, "Generating PM Platform Integration")
+                report.write("\n## PM Platform Integration\n\n")
+                self._write_pm_insights(report, pm_data)
+                progress_service.update(progress_ctx)
+
+            # Recommendations
+            progress_service.set_description(progress_ctx, "Generating Recommendations")
+            report.write("\n## Recommendations\n\n")
+            self._write_recommendations(report, insights, ticket_analysis, focus_data)
+            progress_service.update(progress_ctx)
+
+            # Write to file
+            with open(output_path, "w") as f:
+                f.write(report.getvalue())
 
         return output_path
 

gitflow_analytics/security/__init__.py

@@ -0,0 +1,11 @@
+"""Security analysis module for GitFlow Analytics.
+
+This module provides comprehensive security analysis of git commits using a hybrid approach:
+1. Specialized security tools (Semgrep, Bandit, etc.) for known patterns
+2. LLM analysis for novel vulnerabilities and context-aware security review
+"""
+
+from .config import SecurityConfig
+from .security_analyzer import SecurityAnalyzer
+
+__all__ = ["SecurityAnalyzer", "SecurityConfig"]

gitflow_analytics/security/config.py

@@ -0,0 +1,189 @@
+"""Security configuration module."""
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+
+@dataclass
+class SecretScanningConfig:
+    """Configuration for secret detection."""
+
+    enabled: bool = True
+    patterns: Dict[str, str] = field(
+        default_factory=lambda: {
+            # AWS
+            "aws_access_key": r"AKIA[0-9A-Z]{16}",
+            "aws_secret_key": r"aws['\"][0-9a-zA-Z/+=]{40}['\"]",
+            # GitHub
+            "github_token": r"gh[ps]_[a-zA-Z0-9]{36}",
+            "github_oauth": r"gho_[a-zA-Z0-9]{36}",
+            "github_app_token": r"ghs_[a-zA-Z0-9]{36}",
+            # Generic API Keys
+            "api_key": r"(api[_-]?key|apikey)(.{0,20})?['\"]([0-9a-zA-Z]{32,45})['\"]",
+            "secret": r"(secret|password|passwd|pwd)(.{0,20})?['\"]([0-9a-zA-Z]{8,})['\"]",
+            # Private Keys
+            "private_key": r"-----BEGIN (RSA|DSA|EC|OPENSSH) PRIVATE KEY-----",
+            # Database URLs
+            "db_url": r"(postgres|postgresql|mysql|mongodb|redis)://[^:]+:[^@]+@[^/]+",
+            # JWT
+            "jwt": r"eyJ[A-Za-z0-9-_]+\.eyJ[A-Za-z0-9-_]+\.[A-Za-z0-9-_]+",
+            # Slack
+            "slack_token": r"xox[baprs]-[0-9a-zA-Z]{10,48}",
+            # Google
+            "google_api": r"AIza[0-9A-Za-z\\-_]{35}",
+            # Stripe
+            "stripe_key": r"(sk|pk)_(test|live)_[0-9a-zA-Z]{24,}",
+        }
+    )
+    entropy_threshold: float = 4.5
+    exclude_paths: List[str] = field(
+        default_factory=lambda: [
+            "*.test.*",
+            "*.spec.*",
+            "*_test.go",
+            "test_*.py",
+            "*/tests/*",
+            "*/test/*",
+            "*.md",
+            "*.txt",
+        ]
+    )
+
+
+@dataclass
+class VulnerabilityScanningConfig:
+    """Configuration for vulnerability scanning."""
+
+    enabled: bool = True
+
+    # Tool-specific configurations
+    enable_semgrep: bool = True
+    semgrep_rules: List[str] = field(
+        default_factory=lambda: [
+            "auto",  # Use Semgrep's auto configuration
+            "p/security-audit",
+            "p/owasp-top-ten",
+        ]
+    )
+
+    enable_bandit: bool = True  # Python
+    bandit_severity: str = "medium"  # low, medium, high
+
+    enable_gosec: bool = True  # Go
+    enable_eslint_security: bool = True  # JavaScript/TypeScript
+    enable_brakeman: bool = False  # Ruby on Rails
+
+    # Custom patterns for quick checks
+    vulnerability_patterns: Dict[str, str] = field(
+        default_factory=lambda: {
+            "sql_injection": r"(SELECT|DELETE|INSERT|UPDATE|DROP).*\+.*(?:request|params|input)",
+            "command_injection": r"(exec|eval|system|popen|subprocess).*\+.*(?:request|params|input)",
+            "xss": r"innerHTML\s*=.*(?:request|params|input)",
+            "path_traversal": r"\.\./.*(?:request|params|input)",
+            "weak_crypto": r"(md5|sha1|des|rc4)\s*\(",
+            "hardcoded_sql": r"(SELECT|DELETE|INSERT|UPDATE).*FROM.*WHERE.*=\s*['\"]",
+        }
+    )
+
+
+@dataclass
+class DependencyScanningConfig:
+    """Configuration for dependency vulnerability scanning."""
+
+    enabled: bool = True
+    check_npm: bool = True
+    check_pip: bool = True
+    check_go: bool = True
+    check_ruby: bool = True
+    vulnerability_db: str = "ghsa"  # GitHub Security Advisory Database
+    severity_threshold: str = "medium"  # low, medium, high, critical
+
+
+@dataclass
+class LLMSecurityConfig:
+    """Configuration for LLM-based security analysis."""
+
+    enabled: bool = True
+    model: str = "claude-3-haiku-20240307"  # Fast and cost-effective
+    api_key: Optional[str] = None
+
+    # LLM analysis prompts
+    code_review_prompt: str = """Analyze this code change for security vulnerabilities:
+
+Files changed: {files_changed}
+Lines added:
+{lines_added}
+
+Focus on:
+1. Authentication/authorization issues
+2. Input validation problems
+3. Data exposure risks
+4. Injection vulnerabilities
+5. Cryptographic weaknesses
+6. Any other security concerns
+
+Provide a brief, specific analysis. If no issues found, state "No security issues detected."
+"""
+
+    commit_review_prompt: str = """Review this git commit for security implications:
+
+Message: {message}
+Files: {files}
+Category: {category}
+
+Identify any security-relevant changes or potential risks. Be concise and specific.
+"""
+
+    max_lines_for_llm: int = 500  # Limit lines sent to LLM for cost control
+    confidence_threshold: float = 0.7
+
+
+@dataclass
+class SecurityConfig:
+    """Main security configuration."""
+
+    enabled: bool = False  # Disabled by default, opt-in
+
+    secret_scanning: SecretScanningConfig = field(default_factory=SecretScanningConfig)
+    vulnerability_scanning: VulnerabilityScanningConfig = field(
+        default_factory=VulnerabilityScanningConfig
+    )
+    dependency_scanning: DependencyScanningConfig = field(default_factory=DependencyScanningConfig)
+    llm_security: LLMSecurityConfig = field(default_factory=LLMSecurityConfig)
+
+    # Output configuration
+    generate_sarif: bool = False  # Generate SARIF format for GitHub Security
+    fail_on_critical: bool = False  # Fail analysis if critical issues found
+
+    # Performance
+    max_concurrent_scans: int = 4
+    scan_timeout_seconds: int = 30
+
+    @classmethod
+    def from_dict(cls, data: Dict) -> "SecurityConfig":
+        """Create SecurityConfig from dictionary."""
+        if not data:
+            return cls()
+
+        config = cls(enabled=data.get("enabled", False))
+
+        if "secret_scanning" in data:
+            config.secret_scanning = SecretScanningConfig(**data["secret_scanning"])
+
+        if "vulnerability_scanning" in data:
+            config.vulnerability_scanning = VulnerabilityScanningConfig(
+                **data["vulnerability_scanning"]
+            )
+
+        if "dependency_scanning" in data:
+            config.dependency_scanning = DependencyScanningConfig(**data["dependency_scanning"])
+
+        if "llm_security" in data:
+            config.llm_security = LLMSecurityConfig(**data["llm_security"])
+
+        config.generate_sarif = data.get("generate_sarif", False)
+        config.fail_on_critical = data.get("fail_on_critical", False)
+        config.max_concurrent_scans = data.get("max_concurrent_scans", 4)
+        config.scan_timeout_seconds = data.get("scan_timeout_seconds", 30)
+
+        return config

gitflow_analytics/security/extractors/__init__.py

@@ -0,0 +1,7 @@
+"""Security extractors for analyzing code changes."""
+
+from .dependency_checker import DependencyChecker
+from .secret_detector import SecretDetector
+from .vulnerability_scanner import VulnerabilityScanner
+
+__all__ = ["SecretDetector", "VulnerabilityScanner", "DependencyChecker"]