gitflow-analytics 1.3.6__py3-none-any.whl → 3.3.0__py3-none-any.whl

gitflow_analytics/security/extractors/vulnerability_scanner.py

@@ -0,0 +1,333 @@
+"""Vulnerability scanning using multiple security tools and LLM analysis."""
+
+import json
+import logging
+import re
+import shutil
+import subprocess
+import tempfile
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class VulnerabilityScanner:
+    """Scan code for security vulnerabilities using tools and patterns."""
+
+    def __init__(self, config: Any):
+        """Initialize vulnerability scanner with configuration."""
+        self.config = config
+        self.vulnerability_patterns = {
+            name: re.compile(pattern) for name, pattern in config.vulnerability_patterns.items()
+        }
+
+        # Check which tools are available
+        self.available_tools = self._detect_available_tools()
+
+    def scan_files(self, files_changed: List[str], repo_path: Path) -> List[Dict]:
+        """Scan changed files for vulnerabilities.
+
+        Args:
+            files_changed: List of changed file paths
+            repo_path: Path to repository
+
+        Returns:
+            List of vulnerability findings
+        """
+        findings = []
+
+        # Quick pattern-based scanning
+        pattern_findings = self._scan_with_patterns(files_changed, repo_path)
+        findings.extend(pattern_findings)
+
+        # Tool-based scanning (run in parallel for performance)
+        if self.available_tools:
+            tool_findings = self._scan_with_tools(files_changed, repo_path)
+            findings.extend(tool_findings)
+
+        return findings
+
+    def _detect_available_tools(self) -> Dict[str, bool]:
+        """Detect which security tools are installed."""
+        tools = {}
+
+        # Check for Semgrep
+        if self.config.enable_semgrep:
+            tools["semgrep"] = self._is_tool_available("semgrep")
+            if not tools["semgrep"]:
+                logger.info("Semgrep not found. Install with: pip install semgrep")
+
+        # Check for Bandit (Python)
+        if self.config.enable_bandit:
+            tools["bandit"] = self._is_tool_available("bandit")
+            if not tools["bandit"]:
+                logger.info("Bandit not found. Install with: pip install bandit")
+
+        # Check for gosec (Go)
+        if self.config.enable_gosec:
+            tools["gosec"] = self._is_tool_available("gosec")
+            if not tools["gosec"]:
+                logger.info("Gosec not found. Install from: https://github.com/securego/gosec")
+
+        return tools
+
+    def _is_tool_available(self, tool_name: str) -> bool:
+        """Check if a tool is available in PATH."""
+        return shutil.which(tool_name) is not None
+
+    def _scan_with_patterns(self, files_changed: List[str], repo_path: Path) -> List[Dict]:
+        """Quick pattern-based vulnerability detection."""
+        findings = []
+
+        for file_path in files_changed:
+            full_path = repo_path / file_path
+            if not full_path.exists() or not full_path.is_file():
+                continue
+
+            try:
+                content = full_path.read_text(encoding="utf-8", errors="ignore")
+
+                for vuln_type, pattern in self.vulnerability_patterns.items():
+                    for match in pattern.finditer(content):
+                        line_num = content[: match.start()].count("\n") + 1
+                        finding = {
+                            "type": "vulnerability",
+                            "vulnerability_type": vuln_type,
+                            "severity": self._get_vuln_severity(vuln_type),
+                            "file": file_path,
+                            "line": line_num,
+                            "message": f"Potential {vuln_type.replace('_', ' ')} detected",
+                            "tool": "pattern_matcher",
+                            "confidence": "medium",
+                        }
+                        findings.append(finding)
+            except Exception as e:
+                logger.debug(f"Error scanning {file_path}: {e}")
+
+        return findings
+
+    def _scan_with_tools(self, files_changed: List[str], repo_path: Path) -> List[Dict]:
+        """Run security tools on changed files."""
+        all_findings = []
+
+        # Group files by language for efficient tool execution
+        files_by_language = self._group_files_by_language(files_changed)
+
+        with ThreadPoolExecutor(max_workers=4) as executor:
+            futures = []
+
+            # Run Semgrep if available (works on all languages)
+            if self.available_tools.get("semgrep"):
+                future = executor.submit(self._run_semgrep, files_changed, repo_path)
+                futures.append(("semgrep", future))
+
+            # Run Bandit on Python files
+            if self.available_tools.get("bandit") and files_by_language.get("python"):
+                future = executor.submit(self._run_bandit, files_by_language["python"], repo_path)
+                futures.append(("bandit", future))
+
+            # Run gosec on Go files
+            if self.available_tools.get("gosec") and files_by_language.get("go"):
+                future = executor.submit(self._run_gosec, files_by_language["go"], repo_path)
+                futures.append(("gosec", future))
+
+            # Collect results
+            for tool_name, future in futures:
+                try:
+                    findings = future.result(timeout=30)
+                    all_findings.extend(findings)
+                except Exception as e:
+                    logger.warning(f"Error running {tool_name}: {e}")
+
+        return all_findings
+
+    def _group_files_by_language(self, files: List[str]) -> Dict[str, List[str]]:
+        """Group files by programming language."""
+        groups = {}
+
+        language_extensions = {
+            "python": [".py"],
+            "go": [".go"],
+            "javascript": [".js", ".jsx", ".ts", ".tsx"],
+            "java": [".java"],
+            "ruby": [".rb"],
+        }
+
+        for file_path in files:
+            path = Path(file_path)
+            for language, extensions in language_extensions.items():
+                if path.suffix in extensions:
+                    if language not in groups:
+                        groups[language] = []
+                    groups[language].append(file_path)
+                    break
+
+        return groups
+
+    def _run_semgrep(self, files: List[str], repo_path: Path) -> List[Dict]:
+        """Run Semgrep security scanning."""
+        findings = []
+
+        if not files:
+            return findings
+
+        try:
+            # Create temporary file list for semgrep
+            with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
+                for file_path in files:
+                    f.write(f"{file_path}\n")
+                file_list_path = f.name
+
+            cmd = [
+                "semgrep",
+                "--config=auto",  # Use automatic rules
+                "--json",
+                "--no-error",
+                f"--include-list={file_list_path}",
+                str(repo_path),
+            ]
+
+            result = subprocess.run(cmd, capture_output=True, text=True, cwd=repo_path)
+
+            if result.returncode == 0 and result.stdout:
+                data = json.loads(result.stdout)
+                for finding in data.get("results", []):
+                    findings.append(
+                        {
+                            "type": "vulnerability",
+                            "vulnerability_type": finding.get("check_id", "unknown"),
+                            "severity": self._map_semgrep_severity(
+                                finding.get("extra", {}).get("severity")
+                            ),
+                            "file": Path(finding["path"]).relative_to(repo_path).as_posix(),
+                            "line": finding.get("start", {}).get("line", 0),
+                            "message": finding.get("extra", {}).get(
+                                "message", "Security issue detected"
+                            ),
+                            "tool": "semgrep",
+                            "confidence": "high",
+                        }
+                    )
+
+            # Clean up temp file
+            Path(file_list_path).unlink()
+
+        except Exception as e:
+            logger.warning(f"Error running Semgrep: {e}")
+
+        return findings
+
+    def _run_bandit(self, files: List[str], repo_path: Path) -> List[Dict]:
+        """Run Bandit for Python security scanning."""
+        findings = []
+
+        if not files:
+            return findings
+
+        try:
+            # Bandit expects full paths
+            full_paths = [str(repo_path / f) for f in files if (repo_path / f).exists()]
+
+            if not full_paths:
+                return findings
+
+            cmd = ["bandit", "-f", "json", "-ll", *full_paths]  # Low severity and higher
+
+            result = subprocess.run(cmd, capture_output=True, text=True)
+
+            if result.stdout:
+                data = json.loads(result.stdout)
+                for finding in data.get("results", []):
+                    findings.append(
+                        {
+                            "type": "vulnerability",
+                            "vulnerability_type": finding.get("test_id", "unknown"),
+                            "severity": finding.get("issue_severity", "medium").lower(),
+                            "file": Path(finding["filename"]).relative_to(repo_path).as_posix(),
+                            "line": finding.get("line_number", 0),
+                            "message": finding.get("issue_text", "Security issue detected"),
+                            "tool": "bandit",
+                            "confidence": finding.get("issue_confidence", "medium").lower(),
+                        }
+                    )
+
+        except Exception as e:
+            logger.warning(f"Error running Bandit: {e}")
+
+        return findings
+
+    def _run_gosec(self, files: List[str], repo_path: Path) -> List[Dict]:
+        """Run gosec for Go security scanning."""
+        findings = []
+
+        if not files:
+            return findings
+
+        try:
+            # gosec works on directories, so we scan the whole repo but filter results
+            cmd = ["gosec", "-fmt", "json", "./..."]
+
+            result = subprocess.run(cmd, capture_output=True, text=True, cwd=repo_path)
+
+            if result.stdout:
+                data = json.loads(result.stdout)
+                for finding in data.get("Issues", []):
+                    file_path = Path(finding["file"]).relative_to(repo_path).as_posix()
+
+                    # Only include findings for changed files
+                    if file_path in files:
+                        findings.append(
+                            {
+                                "type": "vulnerability",
+                                "vulnerability_type": finding.get("rule_id", "unknown"),
+                                "severity": self._map_gosec_severity(finding.get("severity")),
+                                "file": file_path,
+                                "line": int(finding.get("line", "0")),
+                                "message": finding.get("details", "Security issue detected"),
+                                "tool": "gosec",
+                                "confidence": finding.get("confidence", "medium").lower(),
+                            }
+                        )
+
+        except Exception as e:
+            logger.warning(f"Error running gosec: {e}")
+
+        return findings
+
+    def _get_vuln_severity(self, vuln_type: str) -> str:
+        """Map vulnerability type to severity."""
+        critical_types = ["sql_injection", "command_injection", "path_traversal"]
+        high_types = ["xss", "weak_crypto"]
+
+        if vuln_type in critical_types:
+            return "critical"
+        elif vuln_type in high_types:
+            return "high"
+        else:
+            return "medium"
+
+    def _map_semgrep_severity(self, severity: Optional[str]) -> str:
+        """Map Semgrep severity to our severity scale."""
+        if not severity:
+            return "medium"
+        severity = severity.upper()
+        if severity == "ERROR":
+            return "critical"
+        elif severity == "WARNING":
+            return "high"
+        else:
+            return "medium"
+
+    def _map_gosec_severity(self, severity: Optional[str]) -> str:
+        """Map gosec severity to our severity scale."""
+        if not severity:
+            return "medium"
+        severity = severity.upper()
+        if severity == "HIGH":
+            return "critical"
+        elif severity == "MEDIUM":
+            return "high"
+        else:
+            return "medium"

gitflow_analytics/security/llm_analyzer.py

@@ -0,0 +1,347 @@
+"""LLM-based security analysis for comprehensive code review."""
+
+import json
+import logging
+import os
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+class LLMSecurityAnalyzer:
+    """Use LLM to analyze code changes for security issues that tools might miss."""
+
+    def __init__(self, config: Any, cache_dir: Optional[Path] = None):
+        """Initialize LLM security analyzer.
+
+        Args:
+            config: LLM security configuration
+            cache_dir: Directory for caching LLM responses
+        """
+        self.config = config
+        self.api_key = (
+            config.api_key or os.getenv("OPENROUTER_API_KEY") or os.getenv("ANTHROPIC_API_KEY")
+        )
+        self.model = config.model
+        self.cache_dir = cache_dir or Path(".gitflow-cache/llm_security")
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+        # Cache LLM responses for 7 days to save costs
+        self.cache_ttl = timedelta(days=7)
+
+    def analyze_commit(self, commit_data: Dict) -> List[Dict]:
+        """Analyze a commit for security issues using LLM.
+
+        Args:
+            commit_data: Commit data with message, files_changed, etc.
+
+        Returns:
+            List of security findings
+        """
+        if not self.api_key:
+            logger.debug("LLM API key not configured, skipping LLM security analysis")
+            return []
+
+        findings = []
+
+        # Check cache first
+        cache_key = self._get_cache_key(commit_data)
+        cached_result = self._get_cached_result(cache_key)
+        if cached_result is not None:
+            return cached_result
+
+        try:
+            # Analyze commit message and metadata
+            commit_findings = self._analyze_commit_message(commit_data)
+            findings.extend(commit_findings)
+
+            # Analyze code changes if available
+            if "diff_content" in commit_data:
+                code_findings = self._analyze_code_changes(commit_data)
+                findings.extend(code_findings)
+
+            # Cache the results
+            self._cache_result(cache_key, findings)
+
+        except Exception as e:
+            logger.warning(f"Error in LLM security analysis: {e}")
+
+        return findings
+
+    def _analyze_commit_message(self, commit_data: Dict) -> List[Dict]:
+        """Analyze commit message for security implications."""
+        prompt = self.config.commit_review_prompt.format(
+            message=commit_data.get("message", ""),
+            files=", ".join(commit_data.get("files_changed", [])),
+            category=commit_data.get("category", "unknown"),
+        )
+
+        response = self._call_llm(prompt)
+        return self._parse_llm_response(response, commit_data)
+
+    def _analyze_code_changes(self, commit_data: Dict) -> List[Dict]:
+        """Analyze actual code changes for security issues."""
+        # Limit the amount of code sent to LLM for cost control
+        lines_added = commit_data.get("diff_content", "")
+        if len(lines_added.split("\n")) > self.config.max_lines_for_llm:
+            lines_added = "\n".join(lines_added.split("\n")[: self.config.max_lines_for_llm])
+            lines_added += "\n... (truncated for analysis)"
+
+        prompt = self.config.code_review_prompt.format(
+            files_changed=", ".join(commit_data.get("files_changed", [])), lines_added=lines_added
+        )
+
+        response = self._call_llm(prompt)
+        return self._parse_llm_response(response, commit_data, is_code_analysis=True)
+
+    def _call_llm(self, prompt: str) -> str:
+        """Call the LLM API with the given prompt."""
+        if self.model.startswith("claude"):
+            return self._call_anthropic(prompt)
+        else:
+            return self._call_openrouter(prompt)
+
+    def _call_anthropic(self, prompt: str) -> str:
+        """Call Anthropic's Claude API."""
+        try:
+            headers = {
+                "x-api-key": self.api_key,
+                "anthropic-version": "2023-06-01",
+                "content-type": "application/json",
+            }
+
+            data = {
+                "model": self.model,
+                "max_tokens": 500,
+                "messages": [{"role": "user", "content": prompt}],
+                "temperature": 0.1,  # Low temperature for consistent analysis
+            }
+
+            with httpx.Client() as client:
+                response = client.post(
+                    "https://api.anthropic.com/v1/messages", headers=headers, json=data, timeout=30
+                )
+
+            if response.status_code == 200:
+                return response.json()["content"][0]["text"]
+            else:
+                logger.warning(f"Claude API error: {response.status_code}")
+                return ""
+
+        except Exception as e:
+            logger.warning(f"Error calling Claude API: {e}")
+            return ""
+
+    def _call_openrouter(self, prompt: str) -> str:
+        """Call OpenRouter API for various LLM models."""
+        try:
+            headers = {
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+            }
+
+            data = {
+                "model": self.model,
+                "messages": [
+                    {
+                        "role": "system",
+                        "content": "You are a security expert analyzing code for vulnerabilities. Be concise and specific.",
+                    },
+                    {"role": "user", "content": prompt},
+                ],
+                "max_tokens": 500,
+                "temperature": 0.1,
+            }
+
+            with httpx.Client() as client:
+                response = client.post(
+                    "https://openrouter.ai/api/v1/chat/completions",
+                    headers=headers,
+                    json=data,
+                    timeout=30,
+                )
+
+            if response.status_code == 200:
+                return response.json()["choices"][0]["message"]["content"]
+            else:
+                logger.warning(f"OpenRouter API error: {response.status_code}")
+                return ""
+
+        except Exception as e:
+            logger.warning(f"Error calling OpenRouter API: {e}")
+            return ""
+
+    def _parse_llm_response(
+        self, response: str, commit_data: Dict, is_code_analysis: bool = False
+    ) -> List[Dict]:
+        """Parse LLM response and extract security findings."""
+        findings = []
+
+        if not response or "no security issues" in response.lower():
+            return findings
+
+        # Extract specific security concerns from the response
+        security_keywords = {
+            "authentication": ("high", "authentication"),
+            "authorization": ("high", "authorization"),
+            "injection": ("critical", "injection"),
+            "sql": ("critical", "sql_injection"),
+            "xss": ("high", "xss"),
+            "csrf": ("high", "csrf"),
+            "exposure": ("high", "data_exposure"),
+            "credential": ("critical", "credential_exposure"),
+            "secret": ("critical", "secret_exposure"),
+            "crypto": ("high", "weak_cryptography"),
+            "validation": ("medium", "input_validation"),
+            "sanitization": ("medium", "input_sanitization"),
+            "permission": ("high", "permission_issue"),
+            "privilege": ("high", "privilege_escalation"),
+            "buffer": ("critical", "buffer_overflow"),
+            "race": ("high", "race_condition"),
+            "session": ("high", "session_management"),
+            "cookie": ("medium", "cookie_security"),
+            "cors": ("medium", "cors_misconfiguration"),
+            "encryption": ("high", "encryption_issue"),
+        }
+
+        # Check for security keywords in the response
+        response_lower = response.lower()
+        found_issues = []
+
+        for keyword, (severity, issue_type) in security_keywords.items():
+            if keyword in response_lower:
+                found_issues.append((severity, issue_type))
+
+        # Create findings based on detected issues
+        if found_issues:
+            # Extract the most severe issue
+            severity_order = {"critical": 0, "high": 1, "medium": 2, "low": 3}
+            found_issues.sort(key=lambda x: severity_order.get(x[0], 999))
+
+            finding = {
+                "type": "security",
+                "source": "llm_analysis",
+                "vulnerability_type": found_issues[0][1],
+                "severity": found_issues[0][0],
+                "commit": commit_data.get("commit_hash_short", "unknown"),
+                "message": self._extract_finding_message(response),
+                "confidence": self._calculate_confidence(response),
+                "analysis_type": "code" if is_code_analysis else "commit",
+                "files": commit_data.get("files_changed", []),
+            }
+
+            findings.append(finding)
+
+        return findings
+
+    def _extract_finding_message(self, response: str) -> str:
+        """Extract a concise finding message from LLM response."""
+        # Take the first meaningful sentence
+        sentences = response.split(".")
+        for sentence in sentences:
+            sentence = sentence.strip()
+            if len(sentence) > 20 and not sentence.lower().startswith(("the", "this", "it")):
+                return sentence + "."
+
+        # Fallback to truncated response
+        return response[:200] + "..." if len(response) > 200 else response
+
+    def _calculate_confidence(self, response: str) -> str:
+        """Calculate confidence level based on LLM response characteristics."""
+        response_lower = response.lower()
+
+        # High confidence indicators
+        high_confidence_words = [
+            "definitely",
+            "clearly",
+            "certain",
+            "obvious",
+            "critical",
+            "severe",
+        ]
+        if any(word in response_lower for word in high_confidence_words):
+            return "high"
+
+        # Low confidence indicators
+        low_confidence_words = ["might", "could", "possibly", "perhaps", "may", "potential"]
+        if any(word in response_lower for word in low_confidence_words):
+            return "medium"
+
+        return "high" if len(response) > 100 else "medium"
+
+    def _get_cache_key(self, commit_data: Dict) -> str:
+        """Generate cache key for commit data."""
+        key_parts = [
+            commit_data.get("commit_hash", ""),
+            str(sorted(commit_data.get("files_changed", []))),
+            commit_data.get("message", "")[:100],
+        ]
+        key_str = "|".join(key_parts)
+        # Simple hash for filename
+        import hashlib
+
+        return hashlib.sha256(key_str.encode()).hexdigest()[:16]
+
+    def _get_cached_result(self, cache_key: str) -> Optional[List[Dict]]:
+        """Get cached result if it exists and is not expired."""
+        cache_file = self.cache_dir / f"{cache_key}.json"
+        if not cache_file.exists():
+            return None
+
+        try:
+            # Check if cache is expired
+            file_time = datetime.fromtimestamp(cache_file.stat().st_mtime)
+            if datetime.now() - file_time > self.cache_ttl:
+                cache_file.unlink()  # Delete expired cache
+                return None
+
+            with open(cache_file) as f:
+                return json.load(f)
+        except Exception as e:
+            logger.debug(f"Error reading cache: {e}")
+            return None
+
+    def _cache_result(self, cache_key: str, result: List[Dict]) -> None:
+        """Cache the analysis result."""
+        cache_file = self.cache_dir / f"{cache_key}.json"
+        try:
+            with open(cache_file, "w") as f:
+                json.dump(result, f)
+        except Exception as e:
+            logger.debug(f"Error writing cache: {e}")
+
+    def generate_security_insights(self, all_findings: List[Dict]) -> str:
+        """Generate high-level security insights from all findings."""
+        if not all_findings:
+            return "No security issues detected in the analyzed period."
+
+        # Aggregate findings
+        by_severity = {}
+        by_type = {}
+
+        for finding in all_findings:
+            severity = finding.get("severity", "unknown")
+            vuln_type = finding.get("vulnerability_type", "unknown")
+
+            by_severity[severity] = by_severity.get(severity, 0) + 1
+            by_type[vuln_type] = by_type.get(vuln_type, 0) + 1
+
+        # Generate insights prompt
+        prompt = f"""Analyze these security findings and provide strategic recommendations:
+
+Findings by severity: {json.dumps(by_severity, indent=2)}
+Findings by type: {json.dumps(by_type, indent=2)}
+
+Provide:
+1. Top 3 security risks to address
+2. Recommended security improvements
+3. Security training needs for the team
+
+Be concise and actionable."""
+
+        response = self._call_llm(prompt)
+        return response if response else "Unable to generate security insights."

gitflow_analytics/security/reports/__init__.py

@@ -0,0 +1,5 @@
+"""Security reporting module."""
+
+from .security_report import SecurityReportGenerator
+
+__all__ = ["SecurityReportGenerator"]