crackerjack 0.31.10__py3-none-any.whl → 0.31.13__py3-none-any.whl

crackerjack/agents/security_agent.py

@@ -1,6 +1,6 @@
-import re
 from pathlib import Path
 
+from ..services.regex_patterns import SAFE_PATTERNS
 from .base import (
     AgentContext,
     FixResult,
@@ -14,22 +14,9 @@ from .base import (
 class SecurityAgent(SubAgent):
     def __init__(self, context: AgentContext) -> None:
         super().__init__(context)
-        self.security_patterns = {
-            "hardcoded_temp_paths": r"(?:/tmp/|/temp/|C:\\temp\\|C:\\tmp\\)",
-            "shell_injection": r"shell=True|os\.system\(|subprocess\.call\([^)]*shell=True",
-            "path_traversal": r"\.\./|\.\.\\",
-            "hardcoded_secrets": r"(?:password|secret|key|token)\s*=\s*['\"][^'\"]+['\"]",
-            "unsafe_yaml": r"yaml\.load\([^)]*\)",
-            "eval_usage": r"\beval\s*\(",
-            "exec_usage": r"\bexec\s*\(",
-            "pickle_usage": r"\bpickle\.loads?\s*\(",
-            "sql_injection": r"(?:execute|query)\s*\([^)]*%[sd]",
-            "weak_crypto": r"(?:md5|sha1)\s*\(",
-            "insecure_random": r"random\.random\(\)|random\.choice\(",
-        }
 
     def get_supported_types(self) -> set[IssueType]:
-        return {IssueType.SECURITY}
+        return {IssueType.SECURITY, IssueType.REGEX_VALIDATION}
 
     async def can_handle(self, issue: Issue) -> float:
         if issue.type not in self.get_supported_types():
@@ -37,6 +24,27 @@ class SecurityAgent(SubAgent):
 
         message_lower = issue.message.lower()
 
+        # High confidence (0.95) for regex validation issues
+        if issue.type == IssueType.REGEX_VALIDATION:
+            return 0.95
+
+        # High confidence (0.95) for regex validation keywords
+        if any(
+            keyword in message_lower
+            for keyword in (
+                "validate-regex-patterns",
+                "raw regex",
+                "regex pattern",
+                r"\g<",
+                "replacement",
+                "unsafe regex",
+                "regex vulnerability",
+                "redos",
+            )
+        ):
+            return 0.95
+
+        # High confidence (0.9) for critical security issues
         if any(
             keyword in message_lower
             for keyword in (
@@ -51,20 +59,36 @@ class SecurityAgent(SubAgent):
                 "b506",
                 "unsafe",
                 "injection",
+                "pickle",
+                "yaml.load",
+                "md5",
+                "sha1",
+                "jwt_secret",
             )
         ):
-            return 1.0
+            return 0.9
+
+        # Enhanced detection using new security patterns
+        enhanced_patterns = [
+            "detect_security_keywords",
+            "detect_crypto_weak_algorithms",
+            "detect_hardcoded_credentials_advanced",
+            "detect_subprocess_shell_injection",
+            "detect_unsafe_pickle_usage",
+        ]
 
-        for pattern in self.security_patterns.values():
-            if re.search(pattern, issue.message, re.IGNORECASE):
+        for pattern_name in enhanced_patterns:
+            if SAFE_PATTERNS[pattern_name].test(issue.message):
                 return 0.9
 
+        # Medium confidence for security-related files
         if issue.file_path and any(
             keyword in issue.file_path.lower()
-            for keyword in ("security", "auth", "crypto", "password")
+            for keyword in ("security", "auth", "crypto", "password", "token", "jwt")
         ):
             return 0.7
 
+        # Base confidence for security type
         if issue.type == IssueType.SECURITY:
             return 0.6
 
@@ -95,7 +119,7 @@ class SecurityAgent(SubAgent):
             )
 
             success = len(fixes_applied) > 0
-            confidence = 0.85 if success else 0.4
+            confidence = 0.95 if success else 0.4
 
             if not success:
                 recommendations = self._get_security_recommendations()
@@ -120,12 +144,16 @@ class SecurityAgent(SubAgent):
         files_modified: list[str],
     ) -> tuple[list[str], list[str]]:
         vulnerability_fix_map = {
+            "regex_validation": self._fix_regex_validation_issues,
             "hardcoded_temp_paths": self._fix_hardcoded_temp_paths,
             "shell_injection": self._fix_shell_injection,
             "hardcoded_secrets": self._fix_hardcoded_secrets,
             "unsafe_yaml": self._fix_unsafe_yaml,
             "eval_usage": self._fix_eval_usage,
             "weak_crypto": self._fix_weak_crypto,
+            "jwt_secrets": self._fix_jwt_secrets,
+            "pickle_usage": self._fix_pickle_usage,
+            "insecure_random": self._fix_insecure_random,
         }
 
         if fix_method := vulnerability_fix_map.get(vulnerability_type):
@@ -155,12 +183,21 @@ class SecurityAgent(SubAgent):
 
     def _get_security_recommendations(self) -> list[str]:
         return [
-            "Use tempfile module for temporary file creation",
-            "Avoid shell=True in subprocess calls",
-            "Use environment variables for secrets",
-            "Implement proper input validation",
-            "Use safe_load() instead of load() for YAML",
-            "Consider using cryptographically secure random module",
+            "Use centralized SAFE_PATTERNS for regex operations to prevent ReDoS attacks",
+            "Avoid raw regex patterns with vulnerable replacement syntax like \\g < 1 >",
+            "Use tempfile module for temporary file creation instead of hardcoded paths",
+            "Avoid shell=True in subprocess calls to prevent command injection",
+            "Store secrets in environment variables using os.getenv(), never hardcode them",
+            "Replace weak cryptographic algorithms (MD5, SHA1, DES, RC4) with stronger alternatives",
+            "Use secrets module instead of random for cryptographically secure operations",
+            "Replace unsafe yaml.load() with yaml.safe_load() to prevent code execution",
+            "Avoid pickle.load() with untrusted data as it can execute arbitrary code",
+            "Use JWT secrets from environment variables, never hardcode them",
+            "Implement proper input validation and sanitization for all user inputs",
+            "Add security comments to document potential risks in legacy code",
+            "Run bandit security scanner regularly to identify new vulnerabilities",
+            "Review all subprocess calls for potential injection vulnerabilities",
+            "Ensure all cryptographic operations use secure algorithms and proper key management",
         ]
 
     def _create_error_fix_result(self, error: Exception) -> FixResult:
@@ -178,6 +215,66 @@ class SecurityAgent(SubAgent):
     def _identify_vulnerability_type(self, issue: Issue) -> str:
         message = issue.message
 
+        # Regex validation issues
+        if self._is_regex_validation_issue(issue):
+            return "regex_validation"
+
+        # Enhanced pattern-based detection
+        pattern_checks = self._check_enhanced_patterns(message)
+        if pattern_checks:
+            return pattern_checks
+
+        # Bandit code-based detection
+        bandit_checks = self._check_bandit_patterns(message)
+        if bandit_checks:
+            return bandit_checks
+
+        # Legacy pattern-based detection
+        legacy_checks = self._check_legacy_patterns(message)
+        if legacy_checks:
+            return legacy_checks
+
+        # JWT and other specific patterns
+        if self._is_jwt_secret_issue(message):
+            return "jwt_secrets"
+
+        return "unknown"
+
+    def _is_regex_validation_issue(self, issue: Issue) -> bool:
+        """Check if issue is related to regex validation."""
+        if issue.type == IssueType.REGEX_VALIDATION:
+            return True
+
+        message_lower = issue.message.lower()
+        return any(
+            keyword in message_lower
+            for keyword in (
+                "validate-regex-patterns",
+                "raw regex",
+                "unsafe regex",
+                r"\g<",
+                "redos",
+            )
+        )
+
+    def _check_enhanced_patterns(self, message: str) -> str | None:
+        """Check enhanced security patterns."""
+        pattern_map = {
+            "detect_crypto_weak_algorithms": "weak_crypto",
+            "detect_hardcoded_credentials_advanced": "hardcoded_secrets",
+            "detect_subprocess_shell_injection": "shell_injection",
+            "detect_unsafe_pickle_usage": "pickle_usage",
+            "detect_regex_redos_vulnerable": "regex_validation",
+        }
+
+        for pattern_name, vulnerability_type in pattern_map.items():
+            if SAFE_PATTERNS[pattern_name].test(message):
+                return vulnerability_type
+
+        return None
+
+    def _check_bandit_patterns(self, message: str) -> str | None:
+        """Check Bandit-specific patterns."""
         if "B108" in message:
             return "hardcoded_temp_paths"
         if "B602" in message or "shell=True" in message:
@@ -186,12 +283,131 @@ class SecurityAgent(SubAgent):
             return "pickle_usage"
         if "B506" in message or "yaml.load" in message:
             return "unsafe_yaml"
+        if any(crypto in message.lower() for crypto in ("md5", "sha1", "des", "rc4")):
+            return "weak_crypto"
 
-        for pattern_name, pattern in self.security_patterns.items():
-            if re.search(pattern, message, re.IGNORECASE):
-                return pattern_name
+        return None
 
-        return "unknown"
+    def _check_legacy_patterns(self, message: str) -> str | None:
+        """Check legacy security patterns."""
+        pattern_map = {
+            "detect_hardcoded_temp_paths_basic": "hardcoded_temp_paths",
+            "detect_hardcoded_secrets": "hardcoded_secrets",
+            "detect_insecure_random_usage": "insecure_random",
+        }
+
+        for pattern_name, vulnerability_type in pattern_map.items():
+            if SAFE_PATTERNS[pattern_name].test(message):
+                return vulnerability_type
+
+        return None
+
+    def _is_jwt_secret_issue(self, message: str) -> bool:
+        """Check if message indicates JWT secret issue."""
+        message_lower = message.lower()
+        return "jwt" in message_lower and (
+            "secret" in message_lower or "hardcoded" in message_lower
+        )
+
+    async def _fix_regex_validation_issues(self, issue: Issue) -> dict[str, list[str]]:
+        """Fix unsafe regex patterns by converting them to use centralized SAFE_PATTERNS."""
+        fixes: list[str] = []
+        files: list[str] = []
+
+        if not issue.file_path:
+            # If no specific file path, scan all Python files in the project
+            await self._fix_regex_patterns_project_wide(fixes, files)
+            return {"fixes": fixes, "files": files}
+
+        file_path = Path(issue.file_path)
+        if not file_path.exists():
+            return {"fixes": fixes, "files": files}
+
+        content = self.context.get_file_content(file_path)
+        if not content:
+            return {"fixes": fixes, "files": files}
+
+        original_content = content
+        content = await self._apply_regex_pattern_fixes(content)
+
+        if content != original_content:
+            if self.context.write_file_content(file_path, content):
+                fixes.append(f"Fixed unsafe regex patterns in {issue.file_path}")
+                files.append(str(file_path))
+                self.log(f"Fixed regex patterns in {issue.file_path}")
+
+        return {"fixes": fixes, "files": files}
+
+    async def _fix_regex_patterns_project_wide(
+        self, fixes: list[str], files: list[str]
+    ) -> None:
+        """Fix regex patterns across the entire project."""
+        try:
+            python_files = self._get_python_files_for_security_scan()
+            await self._process_python_files_for_regex_fixes(python_files, fixes, files)
+        except Exception as e:
+            self.log(f"Error during project-wide regex fixes: {e}", "ERROR")
+
+    def _get_python_files_for_security_scan(self) -> list[Path]:
+        """Get list of Python files that should be scanned for security issues."""
+        python_files = list(self.context.project_path.rglob("*.py"))
+        return [
+            f for f in python_files if not self._should_skip_file_for_security_scan(f)
+        ]
+
+    def _should_skip_file_for_security_scan(self, file_path: Path) -> bool:
+        """Check if a file should be skipped during security scanning."""
+        skip_patterns = [".venv", "__pycache__", ".git"]
+        return any(part in str(file_path) for part in skip_patterns)
+
+    async def _process_python_files_for_regex_fixes(
+        self, python_files: list[Path], fixes: list[str], files: list[str]
+    ) -> None:
+        """Process each Python file for regex fixes."""
+        for file_path in python_files:
+            await self._process_single_file_for_regex_fixes(file_path, fixes, files)
+
+    async def _process_single_file_for_regex_fixes(
+        self, file_path: Path, fixes: list[str], files: list[str]
+    ) -> None:
+        """Process a single file for regex pattern fixes."""
+        content = self.context.get_file_content(file_path)
+        if not content:
+            return
+
+        original_content = content
+        content = await self._apply_regex_pattern_fixes(content)
+
+        if self._should_save_regex_fixes(content, original_content):
+            await self._save_regex_fixes_to_file(file_path, content, fixes, files)
+
+    def _should_save_regex_fixes(self, content: str, original_content: str) -> bool:
+        """Check if regex fixes should be saved."""
+        return content != original_content
+
+    async def _save_regex_fixes_to_file(
+        self, file_path: Path, content: str, fixes: list[str], files: list[str]
+    ) -> None:
+        """Save regex fixes to file and update tracking lists."""
+        if self.context.write_file_content(file_path, content):
+            fixes.append(f"Fixed unsafe regex patterns in {file_path}")
+            files.append(str(file_path))
+            self.log(f"Fixed regex patterns in {file_path}")
+
+    async def _apply_regex_pattern_fixes(self, content: str) -> str:
+        """Apply all regex pattern fixes using SAFE_PATTERNS."""
+        # Import regex fix utilities
+        from crackerjack.services.regex_utils import (
+            replace_unsafe_regex_with_safe_patterns,
+        )
+
+        try:
+            # Apply centralized regex fixes
+            fixed_content = replace_unsafe_regex_with_safe_patterns(content)
+            return fixed_content
+        except Exception as e:
+            self.log(f"Error applying regex fixes: {e}", "ERROR")
+            return content
 
     async def _fix_hardcoded_temp_paths(self, issue: Issue) -> dict[str, list[str]]:
         fixes: list[str] = []
@@ -248,26 +464,26 @@ class SecurityAgent(SubAgent):
         return lines, True
 
     def _replace_hardcoded_temp_paths(self, lines: list[str]) -> tuple[list[str], bool]:
-        replacements = [
-            (r'Path\("/tmp/([^"]+)"\)', r'Path(tempfile.gettempdir()) / "\1"'),
-            (r'"/tmp/([^"]+)"', r'str(Path(tempfile.gettempdir()) / "\1")'),
-            (r"'/tmp/([^']+)'", r"str(Path(tempfile.gettempdir()) / '\1')"),
-            (
-                r'Path\("/test/path"\)',
-                r"Path(tempfile.gettempdir()) / 'test-path'",
-            ),
-            (r'"/test/path"', r'str(Path(tempfile.gettempdir()) / "test-path")'),
-            (r"'/test/path'", r"str(Path(tempfile.gettempdir()) / 'test-path')"),
-        ]
-
-        modified = False
-        for pattern, replacement in replacements:
-            new_content = "\n".join(lines)
-            if re.search(pattern, new_content):
-                lines = re.sub(pattern, replacement, new_content).split("\n")
-                modified = True
+        # Apply temp path replacements using SAFE_PATTERNS
+        new_content = "\n".join(lines)
+
+        # Check if any temp paths need replacement
+        if SAFE_PATTERNS["detect_hardcoded_temp_paths_basic"].test(new_content):
+            # Apply multiple replacement patterns
+            new_content = SAFE_PATTERNS["replace_hardcoded_temp_paths"].apply(
+                new_content
+            )
+            new_content = SAFE_PATTERNS["replace_hardcoded_temp_strings"].apply(
+                new_content
+            )
+            new_content = SAFE_PATTERNS["replace_hardcoded_temp_single_quotes"].apply(
+                new_content
+            )
+            new_content = SAFE_PATTERNS["replace_test_path_patterns"].apply(new_content)
+            lines = new_content.split("\n")
+            return lines, True
 
-        return lines, modified
+        return lines, False
 
     async def _fix_shell_injection(self, issue: Issue) -> dict[str, list[str]]:
         fixes: list[str] = []
@@ -283,23 +499,9 @@ class SecurityAgent(SubAgent):
 
         original_content = content
 
-        patterns = [
-            (
-                r"subprocess\.run\(([^,]+),\s*shell=True\)",
-                r"subprocess.run(\1.split())",
-            ),
-            (
-                r"subprocess\.call\(([^,]+),\s*shell=True\)",
-                r"subprocess.call(\1.split())",
-            ),
-            (
-                r"subprocess\.Popen\(([^,]+),\s*shell=True\)",
-                r"subprocess.Popen(\1.split())",
-            ),
-        ]
+        from crackerjack.services.regex_patterns import apply_security_fixes
 
-        for pattern, replacement in patterns:
-            content = re.sub(pattern, replacement, content)
+        content = apply_security_fixes(content)
 
         if content != original_content:
             if self.context.write_file_content(file_path, content):
@@ -366,18 +568,12 @@ class SecurityAgent(SubAgent):
         return lines, False
 
     def _line_contains_hardcoded_secret(self, line: str) -> bool:
-        return bool(
-            re.search(
-                r'(password|secret|key|token)\s*=\s*[\'"][^\'"]+[\'"]',
-                line,
-                re.IGNORECASE,
-            ),
-        )
+        return SAFE_PATTERNS["detect_hardcoded_secrets"].test(line)
 
     def _replace_hardcoded_secret_with_env_var(self, line: str) -> str:
-        match = re.search(r"(\w+)\s*=", line)
-        if match:
-            var_name = match.group(1)
+        # Extract variable name using safe patterns
+        var_name = SAFE_PATTERNS["extract_variable_name_from_assignment"].apply(line)
+        if var_name != line:  # If pattern matched and extracted something
             env_var_name = var_name.upper()
             return f"{var_name} = os.getenv('{env_var_name}', '')"
         return line
@@ -396,7 +592,9 @@ class SecurityAgent(SubAgent):
 
         original_content = content
 
-        content = re.sub(r"\byaml\.load\(", "yaml.safe_load(", content)
+        from crackerjack.services.regex_patterns import SAFE_PATTERNS
+
+        content = SAFE_PATTERNS["fix_unsafe_yaml_load"].apply(content)
 
         if content != original_content:
             if self.context.write_file_content(file_path, content):
@@ -430,13 +628,10 @@ class SecurityAgent(SubAgent):
 
         original_content = content
 
-        replacements = [
-            (r"\bhashlib\.md5\(", "hashlib.sha256("),
-            (r"\bhashlib\.sha1\(", "hashlib.sha256("),
-        ]
+        from crackerjack.services.regex_patterns import SAFE_PATTERNS
 
-        for pattern, replacement in replacements:
-            content = re.sub(pattern, replacement, content)
+        content = SAFE_PATTERNS["fix_weak_md5_hash"].apply(content)
+        content = SAFE_PATTERNS["fix_weak_sha1_hash"].apply(content)
 
         if content != original_content:
             if self.context.write_file_content(file_path, content):
@@ -446,6 +641,117 @@ class SecurityAgent(SubAgent):
 
         return {"fixes": fixes, "files": files}
 
+    async def _fix_jwt_secrets(self, issue: Issue) -> dict[str, list[str]]:
+        """Fix hardcoded JWT secrets by replacing with environment variables."""
+        fixes: list[str] = []
+        files: list[str] = []
+
+        if not issue.file_path:
+            return {"fixes": fixes, "files": files}
+
+        file_path = Path(issue.file_path)
+        content = self.context.get_file_content(file_path)
+        if not content:
+            return {"fixes": fixes, "files": files}
+
+        original_content = content
+
+        # Apply JWT secret fix pattern
+        content = SAFE_PATTERNS["fix_hardcoded_jwt_secret"].apply(content)
+
+        # Ensure os import is present
+        if "os.getenv" in content and "import os" not in content:
+            lines = content.split("\n")
+            import_index = 0
+            for i, line in enumerate(lines):
+                if line.strip().startswith(("import ", "from ")):
+                    import_index = i + 1
+            lines.insert(import_index, "import os")
+            content = "\n".join(lines)
+
+        if content != original_content:
+            if self.context.write_file_content(file_path, content):
+                fixes.append(f"Fixed hardcoded JWT secrets in {issue.file_path}")
+                files.append(str(file_path))
+                self.log(f"Fixed JWT secrets in {issue.file_path}")
+
+        return {"fixes": fixes, "files": files}
+
+    async def _fix_pickle_usage(self, issue: Issue) -> dict[str, list[str]]:
+        """Fix unsafe pickle usage by documenting the security risk."""
+        fixes: list[str] = []
+        files: list[str] = []
+
+        if not issue.file_path:
+            return {"fixes": fixes, "files": files}
+
+        file_path = Path(issue.file_path)
+        content = self.context.get_file_content(file_path)
+        if not content:
+            return {"fixes": fixes, "files": files}
+
+        # For pickle usage, we document the risk rather than auto-fixing
+        # as pickle is sometimes necessary and the fix depends on context
+        fixes.append(
+            f"Documented unsafe pickle usage in {issue.file_path} - manual review required"
+        )
+
+        # Add a security comment if pickle.load/loads is found
+        if "pickle.load" in content:
+            lines = content.split("\n")
+            for i, line in enumerate(lines):
+                if "pickle.load" in line and "# SECURITY:" not in line:
+                    lines[i] = (
+                        line + "  # SECURITY: pickle.load is unsafe with untrusted data"
+                    )
+                    if self.context.write_file_content(file_path, "\n".join(lines)):
+                        fixes.append(
+                            f"Added security warning for pickle usage in {issue.file_path}"
+                        )
+                        files.append(str(file_path))
+                        self.log(
+                            f"Added security warning for pickle in {issue.file_path}"
+                        )
+                    break
+
+        return {"fixes": fixes, "files": files}
+
+    async def _fix_insecure_random(self, issue: Issue) -> dict[str, list[str]]:
+        """Fix insecure random usage by replacing with secrets module."""
+        fixes: list[str] = []
+        files: list[str] = []
+
+        if not issue.file_path:
+            return {"fixes": fixes, "files": files}
+
+        file_path = Path(issue.file_path)
+        content = self.context.get_file_content(file_path)
+        if not content:
+            return {"fixes": fixes, "files": files}
+
+        original_content = content
+
+        # Apply insecure random fix
+        content = SAFE_PATTERNS["fix_insecure_random_choice"].apply(content)
+
+        # Ensure secrets import
+        if "secrets.choice" in content and "import secrets" not in content:
+            lines = content.split("\n")
+            import_index = 0
+            for i, line in enumerate(lines):
+                if line.strip().startswith(("import ", "from ")):
+                    import_index = i + 1
+            lines.insert(import_index, "import secrets")
+            content = "\n".join(lines)
+
+        if content != original_content:
+            if self.context.write_file_content(file_path, content):
+                fixes.append(f"Fixed insecure random usage in {issue.file_path}")
+                files.append(str(file_path))
+                self.log(f"Fixed insecure random usage in {issue.file_path}")
+
+        return {"fixes": fixes, "files": files}
+
     async def _run_bandit_analysis(self) -> list[str]:
         fixes: list[str] = []
 
@@ -499,12 +805,12 @@ class SecurityAgent(SubAgent):
         return self._remove_debug_prints_with_secrets(content)
 
     async def _fix_insecure_random_usage(self, content: str) -> str:
-        if not re.search(r"random\.(?:random|choice)\(", content):
+        if not SAFE_PATTERNS["detect_insecure_random_usage"].test(content):
             return content
 
         content = self._add_secrets_import_if_needed(content)
 
-        return re.sub(r"random\.choice\(([^)]+)\)", r"secrets.choice(\1)", content)
+        return SAFE_PATTERNS["fix_insecure_random_choice"].apply(content)
 
     def _add_secrets_import_if_needed(self, content: str) -> str:
         if "import secrets" in content:
@@ -518,12 +824,9 @@ class SecurityAgent(SubAgent):
         return "\n".join(lines)
 
     def _remove_debug_prints_with_secrets(self, content: str) -> str:
-        return re.sub(
-            r"print\s*\([^)]*(?:password|secret|key|token)[^)]*\)",
-            "",
-            content,
-            flags=re.IGNORECASE,
-        )
+        from crackerjack.services.regex_patterns import SAFE_PATTERNS
+
+        return SAFE_PATTERNS["remove_debug_prints_with_secrets"].apply(content)
 
 
 agent_registry.register(SecurityAgent)