crackerjack 0.18.2__py3-none-any.whl → 0.45.2__py3-none-any.whl

crackerjack/services/patterns/formatting.py

@@ -0,0 +1,226 @@
+"""Formatting and whitespace normalization patterns."""
+
+from .core import ValidatedPattern
+
+PATTERNS: dict[str, ValidatedPattern] = {
+    "fix_command_spacing": ValidatedPattern(
+        name="fix_command_spacing",
+        pattern=r"python\s*-\s*m\s+(\w+)",
+        replacement=r"python -m \1",
+        description="Fix spacing in 'python -m command' patterns",
+        test_cases=[
+            ("python - m crackerjack", "python -m crackerjack"),
+            ("python -m crackerjack", "python -m crackerjack"),
+            ("python - m pytest", "python -m pytest"),
+            ("other python - m stuff", "other python -m stuff"),
+        ],
+    ),
+    "fix_long_flag_spacing": ValidatedPattern(
+        name="fix_long_flag_spacing",
+        pattern=r"-\s*-\s*(\w+(?: -\w+)*)",
+        replacement=r"--\1",
+        description="Fix spacing in long flags like '--help'",
+        test_cases=[
+            ("- - help", "--help"),
+            ("- - ai-fix", "--ai-fix"),
+            ("--help", "--help"),
+            ("- - start-websocket-server", "--start-websocket-server"),
+        ],
+    ),
+    "fix_short_flag_spacing": ValidatedPattern(
+        name="fix_short_flag_spacing",
+        pattern=r"(?<!\w)-\s+(\w)(?!\w)",
+        replacement=r"-\1",
+        description="Fix spacing in short flags like '-t'",
+        test_cases=[
+            ("python -m crackerjack - t", "python -m crackerjack -t"),
+            ("- q", "-q"),
+            ("-t", "-t"),
+            ("some - x flag", "some -x flag"),
+        ],
+    ),
+    "fix_hyphenated_names": ValidatedPattern(
+        name="fix_hyphenated_names",
+        pattern=r"(\w+)\s*-\s*(\w+)",
+        replacement=r"\1-\2",
+        description="Fix spacing in hyphenated names and identifiers",
+        test_cases=[
+            ("python - pro", "python-pro"),
+            (
+                "pytest - hypothesis - specialist",
+                "pytest-hypothesis - specialist",
+            ),
+            ("backend - architect", "backend-architect"),
+            ("python-pro", "python-pro"),
+            ("end - of - file-fixer", "end-of - file-fixer"),
+        ],
+    ),
+    "fix_hyphenated_names_global": ValidatedPattern(
+        name="fix_hyphenated_names_global",
+        pattern=r"(\w+)\s+-\s+(\w+)",
+        replacement=r"\1-\2",
+        description="Globally fix spacing in hyphenated names (single pass only)",
+        global_replace=True,
+        test_cases=[
+            ("python - pro", "python-pro"),
+            ("end - of - file", "end-of - file"),
+            ("already-hyphenated", "already-hyphenated"),
+            ("start - middle - end", "start-middle - end"),
+        ],
+    ),
+    "fix_spaced_hyphens": ValidatedPattern(
+        name="fix_spaced_hyphens",
+        pattern=r"(\w+)\s+-\s+(\w+)",
+        replacement=r"\1-\2",
+        description="Fix spaced hyphens with spaces around dashes (use apply_iteratively for multi-word)",
+        global_replace=True,
+        test_cases=[
+            ("python - pro", "python-pro"),
+            (
+                "pytest - hypothesis - specialist",
+                "pytest-hypothesis - specialist",
+            ),
+            (
+                "end - of - file - fixer",
+                "end-of - file-fixer",
+            ),
+            ("already-hyphenated", "already-hyphenated"),
+            ("mixed-case with - spaces", "mixed-case with-spaces"),
+        ],
+    ),
+    "fix_debug_log_pattern": ValidatedPattern(
+        name="fix_debug_log_pattern",
+        pattern=r"crackerjack\s*-\s*debug",
+        replacement="crackerjack-debug",
+        description="Fix spacing in debug log patterns",
+        test_cases=[
+            ("crackerjack - debug-12345.log", "crackerjack-debug-12345.log"),
+            ("crackerjack-debug.log", "crackerjack-debug.log"),
+            ("old crackerjack - debug files", "old crackerjack-debug files"),
+        ],
+    ),
+    "fix_job_file_pattern": ValidatedPattern(
+        name="fix_job_file_pattern",
+        pattern=r"job\s*-\s*(\{[^}]+\}|\w+)",
+        replacement=r"job-\1",
+        description="Fix spacing in job file patterns",
+        test_cases=[
+            ("job - {self.web_job_id}.json", "job-{self.web_job_id}.json"),
+            ("job - abc123.json", "job-abc123.json"),
+            ("job-existing.json", "job-existing.json"),
+        ],
+    ),
+    "fix_markdown_bold": ValidatedPattern(
+        name="fix_markdown_bold",
+        pattern=r"\*\s+\*(.+?)\s*\*\s+\*",
+        replacement=r"**\1**",
+        description="Fix spacing in markdown bold patterns",
+        test_cases=[
+            ("* *Bold Text * *", "**Bold Text**"),
+            ("* *🧪 pytest-specialist * *", "**🧪 pytest-specialist**"),
+            ("**Already Bold**", "**Already Bold**"),
+        ],
+    ),
+    "remove_trailing_whitespace": ValidatedPattern(
+        name="remove_trailing_whitespace",
+        pattern=r"[ \t]+$",
+        replacement="",
+        description="Remove trailing whitespace from lines",
+        global_replace=True,
+        test_cases=[
+            ("line with spaces ", "line with spaces"),
+            ("line with tabs\t\t", "line with tabs"),
+            ("normal line", "normal line"),
+            ("mixed \t ", "mixed"),
+            ("", ""),
+        ],
+    ),
+    "normalize_multiple_newlines": ValidatedPattern(
+        name="normalize_multiple_newlines",
+        pattern=r"\n{3,}",
+        replacement="\n\n",
+        description="Normalize multiple consecutive newlines to maximum 2",
+        global_replace=True,
+        test_cases=[
+            ("line1\n\n\nline2", "line1\n\nline2"),
+            ("line1\n\n\n\n\nline2", "line1\n\nline2"),
+            ("line1\n\nline2", "line1\n\nline2"),
+            ("line1\nline2", "line1\nline2"),
+        ],
+    ),
+    "normalize_assert_statements": ValidatedPattern(
+        name="normalize_assert_statements",
+        pattern=r"assert (.+?)\s*==\s*(.+)",
+        replacement=r"assert \1 == \2",
+        description="Normalize spacing around == in assert statements",
+        global_replace=True,
+        test_cases=[
+            ("assert result==expected", "assert result == expected"),
+            ("assert value == other", "assert value == other"),
+            ("assert result== expected", "assert result == expected"),
+            ("assert result ==expected", "assert result == expected"),
+            (
+                "assert result == expected",
+                "assert result == expected",
+            ),
+        ],
+    ),
+    "spacing_after_comma": ValidatedPattern(
+        name="spacing_after_comma",
+        pattern=r",(?! |\n|$)",
+        replacement=r", ",
+        description="Add space after comma if missing (skip if already spaced, at end of line, or end of string)",
+        global_replace=True,
+        test_cases=[
+            ("func(a,b,c)", "func(a, b, c)"),
+            ("func(a, b, c)", "func(a, b, c)"),
+            ("[1,2,3]", "[1, 2, 3]"),
+            ("[1, 2, 3]", "[1, 2, 3]"),
+            ("{'x':1,'y':2,'z':3}", "{'x':1, 'y':2, 'z':3}"),
+            ("trailing,\nnewline", "trailing,\nnewline"),
+            ("end_comma,", "end_comma,"),
+        ],
+    ),
+    "spacing_after_colon": ValidatedPattern(
+        name="spacing_after_colon",
+        pattern=r"(?<!https)(?<!http)(?<!ftp)(?<!file)(?<!: )(\b[a-zA-Z_][a-zA-Z0-9_]*):([a-zA-Z0-9_][^ \n:]*)",
+        replacement=r"\1: \2",
+        global_replace=True,
+        description="Add space after colon if missing (avoid double colons, URLs, and protocols)",
+        test_cases=[
+            ("def func(x: int, y: str): ", "def func(x: int, y: str): "),
+            ("dict_item = {'key': 'value'}", "dict_item = {'key': 'value'}"),
+            ("already: spaced: properly", "already: spaced: properly"),
+            ("class::method", "class::method"),
+            ("https://github.com", "https://github.com"),
+            ("http://example.com", "http://example.com"),
+            ("ftp://server.com", "ftp://server.com"),
+            ("repo:local", "repo: local"),
+        ],
+    ),
+    "multiple_spaces": ValidatedPattern(
+        name="multiple_spaces",
+        pattern=r" {2,}",
+        replacement=r" ",
+        description="Replace multiple spaces with single space",
+        global_replace=True,
+        test_cases=[
+            ("def func( x, y ): ", "def func( x, y ): "),
+            ("single space only", "single space only"),
+            ("lots of spaces", "lots of spaces"),
+            ("\tkeep\ttabs\tbut fix spaces", "\tkeep\ttabs\tbut fix spaces"),
+        ],
+    ),
+    "normalize_whitespace": ValidatedPattern(
+        name="normalize_whitespace",
+        pattern=r"\s+",
+        replacement=r" ",
+        description="Normalize multiple whitespace to single space",
+        global_replace=True,
+        test_cases=[
+            ("import module", "import module"),
+            ("from pkg import item", "from pkg import item"),
+            ("normal text", "normal text"),
+        ],
+    ),
+}

crackerjack/services/patterns/operations.py

@@ -0,0 +1,339 @@
+"""Utility functions for regex pattern operations."""
+
+import re
+
+# Import for type checking - circular import avoided
+from typing import TYPE_CHECKING
+
+from .core import MAX_ITERATIONS, CompiledPatternCache, ValidatedPattern
+
+if TYPE_CHECKING:
+    pass  # SAFE_PATTERNS imported at runtime to avoid circular imports
+
+
+def validate_all_patterns() -> dict[str, bool]:
+    """Validate all patterns in the registry."""
+    from . import SAFE_PATTERNS
+
+    results: dict[str, bool] = {}
+    for name, pattern in SAFE_PATTERNS.items():
+        try:
+            pattern._validate()
+            results[name] = True
+        except ValueError as e:
+            results[name] = False
+            print(f"Pattern '{name}' failed validation: {e}")
+    return results
+
+
+def find_pattern_for_text(text: str) -> list[str]:
+    """Find all patterns that match the given text."""
+    from . import SAFE_PATTERNS
+
+    return [name for name, pattern in SAFE_PATTERNS.items() if pattern.test(text)]
+
+
+def apply_safe_replacement(text: str, pattern_name: str) -> str:
+    """Apply a named pattern to text."""
+    from . import SAFE_PATTERNS
+
+    if pattern_name not in SAFE_PATTERNS:
+        raise ValueError(f"Unknown pattern: {pattern_name}")
+
+    return SAFE_PATTERNS[pattern_name].apply(text)
+
+
+def get_pattern_description(pattern_name: str) -> str:
+    """Get description for a named pattern."""
+    from . import SAFE_PATTERNS
+
+    if pattern_name not in SAFE_PATTERNS:
+        return "Unknown pattern"
+
+    return SAFE_PATTERNS[pattern_name].description
+
+
+def fix_multi_word_hyphenation(text: str) -> str:
+    """Fix multi-word hyphenation iteratively."""
+    from . import SAFE_PATTERNS
+
+    return SAFE_PATTERNS["fix_spaced_hyphens"].apply_iteratively(text)
+
+
+def update_pyproject_version(content: str, new_version: str) -> str:
+    """Update version in pyproject.toml content."""
+    from . import SAFE_PATTERNS
+
+    pattern_obj = SAFE_PATTERNS["update_pyproject_version"]
+
+    temp_pattern = ValidatedPattern(
+        name="temp_version_update",
+        pattern=pattern_obj.pattern,
+        replacement=f"\\g<1>{new_version}\\g<3>",
+        description=f"Update version to {new_version}",
+        test_cases=[
+            ('version = "1.2.3"', f'version = "{new_version}"'),
+        ],
+    )
+
+    return re.compile(pattern_obj.pattern, re.MULTILINE).sub(
+        temp_pattern.replacement, content
+    )
+
+
+def apply_formatting_fixes(content: str) -> str:
+    """Apply common formatting fixes to content."""
+    from . import SAFE_PATTERNS
+
+    pattern = SAFE_PATTERNS["remove_trailing_whitespace"]
+    content = re.compile(pattern.pattern, re.MULTILINE).sub(
+        pattern.replacement, content
+    )
+
+    content = SAFE_PATTERNS["normalize_multiple_newlines"].apply(content)
+
+    return content
+
+
+def apply_security_fixes(content: str) -> str:
+    """Apply common security fixes to content."""
+    from . import SAFE_PATTERNS
+
+    content = SAFE_PATTERNS["fix_subprocess_run_shell"].apply(content)
+    content = SAFE_PATTERNS["fix_subprocess_call_shell"].apply(content)
+    content = SAFE_PATTERNS["fix_subprocess_popen_shell"].apply(content)
+
+    content = SAFE_PATTERNS["fix_unsafe_yaml_load"].apply(content)
+    content = SAFE_PATTERNS["fix_weak_md5_hash"].apply(content)
+    content = SAFE_PATTERNS["fix_weak_sha1_hash"].apply(content)
+    content = SAFE_PATTERNS["fix_insecure_random_choice"].apply(content)
+
+    content = SAFE_PATTERNS["remove_debug_prints_with_secrets"].apply(content)
+
+    return content
+
+
+def apply_test_fixes(content: str) -> str:
+    """Apply common test-related fixes to content."""
+    from . import SAFE_PATTERNS
+
+    return SAFE_PATTERNS["normalize_assert_statements"].apply(content)
+
+
+def is_valid_job_id(job_id: str) -> bool:
+    """Check if a job ID is valid."""
+    from . import SAFE_PATTERNS
+
+    return SAFE_PATTERNS["validate_job_id_alphanumeric"].test(job_id)
+
+
+def remove_coverage_fail_under(addopts: str) -> str:
+    """Remove --cov-fail-under from pytest addopts."""
+    from . import SAFE_PATTERNS
+
+    return SAFE_PATTERNS["remove_coverage_fail_under"].apply(addopts)
+
+
+def update_coverage_requirement(content: str, new_coverage: float) -> str:
+    """Update coverage requirement to a new value."""
+    from . import SAFE_PATTERNS
+
+    pattern_obj = SAFE_PATTERNS["update_coverage_requirement"]
+
+    temp_pattern = ValidatedPattern(
+        name="temp_coverage_update",
+        pattern=pattern_obj.pattern,
+        replacement=f"\\1{new_coverage: .0f}",
+        description=f"Update coverage to {new_coverage}",
+        test_cases=[
+            ("--cov-fail-under=85", f"--cov-fail-under={new_coverage: .0f}"),
+        ],
+    )
+
+    return re.compile(pattern_obj.pattern).sub(temp_pattern.replacement, content)
+
+
+def update_repo_revision(content: str, repo_url: str, new_revision: str) -> str:
+    """Update repository revision in content."""
+    escaped_url = re.escape(repo_url)
+    pattern = rf'("repo": "{escaped_url}".*?"rev": )"([^"]+)"'
+    replacement = rf'\1"{new_revision}"'
+
+    return re.compile(pattern, re.DOTALL).sub(replacement, content)
+
+
+def sanitize_internal_urls(text: str) -> str:
+    """Sanitize all internal URLs in text."""
+    from . import SAFE_PATTERNS
+
+    url_patterns = [
+        "sanitize_localhost_urls",
+        "sanitize_127_urls",
+        "sanitize_any_localhost_urls",
+        "sanitize_ws_localhost_urls",
+        "sanitize_ws_127_urls",
+        "sanitize_simple_localhost_urls",
+        "sanitize_simple_ws_localhost_urls",
+    ]
+
+    result = text
+    for pattern_name in url_patterns:
+        result = SAFE_PATTERNS[pattern_name].apply(result)
+
+    return result
+
+
+def apply_pattern_iteratively(
+    text: str, pattern_name: str, max_iterations: int = MAX_ITERATIONS
+) -> str:
+    """Apply a pattern iteratively until no more changes."""
+    from . import SAFE_PATTERNS
+
+    if pattern_name not in SAFE_PATTERNS:
+        raise ValueError(f"Unknown pattern: {pattern_name}")
+
+    return SAFE_PATTERNS[pattern_name].apply_iteratively(text, max_iterations)
+
+
+def get_all_pattern_stats() -> dict[str, dict[str, float] | dict[str, str]]:
+    """Get performance statistics for all patterns."""
+    from . import SAFE_PATTERNS
+
+    test_text = "python - m crackerjack - t with pytest - hypothesis - specialist"
+    stats: dict[str, dict[str, float] | dict[str, str]] = {}
+
+    for name, pattern in SAFE_PATTERNS.items():
+        try:
+            pattern_stats = pattern.get_performance_stats(test_text, iterations=10)
+            stats[name] = pattern_stats
+        except Exception as e:
+            stats[name] = {"error": str(e)}
+
+    return stats
+
+
+def clear_all_caches() -> None:
+    """Clear all pattern caches."""
+    CompiledPatternCache.clear_cache()
+
+
+def get_cache_info() -> dict[str, int | list[str]]:
+    """Get cache statistics."""
+    return CompiledPatternCache.get_cache_stats()
+
+
+def detect_path_traversal_patterns(path_str: str) -> list[str]:
+    """Detect path traversal patterns in a path string."""
+    from . import SAFE_PATTERNS
+
+    detected = []
+    traversal_patterns = [
+        "detect_directory_traversal_basic",
+        "detect_directory_traversal_backslash",
+        "detect_url_encoded_traversal",
+        "detect_double_url_encoded_traversal",
+    ]
+
+    for pattern_name in traversal_patterns:
+        pattern = SAFE_PATTERNS[pattern_name]
+        if pattern.test(path_str):
+            detected.append(pattern_name)
+
+    return detected
+
+
+def detect_null_byte_patterns(path_str: str) -> list[str]:
+    """Detect null byte patterns in a path string."""
+    from . import SAFE_PATTERNS
+
+    detected = []
+    null_patterns = [
+        "detect_null_bytes_url",
+        "detect_null_bytes_literal",
+        "detect_utf8_overlong_null",
+    ]
+
+    for pattern_name in null_patterns:
+        pattern = SAFE_PATTERNS[pattern_name]
+        if pattern.test(path_str):
+            detected.append(pattern_name)
+
+    return detected
+
+
+def detect_dangerous_directory_patterns(path_str: str) -> list[str]:
+    """Detect dangerous directory access patterns in a path string."""
+    from . import SAFE_PATTERNS
+
+    detected = []
+    dangerous_patterns = [
+        "detect_sys_directory_pattern",
+        "detect_proc_directory_pattern",
+        "detect_etc_directory_pattern",
+        "detect_boot_directory_pattern",
+        "detect_dev_directory_pattern",
+        "detect_root_directory_pattern",
+        "detect_var_log_directory_pattern",
+        "detect_bin_directory_pattern",
+        "detect_sbin_directory_pattern",
+    ]
+
+    for pattern_name in dangerous_patterns:
+        pattern = SAFE_PATTERNS[pattern_name]
+        if pattern.test(path_str):
+            detected.append(pattern_name)
+
+    return detected
+
+
+def detect_suspicious_path_patterns(path_str: str) -> list[str]:
+    """Detect suspicious path patterns in a path string."""
+    from . import SAFE_PATTERNS
+
+    detected = []
+    suspicious_patterns = [
+        "detect_parent_directory_in_path",
+        "detect_suspicious_temp_traversal",
+        "detect_suspicious_var_traversal",
+    ]
+
+    for pattern_name in suspicious_patterns:
+        pattern = SAFE_PATTERNS[pattern_name]
+        if pattern.test(path_str):
+            detected.append(pattern_name)
+
+    return detected
+
+
+def validate_path_security(path_str: str) -> dict[str, list[str]]:
+    """Validate path security by checking for various attack patterns."""
+    return {
+        "traversal_patterns": detect_path_traversal_patterns(path_str),
+        "null_bytes": detect_null_byte_patterns(path_str),
+        "dangerous_directories": detect_dangerous_directory_patterns(path_str),
+        "suspicious_patterns": detect_suspicious_path_patterns(path_str),
+    }
+
+
+class RegexPatternsService:
+    """Service class that implements RegexPatternsProtocol.
+
+    Wraps module-level regex pattern functions to provide a protocol-compliant interface.
+    """
+
+    def update_pyproject_version(self, content: str, new_version: str) -> str:
+        """Update version in pyproject.toml content."""
+        return update_pyproject_version(content, new_version)
+
+    def remove_coverage_fail_under(self, content: str) -> str:
+        """Remove --cov-fail-under from pytest addopts."""
+        return remove_coverage_fail_under(content)
+
+    def update_version_in_changelog(self, content: str, new_version: str) -> str:
+        """Update version in changelog content."""
+        # Placeholder implementation - can be enhanced later
+        return content
+
+    def mask_tokens_in_text(self, text: str) -> str:
+        """Mask sensitive tokens in text."""
+        return sanitize_internal_urls(text)

crackerjack/services/patterns/security/__init__.py

@@ -0,0 +1,23 @@
+"""Security-related patterns for credential detection, path traversal, and unsafe operations.
+
+This module aggregates all security patterns from specialized submodules:
+- credentials: Hardcoded credentials, secrets, tokens, and passwords (9 patterns)
+- path_traversal: Directory traversal and unsafe path access (19 patterns)
+- unsafe_operations: Weak crypto, insecure random, subprocess injection (14 patterns)
+- code_injection: SQL injection, code eval, system commands (8 patterns)
+
+Total: 50 security patterns
+"""
+
+from ..core import ValidatedPattern
+from .code_injection import PATTERNS as CODE_INJECTION_PATTERNS
+from .credentials import PATTERNS as CREDENTIAL_PATTERNS
+from .path_traversal import PATTERNS as PATH_TRAVERSAL_PATTERNS
+from .unsafe_operations import PATTERNS as UNSAFE_OPERATION_PATTERNS
+
+PATTERNS: dict[str, ValidatedPattern] = (
+    CREDENTIAL_PATTERNS
+    | PATH_TRAVERSAL_PATTERNS
+    | UNSAFE_OPERATION_PATTERNS
+    | CODE_INJECTION_PATTERNS
+)