ballpython 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

pycleaner/pipeline.py

@@ -1,5 +1,6 @@
 """
-Cleanup pipeline coordinating syntax healing, import resolution, linting, and formatting.
+Cleanup pipeline coordinating syntax healing, code modernization, dead-code pruning,
+import resolution, linting, and formatting.
 """
 
 from __future__ import annotations
@@ -12,9 +13,18 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
+from pycleaner.cache import ContentAddressableCache
+from pycleaner.dead_code_detector import DeadCodeFixer
 from pycleaner.import_resolver import ImportResolver
 from pycleaner.linter_formatter import LinterFormatter
+from pycleaner.modernizer import Modernizer
 from pycleaner.syntax_healer import SyntaxHealer
+from pycleaner.verifier import (
+    CounterExample,
+    IsolatedDifferentialVerifier,
+    ProofReceipt,
+    VerificationTier,
+)
 
 if TYPE_CHECKING:
     from pycleaner.config import PyCleanerConfig
@@ -30,12 +40,17 @@ class CleanResult:
     changed: bool
     is_valid_python: bool
     syntax_repairs: list[str] = field(default_factory=list)
+    modernize_transforms: list[str] = field(default_factory=list)
+    dead_code_pruned: list[str] = field(default_factory=list)
     resolved_imports: list[str] = field(default_factory=list)
     unresolved_symbols: list[str] = field(default_factory=list)
     lint_changed: bool = False
     format_changed: bool = False
     error: str | None = None
     diagnostics: list[dict[str, str]] = field(default_factory=list)
+    proof_receipts: list[ProofReceipt] = field(default_factory=list)
+    refused_changes: list[CounterExample] = field(default_factory=list)
+    verification_tier: VerificationTier | None = None
 
     @property
     def diff(self) -> str:
@@ -59,10 +74,17 @@ class PipelineOptions:
     """Configurable feature flags and mappings for CleanPipeline."""
 
     enable_syntax_healing: bool = True
+    enable_modernizer: bool = True
+    enable_dead_code_pruning: bool = True
     enable_import_resolution: bool = True
     enable_lint_fixing: bool = True
     enable_formatting: bool = True
     custom_import_map: dict[str, str] | None = None
+    verify_proofs: bool = False
+    proof_iterations: int = 50
+    proof_seed: int | None = None
+    enable_cache: bool = False
+    cache_db_path: str = ".pycleaner/cache.db"
 
 
 class CleanPipeline:
@@ -76,16 +98,36 @@ class CleanPipeline:
     ) -> None:
         opts = options or PipelineOptions(
             enable_syntax_healing=kwargs.get("enable_syntax_healing", True),
+            enable_modernizer=kwargs.get("enable_modernizer", True),
+            enable_dead_code_pruning=kwargs.get("enable_dead_code_pruning", True),
             enable_import_resolution=kwargs.get("enable_import_resolution", True),
             enable_lint_fixing=kwargs.get("enable_lint_fixing", True),
             enable_formatting=kwargs.get("enable_formatting", True),
             custom_import_map=kwargs.get("custom_import_map"),
+            verify_proofs=kwargs.get("verify_proofs", False),
+            proof_iterations=kwargs.get("proof_iterations", 50),
+            proof_seed=kwargs.get("proof_seed"),
+            enable_cache=kwargs.get("enable_cache", False),
+            cache_db_path=kwargs.get("cache_db_path", ".pycleaner/cache.db"),
         )
         self.enable_syntax_healing = opts.enable_syntax_healing
+        self.enable_modernizer = opts.enable_modernizer
+        self.enable_dead_code_pruning = opts.enable_dead_code_pruning
         self.enable_import_resolution = opts.enable_import_resolution
         self.enable_lint_fixing = opts.enable_lint_fixing
         self.enable_formatting = opts.enable_formatting
+        self.verify_proofs = opts.verify_proofs
+        self.proof_iterations = opts.proof_iterations
+        self.proof_seed = opts.proof_seed
+        self.enable_cache = opts.enable_cache
+        self.cache = (
+            ContentAddressableCache(opts.cache_db_path) if opts.enable_cache else None
+        )
         self.config = config
+        self.verifier = IsolatedDifferentialVerifier(
+            iterations=self.proof_iterations,
+            base_seed=self.proof_seed,
+        )
 
         import_map = dict(opts.custom_import_map or {})
         custom_map = getattr(config, "custom_import_map", None)
@@ -93,6 +135,8 @@ class CleanPipeline:
             import_map.update(custom_map)
 
         self.syntax_healer = SyntaxHealer()
+        self.modernizer = Modernizer()
+        self.dead_code_fixer = DeadCodeFixer()
         self.import_resolver = ImportResolver(custom_import_map=import_map)
         self.linter_formatter = LinterFormatter()
 
@@ -118,6 +162,32 @@ class CleanPipeline:
             )
         return current_code, None
 
+    def _stage_modernize(
+        self, current_code: str, filename: str, modernize_transforms: list[str]
+    ) -> str:
+        """Execute Stage 2: Code Modernization."""
+        if not self.enable_modernizer:
+            return current_code
+
+        mod_res = self.modernizer.modernize(current_code, filename=filename)
+        if mod_res.changed:
+            modernize_transforms.extend(mod_res.transformations)
+            return mod_res.code
+        return current_code
+
+    def _stage_dead_code(
+        self, current_code: str, filename: str, dead_code_pruned: list[str]
+    ) -> str:
+        """Execute Stage 3: Dead Code Pruning."""
+        if not self.enable_dead_code_pruning:
+            return current_code
+
+        dc_res = self.dead_code_fixer.fix(current_code, filename=filename)
+        if dc_res.changed:
+            dead_code_pruned.extend(dc_res.pruned_items)
+            return dc_res.code
+        return current_code
+
     def _stage_resolve_imports(
         self,
         current_code: str,
@@ -126,7 +196,7 @@ class CleanPipeline:
         unresolved_symbols: list[str],
         diagnostics: list[dict[str, str]],
     ) -> str:
-        """Execute Stage 2: Missing Import Resolution."""
+        """Execute Stage 4: Missing Import Resolution."""
         if self.enable_import_resolution:
             import_res = self.import_resolver.resolve(current_code, filename=filename)
             if import_res.resolved_imports:
@@ -149,7 +219,7 @@ class CleanPipeline:
     def _stage_lint_format(
         self, current_code: str, filename: str
     ) -> tuple[str, bool, bool]:
-        """Execute Stage 3 & 4: Lint Auto-fixing and Formatting."""
+        """Execute Stage 5: Lint Auto-fixing and Formatting."""
         if not (self.enable_lint_fixing or self.enable_formatting):
             return current_code, False, False
 
@@ -176,6 +246,8 @@ class CleanPipeline:
         """Process in-memory Python source code through the pipeline."""
         current_code = source
         syntax_repairs: list[str] = []
+        modernize_transforms: list[str] = []
+        dead_code_pruned: list[str] = []
         resolved_imports: list[str] = []
         unresolved_symbols: list[str] = []
         lint_changed = False
@@ -190,6 +262,13 @@ class CleanPipeline:
             )
             if error_msg is not None:
                 break
+
+            current_code = self._stage_modernize(
+                current_code, filename, modernize_transforms
+            )
+            current_code = self._stage_dead_code(
+                current_code, filename, dead_code_pruned
+            )
             current_code = self._stage_resolve_imports(
                 current_code,
                 filename,
@@ -204,39 +283,143 @@ class CleanPipeline:
                 break
 
         is_valid, final_error = self._validate_syntax(current_code, filename, error_msg)
+
+        proof_receipts: list[ProofReceipt] = []
+        refused_changes: list[CounterExample] = []
+        verification_tier: VerificationTier | None = None
+
+        if self.verify_proofs and current_code != source and is_valid:
+            target_callables = self._find_modified_callables(source, current_code)
+            if not target_callables:
+                verification_tier = VerificationTier.TIER_B_SUGGESTED
+            else:
+                all_proven = True
+                for func_name in target_callables:
+                    receipt = self.verifier.verify_transformation(
+                        filepath=filename,
+                        original_source=source,
+                        transformed_source=current_code,
+                        target_callable=func_name,
+                    )
+                    proof_receipts.append(receipt)
+                    if receipt.tier == VerificationTier.TIER_C_REFUSED:
+                        all_proven = False
+                        if receipt.counterexample:
+                            refused_changes.append(receipt.counterexample)
+                    elif receipt.tier != VerificationTier.TIER_A_PROVEN:
+                        all_proven = False
+
+                if refused_changes:
+                    verification_tier = VerificationTier.TIER_C_REFUSED
+                    # Absolute rollback guarantee on Tier C refusal
+                    current_code = source
+                elif all_proven and proof_receipts:
+                    verification_tier = VerificationTier.TIER_A_PROVEN
+                else:
+                    verification_tier = VerificationTier.TIER_B_SUGGESTED
+
         return CleanResult(
             path=Path(filename),
             original_code=source,
             cleaned_code=current_code,
-            changed=current_code != source,
+            changed=(current_code != source)
+            and (verification_tier != VerificationTier.TIER_C_REFUSED),
             is_valid_python=is_valid,
             syntax_repairs=syntax_repairs,
+            modernize_transforms=modernize_transforms,
+            dead_code_pruned=dead_code_pruned,
             resolved_imports=resolved_imports,
             unresolved_symbols=unresolved_symbols,
             lint_changed=lint_changed,
             format_changed=format_changed,
             error=final_error,
             diagnostics=diagnostics,
+            proof_receipts=proof_receipts,
+            refused_changes=refused_changes,
+            verification_tier=verification_tier,
         )
 
+    @staticmethod
+    def _find_modified_callables(orig_code: str, clean_code: str) -> list[str]:
+        """Identify functions/methods whose AST has changed."""
+        try:
+            orig_tree = ast.parse(orig_code)
+            clean_tree = ast.parse(clean_code)
+        except SyntaxError:
+            return []
+
+        orig_funcs: dict[str, str] = {}
+        for node in ast.walk(orig_tree):
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                orig_funcs[node.name] = ast.dump(node)
+
+        modified: list[str] = []
+        clean_funcs: list[str] = []
+        for node in ast.walk(clean_tree):
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+                clean_funcs.append(node.name)
+                clean_dump = ast.dump(node)
+                if node.name not in orig_funcs or orig_funcs[node.name] != clean_dump:
+                    if node.name not in modified:
+                        modified.append(node.name)
+
+        if not modified and clean_funcs:
+            return clean_funcs[:3]
+
+        return modified
+
     def process_file(
         self,
         filepath: Path | str,
         apply_changes: bool = True,
         backup: bool = False,
     ) -> CleanResult:
-        """Process a single file on disk and optionally write back updates.
-
-        Args:
-            filepath: Path to the Python file to process.
-            apply_changes: If True, write cleaned code back to disk.
-            backup: If True, create a .pycleaner.bak file before overwriting.
-        """
+        """Process a single file on disk and optionally write back updates."""
         path = Path(filepath).resolve()
         content = path.read_text(encoding="utf-8", errors="replace")
+
+        if self.cache is not None:
+            cached_result = self.cache.get(path, content)
+            if cached_result is not None:
+                if (
+                    apply_changes
+                    and cached_result.changed
+                    and cached_result.is_valid_python
+                    and (
+                        cached_result.verification_tier
+                        != VerificationTier.TIER_C_REFUSED
+                    )
+                ):
+                    if backup:
+                        bak_path = path.with_name(path.name + ".pycleaner.bak")
+                        shutil.copy2(path, bak_path)
+                    path.write_text(cached_result.cleaned_code, encoding="utf-8")
+                return cached_result
+
         result = self.process_source(content, filename=str(path))
 
-        if apply_changes and result.changed and result.is_valid_python:
+        if self.cache is not None:
+            self.cache.set(path, content, result)
+            if result.changed and apply_changes:
+                tier_label = (
+                    result.verification_tier.value
+                    if result.verification_tier
+                    else "UNVERIFIED"
+                )
+                self.cache.record_provenance(
+                    file_path=path,
+                    transformation_type="clean",
+                    verification_tier=tier_label,
+                    seed=self.proof_seed,
+                    diff=result.diff,
+                )
+
+        if (
+            apply_changes
+            and result.changed
+            and result.is_valid_python
+            and (result.verification_tier != VerificationTier.TIER_C_REFUSED)
+        ):
             if backup:
                 bak_path = path.with_name(path.name + ".pycleaner.bak")
                 shutil.copy2(path, bak_path)
@@ -251,14 +434,7 @@ class CleanPipeline:
         backup: bool = False,
         max_workers: int | None = None,
     ) -> list[CleanResult]:
-        """Process multiple files, optionally in parallel.
-
-        Args:
-            filepaths: List of Python file paths.
-            apply_changes: Write cleaned code back to disk.
-            backup: Create .pycleaner.bak before overwriting.
-            max_workers: Max parallel workers. None = sequential. 1+ = parallel.
-        """
+        """Process multiple files, optionally in parallel."""
         if max_workers is not None and max_workers > 1 and len(filepaths) > 1:
             return self._process_parallel(filepaths, apply_changes, backup, max_workers)
         return [
@@ -307,6 +483,8 @@ class CleanPipeline:
                         apply_changes=apply_changes,
                         backup=backup,
                         enable_syntax=self.enable_syntax_healing,
+                        enable_modernize=self.enable_modernizer,
+                        enable_dead_code=self.enable_dead_code_pruning,
                         enable_imports=self.enable_import_resolution,
                         enable_lint=self.enable_lint_fixing,
                         enable_format=self.enable_formatting,
@@ -329,20 +507,30 @@ class _StandaloneWorkerTask:
     apply_changes: bool
     backup: bool
     enable_syntax: bool
+    enable_modernize: bool
+    enable_dead_code: bool
     enable_imports: bool
     enable_lint: bool
     enable_format: bool
     custom_import_map: dict[str, str] | None = None
+    verify_proofs: bool = False
+    proof_iterations: int = 50
+    proof_seed: int | None = None
 
 
 def _process_file_standalone(task: _StandaloneWorkerTask) -> CleanResult:
     """Standalone function for ProcessPoolExecutor (must be module-level and picklable)."""
     pipeline = CleanPipeline(
         enable_syntax_healing=task.enable_syntax,
+        enable_modernizer=task.enable_modernize,
+        enable_dead_code_pruning=task.enable_dead_code,
         enable_import_resolution=task.enable_imports,
         enable_lint_fixing=task.enable_lint,
         enable_formatting=task.enable_format,
         custom_import_map=task.custom_import_map,
+        verify_proofs=task.verify_proofs,
+        proof_iterations=task.proof_iterations,
+        proof_seed=task.proof_seed,
     )
     return pipeline.process_file(
         task.filepath, apply_changes=task.apply_changes, backup=task.backup

pycleaner/security_scanner.py

@@ -9,11 +9,13 @@ and assert statements used for input validation.
 from __future__ import annotations
 
 import ast
-import os
 import re
+from collections.abc import Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
 
+from pycleaner.discovery import collect_project_python_files
+
 
 @dataclass(slots=True)
 class SecurityFinding:
@@ -194,6 +196,35 @@ class _DangerousCallDetector(ast.NodeVisitor):
             return self.source_lines[lineno - 1].strip()
         return ""
 
+    def _is_suppressed(self, lineno: int, category: str) -> bool:
+        if 1 <= lineno <= len(self.source_lines):
+            line = self.source_lines[lineno - 1]
+            if "#" in line:
+                comment = line.split("#", 1)[1].strip().lower()
+                if (
+                    "nosec" in comment
+                    or "noqa" in comment
+                    or "pycleaner: ignore" in comment
+                    or f"ignore[{category.lower()}]" in comment
+                ):
+                    return True
+            if lineno > 1:
+                prev_line = self.source_lines[lineno - 2]
+                if prev_line.strip().startswith("#"):
+                    prev_comment = prev_line.strip().lstrip("#").strip().lower()
+                    if (
+                        "nosec" in prev_comment
+                        or "noqa" in prev_comment
+                        or "pycleaner: ignore" in prev_comment
+                        or f"ignore[{category.lower()}]" in prev_comment
+                    ):
+                        return True
+        return False
+
+    def _add_finding(self, finding: SecurityFinding) -> None:
+        if not self._is_suppressed(finding.lineno, finding.category):
+            self.findings.append(finding)
+
     def visit_Call(self, node: ast.Call) -> None:
         func_name = self._get_call_name(node)
         if func_name:
@@ -212,7 +243,7 @@ class _DangerousCallDetector(ast.NodeVisitor):
     def visit_Assert(self, node: ast.Assert) -> None:
         if self._is_test_path(self.filepath):
             return
-        self.findings.append(
+        self._add_finding(
             SecurityFinding(
                 filepath=self.filepath,
                 lineno=node.lineno,
@@ -250,7 +281,7 @@ class _DangerousCallDetector(ast.NodeVisitor):
         spec = _DANGEROUS_CALL_SPECS.get(func_name)
         if spec:
             severity, category, msg, suggestion = spec
-            self.findings.append(
+            self._add_finding(
                 SecurityFinding(
                     filepath=self.filepath,
                     lineno=node.lineno,
@@ -280,7 +311,7 @@ class _DangerousCallDetector(ast.NodeVisitor):
                 and isinstance(kw.value, ast.Constant)
                 and kw.value.value is True
             ):
-                self.findings.append(
+                self._add_finding(
                     SecurityFinding(
                         filepath=self.filepath,
                         lineno=node.lineno,
@@ -319,7 +350,7 @@ class _DangerousCallDetector(ast.NodeVisitor):
                     and isinstance(kw.value, ast.Constant)
                     and kw.value.value is False
                 ):
-                    self.findings.append(
+                    self._add_finding(
                         SecurityFinding(
                             filepath=self.filepath,
                             lineno=node.lineno,
@@ -340,7 +371,7 @@ class _DangerousCallDetector(ast.NodeVisitor):
                     and isinstance(kw.value, ast.Constant)
                     and kw.value.value is True
                 ):
-                    self.findings.append(
+                    self._add_finding(
                         SecurityFinding(
                             filepath=self.filepath,
                             lineno=node.lineno,
@@ -373,7 +404,7 @@ class _DangerousCallDetector(ast.NodeVisitor):
                 ):
                     has_safe_loader = True
             if not has_safe_loader and func_name == "yaml.load":
-                self.findings.append(
+                self._add_finding(
                     SecurityFinding(
                         filepath=self.filepath,
                         lineno=node.lineno,
@@ -407,7 +438,7 @@ class _DangerousCallDetector(ast.NodeVisitor):
             return
 
         if self._is_dynamic_sql_arg(node.args[0]):
-            self.findings.append(
+            self._add_finding(
                 SecurityFinding(
                     filepath=self.filepath,
                     lineno=node.lineno,
@@ -424,23 +455,6 @@ class _DangerousCallDetector(ast.NodeVisitor):
 class SecurityScanner:
     """Scans Python source code for security vulnerabilities."""
 
-    IGNORE_DIRS = frozenset(
-        {
-            ".git",
-            ".venv",
-            "venv",
-            "env",
-            "__pycache__",
-            "build",
-            "dist",
-            ".tox",
-            ".mypy_cache",
-            ".pytest_cache",
-            ".ruff_cache",
-            "site-packages",
-        }
-    )
-
     def __init__(
         self,
         severity_threshold: str = "LOW",
@@ -479,21 +493,19 @@ class SecurityScanner:
 
         return SecurityReport(findings=filtered, files_scanned=1)
 
-    def _discover_project_py_files(self, root: Path) -> list[Path]:
-        py_files: list[Path] = []
-        for current_root, dirs, filenames in os.walk(root):
-            dirs[:] = [
-                d for d in dirs if d not in self.IGNORE_DIRS and not d.startswith(".")
-            ]
-            for fname in filenames:
-                if fname.endswith(".py"):
-                    py_files.append(Path(current_root) / fname)
-        return py_files
-
-    def scan_project(self, root_dir: Path | str) -> SecurityReport:
+    def _discover_project_py_files(
+        self, root: Path, exclude_patterns: Sequence[str] = ()
+    ) -> list[Path]:
+        return collect_project_python_files(root, exclude_patterns=exclude_patterns)
+
+    def scan_project(
+        self, root_dir: Path | str, exclude_patterns: Sequence[str] = ()
+    ) -> SecurityReport:
         """Scan all Python files in a project for security issues."""
         root = Path(root_dir).resolve()
-        py_files = self._discover_project_py_files(root)
+        py_files = self._discover_project_py_files(
+            root, exclude_patterns=exclude_patterns
+        )
         all_findings: list[SecurityFinding] = []
 
         for fpath in py_files:
@@ -532,6 +544,15 @@ class SecurityScanner:
                 filename, stripped.lower()
             ):
                 continue
+            if "#" in line:
+                comment = line.split("#", 1)[1].strip().lower()
+                if (
+                    "nosec" in comment
+                    or "noqa" in comment
+                    or "pycleaner: ignore" in comment
+                    or "ignore[hardcoded-secret]" in comment
+                ):
+                    continue
 
             for name, pattern, suggestion in _SECRET_PATTERNS:
                 if pattern.search(line):