ballpython 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

pycleaner/dependency_auditor.py

@@ -10,13 +10,16 @@ from __future__ import annotations
 
 import ast
 import importlib.metadata
-import os
 import re
+import shutil
 import sys
+from collections.abc import Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import ClassVar
 
+from pycleaner.discovery import collect_project_python_files
+
 
 @dataclass(slots=True)
 class DependencyAuditReport:
@@ -57,11 +60,30 @@ class DependencyAuditor:
         "OpenGL": "PyOpenGL",
         "attr": "attrs",
         "google": "protobuf",
+        # High-frequency PyPI packages
+        "nmap": "python-nmap",
+        "dns": "dnspython",
+        "telegram": "python-telegram-bot",
+        "discord": "discord.py",
+        "pkg_resources": "setuptools",
+        "setuptools": "setuptools",
+        "pydantic_core": "pydantic-core",
+        "multipart": "python-multipart",
+        "kafka": "kafka-python",
+        "smart_open": "smart-open",
+        "MySQLdb": "mysqlclient",
+        "psycopg2": "psycopg2-binary",
+        "redis": "redis",
+        "playwright": "playwright",
     }
 
-    def __init__(self, root_dir: str | Path) -> None:
+    def __init__(
+        self, root_dir: str | Path, exclude_patterns: Sequence[str] = ()
+    ) -> None:
         self.root_dir = Path(root_dir).resolve()
+        self.exclude_patterns = tuple(exclude_patterns)
         self.stdlib_names = set(sys.stdlib_module_names)
+        self._last_optional_packages: set[str] = set()
         try:
             self.dist_map = importlib.metadata.packages_distributions()
         except AttributeError:
@@ -79,78 +101,106 @@ class DependencyAuditor:
             "pylint",
             "build",
             "twine",
-            "pip",
             "wheel",
             "setuptools",
+            "pip",
+            "pre-commit",
+            "coverage",
+            "pytest-cov",
+            "tox",
+            "nox",
         }
     )
 
     @staticmethod
     def canonicalize_name(name: str) -> str:
-        """Canonicalize package name per PEP 503 (lowercase, dashes instead of underscores)."""
+        """Normalize package name per PEP 503."""
         return re.sub(r"[-_.]+", "-", name).lower()
 
-    @staticmethod
-    def _extract_imports_from_file(filepath: Path) -> set[str]:
-        """Extract root imported module names from a single Python file."""
-        modules: set[str] = set()
+    def _extract_imports_from_file(self, file_path: Path) -> set[str]:
+        imports: set[str] = set()
         try:
-            content = filepath.read_text(encoding="utf-8", errors="replace")
-            tree = ast.parse(content, filename=str(filepath))
+            tree = ast.parse(file_path.read_text(encoding="utf-8", errors="ignore"))
         except SyntaxError:
-            return modules
+            return imports
 
         for node in ast.walk(tree):
             if isinstance(node, ast.Import):
                 for alias in node.names:
-                    modules.add(alias.name.split(".")[0])
-            elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
-                modules.add(node.module.split(".")[0])
-        return modules
-
-    def scan_codebase_imports(self) -> set[str]:
-        """Traverse the project directory and extract all imported root module names."""
-        ignore_dirs = {
-            ".git",
-            ".venv",
-            "venv",
-            "env",
-            "__pycache__",
-            "build",
-            "dist",
-            ".tox",
-            ".mypy_cache",
-            ".pytest_cache",
-            ".ruff_cache",
-            "site-packages",
-        }
-        imported_modules: set[str] = set()
-        for current_root, dirs, files in os.walk(self.root_dir):
-            dirs[:] = [
-                d for d in dirs if d not in ignore_dirs and not d.startswith(".")
+                    root_pkg = alias.name.split(".")[0]
+                    imports.add(root_pkg)
+            elif isinstance(node, ast.ImportFrom):
+                if node.level == 0 and node.module:
+                    root_pkg = node.module.split(".")[0]
+                    imports.add(root_pkg)
+        return imports
+
+    def scan_codebase_imports(
+        self, target_files: Sequence[Path] | None = None
+    ) -> set[str]:
+        """Scan Python files across the project or given targets for imported modules."""
+        if target_files is not None:
+            files_to_scan = [
+                f for f in target_files if f.is_file() and f.suffix == ".py"
             ]
-            for filename in files:
-                if filename.endswith(".py"):
-                    imported_modules.update(
-                        self._extract_imports_from_file(Path(current_root) / filename)
-                    )
+        else:
+            files_to_scan = collect_project_python_files(
+                self.root_dir, exclude_patterns=self.exclude_patterns
+            )
+        imported_modules: set[str] = set()
+        for py_file in files_to_scan:
+            imported_modules.update(self._extract_imports_from_file(py_file))
         return imported_modules
 
     def identify_local_modules(self) -> set[str]:
         """Identify local modules and packages belonging to the current project."""
         local_mods: set[str] = set()
-        search_dirs = [self.root_dir]
 
+        # 1. Project root folder itself is a primary local package / namespace candidate
+        local_mods.add(self.root_dir.name)
+        local_mods.add(self.root_dir.stem)
+
+        # 2. Extract package name from pyproject.toml if present
+        pyproject_file = self.root_dir / "pyproject.toml"
+        if pyproject_file.is_file():
+            try:
+                import tomllib
+            except ImportError:
+                import tomli as tomllib  # type: ignore
+            try:
+                data = tomllib.loads(
+                    pyproject_file.read_text(encoding="utf-8", errors="ignore")
+                )
+                proj_name = data.get("project", {}).get("name") or data.get(
+                    "tool", {}
+                ).get("poetry", {}).get("name")
+                if proj_name:
+                    local_mods.add(str(proj_name))
+                    local_mods.add(str(proj_name).replace("-", "_"))
+            except Exception:
+                pass  # Ignore invalid or non-standard pyproject.toml configuration
+
+        # 3. Discover local .py files and subpackages (including PEP 420 namespace packages)
+        search_dirs = [self.root_dir]
         src_dir = self.root_dir / "src"
         if src_dir.is_dir():
             search_dirs.append(src_dir)
 
         for sdir in search_dirs:
-            for item in sdir.iterdir():
-                if item.is_file() and item.suffix == ".py":
-                    local_mods.add(item.stem)
-                elif item.is_dir() and (item / "__init__.py").exists():
-                    local_mods.add(item.name)
+            try:
+                for item in sdir.iterdir():
+                    if item.is_file() and item.suffix == ".py":
+                        local_mods.add(item.stem)
+                    elif item.is_dir() and not item.name.startswith((".", "__")):
+                        # Standard package or PEP 420 namespace package with python files inside
+                        if (item / "__init__.py").exists() or any(
+                            sub.suffix == ".py"
+                            for sub in item.iterdir()
+                            if sub.is_file()
+                        ):
+                            local_mods.add(item.name)
+            except OSError:
+                continue
 
         return local_mods
 
@@ -171,22 +221,23 @@ class DependencyAuditor:
         if not req_file.exists():
             return declared
 
-        for line in req_file.read_text(encoding="utf-8").splitlines():
+        for line in req_file.read_text(encoding="utf-8", errors="ignore").splitlines():
             line = line.strip()
             if not line or line.startswith(("#", "-")):
                 continue
-            pkg_match = re.match(r"^([a-zA-Z0-9_\-\.]+)", line)
+            pkg_match = re.match(r"^([a-zA-Z0-9_.-]+)", line)
             if pkg_match:
                 pkg_name = pkg_match.group(1)
                 canon = self.canonicalize_name(pkg_name)
                 declared[canon] = line
         return declared
 
-    def _read_pyproject_toml(self) -> dict[str, str]:
+    def _read_pyproject_toml(self) -> tuple[dict[str, str], set[str]]:
         declared: dict[str, str] = {}
+        optional_pkgs: set[str] = set()
         pyproject_file = self.root_dir / "pyproject.toml"
         if not pyproject_file.exists():
-            return declared
+            return declared, optional_pkgs
 
         try:
             import tomllib
@@ -194,23 +245,45 @@ class DependencyAuditor:
             import tomli as tomllib  # type: ignore
 
         try:
-            data = tomllib.loads(pyproject_file.read_text(encoding="utf-8"))
-            project_deps = data.get("project", {}).get("dependencies", [])
-            for dep in project_deps:
-                pkg_match = re.match(r"^([a-zA-Z0-9_\-\.]+)", dep.strip())
-                if pkg_match:
-                    pkg_name = pkg_match.group(1)
-                    canon = self.canonicalize_name(pkg_name)
-                    declared[canon] = dep
-        except tomllib.TOMLDecodeError:
-            # Fall back to requirements.txt if pyproject.toml is malformed
-            pass
-        return declared
+            data = tomllib.loads(
+                pyproject_file.read_text(encoding="utf-8", errors="ignore")
+            )
+            project_data = data.get("project", {})
+            for dep in project_data.get("dependencies", []):
+                if isinstance(dep, str):
+                    pkg_match = re.match(r"^([a-zA-Z0-9_.-]+)", dep.strip())
+                    if pkg_match:
+                        canon = self.canonicalize_name(pkg_match.group(1))
+                        declared[canon] = dep
+
+            for opt_list in project_data.get("optional-dependencies", {}).values():
+                if isinstance(opt_list, list):
+                    for dep in opt_list:
+                        if isinstance(dep, str):
+                            pkg_match = re.match(r"^([a-zA-Z0-9_.-]+)", dep.strip())
+                            if pkg_match:
+                                canon = self.canonicalize_name(pkg_match.group(1))
+                                declared[canon] = dep
+                                optional_pkgs.add(canon)
+
+            for grp_list in data.get("dependency-groups", {}).values():
+                if isinstance(grp_list, list):
+                    for dep in grp_list:
+                        if isinstance(dep, str):
+                            pkg_match = re.match(r"^([a-zA-Z0-9_.-]+)", dep.strip())
+                            if pkg_match:
+                                canon = self.canonicalize_name(pkg_match.group(1))
+                                declared[canon] = dep
+                                optional_pkgs.add(canon)
+        except Exception:
+            pass  # Best-effort reading; continue if pyproject.toml is unparseable
+        return declared, optional_pkgs
 
     def read_declared_dependencies(self) -> dict[str, str]:
         """Read declared dependencies from requirements.txt or pyproject.toml."""
         declared = self._read_requirements_txt()
-        declared.update(self._read_pyproject_toml())
+        pyproj_declared, self._last_optional_packages = self._read_pyproject_toml()
+        declared.update(pyproj_declared)
         return declared
 
     def _resolve_expected_distributions(
@@ -224,17 +297,23 @@ class DependencyAuditor:
         return expected_dists
 
     def _find_missing_and_unused(
-        self, expected_dists: dict[str, str], declared: dict[str, str]
+        self,
+        expected_dists: dict[str, str],
+        declared: dict[str, str],
+        optional_packages: set[str] | None = None,
     ) -> tuple[set[str], set[str]]:
         missing_packages = {
             orig_name
             for canon, orig_name in expected_dists.items()
             if canon not in declared
         }
+        optional_set = optional_packages or set()
         unused_packages = {
             orig_line
             for canon, orig_line in declared.items()
-            if canon not in expected_dists and canon not in self._KNOWN_DEV_TOOLS
+            if canon not in expected_dists
+            and canon not in self._KNOWN_DEV_TOOLS
+            and canon not in optional_set
         }
         return missing_packages, unused_packages
 
@@ -252,14 +331,19 @@ class DependencyAuditor:
                 f"Unused dependencies (declared but not imported): {', '.join(sorted(unused))}"
             )
         if fixed:
-            details.append("Updated requirements.txt successfully")
+            details.append(
+                "Updated requirements.txt successfully (backup saved as requirements.txt.bak)"
+            )
         return details
 
     def audit(
-        self, fix: bool = False, prune_unused: bool = False
+        self,
+        fix: bool = False,
+        prune_unused: bool = False,
+        target_files: Sequence[Path] | None = None,
     ) -> DependencyAuditReport:
         """Audit dependencies and optionally update requirements.txt."""
-        all_imports = self.scan_codebase_imports()
+        all_imports = self.scan_codebase_imports(target_files=target_files)
         local_mods = self.identify_local_modules()
 
         third_party_mods = {
@@ -271,7 +355,7 @@ class DependencyAuditor:
         declared = self.read_declared_dependencies()
 
         missing_packages, unused_packages = self._find_missing_and_unused(
-            expected_dists, declared
+            expected_dists, declared, optional_packages=self._last_optional_packages
         )
 
         fixed_reqs = False
@@ -297,16 +381,32 @@ class DependencyAuditor:
     def _update_requirements(
         self, missing: set[str], unused_to_remove: set[str]
     ) -> bool:
-        """Append missing dependencies and remove unused ones in requirements.txt."""
+        """Append missing dependencies and remove unused ones in requirements.txt with safety checks."""
         req_file = self.root_dir / "requirements.txt"
         existing_lines: list[str] = []
         if req_file.exists():
-            existing_lines = req_file.read_text(encoding="utf-8").splitlines()
+            # Create a backup before making any changes
+            backup_file = req_file.with_suffix(".txt.bak")
+            try:
+                shutil.copy2(req_file, backup_file)
+            except OSError:
+                pass  # Continue if backup creation fails
+            existing_lines = req_file.read_text(
+                encoding="utf-8", errors="ignore"
+            ).splitlines()
 
         remove_canons = {
             self.canonicalize_name(m.group(1))
             for u in unused_to_remove
-            if (m := re.match(r"^([a-zA-Z0-9_\-\.]+)", u))
+            if (m := re.match(r"^([a-zA-Z0-9_.-]+)", u))
+        }
+
+        # Filter out any local module from being appended to requirements.txt
+        local_canons = {
+            self.canonicalize_name(m) for m in self.identify_local_modules()
+        }
+        safe_missing = {
+            pkg for pkg in missing if self.canonicalize_name(pkg) not in local_canons
         }
 
         new_lines: list[str] = []
@@ -316,7 +416,7 @@ class DependencyAuditor:
                 new_lines.append(line)
                 continue
 
-            pkg_match = re.match(r"^([a-zA-Z0-9_\-\.]+)", stripped)
+            pkg_match = re.match(r"^([a-zA-Z0-9_.-]+)", stripped)
             if pkg_match:
                 pkg_name = pkg_match.group(1)
                 canon = self.canonicalize_name(pkg_name)
@@ -325,7 +425,7 @@ class DependencyAuditor:
             new_lines.append(line)
 
         # Append missing packages
-        new_lines.extend(sorted(missing))
+        new_lines.extend(sorted(safe_missing))
 
         req_file.write_text("\n".join(new_lines) + "\n", encoding="utf-8")
         return True

pycleaner/discovery.py

@@ -0,0 +1,206 @@
+"""
+Centralized file discovery, traversal, and security/safety filtering engine.
+
+Enforces strict exclusion of virtual environments, version control roots,
+cache artifacts, and sensitive files (.env, secrets, keys, lockfiles) across
+all pycleaner analysis, healing, and formatting phases.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import os
+from collections.abc import Sequence
+from pathlib import Path
+
+# Directories that must NEVER be traversed or inspected under any circumstance
+DEFAULT_IGNORED_DIRS: frozenset[str] = frozenset(
+    {
+        ".git",
+        ".svn",
+        ".hg",
+        ".venv",
+        "venv",
+        "env",
+        ".env",
+        "ENV",
+        ".tox",
+        ".nox",
+        ".direnv",
+        "site-packages",
+        "__pycache__",
+        ".mypy_cache",
+        ".ruff_cache",
+        ".pytest_cache",
+        ".pytest_temp",
+        ".release-test",
+        "build",
+        "dist",
+        ".eggs",
+        ".cache",
+        ".idea",
+        ".vscode",
+    }
+)
+
+# Patterns for sensitive, secret, or lock files that must NEVER be cleaned or formatted
+PROTECTED_FILE_PATTERNS: tuple[str, ...] = (
+    ".env*",
+    "*.env",
+    "*secret*",
+    "*credential*",
+    "*.pem",
+    "*.key",
+    "*.cert",
+    "*.crt",
+    "*.pfx",
+    "poetry.lock",
+    "Pipfile.lock",
+    "uv.lock",
+    "pdm.lock",
+    "package-lock.json",
+    "yarn.lock",
+    "pnpm-lock.yaml",
+)
+
+
+def is_ignored_directory(name: str) -> bool:
+    """Check if a directory name should be excluded from file discovery."""
+    if name in DEFAULT_IGNORED_DIRS:
+        return True
+    if name.endswith(".egg-info"):
+        return True
+    if name.startswith(".") and name not in (".", ".."):
+        return True
+    return False
+
+
+def is_protected_file(path: Path | str) -> bool:
+    """Check if a file is protected and must not be touched by formatting or cleaning."""
+    name = Path(path).name
+    for pattern in PROTECTED_FILE_PATTERNS:
+        if fnmatch.fnmatch(name, pattern):
+            return True
+    return False
+
+
+def find_project_root(path: Path | str | None = None) -> Path:
+    """
+    Find the project root containing pyproject.toml, setup.py, requirements.txt, or .git.
+    Traverses upwards from the given path (or current working directory).
+    """
+    start = Path(path).resolve() if path else Path.cwd()
+    if start.is_file():
+        start = start.parent
+
+    for candidate in [start, *start.parents]:
+        if (
+            (candidate / "pyproject.toml").is_file()
+            or (candidate / "setup.py").is_file()
+            or (candidate / "setup.cfg").is_file()
+            or (candidate / "requirements.txt").is_file()
+            or (candidate / ".git").is_dir()
+        ):
+            return candidate
+
+    return start
+
+
+def load_gitignore_patterns(project_root: Path) -> list[str]:
+    """Parse .gitignore rules from the project root if present."""
+    gitignore_file = project_root / ".gitignore"
+    if not gitignore_file.is_file():
+        return []
+
+    patterns: list[str] = []
+    try:
+        for line in gitignore_file.read_text(
+            encoding="utf-8", errors="ignore"
+        ).splitlines():
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            patterns.append(line)
+    except OSError:
+        return []
+    return patterns
+
+
+def matches_gitignore(rel_path: str, patterns: Sequence[str]) -> bool:
+    """Check if a normalized relative POSIX path matches any gitignore pattern."""
+    normalized = Path(rel_path).as_posix()
+    for pat in patterns:
+        pat_clean = pat.rstrip("/")
+        if fnmatch.fnmatch(normalized, pat) or fnmatch.fnmatch(normalized, pat_clean):
+            return True
+        if fnmatch.fnmatch(normalized, f"*/{pat_clean}") or fnmatch.fnmatch(
+            normalized, f"*/{pat}"
+        ):
+            return True
+        for part in normalized.split("/"):
+            if fnmatch.fnmatch(part, pat_clean):
+                return True
+    return False
+
+
+def collect_project_python_files(
+    root: Path | str,
+    exclude_patterns: Sequence[str] = (),
+    respect_gitignore: bool = True,
+) -> list[Path]:
+    """
+    Canonical file discovery engine.
+
+    Recursively scans the target root, filtering out all virtual environments,
+    caches, version-control folders, gitignored files, protected files, and
+    custom exclude patterns. Returns sorted, resolved absolute paths.
+    """
+    target_root = Path(root).resolve()
+    if target_root.is_file():
+        if target_root.suffix == ".py" and not is_protected_file(target_root):
+            return [target_root]
+        return []
+
+    gitignore_rules = load_gitignore_patterns(target_root) if respect_gitignore else []
+    python_files: list[Path] = []
+
+    for current_root, dirs, filenames in os.walk(target_root):
+        # Prune ignored directories in-place to prevent os.walk from recursing into them
+        dirs[:] = [
+            d
+            for d in dirs
+            if not is_ignored_directory(d)
+            and not (respect_gitignore and matches_gitignore(d, gitignore_rules))
+        ]
+
+        for fname in filenames:
+            if not fname.endswith(".py"):
+                continue
+
+            file_path = Path(current_root) / fname
+            if is_protected_file(file_path):
+                continue
+
+            try:
+                rel_path = str(file_path.relative_to(target_root))
+            except ValueError:
+                rel_path = fname
+
+            if respect_gitignore and matches_gitignore(rel_path, gitignore_rules):
+                continue
+
+            if exclude_patterns:
+                rel_posix = Path(rel_path).as_posix()
+                rel_parts = Path(rel_path).parts
+                if any(
+                    fnmatch.fnmatch(rel_posix, pat)
+                    or fnmatch.fnmatch(rel_posix, f"*/{pat}")
+                    or fnmatch.fnmatch(fname, pat)
+                    or pat.rstrip("/\\") in rel_parts
+                    for pat in exclude_patterns
+                ):
+                    continue
+
+            python_files.append(file_path.resolve())
+
+    return sorted(python_files)