ballpython 2.0.0__py3-none-any.whl → 2.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ballpython/py.typed +0 -0
- ballpython-2.0.3.dist-info/METADATA +103 -0
- ballpython-2.0.3.dist-info/RECORD +34 -0
- pycleaner/__init__.py +39 -2
- pycleaner/baseline.py +127 -0
- pycleaner/cache.py +282 -0
- pycleaner/cli.py +644 -43
- pycleaner/complexity_analyzer.py +16 -28
- pycleaner/config.py +8 -1
- pycleaner/dead_code_detector.py +275 -38
- pycleaner/dependency_auditor.py +175 -75
- pycleaner/discovery.py +206 -0
- pycleaner/explanations.py +323 -0
- pycleaner/frameworks/__init__.py +115 -0
- pycleaner/frameworks/plugins.py +290 -0
- pycleaner/modernizer.py +349 -0
- pycleaner/pipeline.py +208 -20
- pycleaner/py.typed +0 -0
- pycleaner/security_scanner.py +64 -38
- pycleaner/syntax_healer.py +188 -10
- pycleaner/taint_engine.py +8 -8
- pycleaner/test_generator.py +124 -35
- pycleaner/type_checker.py +29 -33
- pycleaner/verifier.py +495 -0
- ballpython-2.0.0.dist-info/METADATA +0 -322
- ballpython-2.0.0.dist-info/RECORD +0 -24
- {ballpython-2.0.0.dist-info → ballpython-2.0.3.dist-info}/WHEEL +0 -0
- {ballpython-2.0.0.dist-info → ballpython-2.0.3.dist-info}/entry_points.txt +0 -0
- {ballpython-2.0.0.dist-info → ballpython-2.0.3.dist-info}/top_level.txt +0 -0
ballpython/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ballpython
|
|
3
|
+
Version: 2.0.3
|
|
4
|
+
Summary: The Ultimate Static Python Intelligence, Healing, Type Verification, and Security Suite
|
|
5
|
+
Author: Developer
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: ruff>=0.1.0
|
|
10
|
+
Requires-Dist: rich>=13.0.0
|
|
11
|
+
Requires-Dist: tomli>=1.1.0; python_version < "3.11"
|
|
12
|
+
Provides-Extra: dev
|
|
13
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
14
|
+
Requires-Dist: black>=24.0.0; extra == "dev"
|
|
15
|
+
Requires-Dist: isort>=5.13.0; extra == "dev"
|
|
16
|
+
Provides-Extra: security
|
|
17
|
+
Requires-Dist: cryptography>=41.0.0; extra == "security"
|
|
18
|
+
Requires-Dist: defusedxml>=0.7.1; extra == "security"
|
|
19
|
+
|
|
20
|
+
# ballpython
|
|
21
|
+
|
|
22
|
+
[](https://pypi.org/project/ballpython/)
|
|
23
|
+
[](https://pypi.org/project/ballpython/)
|
|
24
|
+
[](https://github.com/moonrox420/ball-python/actions/workflows/ci.yml)
|
|
25
|
+
[](https://opensource.org/licenses/MIT)
|
|
26
|
+
|
|
27
|
+
**The zero-config Python code cleaner, syntax healer, and quality suite.**
|
|
28
|
+
|
|
29
|
+
`ballpython` automatically repairs broken syntax, resolves missing imports, prunes dead code, scans for security vulnerabilities, and formats your code in a single command. Zero LLM dependencies, 100% offline, and fast.
|
|
30
|
+
|
|
31
|
+
Both `ballpython` and `pycleaner` CLI commands are identical.
|
|
32
|
+
|
|
33
|
+
---
|
|
34
|
+
|
|
35
|
+
## Installation
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install ballpython
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
---
|
|
42
|
+
|
|
43
|
+
## Quickstart
|
|
44
|
+
|
|
45
|
+
### 1. Fix and format your code
|
|
46
|
+
Run `ballpython` on any file or directory. It automatically heals syntax errors, resolves missing imports, removes unused variables, and applies canonical formatting:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
ballpython .
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
### 2. Check code in CI (Dry Run)
|
|
53
|
+
Check for syntax errors, security vulnerabilities, dead code, and linting violations without modifying any files on disk:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
ballpython check .
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
|
|
61
|
+
## What It Does Automatically
|
|
62
|
+
|
|
63
|
+
When you run `ballpython`, all engines run under the hood in one pass:
|
|
64
|
+
|
|
65
|
+
* **Syntax Healing:** Repairs missing statement colons (`def`, `class`, `if`), accidental `=` assignments in conditionals, unclosed parentheses/brackets, tab/space mixups, and legacy Python 2 except/print statements.
|
|
66
|
+
* **Smart Import Resolution:** Resolves undefined symbols to standard library modules and common aliases, inserts `TYPE_CHECKING` guards, and prunes unused imports.
|
|
67
|
+
* **Formatting & Modernization:** Formats code to PEP 8 standards (powered by `ruff`), modernizes legacy type annotations (PEP 585/604), and removes redundant `object` inheritance.
|
|
68
|
+
* **Dead Code Pruning:** Discovers and eliminates unreachable code, empty pass blocks, and unused private definitions.
|
|
69
|
+
* **Security & Taint Audit:** Detects hardcoded API secrets, SQL/command injection patterns, and unsafe `eval`/`exec`/`pickle` calls.
|
|
70
|
+
* **Type Verification:** Verifies function call signatures against bundled Typeshed stubs.
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
## Pre-Commit Hook
|
|
75
|
+
|
|
76
|
+
Add `ballpython` to your `.pre-commit-config.yaml`:
|
|
77
|
+
|
|
78
|
+
```yaml
|
|
79
|
+
repos:
|
|
80
|
+
- repo: https://github.com/moonrox420/ball-python
|
|
81
|
+
rev: v2.0.3
|
|
82
|
+
hooks:
|
|
83
|
+
- id: pycleaner
|
|
84
|
+
args: ["check"]
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
89
|
+
## Configuration
|
|
90
|
+
|
|
91
|
+
Optional configuration can be placed in `pyproject.toml` under `[tool.pycleaner]`:
|
|
92
|
+
|
|
93
|
+
```toml
|
|
94
|
+
[tool.pycleaner]
|
|
95
|
+
exclude = ["migrations/", "generated/"]
|
|
96
|
+
line-length = 88
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## License
|
|
102
|
+
|
|
103
|
+
MIT License.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
ballpython/__init__.py,sha256=d8ONxgaz9jml_fHPVvg867Vt5jGRhGTqzWPtux3WJOc,206
|
|
2
|
+
ballpython/__main__.py,sha256=aCRr0vnZfKYHDrPHBsYSjX5xTxtFJpTTSPVUSZTBK-I,136
|
|
3
|
+
ballpython/cli.py,sha256=LllCLRAQQ-fRR_lIXwmd2e3Q7DLCIvf-5jHBXsAhJWs,125
|
|
4
|
+
ballpython/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
pycleaner/__init__.py,sha256=dK4jaytHQuXqaK3pqum6o8z8JrRLU-XHUfv_NBOtdxs,2894
|
|
6
|
+
pycleaner/__main__.py,sha256=8QBFrPyxId563F6cipPMNVEBTG8TIZUgxQfp8gk1VSI,135
|
|
7
|
+
pycleaner/baseline.py,sha256=DS32C8_G8khYYNJo42FVv6Cc58lJj7NUmPOzWNfQwlU,3856
|
|
8
|
+
pycleaner/cache.py,sha256=WhCiMkuL6ryWVr3l3XrYLLoI7A3bRF8jNxpOyedNW8M,10761
|
|
9
|
+
pycleaner/cli.py,sha256=NhMMmLcf7RMuKjc0rjCmSGYn5ns_yib2rUbXRJDVaYY,73034
|
|
10
|
+
pycleaner/complexity_analyzer.py,sha256=uGC40cHCcr1tshzfIUw56sodEDySMFR3kGmjo7nq_1c,14977
|
|
11
|
+
pycleaner/config.py,sha256=XhloyaY_9pdwbQuc3QaAPCerLeXz-i1da4xT50h-pjY,8514
|
|
12
|
+
pycleaner/dead_code_detector.py,sha256=1DwtBSZL1ug1OI9-6jcJyxkhqHmJ0F8ZR19POI2zO3c,26549
|
|
13
|
+
pycleaner/dependency_auditor.py,sha256=yQhpN_nfJhEFaBPKVTzndpGas_U2OHyKP_Pifa5YrWA,16070
|
|
14
|
+
pycleaner/discovery.py,sha256=dt3OmHg886hy_Z0p80ccMyDkITQxz4biaJY_zB3MU_g,6151
|
|
15
|
+
pycleaner/explanations.py,sha256=2OOU_l3YL5o_Br_6fP6MF8HlY75-1JRmw3eTYT2lu8M,14219
|
|
16
|
+
pycleaner/import_resolver.py,sha256=JnAysZeYtXlmmM0LEnb9HBdkHnEsNwwp4QlxebRxGNE,32189
|
|
17
|
+
pycleaner/linter_formatter.py,sha256=LB5j66OCASU8WG8QgRFpSX7C2c7_UNMBgkJh2hSWxAY,20857
|
|
18
|
+
pycleaner/modernizer.py,sha256=L9hxXXfdyTwf8itDO58y6bW5LobGo6dg8q537QdiYbM,12946
|
|
19
|
+
pycleaner/pipeline.py,sha256=ZG7_9PNz0jBadFrZoiI7juLtUJXglGQhLgQhrLJqEac,20612
|
|
20
|
+
pycleaner/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
+
pycleaner/security_scanner.py,sha256=h7WSPYf40X-PBlJVOsJL-KaknDaOcWCMhcPwXkUvlbA,21986
|
|
22
|
+
pycleaner/syntax_healer.py,sha256=TlIMKIJhaX9GED3e2JNzNMJ5aXmL9eLJMCu7xKSjdv8,26711
|
|
23
|
+
pycleaner/taint_engine.py,sha256=96cZdDpSPbo5b13zIZY18f_4GJm_tw0clqCZMBtjXMQ,26481
|
|
24
|
+
pycleaner/test_generator.py,sha256=TJsNJSojc6eIsaKZkwMlpnb-MKHuDpNLZCPnMlO-a8M,19663
|
|
25
|
+
pycleaner/type_checker.py,sha256=h09QI_8MRHv83s23myjSYFdMTsWoQ900ffBJHtRuhmE,34087
|
|
26
|
+
pycleaner/typeshed_resolver.py,sha256=N-kWU_CI5DG59lQMkt7wMW9sHYjNCBdfFS42b4y2Kr4,12742
|
|
27
|
+
pycleaner/verifier.py,sha256=pOfSLRNHe4qDUYJG8mxDxjY6Ph8QKBdJZmqBtoJX87w,17937
|
|
28
|
+
pycleaner/frameworks/__init__.py,sha256=7MLgg469hCNtQR6dsE5gZcAZkCywJHz9zOPrbH8NmrE,3415
|
|
29
|
+
pycleaner/frameworks/plugins.py,sha256=K3J4mVJw2MR_pa6Mhmo5oZOX8pUesL8lx9LUo4RLrNo,9390
|
|
30
|
+
ballpython-2.0.3.dist-info/METADATA,sha256=K1nMS6fIKnHpTsJ0mRLaKNv_o1TFuzb7SI_KgywGCaw,3466
|
|
31
|
+
ballpython-2.0.3.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
32
|
+
ballpython-2.0.3.dist-info/entry_points.txt,sha256=6ux-5UxoVSmlK5p9M-GDVt24sCdwTQq9wgRDkpMIYU8,81
|
|
33
|
+
ballpython-2.0.3.dist-info/top_level.txt,sha256=u6lmy8RWZ_saxCZAQTL7EijB4sunegOiVyk5fHjpwLk,21
|
|
34
|
+
ballpython-2.0.3.dist-info/RECORD,,
|
pycleaner/__init__.py
CHANGED
|
@@ -10,19 +10,38 @@ dataflow taint vulnerabilities, and synthesizes automated behavioral test suites
|
|
|
10
10
|
|
|
11
11
|
from __future__ import annotations
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
from pycleaner.discovery import (
|
|
14
|
+
DEFAULT_IGNORED_DIRS,
|
|
15
|
+
PROTECTED_FILE_PATTERNS,
|
|
16
|
+
collect_project_python_files,
|
|
17
|
+
is_ignored_directory,
|
|
18
|
+
is_protected_file,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
__version__ = "2.0.3"
|
|
14
22
|
__all__ = [
|
|
23
|
+
"BaselineFingerprint",
|
|
24
|
+
"BaselineManager",
|
|
15
25
|
"CleanPipeline",
|
|
16
26
|
"CleanResult",
|
|
17
27
|
"ComplexityAnalyzer",
|
|
18
28
|
"ComplexityReport",
|
|
29
|
+
"ContentAddressableCache",
|
|
30
|
+
"CounterExample",
|
|
19
31
|
"DeadCodeDetector",
|
|
32
|
+
"DeadCodeFixResult",
|
|
33
|
+
"DeadCodeFixer",
|
|
20
34
|
"DeadCodeReport",
|
|
21
35
|
"DependencyAuditor",
|
|
22
36
|
"GeneratedTestSuite",
|
|
23
37
|
"ImportResolver",
|
|
38
|
+
"IsolatedDifferentialVerifier",
|
|
24
39
|
"LinterFormatter",
|
|
40
|
+
"ModernizeResult",
|
|
41
|
+
"Modernizer",
|
|
42
|
+
"ProofReceipt",
|
|
25
43
|
"PyCleanerConfig",
|
|
44
|
+
"RuleExplanation",
|
|
26
45
|
"SecurityReport",
|
|
27
46
|
"SecurityScanner",
|
|
28
47
|
"SyntaxHealer",
|
|
@@ -35,15 +54,27 @@ __all__ = [
|
|
|
35
54
|
"TypeFinding",
|
|
36
55
|
"TypeReport",
|
|
37
56
|
"TypeshedResolver",
|
|
57
|
+
"VerificationTier",
|
|
58
|
+
"get_explanation",
|
|
59
|
+
"list_rules",
|
|
38
60
|
"load_config",
|
|
39
61
|
]
|
|
40
62
|
|
|
63
|
+
from pycleaner.baseline import BaselineFingerprint, BaselineManager
|
|
64
|
+
from pycleaner.cache import ContentAddressableCache
|
|
41
65
|
from pycleaner.complexity_analyzer import ComplexityAnalyzer, ComplexityReport
|
|
42
66
|
from pycleaner.config import PyCleanerConfig, load_config
|
|
43
|
-
from pycleaner.dead_code_detector import
|
|
67
|
+
from pycleaner.dead_code_detector import (
|
|
68
|
+
DeadCodeDetector,
|
|
69
|
+
DeadCodeFixer,
|
|
70
|
+
DeadCodeFixResult,
|
|
71
|
+
DeadCodeReport,
|
|
72
|
+
)
|
|
44
73
|
from pycleaner.dependency_auditor import DependencyAuditor
|
|
74
|
+
from pycleaner.explanations import RuleExplanation, get_explanation, list_rules
|
|
45
75
|
from pycleaner.import_resolver import ImportResolver
|
|
46
76
|
from pycleaner.linter_formatter import LinterFormatter
|
|
77
|
+
from pycleaner.modernizer import Modernizer, ModernizeResult
|
|
47
78
|
from pycleaner.pipeline import CleanPipeline, CleanResult
|
|
48
79
|
from pycleaner.security_scanner import SecurityReport, SecurityScanner
|
|
49
80
|
from pycleaner.syntax_healer import SyntaxHealer
|
|
@@ -51,3 +82,9 @@ from pycleaner.taint_engine import TaintEngine, TaintFinding, TaintReport
|
|
|
51
82
|
from pycleaner.test_generator import GeneratedTestSuite, TestCase, TestGenerator
|
|
52
83
|
from pycleaner.type_checker import TypeChecker, TypeFinding, TypeReport
|
|
53
84
|
from pycleaner.typeshed_resolver import TypeshedResolver
|
|
85
|
+
from pycleaner.verifier import (
|
|
86
|
+
CounterExample,
|
|
87
|
+
IsolatedDifferentialVerifier,
|
|
88
|
+
ProofReceipt,
|
|
89
|
+
VerificationTier,
|
|
90
|
+
)
|
pycleaner/baseline.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""
|
|
2
|
+
pycleaner.baseline
|
|
3
|
+
==================
|
|
4
|
+
|
|
5
|
+
Technical Debt Ratchet and Baseline Management Engine.
|
|
6
|
+
|
|
7
|
+
Enables incremental adoption of PyCleaner on legacy codebases by snapshotting
|
|
8
|
+
existing diagnostic violations into `.pycleaner/baseline.json`. Future CI/CD
|
|
9
|
+
checks ensure that existing technical debt can only decrease, never increase.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
import time
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class BaselineFingerprint:
|
|
24
|
+
rule: str
|
|
25
|
+
file: str
|
|
26
|
+
line: int
|
|
27
|
+
symbol: str
|
|
28
|
+
|
|
29
|
+
def to_hash(self) -> str:
|
|
30
|
+
raw = f"{self.rule}:{self.file}:{self.line}:{self.symbol}"
|
|
31
|
+
return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16]
|
|
32
|
+
|
|
33
|
+
def to_dict(self) -> dict[str, Any]:
|
|
34
|
+
return {
|
|
35
|
+
"rule": self.rule,
|
|
36
|
+
"file": self.file,
|
|
37
|
+
"line": self.line,
|
|
38
|
+
"symbol": self.symbol,
|
|
39
|
+
"hash": self.to_hash(),
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class BaselineManager:
|
|
44
|
+
"""Manages baseline generation, loading, and ratchet comparison."""
|
|
45
|
+
|
|
46
|
+
def __init__(self, baseline_path: Path | str = ".pycleaner/baseline.json") -> None:
|
|
47
|
+
self.baseline_path = Path(baseline_path)
|
|
48
|
+
|
|
49
|
+
def load_fingerprints(self) -> set[str]:
|
|
50
|
+
"""Loads baseline issue hashes from disk."""
|
|
51
|
+
if not self.baseline_path.exists():
|
|
52
|
+
return set()
|
|
53
|
+
try:
|
|
54
|
+
data = json.loads(self.baseline_path.read_text(encoding="utf-8"))
|
|
55
|
+
fingerprints = data.get("fingerprints", [])
|
|
56
|
+
return {
|
|
57
|
+
fp.get("hash")
|
|
58
|
+
for fp in fingerprints
|
|
59
|
+
if isinstance(fp, dict) and fp.get("hash")
|
|
60
|
+
}
|
|
61
|
+
except json.JSONDecodeError:
|
|
62
|
+
return set()
|
|
63
|
+
|
|
64
|
+
def save_baseline(
|
|
65
|
+
self,
|
|
66
|
+
fingerprints: list[BaselineFingerprint],
|
|
67
|
+
root_dir: Path,
|
|
68
|
+
) -> Path:
|
|
69
|
+
"""Serializes fingerprints to the baseline JSON file."""
|
|
70
|
+
self.baseline_path.parent.mkdir(parents=True, exist_ok=True)
|
|
71
|
+
payload = {
|
|
72
|
+
"version": 1,
|
|
73
|
+
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
74
|
+
"root_dir": str(root_dir.resolve()),
|
|
75
|
+
"issue_count": len(fingerprints),
|
|
76
|
+
"fingerprints": [fp.to_dict() for fp in fingerprints],
|
|
77
|
+
}
|
|
78
|
+
self.baseline_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
|
79
|
+
return self.baseline_path
|
|
80
|
+
|
|
81
|
+
@staticmethod
|
|
82
|
+
def create_fingerprint(
|
|
83
|
+
rule: str,
|
|
84
|
+
file_path: Path | str,
|
|
85
|
+
line: int,
|
|
86
|
+
symbol: str,
|
|
87
|
+
root_dir: Path | None = None,
|
|
88
|
+
) -> BaselineFingerprint:
|
|
89
|
+
"""Constructs a deterministic fingerprint with normalized relative path."""
|
|
90
|
+
p = Path(file_path)
|
|
91
|
+
if root_dir is not None:
|
|
92
|
+
try:
|
|
93
|
+
rel = str(p.resolve().relative_to(root_dir.resolve())).replace(
|
|
94
|
+
"\\", "/"
|
|
95
|
+
)
|
|
96
|
+
except ValueError:
|
|
97
|
+
rel = p.name
|
|
98
|
+
else:
|
|
99
|
+
rel = p.name
|
|
100
|
+
|
|
101
|
+
return BaselineFingerprint(
|
|
102
|
+
rule=rule,
|
|
103
|
+
file=rel,
|
|
104
|
+
line=line,
|
|
105
|
+
symbol=symbol,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
def filter_new_issues(
|
|
109
|
+
self,
|
|
110
|
+
issues: list[BaselineFingerprint],
|
|
111
|
+
) -> tuple[list[BaselineFingerprint], list[BaselineFingerprint]]:
|
|
112
|
+
"""
|
|
113
|
+
Separates issues into (tolerated_baseline_issues, new_debt_issues).
|
|
114
|
+
Returns:
|
|
115
|
+
(tolerated, new_debt)
|
|
116
|
+
"""
|
|
117
|
+
known_hashes = self.load_fingerprints()
|
|
118
|
+
tolerated: list[BaselineFingerprint] = []
|
|
119
|
+
new_debt: list[BaselineFingerprint] = []
|
|
120
|
+
|
|
121
|
+
for issue in issues:
|
|
122
|
+
if issue.to_hash() in known_hashes:
|
|
123
|
+
tolerated.append(issue)
|
|
124
|
+
else:
|
|
125
|
+
new_debt.append(issue)
|
|
126
|
+
|
|
127
|
+
return tolerated, new_debt
|
pycleaner/cache.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
"""
|
|
2
|
+
pycleaner.cache
|
|
3
|
+
===============
|
|
4
|
+
|
|
5
|
+
Content-Addressable Incremental Cache and Provenance Ledger for PyCleaner.
|
|
6
|
+
|
|
7
|
+
Provides:
|
|
8
|
+
- SQLite-backed, WAL-journaled content-addressable caching keyed by SHA-256(content + config).
|
|
9
|
+
- Dependency invalidation tracking via AST-extracted import graphs.
|
|
10
|
+
- Audit provenance ledger recording every verified repair with seed and diff receipts.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import ast
|
|
16
|
+
import hashlib
|
|
17
|
+
import json
|
|
18
|
+
import sqlite3
|
|
19
|
+
import time
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import TYPE_CHECKING, Any
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from pycleaner.pipeline import CleanResult
|
|
25
|
+
from pycleaner.verifier import VerificationTier
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def compute_content_hash(source: str, config_hash: str = "") -> str:
|
|
29
|
+
"""Computes a deterministic SHA-256 hash of the source code and configuration."""
|
|
30
|
+
hasher = hashlib.sha256()
|
|
31
|
+
hasher.update(source.encode("utf-8"))
|
|
32
|
+
if config_hash:
|
|
33
|
+
hasher.update(config_hash.encode("utf-8"))
|
|
34
|
+
return hasher.hexdigest()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def extract_file_dependencies(source: str) -> list[str]:
|
|
38
|
+
"""Extracts imported module names using AST analysis for dependency tracking."""
|
|
39
|
+
deps: list[str] = []
|
|
40
|
+
try:
|
|
41
|
+
tree = ast.parse(source)
|
|
42
|
+
for node in ast.walk(tree):
|
|
43
|
+
if isinstance(node, ast.Import):
|
|
44
|
+
for alias in node.names:
|
|
45
|
+
deps.append(alias.name.split(".")[0])
|
|
46
|
+
elif isinstance(node, ast.ImportFrom) and node.module:
|
|
47
|
+
deps.append(node.module.split(".")[0])
|
|
48
|
+
except SyntaxError:
|
|
49
|
+
pass # Best-effort dependency extraction; syntax errors handled downstream
|
|
50
|
+
return sorted(set(deps))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class ContentAddressableCache:
|
|
54
|
+
"""
|
|
55
|
+
High-performance incremental cache backed by SQLite with write-ahead logging (WAL).
|
|
56
|
+
Guarantees sub-millisecond cache lookups for unchanged files.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def __init__(self, db_path: Path | str = ".pycleaner/cache.db") -> None:
|
|
60
|
+
self.db_path = Path(db_path)
|
|
61
|
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
62
|
+
self._init_db()
|
|
63
|
+
|
|
64
|
+
def _get_connection(self) -> sqlite3.Connection:
|
|
65
|
+
conn = sqlite3.connect(str(self.db_path), timeout=10.0)
|
|
66
|
+
conn.execute("PRAGMA journal_mode=WAL;")
|
|
67
|
+
conn.execute("PRAGMA synchronous=NORMAL;")
|
|
68
|
+
conn.row_factory = sqlite3.Row
|
|
69
|
+
return conn
|
|
70
|
+
|
|
71
|
+
def _init_db(self) -> None:
|
|
72
|
+
with self._get_connection() as conn:
|
|
73
|
+
conn.execute("""
|
|
74
|
+
CREATE TABLE IF NOT EXISTS file_cache (
|
|
75
|
+
content_hash TEXT PRIMARY KEY,
|
|
76
|
+
file_path TEXT NOT NULL,
|
|
77
|
+
config_hash TEXT NOT NULL,
|
|
78
|
+
cleaned_code TEXT NOT NULL,
|
|
79
|
+
is_valid_python INTEGER NOT NULL,
|
|
80
|
+
changed INTEGER NOT NULL,
|
|
81
|
+
error TEXT,
|
|
82
|
+
syntax_repairs TEXT NOT NULL,
|
|
83
|
+
modernize_transforms TEXT NOT NULL,
|
|
84
|
+
dead_code_pruned TEXT NOT NULL,
|
|
85
|
+
resolved_imports TEXT NOT NULL,
|
|
86
|
+
unresolved_symbols TEXT NOT NULL,
|
|
87
|
+
lint_changed INTEGER NOT NULL,
|
|
88
|
+
format_changed INTEGER NOT NULL,
|
|
89
|
+
verification_tier TEXT,
|
|
90
|
+
dependencies TEXT NOT NULL,
|
|
91
|
+
updated_at REAL NOT NULL
|
|
92
|
+
);
|
|
93
|
+
""")
|
|
94
|
+
conn.execute("""
|
|
95
|
+
CREATE INDEX IF NOT EXISTS idx_file_cache_path
|
|
96
|
+
ON file_cache (file_path);
|
|
97
|
+
""")
|
|
98
|
+
conn.execute("""
|
|
99
|
+
CREATE TABLE IF NOT EXISTS provenance_ledger (
|
|
100
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
101
|
+
timestamp TEXT NOT NULL,
|
|
102
|
+
file_path TEXT NOT NULL,
|
|
103
|
+
transformation_type TEXT NOT NULL,
|
|
104
|
+
verification_tier TEXT NOT NULL,
|
|
105
|
+
seed INTEGER,
|
|
106
|
+
diff TEXT NOT NULL
|
|
107
|
+
);
|
|
108
|
+
""")
|
|
109
|
+
conn.execute("""
|
|
110
|
+
CREATE INDEX IF NOT EXISTS idx_provenance_path
|
|
111
|
+
ON provenance_ledger (file_path);
|
|
112
|
+
""")
|
|
113
|
+
conn.commit()
|
|
114
|
+
|
|
115
|
+
def get(
|
|
116
|
+
self,
|
|
117
|
+
file_path: Path | str,
|
|
118
|
+
source: str,
|
|
119
|
+
config_hash: str = "",
|
|
120
|
+
) -> CleanResult | None:
|
|
121
|
+
"""
|
|
122
|
+
Retrieves a cached CleanResult if content_hash matches.
|
|
123
|
+
Returns None on cache miss.
|
|
124
|
+
"""
|
|
125
|
+
expected_hash = compute_content_hash(source, config_hash)
|
|
126
|
+
from pycleaner.pipeline import CleanResult
|
|
127
|
+
|
|
128
|
+
with self._get_connection() as conn:
|
|
129
|
+
cursor = conn.execute(
|
|
130
|
+
"""
|
|
131
|
+
SELECT * FROM file_cache
|
|
132
|
+
WHERE content_hash = ?;
|
|
133
|
+
""",
|
|
134
|
+
(expected_hash,),
|
|
135
|
+
)
|
|
136
|
+
row = cursor.fetchone()
|
|
137
|
+
if row is None:
|
|
138
|
+
return None
|
|
139
|
+
|
|
140
|
+
tier = None
|
|
141
|
+
if row["verification_tier"]:
|
|
142
|
+
try:
|
|
143
|
+
tier = VerificationTier(row["verification_tier"])
|
|
144
|
+
except ValueError:
|
|
145
|
+
tier = None
|
|
146
|
+
|
|
147
|
+
return CleanResult(
|
|
148
|
+
path=Path(row["file_path"]),
|
|
149
|
+
original_code=source,
|
|
150
|
+
cleaned_code=row["cleaned_code"],
|
|
151
|
+
changed=bool(row["changed"]),
|
|
152
|
+
is_valid_python=bool(row["is_valid_python"]),
|
|
153
|
+
syntax_repairs=json.loads(row["syntax_repairs"]),
|
|
154
|
+
modernize_transforms=json.loads(row["modernize_transforms"]),
|
|
155
|
+
dead_code_pruned=json.loads(row["dead_code_pruned"]),
|
|
156
|
+
resolved_imports=json.loads(row["resolved_imports"]),
|
|
157
|
+
unresolved_symbols=json.loads(row["unresolved_symbols"]),
|
|
158
|
+
lint_changed=bool(row["lint_changed"]),
|
|
159
|
+
format_changed=bool(row["format_changed"]),
|
|
160
|
+
error=row["error"],
|
|
161
|
+
verification_tier=tier,
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
def set(
|
|
165
|
+
self,
|
|
166
|
+
file_path: Path | str,
|
|
167
|
+
source: str,
|
|
168
|
+
result: CleanResult,
|
|
169
|
+
config_hash: str = "",
|
|
170
|
+
) -> None:
|
|
171
|
+
"""Stores or updates the CleanResult in the incremental cache keyed by content_hash."""
|
|
172
|
+
canonical_path = str(Path(file_path).resolve())
|
|
173
|
+
content_hash = compute_content_hash(source, config_hash)
|
|
174
|
+
deps = extract_file_dependencies(source)
|
|
175
|
+
tier_str = result.verification_tier.value if result.verification_tier else None
|
|
176
|
+
|
|
177
|
+
with self._get_connection() as conn:
|
|
178
|
+
conn.execute(
|
|
179
|
+
"""
|
|
180
|
+
INSERT OR REPLACE INTO file_cache (
|
|
181
|
+
content_hash, file_path, config_hash, cleaned_code,
|
|
182
|
+
is_valid_python, changed, error, syntax_repairs,
|
|
183
|
+
modernize_transforms, dead_code_pruned, resolved_imports,
|
|
184
|
+
unresolved_symbols, lint_changed, format_changed,
|
|
185
|
+
verification_tier, dependencies, updated_at
|
|
186
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
|
|
187
|
+
""",
|
|
188
|
+
(
|
|
189
|
+
content_hash,
|
|
190
|
+
canonical_path,
|
|
191
|
+
config_hash,
|
|
192
|
+
result.cleaned_code,
|
|
193
|
+
1 if result.is_valid_python else 0,
|
|
194
|
+
1 if result.changed else 0,
|
|
195
|
+
result.error,
|
|
196
|
+
json.dumps(result.syntax_repairs),
|
|
197
|
+
json.dumps(result.modernize_transforms),
|
|
198
|
+
json.dumps(result.dead_code_pruned),
|
|
199
|
+
json.dumps(result.resolved_imports),
|
|
200
|
+
json.dumps(result.unresolved_symbols),
|
|
201
|
+
1 if result.lint_changed else 0,
|
|
202
|
+
1 if result.format_changed else 0,
|
|
203
|
+
tier_str,
|
|
204
|
+
json.dumps(deps),
|
|
205
|
+
time.time(),
|
|
206
|
+
),
|
|
207
|
+
)
|
|
208
|
+
conn.commit()
|
|
209
|
+
|
|
210
|
+
def record_provenance(
|
|
211
|
+
self,
|
|
212
|
+
file_path: Path | str,
|
|
213
|
+
transformation_type: str,
|
|
214
|
+
verification_tier: str,
|
|
215
|
+
seed: int | None,
|
|
216
|
+
diff: str,
|
|
217
|
+
) -> None:
|
|
218
|
+
"""Records an immutable audit entry into the provenance ledger."""
|
|
219
|
+
canonical_path = str(Path(file_path).resolve())
|
|
220
|
+
now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
221
|
+
with self._get_connection() as conn:
|
|
222
|
+
conn.execute(
|
|
223
|
+
"""
|
|
224
|
+
INSERT INTO provenance_ledger (
|
|
225
|
+
timestamp, file_path, transformation_type,
|
|
226
|
+
verification_tier, seed, diff
|
|
227
|
+
) VALUES (?, ?, ?, ?, ?, ?);
|
|
228
|
+
""",
|
|
229
|
+
(
|
|
230
|
+
now,
|
|
231
|
+
canonical_path,
|
|
232
|
+
transformation_type,
|
|
233
|
+
verification_tier,
|
|
234
|
+
seed,
|
|
235
|
+
diff,
|
|
236
|
+
),
|
|
237
|
+
)
|
|
238
|
+
conn.commit()
|
|
239
|
+
|
|
240
|
+
def get_provenance(self, limit: int = 50) -> list[dict[str, Any]]:
|
|
241
|
+
"""Returns the most recent records from the provenance ledger."""
|
|
242
|
+
with self._get_connection() as conn:
|
|
243
|
+
cursor = conn.execute(
|
|
244
|
+
"""
|
|
245
|
+
SELECT timestamp, file_path, transformation_type,
|
|
246
|
+
verification_tier, seed, diff
|
|
247
|
+
FROM provenance_ledger
|
|
248
|
+
ORDER BY id DESC
|
|
249
|
+
LIMIT ?;
|
|
250
|
+
""",
|
|
251
|
+
(limit,),
|
|
252
|
+
)
|
|
253
|
+
return [dict(row) for row in cursor.fetchall()]
|
|
254
|
+
|
|
255
|
+
def invalidate(self, file_path: Path | str) -> None:
|
|
256
|
+
"""Invalidates the cache entry for a specific file path."""
|
|
257
|
+
canonical_path = str(Path(file_path).resolve())
|
|
258
|
+
with self._get_connection() as conn:
|
|
259
|
+
conn.execute(
|
|
260
|
+
"DELETE FROM file_cache WHERE file_path = ?;", (canonical_path,)
|
|
261
|
+
)
|
|
262
|
+
conn.commit()
|
|
263
|
+
|
|
264
|
+
def clear(self) -> None:
|
|
265
|
+
"""Clears all entries from the cache database."""
|
|
266
|
+
with self._get_connection() as conn:
|
|
267
|
+
conn.execute("DELETE FROM file_cache;")
|
|
268
|
+
conn.commit()
|
|
269
|
+
|
|
270
|
+
def get_stats(self) -> dict[str, Any]:
|
|
271
|
+
"""Returns statistics on cache utilization and database storage."""
|
|
272
|
+
with self._get_connection() as conn:
|
|
273
|
+
cursor = conn.execute("SELECT COUNT(*) as cnt FROM file_cache;")
|
|
274
|
+
total_entries = cursor.fetchone()["cnt"]
|
|
275
|
+
cursor = conn.execute("SELECT COUNT(*) as cnt FROM provenance_ledger;")
|
|
276
|
+
provenance_entries = cursor.fetchone()["cnt"]
|
|
277
|
+
db_size = self.db_path.stat().st_size if self.db_path.exists() else 0
|
|
278
|
+
return {
|
|
279
|
+
"file_cache_entries": total_entries,
|
|
280
|
+
"provenance_entries": provenance_entries,
|
|
281
|
+
"db_size_bytes": db_size,
|
|
282
|
+
}
|