ballpython 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ballpython/py.typed +0 -0
- {ballpython-2.0.0.dist-info → ballpython-2.0.2.dist-info}/METADATA +13 -7
- ballpython-2.0.2.dist-info/RECORD +34 -0
- pycleaner/__init__.py +39 -2
- pycleaner/baseline.py +127 -0
- pycleaner/cache.py +282 -0
- pycleaner/cli.py +644 -43
- pycleaner/complexity_analyzer.py +16 -28
- pycleaner/config.py +8 -1
- pycleaner/dead_code_detector.py +275 -38
- pycleaner/dependency_auditor.py +175 -75
- pycleaner/discovery.py +206 -0
- pycleaner/explanations.py +323 -0
- pycleaner/frameworks/__init__.py +115 -0
- pycleaner/frameworks/plugins.py +290 -0
- pycleaner/modernizer.py +349 -0
- pycleaner/pipeline.py +208 -20
- pycleaner/py.typed +0 -0
- pycleaner/security_scanner.py +59 -38
- pycleaner/syntax_healer.py +188 -10
- pycleaner/taint_engine.py +8 -8
- pycleaner/test_generator.py +124 -35
- pycleaner/type_checker.py +29 -33
- pycleaner/verifier.py +495 -0
- ballpython-2.0.0.dist-info/RECORD +0 -24
- {ballpython-2.0.0.dist-info → ballpython-2.0.2.dist-info}/WHEEL +0 -0
- {ballpython-2.0.0.dist-info → ballpython-2.0.2.dist-info}/entry_points.txt +0 -0
- {ballpython-2.0.0.dist-info → ballpython-2.0.2.dist-info}/top_level.txt +0 -0
ballpython/py.typed
ADDED
|
File without changes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ballpython
|
|
3
|
-
Version: 2.0.
|
|
3
|
+
Version: 2.0.2
|
|
4
4
|
Summary: The Ultimate Static Python Intelligence, Healing, Type Verification, and Security Suite
|
|
5
5
|
Author: Developer
|
|
6
6
|
License-Expression: MIT
|
|
@@ -47,6 +47,7 @@ Zero external LLM dependencies, zero mock modes, and built for deterministic dev
|
|
|
47
47
|
- Modernizes deprecated syntax via `pyupgrade` (`UP`).
|
|
48
48
|
- Applies deterministic PEP 8 formatting (`ruff format` / `black` / pure-Python formatter).
|
|
49
49
|
- Wraps long `from ... import (...)` statements exceeding configured `line-length`.
|
|
50
|
+
- Modernizes legacy typing to PEP 585/604 syntax (`List[T]` -> `list[T]`, `Optional[T]` -> `T | None`), singleton comparisons, and `class X(object)` inheritance via `Modernizer` (disable with `--no-modernize`).
|
|
50
51
|
|
|
51
52
|
4. **Dead Code Detector (`DeadCodeDetector`)**:
|
|
52
53
|
- Builds a cross-file symbol definition and reference graph across a project.
|
|
@@ -54,6 +55,7 @@ Zero external LLM dependencies, zero mock modes, and built for deterministic dev
|
|
|
54
55
|
- Respects public exports in `__all__` and framework decorators (`@app.route`, `@pytest.fixture`, `@abstractmethod`, etc.).
|
|
55
56
|
- Detects unreachable code following unconditional `return`, `raise`, `break`, `continue`, or `sys.exit()`.
|
|
56
57
|
- Identifies empty pass blocks with no comments.
|
|
58
|
+
- Auto-prunes unreachable code, redundant `pass` statements, and `if False:` branches via `DeadCodeFixer` (enable with `dead-code --fix`, disable in the pipeline with `--no-dead-code`).
|
|
57
59
|
|
|
58
60
|
5. **Static Security Scanner (`SecurityScanner`)**:
|
|
59
61
|
- Detects dangerous function calls: `eval()`, `exec()`, `compile()`, `pickle.loads()`, `os.system()`, and unsafe `yaml.load()` lacking `SafeLoader`.
|
|
@@ -103,12 +105,13 @@ pip install -e ".[dev,security]"
|
|
|
103
105
|
### Subcommands
|
|
104
106
|
|
|
105
107
|
#### `fix` (Default Action)
|
|
106
|
-
Heals syntax,
|
|
108
|
+
Heals syntax, modernizes legacy typing (PEP 585/604), prunes dead code, resolves imports, fixes lint violations, and formats code:
|
|
107
109
|
```bash
|
|
108
110
|
py -m pycleaner fix src/
|
|
109
111
|
py -m pycleaner fix --diff path/to/script.py
|
|
110
112
|
py -m pycleaner fix --no-backup src/
|
|
111
113
|
py -m pycleaner fix --parallel --workers 4 src/
|
|
114
|
+
py -m pycleaner fix --no-modernize --no-dead-code src/
|
|
112
115
|
```
|
|
113
116
|
|
|
114
117
|
#### `check` (Dry-Run CI Verification)
|
|
@@ -139,6 +142,7 @@ Finds unused functions, unused classes, empty pass branches, and dead code:
|
|
|
139
142
|
```bash
|
|
140
143
|
py -m pycleaner dead-code .
|
|
141
144
|
py -m pycleaner dead-code --json .
|
|
145
|
+
py -m pycleaner dead-code --fix .
|
|
142
146
|
```
|
|
143
147
|
|
|
144
148
|
#### `audit` (Project Dependency Verification)
|
|
@@ -188,7 +192,7 @@ pycleaner path/to/file.py # Equivalent to: pycleaner fix path/to/file.p
|
|
|
188
192
|
|
|
189
193
|
## Configuration
|
|
190
194
|
|
|
191
|
-
`pycleaner` automatically reads configuration from `pyproject.toml` under `[tool.pycleaner]` or from `.pycleaner.toml`.
|
|
195
|
+
`pycleaner` automatically reads configuration from `pyproject.toml` under `[tool.pycleaner]` or from `.pycleaner.toml`. An explicit file can be forced with `pycleaner --config path/to/pyproject.toml <command> ...`, which overrides target-path auto-discovery.
|
|
192
196
|
|
|
193
197
|
### `pyproject.toml` Example
|
|
194
198
|
|
|
@@ -294,12 +298,14 @@ for func in violations:
|
|
|
294
298
|
from pycleaner import DeadCodeDetector
|
|
295
299
|
|
|
296
300
|
detector = DeadCodeDetector()
|
|
297
|
-
report = detector.
|
|
301
|
+
report = detector.scan_project("src/")
|
|
298
302
|
|
|
299
|
-
|
|
300
|
-
print(f"Unused {item.kind} '{item.name}' at {item.filepath}:{item.lineno}")
|
|
303
|
+
print(f"{report.count} dead code item(s) across {report.files_scanned} file(s).")
|
|
301
304
|
|
|
302
|
-
for item in report.
|
|
305
|
+
for item in report.items:
|
|
306
|
+
print(f"Dead {item.kind} '{item.name}' at {item.filepath}:{item.lineno}")
|
|
307
|
+
|
|
308
|
+
for item in report.by_kind("unreachable"):
|
|
303
309
|
print(f"Unreachable code at {item.filepath}:{item.lineno} ({item.reason})")
|
|
304
310
|
```
|
|
305
311
|
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
ballpython/__init__.py,sha256=d8ONxgaz9jml_fHPVvg867Vt5jGRhGTqzWPtux3WJOc,206
|
|
2
|
+
ballpython/__main__.py,sha256=aCRr0vnZfKYHDrPHBsYSjX5xTxtFJpTTSPVUSZTBK-I,136
|
|
3
|
+
ballpython/cli.py,sha256=LllCLRAQQ-fRR_lIXwmd2e3Q7DLCIvf-5jHBXsAhJWs,125
|
|
4
|
+
ballpython/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
pycleaner/__init__.py,sha256=GMc3bYj3wg2AM_SZn_MRDq4i0f0PTaVq_ITR5spiOgI,2894
|
|
6
|
+
pycleaner/__main__.py,sha256=8QBFrPyxId563F6cipPMNVEBTG8TIZUgxQfp8gk1VSI,135
|
|
7
|
+
pycleaner/baseline.py,sha256=DS32C8_G8khYYNJo42FVv6Cc58lJj7NUmPOzWNfQwlU,3856
|
|
8
|
+
pycleaner/cache.py,sha256=WhCiMkuL6ryWVr3l3XrYLLoI7A3bRF8jNxpOyedNW8M,10761
|
|
9
|
+
pycleaner/cli.py,sha256=NhMMmLcf7RMuKjc0rjCmSGYn5ns_yib2rUbXRJDVaYY,73034
|
|
10
|
+
pycleaner/complexity_analyzer.py,sha256=uGC40cHCcr1tshzfIUw56sodEDySMFR3kGmjo7nq_1c,14977
|
|
11
|
+
pycleaner/config.py,sha256=XhloyaY_9pdwbQuc3QaAPCerLeXz-i1da4xT50h-pjY,8514
|
|
12
|
+
pycleaner/dead_code_detector.py,sha256=1DwtBSZL1ug1OI9-6jcJyxkhqHmJ0F8ZR19POI2zO3c,26549
|
|
13
|
+
pycleaner/dependency_auditor.py,sha256=yQhpN_nfJhEFaBPKVTzndpGas_U2OHyKP_Pifa5YrWA,16070
|
|
14
|
+
pycleaner/discovery.py,sha256=dt3OmHg886hy_Z0p80ccMyDkITQxz4biaJY_zB3MU_g,6151
|
|
15
|
+
pycleaner/explanations.py,sha256=2OOU_l3YL5o_Br_6fP6MF8HlY75-1JRmw3eTYT2lu8M,14219
|
|
16
|
+
pycleaner/import_resolver.py,sha256=JnAysZeYtXlmmM0LEnb9HBdkHnEsNwwp4QlxebRxGNE,32189
|
|
17
|
+
pycleaner/linter_formatter.py,sha256=LB5j66OCASU8WG8QgRFpSX7C2c7_UNMBgkJh2hSWxAY,20857
|
|
18
|
+
pycleaner/modernizer.py,sha256=L9hxXXfdyTwf8itDO58y6bW5LobGo6dg8q537QdiYbM,12946
|
|
19
|
+
pycleaner/pipeline.py,sha256=ZG7_9PNz0jBadFrZoiI7juLtUJXglGQhLgQhrLJqEac,20612
|
|
20
|
+
pycleaner/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
|
+
pycleaner/security_scanner.py,sha256=9mAWc8JxcWavwpJP9aYeiDSfzYrH1eZLmeZ54xRouOY,21841
|
|
22
|
+
pycleaner/syntax_healer.py,sha256=TlIMKIJhaX9GED3e2JNzNMJ5aXmL9eLJMCu7xKSjdv8,26711
|
|
23
|
+
pycleaner/taint_engine.py,sha256=96cZdDpSPbo5b13zIZY18f_4GJm_tw0clqCZMBtjXMQ,26481
|
|
24
|
+
pycleaner/test_generator.py,sha256=TJsNJSojc6eIsaKZkwMlpnb-MKHuDpNLZCPnMlO-a8M,19663
|
|
25
|
+
pycleaner/type_checker.py,sha256=h09QI_8MRHv83s23myjSYFdMTsWoQ900ffBJHtRuhmE,34087
|
|
26
|
+
pycleaner/typeshed_resolver.py,sha256=N-kWU_CI5DG59lQMkt7wMW9sHYjNCBdfFS42b4y2Kr4,12742
|
|
27
|
+
pycleaner/verifier.py,sha256=pOfSLRNHe4qDUYJG8mxDxjY6Ph8QKBdJZmqBtoJX87w,17937
|
|
28
|
+
pycleaner/frameworks/__init__.py,sha256=7MLgg469hCNtQR6dsE5gZcAZkCywJHz9zOPrbH8NmrE,3415
|
|
29
|
+
pycleaner/frameworks/plugins.py,sha256=K3J4mVJw2MR_pa6Mhmo5oZOX8pUesL8lx9LUo4RLrNo,9390
|
|
30
|
+
ballpython-2.0.2.dist-info/METADATA,sha256=tpostFTk54cwmBz56dghrw0fOAZVcsJPKA1N9NMUtZ4,12459
|
|
31
|
+
ballpython-2.0.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
32
|
+
ballpython-2.0.2.dist-info/entry_points.txt,sha256=6ux-5UxoVSmlK5p9M-GDVt24sCdwTQq9wgRDkpMIYU8,81
|
|
33
|
+
ballpython-2.0.2.dist-info/top_level.txt,sha256=u6lmy8RWZ_saxCZAQTL7EijB4sunegOiVyk5fHjpwLk,21
|
|
34
|
+
ballpython-2.0.2.dist-info/RECORD,,
|
pycleaner/__init__.py
CHANGED
|
@@ -10,19 +10,38 @@ dataflow taint vulnerabilities, and synthesizes automated behavioral test suites
|
|
|
10
10
|
|
|
11
11
|
from __future__ import annotations
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
from pycleaner.discovery import (
|
|
14
|
+
DEFAULT_IGNORED_DIRS,
|
|
15
|
+
PROTECTED_FILE_PATTERNS,
|
|
16
|
+
collect_project_python_files,
|
|
17
|
+
is_ignored_directory,
|
|
18
|
+
is_protected_file,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
__version__ = "2.0.2"
|
|
14
22
|
__all__ = [
|
|
23
|
+
"BaselineFingerprint",
|
|
24
|
+
"BaselineManager",
|
|
15
25
|
"CleanPipeline",
|
|
16
26
|
"CleanResult",
|
|
17
27
|
"ComplexityAnalyzer",
|
|
18
28
|
"ComplexityReport",
|
|
29
|
+
"ContentAddressableCache",
|
|
30
|
+
"CounterExample",
|
|
19
31
|
"DeadCodeDetector",
|
|
32
|
+
"DeadCodeFixResult",
|
|
33
|
+
"DeadCodeFixer",
|
|
20
34
|
"DeadCodeReport",
|
|
21
35
|
"DependencyAuditor",
|
|
22
36
|
"GeneratedTestSuite",
|
|
23
37
|
"ImportResolver",
|
|
38
|
+
"IsolatedDifferentialVerifier",
|
|
24
39
|
"LinterFormatter",
|
|
40
|
+
"ModernizeResult",
|
|
41
|
+
"Modernizer",
|
|
42
|
+
"ProofReceipt",
|
|
25
43
|
"PyCleanerConfig",
|
|
44
|
+
"RuleExplanation",
|
|
26
45
|
"SecurityReport",
|
|
27
46
|
"SecurityScanner",
|
|
28
47
|
"SyntaxHealer",
|
|
@@ -35,15 +54,27 @@ __all__ = [
|
|
|
35
54
|
"TypeFinding",
|
|
36
55
|
"TypeReport",
|
|
37
56
|
"TypeshedResolver",
|
|
57
|
+
"VerificationTier",
|
|
58
|
+
"get_explanation",
|
|
59
|
+
"list_rules",
|
|
38
60
|
"load_config",
|
|
39
61
|
]
|
|
40
62
|
|
|
63
|
+
from pycleaner.baseline import BaselineFingerprint, BaselineManager
|
|
64
|
+
from pycleaner.cache import ContentAddressableCache
|
|
41
65
|
from pycleaner.complexity_analyzer import ComplexityAnalyzer, ComplexityReport
|
|
42
66
|
from pycleaner.config import PyCleanerConfig, load_config
|
|
43
|
-
from pycleaner.dead_code_detector import
|
|
67
|
+
from pycleaner.dead_code_detector import (
|
|
68
|
+
DeadCodeDetector,
|
|
69
|
+
DeadCodeFixer,
|
|
70
|
+
DeadCodeFixResult,
|
|
71
|
+
DeadCodeReport,
|
|
72
|
+
)
|
|
44
73
|
from pycleaner.dependency_auditor import DependencyAuditor
|
|
74
|
+
from pycleaner.explanations import RuleExplanation, get_explanation, list_rules
|
|
45
75
|
from pycleaner.import_resolver import ImportResolver
|
|
46
76
|
from pycleaner.linter_formatter import LinterFormatter
|
|
77
|
+
from pycleaner.modernizer import Modernizer, ModernizeResult
|
|
47
78
|
from pycleaner.pipeline import CleanPipeline, CleanResult
|
|
48
79
|
from pycleaner.security_scanner import SecurityReport, SecurityScanner
|
|
49
80
|
from pycleaner.syntax_healer import SyntaxHealer
|
|
@@ -51,3 +82,9 @@ from pycleaner.taint_engine import TaintEngine, TaintFinding, TaintReport
|
|
|
51
82
|
from pycleaner.test_generator import GeneratedTestSuite, TestCase, TestGenerator
|
|
52
83
|
from pycleaner.type_checker import TypeChecker, TypeFinding, TypeReport
|
|
53
84
|
from pycleaner.typeshed_resolver import TypeshedResolver
|
|
85
|
+
from pycleaner.verifier import (
|
|
86
|
+
CounterExample,
|
|
87
|
+
IsolatedDifferentialVerifier,
|
|
88
|
+
ProofReceipt,
|
|
89
|
+
VerificationTier,
|
|
90
|
+
)
|
pycleaner/baseline.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""
|
|
2
|
+
pycleaner.baseline
|
|
3
|
+
==================
|
|
4
|
+
|
|
5
|
+
Technical Debt Ratchet and Baseline Management Engine.
|
|
6
|
+
|
|
7
|
+
Enables incremental adoption of PyCleaner on legacy codebases by snapshotting
|
|
8
|
+
existing diagnostic violations into `.pycleaner/baseline.json`. Future CI/CD
|
|
9
|
+
checks ensure that existing technical debt can only decrease, never increase.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import json
|
|
16
|
+
import time
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class BaselineFingerprint:
|
|
24
|
+
rule: str
|
|
25
|
+
file: str
|
|
26
|
+
line: int
|
|
27
|
+
symbol: str
|
|
28
|
+
|
|
29
|
+
def to_hash(self) -> str:
|
|
30
|
+
raw = f"{self.rule}:{self.file}:{self.line}:{self.symbol}"
|
|
31
|
+
return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16]
|
|
32
|
+
|
|
33
|
+
def to_dict(self) -> dict[str, Any]:
|
|
34
|
+
return {
|
|
35
|
+
"rule": self.rule,
|
|
36
|
+
"file": self.file,
|
|
37
|
+
"line": self.line,
|
|
38
|
+
"symbol": self.symbol,
|
|
39
|
+
"hash": self.to_hash(),
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class BaselineManager:
|
|
44
|
+
"""Manages baseline generation, loading, and ratchet comparison."""
|
|
45
|
+
|
|
46
|
+
def __init__(self, baseline_path: Path | str = ".pycleaner/baseline.json") -> None:
|
|
47
|
+
self.baseline_path = Path(baseline_path)
|
|
48
|
+
|
|
49
|
+
def load_fingerprints(self) -> set[str]:
|
|
50
|
+
"""Loads baseline issue hashes from disk."""
|
|
51
|
+
if not self.baseline_path.exists():
|
|
52
|
+
return set()
|
|
53
|
+
try:
|
|
54
|
+
data = json.loads(self.baseline_path.read_text(encoding="utf-8"))
|
|
55
|
+
fingerprints = data.get("fingerprints", [])
|
|
56
|
+
return {
|
|
57
|
+
fp.get("hash")
|
|
58
|
+
for fp in fingerprints
|
|
59
|
+
if isinstance(fp, dict) and fp.get("hash")
|
|
60
|
+
}
|
|
61
|
+
except json.JSONDecodeError:
|
|
62
|
+
return set()
|
|
63
|
+
|
|
64
|
+
def save_baseline(
|
|
65
|
+
self,
|
|
66
|
+
fingerprints: list[BaselineFingerprint],
|
|
67
|
+
root_dir: Path,
|
|
68
|
+
) -> Path:
|
|
69
|
+
"""Serializes fingerprints to the baseline JSON file."""
|
|
70
|
+
self.baseline_path.parent.mkdir(parents=True, exist_ok=True)
|
|
71
|
+
payload = {
|
|
72
|
+
"version": 1,
|
|
73
|
+
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
74
|
+
"root_dir": str(root_dir.resolve()),
|
|
75
|
+
"issue_count": len(fingerprints),
|
|
76
|
+
"fingerprints": [fp.to_dict() for fp in fingerprints],
|
|
77
|
+
}
|
|
78
|
+
self.baseline_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
|
79
|
+
return self.baseline_path
|
|
80
|
+
|
|
81
|
+
@staticmethod
|
|
82
|
+
def create_fingerprint(
|
|
83
|
+
rule: str,
|
|
84
|
+
file_path: Path | str,
|
|
85
|
+
line: int,
|
|
86
|
+
symbol: str,
|
|
87
|
+
root_dir: Path | None = None,
|
|
88
|
+
) -> BaselineFingerprint:
|
|
89
|
+
"""Constructs a deterministic fingerprint with normalized relative path."""
|
|
90
|
+
p = Path(file_path)
|
|
91
|
+
if root_dir is not None:
|
|
92
|
+
try:
|
|
93
|
+
rel = str(p.resolve().relative_to(root_dir.resolve())).replace(
|
|
94
|
+
"\\", "/"
|
|
95
|
+
)
|
|
96
|
+
except ValueError:
|
|
97
|
+
rel = p.name
|
|
98
|
+
else:
|
|
99
|
+
rel = p.name
|
|
100
|
+
|
|
101
|
+
return BaselineFingerprint(
|
|
102
|
+
rule=rule,
|
|
103
|
+
file=rel,
|
|
104
|
+
line=line,
|
|
105
|
+
symbol=symbol,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
def filter_new_issues(
|
|
109
|
+
self,
|
|
110
|
+
issues: list[BaselineFingerprint],
|
|
111
|
+
) -> tuple[list[BaselineFingerprint], list[BaselineFingerprint]]:
|
|
112
|
+
"""
|
|
113
|
+
Separates issues into (tolerated_baseline_issues, new_debt_issues).
|
|
114
|
+
Returns:
|
|
115
|
+
(tolerated, new_debt)
|
|
116
|
+
"""
|
|
117
|
+
known_hashes = self.load_fingerprints()
|
|
118
|
+
tolerated: list[BaselineFingerprint] = []
|
|
119
|
+
new_debt: list[BaselineFingerprint] = []
|
|
120
|
+
|
|
121
|
+
for issue in issues:
|
|
122
|
+
if issue.to_hash() in known_hashes:
|
|
123
|
+
tolerated.append(issue)
|
|
124
|
+
else:
|
|
125
|
+
new_debt.append(issue)
|
|
126
|
+
|
|
127
|
+
return tolerated, new_debt
|
pycleaner/cache.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
"""
|
|
2
|
+
pycleaner.cache
|
|
3
|
+
===============
|
|
4
|
+
|
|
5
|
+
Content-Addressable Incremental Cache and Provenance Ledger for PyCleaner.
|
|
6
|
+
|
|
7
|
+
Provides:
|
|
8
|
+
- SQLite-backed, WAL-journaled content-addressable caching keyed by SHA-256(content + config).
|
|
9
|
+
- Dependency invalidation tracking via AST-extracted import graphs.
|
|
10
|
+
- Audit provenance ledger recording every verified repair with seed and diff receipts.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import ast
|
|
16
|
+
import hashlib
|
|
17
|
+
import json
|
|
18
|
+
import sqlite3
|
|
19
|
+
import time
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import TYPE_CHECKING, Any
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from pycleaner.pipeline import CleanResult
|
|
25
|
+
from pycleaner.verifier import VerificationTier
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def compute_content_hash(source: str, config_hash: str = "") -> str:
|
|
29
|
+
"""Computes a deterministic SHA-256 hash of the source code and configuration."""
|
|
30
|
+
hasher = hashlib.sha256()
|
|
31
|
+
hasher.update(source.encode("utf-8"))
|
|
32
|
+
if config_hash:
|
|
33
|
+
hasher.update(config_hash.encode("utf-8"))
|
|
34
|
+
return hasher.hexdigest()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def extract_file_dependencies(source: str) -> list[str]:
|
|
38
|
+
"""Extracts imported module names using AST analysis for dependency tracking."""
|
|
39
|
+
deps: list[str] = []
|
|
40
|
+
try:
|
|
41
|
+
tree = ast.parse(source)
|
|
42
|
+
for node in ast.walk(tree):
|
|
43
|
+
if isinstance(node, ast.Import):
|
|
44
|
+
for alias in node.names:
|
|
45
|
+
deps.append(alias.name.split(".")[0])
|
|
46
|
+
elif isinstance(node, ast.ImportFrom) and node.module:
|
|
47
|
+
deps.append(node.module.split(".")[0])
|
|
48
|
+
except SyntaxError:
|
|
49
|
+
pass # Best-effort dependency extraction; syntax errors handled downstream
|
|
50
|
+
return sorted(set(deps))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class ContentAddressableCache:
|
|
54
|
+
"""
|
|
55
|
+
High-performance incremental cache backed by SQLite with write-ahead logging (WAL).
|
|
56
|
+
Guarantees sub-millisecond cache lookups for unchanged files.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def __init__(self, db_path: Path | str = ".pycleaner/cache.db") -> None:
|
|
60
|
+
self.db_path = Path(db_path)
|
|
61
|
+
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
62
|
+
self._init_db()
|
|
63
|
+
|
|
64
|
+
def _get_connection(self) -> sqlite3.Connection:
|
|
65
|
+
conn = sqlite3.connect(str(self.db_path), timeout=10.0)
|
|
66
|
+
conn.execute("PRAGMA journal_mode=WAL;")
|
|
67
|
+
conn.execute("PRAGMA synchronous=NORMAL;")
|
|
68
|
+
conn.row_factory = sqlite3.Row
|
|
69
|
+
return conn
|
|
70
|
+
|
|
71
|
+
def _init_db(self) -> None:
|
|
72
|
+
with self._get_connection() as conn:
|
|
73
|
+
conn.execute("""
|
|
74
|
+
CREATE TABLE IF NOT EXISTS file_cache (
|
|
75
|
+
content_hash TEXT PRIMARY KEY,
|
|
76
|
+
file_path TEXT NOT NULL,
|
|
77
|
+
config_hash TEXT NOT NULL,
|
|
78
|
+
cleaned_code TEXT NOT NULL,
|
|
79
|
+
is_valid_python INTEGER NOT NULL,
|
|
80
|
+
changed INTEGER NOT NULL,
|
|
81
|
+
error TEXT,
|
|
82
|
+
syntax_repairs TEXT NOT NULL,
|
|
83
|
+
modernize_transforms TEXT NOT NULL,
|
|
84
|
+
dead_code_pruned TEXT NOT NULL,
|
|
85
|
+
resolved_imports TEXT NOT NULL,
|
|
86
|
+
unresolved_symbols TEXT NOT NULL,
|
|
87
|
+
lint_changed INTEGER NOT NULL,
|
|
88
|
+
format_changed INTEGER NOT NULL,
|
|
89
|
+
verification_tier TEXT,
|
|
90
|
+
dependencies TEXT NOT NULL,
|
|
91
|
+
updated_at REAL NOT NULL
|
|
92
|
+
);
|
|
93
|
+
""")
|
|
94
|
+
conn.execute("""
|
|
95
|
+
CREATE INDEX IF NOT EXISTS idx_file_cache_path
|
|
96
|
+
ON file_cache (file_path);
|
|
97
|
+
""")
|
|
98
|
+
conn.execute("""
|
|
99
|
+
CREATE TABLE IF NOT EXISTS provenance_ledger (
|
|
100
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
101
|
+
timestamp TEXT NOT NULL,
|
|
102
|
+
file_path TEXT NOT NULL,
|
|
103
|
+
transformation_type TEXT NOT NULL,
|
|
104
|
+
verification_tier TEXT NOT NULL,
|
|
105
|
+
seed INTEGER,
|
|
106
|
+
diff TEXT NOT NULL
|
|
107
|
+
);
|
|
108
|
+
""")
|
|
109
|
+
conn.execute("""
|
|
110
|
+
CREATE INDEX IF NOT EXISTS idx_provenance_path
|
|
111
|
+
ON provenance_ledger (file_path);
|
|
112
|
+
""")
|
|
113
|
+
conn.commit()
|
|
114
|
+
|
|
115
|
+
def get(
|
|
116
|
+
self,
|
|
117
|
+
file_path: Path | str,
|
|
118
|
+
source: str,
|
|
119
|
+
config_hash: str = "",
|
|
120
|
+
) -> CleanResult | None:
|
|
121
|
+
"""
|
|
122
|
+
Retrieves a cached CleanResult if content_hash matches.
|
|
123
|
+
Returns None on cache miss.
|
|
124
|
+
"""
|
|
125
|
+
expected_hash = compute_content_hash(source, config_hash)
|
|
126
|
+
from pycleaner.pipeline import CleanResult
|
|
127
|
+
|
|
128
|
+
with self._get_connection() as conn:
|
|
129
|
+
cursor = conn.execute(
|
|
130
|
+
"""
|
|
131
|
+
SELECT * FROM file_cache
|
|
132
|
+
WHERE content_hash = ?;
|
|
133
|
+
""",
|
|
134
|
+
(expected_hash,),
|
|
135
|
+
)
|
|
136
|
+
row = cursor.fetchone()
|
|
137
|
+
if row is None:
|
|
138
|
+
return None
|
|
139
|
+
|
|
140
|
+
tier = None
|
|
141
|
+
if row["verification_tier"]:
|
|
142
|
+
try:
|
|
143
|
+
tier = VerificationTier(row["verification_tier"])
|
|
144
|
+
except ValueError:
|
|
145
|
+
tier = None
|
|
146
|
+
|
|
147
|
+
return CleanResult(
|
|
148
|
+
path=Path(row["file_path"]),
|
|
149
|
+
original_code=source,
|
|
150
|
+
cleaned_code=row["cleaned_code"],
|
|
151
|
+
changed=bool(row["changed"]),
|
|
152
|
+
is_valid_python=bool(row["is_valid_python"]),
|
|
153
|
+
syntax_repairs=json.loads(row["syntax_repairs"]),
|
|
154
|
+
modernize_transforms=json.loads(row["modernize_transforms"]),
|
|
155
|
+
dead_code_pruned=json.loads(row["dead_code_pruned"]),
|
|
156
|
+
resolved_imports=json.loads(row["resolved_imports"]),
|
|
157
|
+
unresolved_symbols=json.loads(row["unresolved_symbols"]),
|
|
158
|
+
lint_changed=bool(row["lint_changed"]),
|
|
159
|
+
format_changed=bool(row["format_changed"]),
|
|
160
|
+
error=row["error"],
|
|
161
|
+
verification_tier=tier,
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
def set(
|
|
165
|
+
self,
|
|
166
|
+
file_path: Path | str,
|
|
167
|
+
source: str,
|
|
168
|
+
result: CleanResult,
|
|
169
|
+
config_hash: str = "",
|
|
170
|
+
) -> None:
|
|
171
|
+
"""Stores or updates the CleanResult in the incremental cache keyed by content_hash."""
|
|
172
|
+
canonical_path = str(Path(file_path).resolve())
|
|
173
|
+
content_hash = compute_content_hash(source, config_hash)
|
|
174
|
+
deps = extract_file_dependencies(source)
|
|
175
|
+
tier_str = result.verification_tier.value if result.verification_tier else None
|
|
176
|
+
|
|
177
|
+
with self._get_connection() as conn:
|
|
178
|
+
conn.execute(
|
|
179
|
+
"""
|
|
180
|
+
INSERT OR REPLACE INTO file_cache (
|
|
181
|
+
content_hash, file_path, config_hash, cleaned_code,
|
|
182
|
+
is_valid_python, changed, error, syntax_repairs,
|
|
183
|
+
modernize_transforms, dead_code_pruned, resolved_imports,
|
|
184
|
+
unresolved_symbols, lint_changed, format_changed,
|
|
185
|
+
verification_tier, dependencies, updated_at
|
|
186
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);
|
|
187
|
+
""",
|
|
188
|
+
(
|
|
189
|
+
content_hash,
|
|
190
|
+
canonical_path,
|
|
191
|
+
config_hash,
|
|
192
|
+
result.cleaned_code,
|
|
193
|
+
1 if result.is_valid_python else 0,
|
|
194
|
+
1 if result.changed else 0,
|
|
195
|
+
result.error,
|
|
196
|
+
json.dumps(result.syntax_repairs),
|
|
197
|
+
json.dumps(result.modernize_transforms),
|
|
198
|
+
json.dumps(result.dead_code_pruned),
|
|
199
|
+
json.dumps(result.resolved_imports),
|
|
200
|
+
json.dumps(result.unresolved_symbols),
|
|
201
|
+
1 if result.lint_changed else 0,
|
|
202
|
+
1 if result.format_changed else 0,
|
|
203
|
+
tier_str,
|
|
204
|
+
json.dumps(deps),
|
|
205
|
+
time.time(),
|
|
206
|
+
),
|
|
207
|
+
)
|
|
208
|
+
conn.commit()
|
|
209
|
+
|
|
210
|
+
def record_provenance(
|
|
211
|
+
self,
|
|
212
|
+
file_path: Path | str,
|
|
213
|
+
transformation_type: str,
|
|
214
|
+
verification_tier: str,
|
|
215
|
+
seed: int | None,
|
|
216
|
+
diff: str,
|
|
217
|
+
) -> None:
|
|
218
|
+
"""Records an immutable audit entry into the provenance ledger."""
|
|
219
|
+
canonical_path = str(Path(file_path).resolve())
|
|
220
|
+
now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
221
|
+
with self._get_connection() as conn:
|
|
222
|
+
conn.execute(
|
|
223
|
+
"""
|
|
224
|
+
INSERT INTO provenance_ledger (
|
|
225
|
+
timestamp, file_path, transformation_type,
|
|
226
|
+
verification_tier, seed, diff
|
|
227
|
+
) VALUES (?, ?, ?, ?, ?, ?);
|
|
228
|
+
""",
|
|
229
|
+
(
|
|
230
|
+
now,
|
|
231
|
+
canonical_path,
|
|
232
|
+
transformation_type,
|
|
233
|
+
verification_tier,
|
|
234
|
+
seed,
|
|
235
|
+
diff,
|
|
236
|
+
),
|
|
237
|
+
)
|
|
238
|
+
conn.commit()
|
|
239
|
+
|
|
240
|
+
def get_provenance(self, limit: int = 50) -> list[dict[str, Any]]:
|
|
241
|
+
"""Returns the most recent records from the provenance ledger."""
|
|
242
|
+
with self._get_connection() as conn:
|
|
243
|
+
cursor = conn.execute(
|
|
244
|
+
"""
|
|
245
|
+
SELECT timestamp, file_path, transformation_type,
|
|
246
|
+
verification_tier, seed, diff
|
|
247
|
+
FROM provenance_ledger
|
|
248
|
+
ORDER BY id DESC
|
|
249
|
+
LIMIT ?;
|
|
250
|
+
""",
|
|
251
|
+
(limit,),
|
|
252
|
+
)
|
|
253
|
+
return [dict(row) for row in cursor.fetchall()]
|
|
254
|
+
|
|
255
|
+
def invalidate(self, file_path: Path | str) -> None:
|
|
256
|
+
"""Invalidates the cache entry for a specific file path."""
|
|
257
|
+
canonical_path = str(Path(file_path).resolve())
|
|
258
|
+
with self._get_connection() as conn:
|
|
259
|
+
conn.execute(
|
|
260
|
+
"DELETE FROM file_cache WHERE file_path = ?;", (canonical_path,)
|
|
261
|
+
)
|
|
262
|
+
conn.commit()
|
|
263
|
+
|
|
264
|
+
def clear(self) -> None:
|
|
265
|
+
"""Clears all entries from the cache database."""
|
|
266
|
+
with self._get_connection() as conn:
|
|
267
|
+
conn.execute("DELETE FROM file_cache;")
|
|
268
|
+
conn.commit()
|
|
269
|
+
|
|
270
|
+
def get_stats(self) -> dict[str, Any]:
|
|
271
|
+
"""Returns statistics on cache utilization and database storage."""
|
|
272
|
+
with self._get_connection() as conn:
|
|
273
|
+
cursor = conn.execute("SELECT COUNT(*) as cnt FROM file_cache;")
|
|
274
|
+
total_entries = cursor.fetchone()["cnt"]
|
|
275
|
+
cursor = conn.execute("SELECT COUNT(*) as cnt FROM provenance_ledger;")
|
|
276
|
+
provenance_entries = cursor.fetchone()["cnt"]
|
|
277
|
+
db_size = self.db_path.stat().st_size if self.db_path.exists() else 0
|
|
278
|
+
return {
|
|
279
|
+
"file_cache_entries": total_entries,
|
|
280
|
+
"provenance_entries": provenance_entries,
|
|
281
|
+
"db_size_bytes": db_size,
|
|
282
|
+
}
|