ballpython 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ballpython-2.0.0/PKG-INFO +322 -0
- ballpython-2.0.0/README.md +303 -0
- ballpython-2.0.0/ballpython/__init__.py +9 -0
- ballpython-2.0.0/ballpython/__main__.py +8 -0
- ballpython-2.0.0/ballpython/cli.py +7 -0
- ballpython-2.0.0/ballpython.egg-info/PKG-INFO +322 -0
- ballpython-2.0.0/ballpython.egg-info/SOURCES.txt +40 -0
- ballpython-2.0.0/ballpython.egg-info/dependency_links.txt +1 -0
- ballpython-2.0.0/ballpython.egg-info/entry_points.txt +3 -0
- ballpython-2.0.0/ballpython.egg-info/requires.txt +14 -0
- ballpython-2.0.0/ballpython.egg-info/top_level.txt +2 -0
- ballpython-2.0.0/pycleaner/__init__.py +53 -0
- ballpython-2.0.0/pycleaner/__main__.py +8 -0
- ballpython-2.0.0/pycleaner/cli.py +1548 -0
- ballpython-2.0.0/pycleaner/complexity_analyzer.py +473 -0
- ballpython-2.0.0/pycleaner/config.py +254 -0
- ballpython-2.0.0/pycleaner/dead_code_detector.py +515 -0
- ballpython-2.0.0/pycleaner/dependency_auditor.py +331 -0
- ballpython-2.0.0/pycleaner/import_resolver.py +832 -0
- ballpython-2.0.0/pycleaner/linter_formatter.py +590 -0
- ballpython-2.0.0/pycleaner/pipeline.py +349 -0
- ballpython-2.0.0/pycleaner/security_scanner.py +563 -0
- ballpython-2.0.0/pycleaner/syntax_healer.py +577 -0
- ballpython-2.0.0/pycleaner/taint_engine.py +720 -0
- ballpython-2.0.0/pycleaner/test_generator.py +444 -0
- ballpython-2.0.0/pycleaner/type_checker.py +989 -0
- ballpython-2.0.0/pycleaner/typeshed_resolver.py +395 -0
- ballpython-2.0.0/pyproject.toml +64 -0
- ballpython-2.0.0/setup.cfg +4 -0
- ballpython-2.0.0/tests/test_cli.py +306 -0
- ballpython-2.0.0/tests/test_complexity_analyzer.py +166 -0
- ballpython-2.0.0/tests/test_config.py +121 -0
- ballpython-2.0.0/tests/test_dead_code_detector.py +140 -0
- ballpython-2.0.0/tests/test_dependency_auditor.py +63 -0
- ballpython-2.0.0/tests/test_import_resolver.py +117 -0
- ballpython-2.0.0/tests/test_linter_formatter.py +113 -0
- ballpython-2.0.0/tests/test_pipeline.py +128 -0
- ballpython-2.0.0/tests/test_security_scanner.py +212 -0
- ballpython-2.0.0/tests/test_syntax_healer.py +164 -0
- ballpython-2.0.0/tests/test_taint_engine.py +131 -0
- ballpython-2.0.0/tests/test_test_generator.py +79 -0
- ballpython-2.0.0/tests/test_type_checker.py +155 -0
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ballpython
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: The Ultimate Static Python Intelligence, Healing, Type Verification, and Security Suite
|
|
5
|
+
Author: Developer
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: ruff>=0.1.0
|
|
10
|
+
Requires-Dist: rich>=13.0.0
|
|
11
|
+
Requires-Dist: tomli>=1.1.0; python_version < "3.11"
|
|
12
|
+
Provides-Extra: dev
|
|
13
|
+
Requires-Dist: pytest>=8.0.0; extra == "dev"
|
|
14
|
+
Requires-Dist: black>=24.0.0; extra == "dev"
|
|
15
|
+
Requires-Dist: isort>=5.13.0; extra == "dev"
|
|
16
|
+
Provides-Extra: security
|
|
17
|
+
Requires-Dist: cryptography>=41.0.0; extra == "security"
|
|
18
|
+
Requires-Dist: defusedxml>=0.7.1; extra == "security"
|
|
19
|
+
|
|
20
|
+
# pycleaner — Ultimate Static Python Code Quality Suite
|
|
21
|
+
|
|
22
|
+
A production-grade, offline Python static analysis, syntax healing, linting, formatting, security scanning, complexity evaluation, dead code detection, and dependency auditing suite.
|
|
23
|
+
|
|
24
|
+
Zero external LLM dependencies, zero mock modes, and built for deterministic developer workflows and CI/CD pipelines.
|
|
25
|
+
|
|
26
|
+
---
|
|
27
|
+
|
|
28
|
+
## Key Capabilities
|
|
29
|
+
|
|
30
|
+
1. **Syntax Healer (`SyntaxHealer`)**:
|
|
31
|
+
- Statically repairs missing colons on compound statement headers (`def`, `class`, `if`, `elif`, `else`, `for`, `while`, `try`, `except`, `finally`, `with`, `async def`, `match`, `case`).
|
|
32
|
+
- Corrects accidental single `=` assignments in conditionals (`if x = 1:` $\to$ `if x == 1:`) without corrupting keyword arguments (`func(x=1)`), augmented assignments, or walrus expressions (`x := 1`).
|
|
33
|
+
- Modernizes legacy Python 2 statements (`print "..."` $\to$ `print(...)`, `except Error, e:` $\to$ `except Error as e:`, and multi-exception tuples `except (E1, E2), e:` on Python 3.12+).
|
|
34
|
+
- Automatically closes unclosed parentheses, brackets, and braces.
|
|
35
|
+
- Normalizes mixed tab characters into 4 spaces.
|
|
36
|
+
|
|
37
|
+
2. **Missing Import Resolver (`ImportResolver`)**:
|
|
38
|
+
- Walks Python AST to extract undefined loaded symbols across local, function, class, and comprehension scopes.
|
|
39
|
+
- Resolves undefined identifiers to standard library modules (`os`, `sys`, `json`, `re`, `subprocess`, `pathlib`, etc.), collections, typing constructs, dataclasses, concurrent primitives (`ThreadPoolExecutor`, `ProcessPoolExecutor`, `Lock`, `Queue`), and popular aliases (`np`, `pd`, `plt`, `sns`, `tf`, `nn`).
|
|
40
|
+
- **`TYPE_CHECKING` Awareness**: Injects typing-only imports into `if TYPE_CHECKING:` guards and adds `from __future__ import annotations` when symbols are used strictly in type annotations.
|
|
41
|
+
- **Custom Import Mapping**: Configurable mapping from symbol names to exact import statements.
|
|
42
|
+
|
|
43
|
+
3. **Linter & Formatter (`LinterFormatter`)**:
|
|
44
|
+
- Integrates Rust-based `ruff` via in-memory stream processing (`--stdin-filename`).
|
|
45
|
+
- Automatically removes unused imports (`F401`) and unused variables (`F841`).
|
|
46
|
+
- Sorts and groups imports via `isort` / `ruff` or an internal pure-Python 3-tier sorter (stdlib $\to$ third-party $\to$ local).
|
|
47
|
+
- Modernizes deprecated syntax via `pyupgrade` (`UP`).
|
|
48
|
+
- Applies deterministic PEP 8 formatting (`ruff format` / `black` / pure-Python formatter).
|
|
49
|
+
- Wraps long `from ... import (...)` statements exceeding configured `line-length`.
|
|
50
|
+
|
|
51
|
+
4. **Dead Code Detector (`DeadCodeDetector`)**:
|
|
52
|
+
- Builds a cross-file symbol definition and reference graph across a project.
|
|
53
|
+
- Identifies unused functions, methods, classes, module-level constants, and class attributes.
|
|
54
|
+
- Respects public exports in `__all__` and framework decorators (`@app.route`, `@pytest.fixture`, `@abstractmethod`, etc.).
|
|
55
|
+
- Detects unreachable code following unconditional `return`, `raise`, `break`, `continue`, or `sys.exit()`.
|
|
56
|
+
- Identifies empty pass blocks with no comments.
|
|
57
|
+
|
|
58
|
+
5. **Static Security Scanner (`SecurityScanner`)**:
|
|
59
|
+
- Detects dangerous function calls: `eval()`, `exec()`, `compile()`, `pickle.loads()`, `os.system()`, and unsafe `yaml.load()` lacking `SafeLoader`.
|
|
60
|
+
- Flags SQL injection patterns (string concatenation, f-strings, `%`-formatting, or `.format()` inside `cursor.execute()`).
|
|
61
|
+
- Flags subprocess command injection (`shell=True`).
|
|
62
|
+
- Flags insecure transport (`verify=False` in `requests` or `httpx`).
|
|
63
|
+
- Scans for hardcoded secrets: AWS access/secret keys, GitHub tokens, Slack tokens, JWT tokens, private key headers, and generic passwords/API keys.
|
|
64
|
+
- Flags production `assert` statements used for input validation (which get stripped under `python -O`).
|
|
65
|
+
|
|
66
|
+
6. **Complexity Analyzer (`ComplexityAnalyzer`)**:
|
|
67
|
+
- McCabe Cyclomatic Complexity per function ($E - N + 2$).
|
|
68
|
+
- Sonar-style Cognitive Complexity scoring (penalizing nested control flow, compound boolean conditions, and break in linear flow).
|
|
69
|
+
- Metrics tracked: line count, argument count, return statement count, and maximum nesting depth.
|
|
70
|
+
- Configurable thresholds with tabular and JSON reporting.
|
|
71
|
+
|
|
72
|
+
7. **Dependency Auditor (`DependencyAuditor`)**:
|
|
73
|
+
- Statically scans all `.py` files across a repository to discover external third-party imports.
|
|
74
|
+
- Maps module import names to PyPI distribution package names (e.g., `yaml` $\to$ `PyYAML`, `PIL` $\to$ `pillow`, `cv2` $\to$ `opencv-python`).
|
|
75
|
+
- Compares imported dependencies against `requirements.txt` and `pyproject.toml`.
|
|
76
|
+
- Synchronizes `requirements.txt` with `--fix-deps` and prunes unused packages with `--prune-deps`.
|
|
77
|
+
|
|
78
|
+
8. **Safety & Concurrency**:
|
|
79
|
+
- Backup creation (`.pycleaner.bak`) before overwriting files (enabled by default).
|
|
80
|
+
- Parallel multi-core file processing via `ProcessPoolExecutor`.
|
|
81
|
+
- Continuous file watching (`watch` mode) with automatic re-healing on save.
|
|
82
|
+
- Pre-commit configuration generator (`hook` subcommand).
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## Installation
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
# Editable install
|
|
90
|
+
pip install -e .
|
|
91
|
+
|
|
92
|
+
# Core dependencies
|
|
93
|
+
pip install ruff rich
|
|
94
|
+
|
|
95
|
+
# Optional security and development packages
|
|
96
|
+
pip install -e ".[dev,security]"
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## CLI Usage
|
|
102
|
+
|
|
103
|
+
### Subcommands
|
|
104
|
+
|
|
105
|
+
#### `fix` (Default Action)
|
|
106
|
+
Heals syntax, resolves imports, prunes unused imports/variables, fixes lint violations, and formats code:
|
|
107
|
+
```bash
|
|
108
|
+
py -m pycleaner fix src/
|
|
109
|
+
py -m pycleaner fix --diff path/to/script.py
|
|
110
|
+
py -m pycleaner fix --no-backup src/
|
|
111
|
+
py -m pycleaner fix --parallel --workers 4 src/
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
#### `check` (Dry-Run CI Verification)
|
|
115
|
+
Inspects files without writing modifications to disk. Exits with code `1` if changes or errors are detected:
|
|
116
|
+
```bash
|
|
117
|
+
py -m pycleaner check src/
|
|
118
|
+
py -m pycleaner check --diff src/
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
#### `scan` (Security Vulnerability Audit)
|
|
122
|
+
Scans Python files for hardcoded secrets, dangerous calls, and injection vulnerabilities:
|
|
123
|
+
```bash
|
|
124
|
+
py -m pycleaner scan .
|
|
125
|
+
py -m pycleaner scan --severity HIGH .
|
|
126
|
+
py -m pycleaner scan --json .
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
#### `complexity` (Function Complexity Metrics)
|
|
130
|
+
Calculates Cyclomatic Complexity, Cognitive Complexity, and architectural metrics:
|
|
131
|
+
```bash
|
|
132
|
+
py -m pycleaner complexity .
|
|
133
|
+
py -m pycleaner complexity --max-cyclomatic 10 --max-cognitive 15 .
|
|
134
|
+
py -m pycleaner complexity --json .
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
#### `dead-code` (Unused Symbols & Unreachable Branches)
|
|
138
|
+
Finds unused functions, unused classes, empty pass branches, and dead code:
|
|
139
|
+
```bash
|
|
140
|
+
py -m pycleaner dead-code .
|
|
141
|
+
py -m pycleaner dead-code --json .
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
#### `audit` (Project Dependency Verification)
|
|
145
|
+
Audits imported third-party libraries against `requirements.txt`:
|
|
146
|
+
```bash
|
|
147
|
+
# Audit only
|
|
148
|
+
py -m pycleaner audit .
|
|
149
|
+
|
|
150
|
+
# Automatically append missing dependencies
|
|
151
|
+
py -m pycleaner audit --fix-deps .
|
|
152
|
+
|
|
153
|
+
# Append missing and remove unimported dependencies
|
|
154
|
+
py -m pycleaner audit --fix-deps --prune-deps .
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
#### `all` (Complete Quality Sweep)
|
|
158
|
+
Executes all engines in one command (fix + dependency audit + security scan + complexity + dead code):
|
|
159
|
+
```bash
|
|
160
|
+
py -m pycleaner all .
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
#### `watch` (Continuous File Watcher)
|
|
164
|
+
Watches files for filesystem modifications and auto-cleans on save:
|
|
165
|
+
```bash
|
|
166
|
+
py -m pycleaner watch src/ --interval 1.0
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
#### `hook` (Pre-Commit Integration)
|
|
170
|
+
Outputs a ready-to-use `.pre-commit-hooks.yaml` configuration block:
|
|
171
|
+
```bash
|
|
172
|
+
py -m pycleaner hook
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
---
|
|
176
|
+
|
|
177
|
+
### Backward-Compatible Flat Invocations
|
|
178
|
+
|
|
179
|
+
Legacy invocations continue to work seamlessly:
|
|
180
|
+
```bash
|
|
181
|
+
pycleaner -a . # Equivalent to: pycleaner all .
|
|
182
|
+
pycleaner --check --diff src/ # Equivalent to: pycleaner check --diff src/
|
|
183
|
+
pycleaner --deps-only . # Equivalent to: pycleaner audit .
|
|
184
|
+
pycleaner path/to/file.py # Equivalent to: pycleaner fix path/to/file.py
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
---
|
|
188
|
+
|
|
189
|
+
## Configuration
|
|
190
|
+
|
|
191
|
+
`pycleaner` automatically reads configuration from `pyproject.toml` under `[tool.pycleaner]` or from `.pycleaner.toml`.
|
|
192
|
+
|
|
193
|
+
### `pyproject.toml` Example
|
|
194
|
+
|
|
195
|
+
```toml
|
|
196
|
+
[tool.pycleaner]
|
|
197
|
+
# File targeting
|
|
198
|
+
exclude = ["migrations/", "generated/", "*_pb2.py", "*_pb2_grpc.py"]
|
|
199
|
+
include = ["src/", "tests/"]
|
|
200
|
+
|
|
201
|
+
# Syntax healing
|
|
202
|
+
fix-py2-syntax = true
|
|
203
|
+
fix-conditional-assignments = true
|
|
204
|
+
|
|
205
|
+
# Import resolution
|
|
206
|
+
custom-import-map = { "logger" = "from myapp.core.logging import logger" }
|
|
207
|
+
auto-add-future-annotations = false
|
|
208
|
+
|
|
209
|
+
# Linting & formatting
|
|
210
|
+
line-length = 88
|
|
211
|
+
select-rules = "F401,F841,I,UP,E,W,B,SIM,RUF"
|
|
212
|
+
|
|
213
|
+
# Dead code
|
|
214
|
+
ignore-decorators = ["@app.route", "@pytest.fixture", "@override"]
|
|
215
|
+
ignore-names = ["_*", "test_*"]
|
|
216
|
+
|
|
217
|
+
# Security
|
|
218
|
+
security-severity-threshold = "LOW"
|
|
219
|
+
ignore-security-rules = []
|
|
220
|
+
|
|
221
|
+
# Complexity thresholds
|
|
222
|
+
max-cyclomatic-complexity = 10
|
|
223
|
+
max-cognitive-complexity = 15
|
|
224
|
+
max-function-length = 50
|
|
225
|
+
max-arguments = 5
|
|
226
|
+
|
|
227
|
+
# Behavior
|
|
228
|
+
backup = true
|
|
229
|
+
parallel = false
|
|
230
|
+
max-workers = 4
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
---
|
|
234
|
+
|
|
235
|
+
## Programmatic API
|
|
236
|
+
|
|
237
|
+
### 1. Cleaning Source Code or Files
|
|
238
|
+
```python
|
|
239
|
+
from pathlib import Path
|
|
240
|
+
from pycleaner import CleanPipeline, load_config
|
|
241
|
+
|
|
242
|
+
# Load config with project overrides
|
|
243
|
+
config = load_config(project_root=".")
|
|
244
|
+
pipeline = CleanPipeline(config=config)
|
|
245
|
+
|
|
246
|
+
# In-memory code processing
|
|
247
|
+
dirty_code = """
|
|
248
|
+
import math
|
|
249
|
+
|
|
250
|
+
def calculate(x)
|
|
251
|
+
if x = 0
|
|
252
|
+
return Path('.')
|
|
253
|
+
return Path(str(x))
|
|
254
|
+
"""
|
|
255
|
+
result = pipeline.process_source(dirty_code, filename="example.py")
|
|
256
|
+
print("Cleaned code:\n", result.cleaned_code)
|
|
257
|
+
print("Repairs applied:", result.syntax_repairs)
|
|
258
|
+
print("Imports added:", result.resolved_imports)
|
|
259
|
+
|
|
260
|
+
# File processing with automatic backup
|
|
261
|
+
file_result = pipeline.process_file("example.py", apply_changes=True, backup=True)
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
### 2. Security Vulnerability Scanning
|
|
265
|
+
```python
|
|
266
|
+
from pycleaner import SecurityScanner
|
|
267
|
+
|
|
268
|
+
scanner = SecurityScanner(severity_threshold="MEDIUM")
|
|
269
|
+
report = scanner.scan_project("src/")
|
|
270
|
+
|
|
271
|
+
for finding in report.findings:
|
|
272
|
+
print(f"[{finding.severity}] {finding.category} at {finding.filepath}:{finding.lineno}")
|
|
273
|
+
print(f" Message: {finding.message}")
|
|
274
|
+
print(f" Fix: {finding.suggestion}")
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
### 3. Complexity Analysis
|
|
278
|
+
```python
|
|
279
|
+
from pycleaner import ComplexityAnalyzer
|
|
280
|
+
|
|
281
|
+
analyzer = ComplexityAnalyzer()
|
|
282
|
+
report = analyzer.analyze_project("src/")
|
|
283
|
+
|
|
284
|
+
print(f"Scanned {report.files_scanned} files, {report.count} functions.")
|
|
285
|
+
print(f"Average Cyclomatic: {report.average_cyclomatic:.2f}")
|
|
286
|
+
|
|
287
|
+
violations = report.above_threshold(max_cyclomatic=10, max_cognitive=15)
|
|
288
|
+
for func in violations:
|
|
289
|
+
print(f"{func.qualified_name} (CC={func.cyclomatic}, Cog={func.cognitive})")
|
|
290
|
+
```
|
|
291
|
+
|
|
292
|
+
### 4. Dead Code Detection
|
|
293
|
+
```python
|
|
294
|
+
from pycleaner import DeadCodeDetector
|
|
295
|
+
|
|
296
|
+
detector = DeadCodeDetector()
|
|
297
|
+
report = detector.analyze_project("src/")
|
|
298
|
+
|
|
299
|
+
for item in report.unused_symbols:
|
|
300
|
+
print(f"Unused {item.kind} '{item.name}' at {item.filepath}:{item.lineno}")
|
|
301
|
+
|
|
302
|
+
for item in report.unreachable_code:
|
|
303
|
+
print(f"Unreachable code at {item.filepath}:{item.lineno} ({item.reason})")
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
---
|
|
307
|
+
|
|
308
|
+
## Exit Codes
|
|
309
|
+
|
|
310
|
+
| Exit Code | Meaning |
|
|
311
|
+
|:---:|:---|
|
|
312
|
+
| `0` | Clean run: all files valid, no errors, or changes cleanly written |
|
|
313
|
+
| `1` | Issues found: check mode detected modifications, errors occurred, or critical security findings flagged |
|
|
314
|
+
| `2` | Configuration error: invalid TOML syntax, invalid configuration keys, or mismatched types |
|
|
315
|
+
|
|
316
|
+
---
|
|
317
|
+
|
|
318
|
+
## License
|
|
319
|
+
|
|
320
|
+
MIT License.
|
|
321
|
+
# ball-python
|
|
322
|
+
# ball-python
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
# pycleaner — Ultimate Static Python Code Quality Suite
|
|
2
|
+
|
|
3
|
+
A production-grade, offline Python static analysis, syntax healing, linting, formatting, security scanning, complexity evaluation, dead code detection, and dependency auditing suite.
|
|
4
|
+
|
|
5
|
+
Zero external LLM dependencies, zero mock modes, and built for deterministic developer workflows and CI/CD pipelines.
|
|
6
|
+
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
## Key Capabilities
|
|
10
|
+
|
|
11
|
+
1. **Syntax Healer (`SyntaxHealer`)**:
|
|
12
|
+
- Statically repairs missing colons on compound statement headers (`def`, `class`, `if`, `elif`, `else`, `for`, `while`, `try`, `except`, `finally`, `with`, `async def`, `match`, `case`).
|
|
13
|
+
- Corrects accidental single `=` assignments in conditionals (`if x = 1:` $\to$ `if x == 1:`) without corrupting keyword arguments (`func(x=1)`), augmented assignments, or walrus expressions (`x := 1`).
|
|
14
|
+
- Modernizes legacy Python 2 statements (`print "..."` $\to$ `print(...)`, `except Error, e:` $\to$ `except Error as e:`, and multi-exception tuples `except (E1, E2), e:` on Python 3.12+).
|
|
15
|
+
- Automatically closes unclosed parentheses, brackets, and braces.
|
|
16
|
+
- Normalizes mixed tab characters into 4 spaces.
|
|
17
|
+
|
|
18
|
+
2. **Missing Import Resolver (`ImportResolver`)**:
|
|
19
|
+
- Walks Python AST to extract undefined loaded symbols across local, function, class, and comprehension scopes.
|
|
20
|
+
- Resolves undefined identifiers to standard library modules (`os`, `sys`, `json`, `re`, `subprocess`, `pathlib`, etc.), collections, typing constructs, dataclasses, concurrent primitives (`ThreadPoolExecutor`, `ProcessPoolExecutor`, `Lock`, `Queue`), and popular aliases (`np`, `pd`, `plt`, `sns`, `tf`, `nn`).
|
|
21
|
+
- **`TYPE_CHECKING` Awareness**: Injects typing-only imports into `if TYPE_CHECKING:` guards and adds `from __future__ import annotations` when symbols are used strictly in type annotations.
|
|
22
|
+
- **Custom Import Mapping**: Configurable mapping from symbol names to exact import statements.
|
|
23
|
+
|
|
24
|
+
3. **Linter & Formatter (`LinterFormatter`)**:
|
|
25
|
+
- Integrates Rust-based `ruff` via in-memory stream processing (`--stdin-filename`).
|
|
26
|
+
- Automatically removes unused imports (`F401`) and unused variables (`F841`).
|
|
27
|
+
- Sorts and groups imports via `isort` / `ruff` or an internal pure-Python 3-tier sorter (stdlib $\to$ third-party $\to$ local).
|
|
28
|
+
- Modernizes deprecated syntax via `pyupgrade` (`UP`).
|
|
29
|
+
- Applies deterministic PEP 8 formatting (`ruff format` / `black` / pure-Python formatter).
|
|
30
|
+
- Wraps long `from ... import (...)` statements exceeding configured `line-length`.
|
|
31
|
+
|
|
32
|
+
4. **Dead Code Detector (`DeadCodeDetector`)**:
|
|
33
|
+
- Builds a cross-file symbol definition and reference graph across a project.
|
|
34
|
+
- Identifies unused functions, methods, classes, module-level constants, and class attributes.
|
|
35
|
+
- Respects public exports in `__all__` and framework decorators (`@app.route`, `@pytest.fixture`, `@abstractmethod`, etc.).
|
|
36
|
+
- Detects unreachable code following unconditional `return`, `raise`, `break`, `continue`, or `sys.exit()`.
|
|
37
|
+
- Identifies empty pass blocks with no comments.
|
|
38
|
+
|
|
39
|
+
5. **Static Security Scanner (`SecurityScanner`)**:
|
|
40
|
+
- Detects dangerous function calls: `eval()`, `exec()`, `compile()`, `pickle.loads()`, `os.system()`, and unsafe `yaml.load()` lacking `SafeLoader`.
|
|
41
|
+
- Flags SQL injection patterns (string concatenation, f-strings, `%`-formatting, or `.format()` inside `cursor.execute()`).
|
|
42
|
+
- Flags subprocess command injection (`shell=True`).
|
|
43
|
+
- Flags insecure transport (`verify=False` in `requests` or `httpx`).
|
|
44
|
+
- Scans for hardcoded secrets: AWS access/secret keys, GitHub tokens, Slack tokens, JWT tokens, private key headers, and generic passwords/API keys.
|
|
45
|
+
- Flags production `assert` statements used for input validation (which get stripped under `python -O`).
|
|
46
|
+
|
|
47
|
+
6. **Complexity Analyzer (`ComplexityAnalyzer`)**:
|
|
48
|
+
- McCabe Cyclomatic Complexity per function ($E - N + 2$).
|
|
49
|
+
- Sonar-style Cognitive Complexity scoring (penalizing nested control flow, compound boolean conditions, and break in linear flow).
|
|
50
|
+
- Metrics tracked: line count, argument count, return statement count, and maximum nesting depth.
|
|
51
|
+
- Configurable thresholds with tabular and JSON reporting.
|
|
52
|
+
|
|
53
|
+
7. **Dependency Auditor (`DependencyAuditor`)**:
|
|
54
|
+
- Statically scans all `.py` files across a repository to discover external third-party imports.
|
|
55
|
+
- Maps module import names to PyPI distribution package names (e.g., `yaml` $\to$ `PyYAML`, `PIL` $\to$ `pillow`, `cv2` $\to$ `opencv-python`).
|
|
56
|
+
- Compares imported dependencies against `requirements.txt` and `pyproject.toml`.
|
|
57
|
+
- Synchronizes `requirements.txt` with `--fix-deps` and prunes unused packages with `--prune-deps`.
|
|
58
|
+
|
|
59
|
+
8. **Safety & Concurrency**:
|
|
60
|
+
- Backup creation (`.pycleaner.bak`) before overwriting files (enabled by default).
|
|
61
|
+
- Parallel multi-core file processing via `ProcessPoolExecutor`.
|
|
62
|
+
- Continuous file watching (`watch` mode) with automatic re-healing on save.
|
|
63
|
+
- Pre-commit configuration generator (`hook` subcommand).
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## Installation
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
# Editable install
|
|
71
|
+
pip install -e .
|
|
72
|
+
|
|
73
|
+
# Core dependencies
|
|
74
|
+
pip install ruff rich
|
|
75
|
+
|
|
76
|
+
# Optional security and development packages
|
|
77
|
+
pip install -e ".[dev,security]"
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
## CLI Usage
|
|
83
|
+
|
|
84
|
+
### Subcommands
|
|
85
|
+
|
|
86
|
+
#### `fix` (Default Action)
|
|
87
|
+
Heals syntax, resolves imports, prunes unused imports/variables, fixes lint violations, and formats code:
|
|
88
|
+
```bash
|
|
89
|
+
py -m pycleaner fix src/
|
|
90
|
+
py -m pycleaner fix --diff path/to/script.py
|
|
91
|
+
py -m pycleaner fix --no-backup src/
|
|
92
|
+
py -m pycleaner fix --parallel --workers 4 src/
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
#### `check` (Dry-Run CI Verification)
|
|
96
|
+
Inspects files without writing modifications to disk. Exits with code `1` if changes or errors are detected:
|
|
97
|
+
```bash
|
|
98
|
+
py -m pycleaner check src/
|
|
99
|
+
py -m pycleaner check --diff src/
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
#### `scan` (Security Vulnerability Audit)
|
|
103
|
+
Scans Python files for hardcoded secrets, dangerous calls, and injection vulnerabilities:
|
|
104
|
+
```bash
|
|
105
|
+
py -m pycleaner scan .
|
|
106
|
+
py -m pycleaner scan --severity HIGH .
|
|
107
|
+
py -m pycleaner scan --json .
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
#### `complexity` (Function Complexity Metrics)
|
|
111
|
+
Calculates Cyclomatic Complexity, Cognitive Complexity, and architectural metrics:
|
|
112
|
+
```bash
|
|
113
|
+
py -m pycleaner complexity .
|
|
114
|
+
py -m pycleaner complexity --max-cyclomatic 10 --max-cognitive 15 .
|
|
115
|
+
py -m pycleaner complexity --json .
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
#### `dead-code` (Unused Symbols & Unreachable Branches)
|
|
119
|
+
Finds unused functions, unused classes, empty pass branches, and dead code:
|
|
120
|
+
```bash
|
|
121
|
+
py -m pycleaner dead-code .
|
|
122
|
+
py -m pycleaner dead-code --json .
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
#### `audit` (Project Dependency Verification)
|
|
126
|
+
Audits imported third-party libraries against `requirements.txt`:
|
|
127
|
+
```bash
|
|
128
|
+
# Audit only
|
|
129
|
+
py -m pycleaner audit .
|
|
130
|
+
|
|
131
|
+
# Automatically append missing dependencies
|
|
132
|
+
py -m pycleaner audit --fix-deps .
|
|
133
|
+
|
|
134
|
+
# Append missing and remove unimported dependencies
|
|
135
|
+
py -m pycleaner audit --fix-deps --prune-deps .
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
#### `all` (Complete Quality Sweep)
|
|
139
|
+
Executes all engines in one command (fix + dependency audit + security scan + complexity + dead code):
|
|
140
|
+
```bash
|
|
141
|
+
py -m pycleaner all .
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
#### `watch` (Continuous File Watcher)
|
|
145
|
+
Watches files for filesystem modifications and auto-cleans on save:
|
|
146
|
+
```bash
|
|
147
|
+
py -m pycleaner watch src/ --interval 1.0
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
#### `hook` (Pre-Commit Integration)
|
|
151
|
+
Outputs a ready-to-use `.pre-commit-hooks.yaml` configuration block:
|
|
152
|
+
```bash
|
|
153
|
+
py -m pycleaner hook
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
---
|
|
157
|
+
|
|
158
|
+
### Backward-Compatible Flat Invocations
|
|
159
|
+
|
|
160
|
+
Legacy invocations continue to work seamlessly:
|
|
161
|
+
```bash
|
|
162
|
+
pycleaner -a . # Equivalent to: pycleaner all .
|
|
163
|
+
pycleaner --check --diff src/ # Equivalent to: pycleaner check --diff src/
|
|
164
|
+
pycleaner --deps-only . # Equivalent to: pycleaner audit .
|
|
165
|
+
pycleaner path/to/file.py # Equivalent to: pycleaner fix path/to/file.py
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
---
|
|
169
|
+
|
|
170
|
+
## Configuration
|
|
171
|
+
|
|
172
|
+
`pycleaner` automatically reads configuration from `pyproject.toml` under `[tool.pycleaner]` or from `.pycleaner.toml`.
|
|
173
|
+
|
|
174
|
+
### `pyproject.toml` Example
|
|
175
|
+
|
|
176
|
+
```toml
|
|
177
|
+
[tool.pycleaner]
|
|
178
|
+
# File targeting
|
|
179
|
+
exclude = ["migrations/", "generated/", "*_pb2.py", "*_pb2_grpc.py"]
|
|
180
|
+
include = ["src/", "tests/"]
|
|
181
|
+
|
|
182
|
+
# Syntax healing
|
|
183
|
+
fix-py2-syntax = true
|
|
184
|
+
fix-conditional-assignments = true
|
|
185
|
+
|
|
186
|
+
# Import resolution
|
|
187
|
+
custom-import-map = { "logger" = "from myapp.core.logging import logger" }
|
|
188
|
+
auto-add-future-annotations = false
|
|
189
|
+
|
|
190
|
+
# Linting & formatting
|
|
191
|
+
line-length = 88
|
|
192
|
+
select-rules = "F401,F841,I,UP,E,W,B,SIM,RUF"
|
|
193
|
+
|
|
194
|
+
# Dead code
|
|
195
|
+
ignore-decorators = ["@app.route", "@pytest.fixture", "@override"]
|
|
196
|
+
ignore-names = ["_*", "test_*"]
|
|
197
|
+
|
|
198
|
+
# Security
|
|
199
|
+
security-severity-threshold = "LOW"
|
|
200
|
+
ignore-security-rules = []
|
|
201
|
+
|
|
202
|
+
# Complexity thresholds
|
|
203
|
+
max-cyclomatic-complexity = 10
|
|
204
|
+
max-cognitive-complexity = 15
|
|
205
|
+
max-function-length = 50
|
|
206
|
+
max-arguments = 5
|
|
207
|
+
|
|
208
|
+
# Behavior
|
|
209
|
+
backup = true
|
|
210
|
+
parallel = false
|
|
211
|
+
max-workers = 4
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
---
|
|
215
|
+
|
|
216
|
+
## Programmatic API
|
|
217
|
+
|
|
218
|
+
### 1. Cleaning Source Code or Files
|
|
219
|
+
```python
|
|
220
|
+
from pathlib import Path
|
|
221
|
+
from pycleaner import CleanPipeline, load_config
|
|
222
|
+
|
|
223
|
+
# Load config with project overrides
|
|
224
|
+
config = load_config(project_root=".")
|
|
225
|
+
pipeline = CleanPipeline(config=config)
|
|
226
|
+
|
|
227
|
+
# In-memory code processing
|
|
228
|
+
dirty_code = """
|
|
229
|
+
import math
|
|
230
|
+
|
|
231
|
+
def calculate(x)
|
|
232
|
+
if x = 0
|
|
233
|
+
return Path('.')
|
|
234
|
+
return Path(str(x))
|
|
235
|
+
"""
|
|
236
|
+
result = pipeline.process_source(dirty_code, filename="example.py")
|
|
237
|
+
print("Cleaned code:\n", result.cleaned_code)
|
|
238
|
+
print("Repairs applied:", result.syntax_repairs)
|
|
239
|
+
print("Imports added:", result.resolved_imports)
|
|
240
|
+
|
|
241
|
+
# File processing with automatic backup
|
|
242
|
+
file_result = pipeline.process_file("example.py", apply_changes=True, backup=True)
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
### 2. Security Vulnerability Scanning
|
|
246
|
+
```python
|
|
247
|
+
from pycleaner import SecurityScanner
|
|
248
|
+
|
|
249
|
+
scanner = SecurityScanner(severity_threshold="MEDIUM")
|
|
250
|
+
report = scanner.scan_project("src/")
|
|
251
|
+
|
|
252
|
+
for finding in report.findings:
|
|
253
|
+
print(f"[{finding.severity}] {finding.category} at {finding.filepath}:{finding.lineno}")
|
|
254
|
+
print(f" Message: {finding.message}")
|
|
255
|
+
print(f" Fix: {finding.suggestion}")
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
### 3. Complexity Analysis
|
|
259
|
+
```python
|
|
260
|
+
from pycleaner import ComplexityAnalyzer
|
|
261
|
+
|
|
262
|
+
analyzer = ComplexityAnalyzer()
|
|
263
|
+
report = analyzer.analyze_project("src/")
|
|
264
|
+
|
|
265
|
+
print(f"Scanned {report.files_scanned} files, {report.count} functions.")
|
|
266
|
+
print(f"Average Cyclomatic: {report.average_cyclomatic:.2f}")
|
|
267
|
+
|
|
268
|
+
violations = report.above_threshold(max_cyclomatic=10, max_cognitive=15)
|
|
269
|
+
for func in violations:
|
|
270
|
+
print(f"{func.qualified_name} (CC={func.cyclomatic}, Cog={func.cognitive})")
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
### 4. Dead Code Detection
|
|
274
|
+
```python
|
|
275
|
+
from pycleaner import DeadCodeDetector
|
|
276
|
+
|
|
277
|
+
detector = DeadCodeDetector()
|
|
278
|
+
report = detector.analyze_project("src/")
|
|
279
|
+
|
|
280
|
+
for item in report.unused_symbols:
|
|
281
|
+
print(f"Unused {item.kind} '{item.name}' at {item.filepath}:{item.lineno}")
|
|
282
|
+
|
|
283
|
+
for item in report.unreachable_code:
|
|
284
|
+
print(f"Unreachable code at {item.filepath}:{item.lineno} ({item.reason})")
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
---
|
|
288
|
+
|
|
289
|
+
## Exit Codes
|
|
290
|
+
|
|
291
|
+
| Exit Code | Meaning |
|
|
292
|
+
|:---:|:---|
|
|
293
|
+
| `0` | Clean run: all files valid, no errors, or changes cleanly written |
|
|
294
|
+
| `1` | Issues found: check mode detected modifications, errors occurred, or critical security findings flagged |
|
|
295
|
+
| `2` | Configuration error: invalid TOML syntax, invalid configuration keys, or mismatched types |
|
|
296
|
+
|
|
297
|
+
---
|
|
298
|
+
|
|
299
|
+
## License
|
|
300
|
+
|
|
301
|
+
MIT License.
|
|
302
|
+
# ball-python
|
|
303
|
+
# ball-python
|