python-code-quality 0.2.3__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- py_cq/cli.py +9 -11
- py_cq/config/config.toml +4 -0
- py_cq/context_hash.py +58 -15
- py_cq/execution_engine.py +152 -59
- py_cq/localtypes.py +3 -1
- py_cq/parsers/common.py +2 -5
- py_cq/parsers/coverageparser.py +7 -18
- py_cq/parsers/interrogateparser.py +20 -21
- py_cq/parsers/pytestparser.py +1 -1
- py_cq/source_file.py +199 -0
- py_cq/table_formatter.py +1 -1
- py_cq/tool_registry.py +1 -0
- {python_code_quality-0.2.3.dist-info → python_code_quality-0.3.0.dist-info}/METADATA +116 -103
- {python_code_quality-0.2.3.dist-info → python_code_quality-0.3.0.dist-info}/RECORD +16 -15
- {python_code_quality-0.2.3.dist-info → python_code_quality-0.3.0.dist-info}/WHEEL +1 -1
- {python_code_quality-0.2.3.dist-info → python_code_quality-0.3.0.dist-info}/entry_points.txt +0 -0
py_cq/cli.py
CHANGED
|
@@ -17,7 +17,7 @@ import logging
|
|
|
17
17
|
import time
|
|
18
18
|
import tomllib
|
|
19
19
|
from enum import Enum
|
|
20
|
-
from importlib.metadata import requires, version
|
|
20
|
+
from importlib.metadata import PackageNotFoundError, requires, version
|
|
21
21
|
from pathlib import Path
|
|
22
22
|
|
|
23
23
|
import tomlkit
|
|
@@ -88,7 +88,7 @@ def _version_callback(value: bool) -> None:
|
|
|
88
88
|
dep_name = re.split(r"[>=<!;\s\[]", req)[0]
|
|
89
89
|
try:
|
|
90
90
|
dep_versions.append((dep_name, version(dep_name)))
|
|
91
|
-
except
|
|
91
|
+
except PackageNotFoundError:
|
|
92
92
|
pass
|
|
93
93
|
typer.echo(f"{pkg} v{pkg_version}")
|
|
94
94
|
for dep_name, dep_ver in sorted(dep_versions):
|
|
@@ -120,7 +120,7 @@ console = Console()
|
|
|
120
120
|
@app.command()
|
|
121
121
|
def check(
|
|
122
122
|
path: str = typer.Argument(".", help="Path to Python file or project directory"),
|
|
123
|
-
output: OutputMode = typer.Option(
|
|
123
|
+
output: OutputMode = typer.Option( # noqa: B008
|
|
124
124
|
OutputMode.TABLE,
|
|
125
125
|
"--output",
|
|
126
126
|
"-o",
|
|
@@ -160,13 +160,13 @@ def check(
|
|
|
160
160
|
limit: int = typer.Option(
|
|
161
161
|
1, "--limit", help="Number of issues to show with -o llm (default: 1)"
|
|
162
162
|
),
|
|
163
|
-
silence: list[str] = typer.Option(
|
|
163
|
+
silence: list[str] = typer.Option( # noqa: B008
|
|
164
164
|
[],
|
|
165
165
|
"--silence",
|
|
166
166
|
"-s",
|
|
167
167
|
help="Silence issues from -o llm output (e.g. -s src/foo.py or -s src/foo.py:42:E501)",
|
|
168
168
|
),
|
|
169
|
-
order: OrderMode = typer.Option(
|
|
169
|
+
order: OrderMode = typer.Option( # noqa: B008
|
|
170
170
|
OrderMode.SEVERITY,
|
|
171
171
|
"--order",
|
|
172
172
|
help="Issue selection order for -o llm/llm-json: severity (most severe first) or phase (first non-clean phase, in dependency order)",
|
|
@@ -186,12 +186,10 @@ def check(
|
|
|
186
186
|
)
|
|
187
187
|
raise typer.Exit(0)
|
|
188
188
|
|
|
189
|
-
if path_obj.is_file():
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
if not (path_obj / "pyproject.toml").exists():
|
|
194
|
-
raise typer.BadParameter(f"Directory must contain pyproject.toml: {path}")
|
|
189
|
+
if path_obj.is_file() and path_obj.suffix != ".py":
|
|
190
|
+
raise typer.BadParameter(f"File must be a Python file (.py): {path}")
|
|
191
|
+
elif path_obj.is_dir() and not (path_obj / "pyproject.toml").exists():
|
|
192
|
+
raise typer.BadParameter(f"Directory must contain pyproject.toml: {path}")
|
|
195
193
|
log.setLevel(log_level)
|
|
196
194
|
|
|
197
195
|
only_list = [t.strip() for t in only.split(",")] if only else None
|
py_cq/config/config.toml
CHANGED
|
@@ -4,6 +4,7 @@ parser = "CompileParser"
|
|
|
4
4
|
order = 1
|
|
5
5
|
warning_threshold = 0.9999
|
|
6
6
|
error_threshold = 0.9999
|
|
7
|
+
cache_invariant = "ast"
|
|
7
8
|
|
|
8
9
|
[python.ruff]
|
|
9
10
|
command = "{python} -m ruff check --output-format concise --no-cache \"{context_path}\" --exclude .claude{exclude}"
|
|
@@ -42,6 +43,7 @@ error_threshold = 1.0
|
|
|
42
43
|
run_in_target_env = true
|
|
43
44
|
extra_deps = ["pytest"]
|
|
44
45
|
skip_for_file = true
|
|
46
|
+
cache_invariant = "ast"
|
|
45
47
|
|
|
46
48
|
[python.coverage]
|
|
47
49
|
command = "{python} -m coverage run --omit=*/tests/*,*/test_*.py -m pytest \"{context_path}\" --ignore .claude{exclude} && {python} -m coverage report --show-missing --omit=*/tests/*,*/test_*.py"
|
|
@@ -54,6 +56,7 @@ run_in_target_env = true
|
|
|
54
56
|
extra_deps = ["coverage", "pytest"]
|
|
55
57
|
skip_for_file = true
|
|
56
58
|
skip_if = "pytest"
|
|
59
|
+
cache_invariant = "ast"
|
|
57
60
|
|
|
58
61
|
[python.vulture]
|
|
59
62
|
command = "{python} -m vulture \"{context_path}\" --min-confidence 80 --exclude .venv,dist,.*_cache,docs,.git,.claude{exclude}"
|
|
@@ -62,6 +65,7 @@ parser = "VultureParser"
|
|
|
62
65
|
order = 7
|
|
63
66
|
warning_threshold = 0.9999
|
|
64
67
|
error_threshold = 0.8
|
|
68
|
+
cache_invariant = "ast"
|
|
65
69
|
|
|
66
70
|
[python.interrogate]
|
|
67
71
|
command = "{python} -m interrogate \"{context_path}\" -e .claude{exclude} -v --fail-under 0"
|
py_cq/context_hash.py
CHANGED
|
@@ -12,23 +12,56 @@ change-detection in larger systems:
|
|
|
12
12
|
a file or directory. For a file it hashes its path, size, and modification
|
|
13
13
|
time; for a directory it aggregates the signatures of all contained files.
|
|
14
14
|
|
|
15
|
+
When called with ``normalize=True`` the hash is derived from the AST of each
|
|
16
|
+
``.py`` file with docstrings stripped, so changes that cannot affect runtime
|
|
17
|
+
behaviour (docstrings, comments, formatting, mtime-only touches such as a
|
|
18
|
+
``git checkout``) leave the hash unchanged. This is used as the cache key for
|
|
19
|
+
tools whose results only depend on executable code (pytest, coverage).
|
|
20
|
+
|
|
15
21
|
These functions provide deterministic fingerprints that can be used for
|
|
16
22
|
file integrity verification, caching, and change-detection logic.
|
|
17
23
|
"""
|
|
18
24
|
|
|
19
25
|
import hashlib
|
|
20
26
|
import os
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
from py_cq.source_file import get_source
|
|
30
|
+
|
|
31
|
+
_ENV_FILES = {".python-version", "pyproject.toml", "uv.lock"}
|
|
32
|
+
_SKIP_DIRS = {".venv", "venv", "__pycache__", ".git"}
|
|
21
33
|
|
|
22
34
|
|
|
23
|
-
def
|
|
35
|
+
def _normalized_sig(path: str) -> str:
|
|
36
|
+
"""Return a docstring/comment/format-invariant signature for a ``.py`` file.
|
|
37
|
+
|
|
38
|
+
Backed by the shared :class:`~py_cq.source_file.SourceFile` cache, so the
|
|
39
|
+
AST parsed here is reused by the coverage and interrogate parsers. Falls
|
|
40
|
+
back to a byte digest when the file can't be parsed (e.g. a mid-edit syntax
|
|
41
|
+
error), so callers always get a usable signature.
|
|
42
|
+
"""
|
|
43
|
+
return f"{path}:{get_source(path).norm_digest}"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _byte_sig(path: str) -> str:
|
|
47
|
+
"""Return a path + content digest, independent of mtime."""
|
|
48
|
+
return f"{path}:{hashlib.md5(Path(path).read_bytes()).hexdigest()}" # nosec
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def get_sigs(path: str, *, normalize: bool = False):
|
|
24
52
|
"""Recursively scans a directory tree and returns a list of signatures for all Python files.
|
|
25
53
|
|
|
26
54
|
The signature format is ``<file_path>:<size_bytes>:<mtime>`` where ``mtime`` is the
|
|
27
55
|
last modification timestamp retrieved from ``os.stat``. The traversal skips
|
|
28
56
|
directories named ``.venv``, ``venv`` and ``__pycache__``.
|
|
29
57
|
|
|
58
|
+
When ``normalize`` is ``True``, ``.py`` files are fingerprinted by their
|
|
59
|
+
docstring-stripped AST and env files by their byte content, so the result is
|
|
60
|
+
invariant to docstrings, comments, formatting, and mtime-only changes.
|
|
61
|
+
|
|
30
62
|
Args:
|
|
31
63
|
path (str): The root directory to scan.
|
|
64
|
+
normalize (bool): Use AST/content fingerprints instead of size+mtime.
|
|
32
65
|
|
|
33
66
|
Returns:
|
|
34
67
|
list[str]: A signature string for each ``.py`` file found.
|
|
@@ -41,7 +74,6 @@ def get_sigs(path: str):
|
|
|
41
74
|
>>> get_sigs('/tmp/project')
|
|
42
75
|
['/tmp/project/main.py:1024:1680000000.0', ...]
|
|
43
76
|
"""
|
|
44
|
-
_ENV_FILES = {".python-version", "pyproject.toml", "uv.lock"}
|
|
45
77
|
items = []
|
|
46
78
|
with os.scandir(path) as entries:
|
|
47
79
|
for entry in entries:
|
|
@@ -50,27 +82,35 @@ def get_sigs(path: str):
|
|
|
50
82
|
if entry.is_file(follow_symlinks=False) and (
|
|
51
83
|
entry.name.endswith(".py") or entry.name in _ENV_FILES
|
|
52
84
|
):
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
85
|
+
if normalize:
|
|
86
|
+
if entry.name.endswith(".py"):
|
|
87
|
+
items.append(_normalized_sig(entry.path))
|
|
88
|
+
else:
|
|
89
|
+
items.append(_byte_sig(entry.path))
|
|
90
|
+
else:
|
|
91
|
+
stat_info = entry.stat(follow_symlinks=False)
|
|
92
|
+
items.append(
|
|
93
|
+
f"{entry.path}:{stat_info.st_size}:{stat_info.st_mtime}"
|
|
94
|
+
)
|
|
95
|
+
if entry.is_dir(follow_symlinks=False) and entry.name not in _SKIP_DIRS:
|
|
96
|
+
items.extend(get_sigs(entry.path, normalize=normalize))
|
|
62
97
|
return items
|
|
63
98
|
|
|
64
99
|
|
|
65
|
-
def get_context_hash(path: str) -> str:
|
|
100
|
+
def get_context_hash(path: str, *, normalize: bool = False) -> str:
|
|
66
101
|
"""Compute an MD5 hash that uniquely identifies a file or directory.
|
|
67
102
|
|
|
68
103
|
The hash is derived from a signature string. For a file, the signature consists of
|
|
69
104
|
its path, size, and modification time. For a directory, the signature is the
|
|
70
105
|
concatenation of the signatures of all files within it (recursively).
|
|
71
106
|
|
|
107
|
+
When ``normalize`` is ``True`` the signatures are derived from each ``.py``
|
|
108
|
+
file's docstring-stripped AST, making the hash invariant to docstrings,
|
|
109
|
+
comments, formatting, and mtime-only changes.
|
|
110
|
+
|
|
72
111
|
Args:
|
|
73
112
|
path (str): The filesystem path to hash.
|
|
113
|
+
normalize (bool): Use AST/content fingerprints instead of size+mtime.
|
|
74
114
|
|
|
75
115
|
Returns:
|
|
76
116
|
str: The hexadecimal MD5 digest.
|
|
@@ -84,10 +124,13 @@ def get_context_hash(path: str) -> str:
|
|
|
84
124
|
"""
|
|
85
125
|
h = hashlib.md5() # nosec
|
|
86
126
|
if os.path.isfile(path):
|
|
87
|
-
|
|
88
|
-
|
|
127
|
+
if normalize and path.endswith(".py"):
|
|
128
|
+
h.update(_normalized_sig(path).encode())
|
|
129
|
+
else:
|
|
130
|
+
s = os.stat(path)
|
|
131
|
+
h.update(f"{path}:{s.st_size}:{s.st_mtime}".encode())
|
|
89
132
|
elif os.path.isdir(path):
|
|
90
|
-
for sig in sorted(get_sigs(path)):
|
|
133
|
+
for sig in sorted(get_sigs(path, normalize=normalize)):
|
|
91
134
|
h.update(sig.encode())
|
|
92
135
|
else:
|
|
93
136
|
h.update(b"empty")
|
py_cq/execution_engine.py
CHANGED
|
@@ -16,9 +16,11 @@ import logging
|
|
|
16
16
|
import os
|
|
17
17
|
import shlex
|
|
18
18
|
import shutil
|
|
19
|
+
import signal
|
|
19
20
|
import subprocess
|
|
20
21
|
import sys
|
|
21
22
|
import tempfile
|
|
23
|
+
import threading
|
|
22
24
|
import time
|
|
23
25
|
from collections.abc import Collection
|
|
24
26
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
@@ -102,6 +104,80 @@ def _build_exclude_str(
|
|
|
102
104
|
return "".join(parts)
|
|
103
105
|
|
|
104
106
|
|
|
107
|
+
def _terminate_process_tree(proc: "subprocess.Popen") -> None:
|
|
108
|
+
"""Best-effort kill of a shell subprocess and all its children, cross-platform.
|
|
109
|
+
|
|
110
|
+
``shell=True`` spawns a shell that spawns ``uv``/``python``, so killing the
|
|
111
|
+
immediate child is not enough - the whole tree must go.
|
|
112
|
+
"""
|
|
113
|
+
if proc.poll() is not None:
|
|
114
|
+
return
|
|
115
|
+
if sys.platform == "win32":
|
|
116
|
+
subprocess.run(
|
|
117
|
+
["taskkill", "/F", "/T", "/PID", str(proc.pid)],
|
|
118
|
+
capture_output=True,
|
|
119
|
+
check=False,
|
|
120
|
+
) # nosec
|
|
121
|
+
else:
|
|
122
|
+
try:
|
|
123
|
+
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
|
124
|
+
except (ProcessLookupError, PermissionError):
|
|
125
|
+
proc.kill()
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _run_command(
|
|
129
|
+
command: str, run_env: dict, cancel_event: threading.Event | None
|
|
130
|
+
) -> tuple[str, str, int, bool]:
|
|
131
|
+
"""Run *command* in a shell and return ``(stdout, stderr, returncode, cancelled)``.
|
|
132
|
+
|
|
133
|
+
When *cancel_event* fires mid-run the process tree is terminated and
|
|
134
|
+
``cancelled`` is True (captured output is discarded). With no event this is
|
|
135
|
+
a plain blocking ``subprocess.run`` (no poll overhead for the common path).
|
|
136
|
+
"""
|
|
137
|
+
if cancel_event is None:
|
|
138
|
+
result = subprocess.run(
|
|
139
|
+
command,
|
|
140
|
+
capture_output=True,
|
|
141
|
+
text=True,
|
|
142
|
+
shell=True,
|
|
143
|
+
encoding="utf-8",
|
|
144
|
+
errors="replace",
|
|
145
|
+
env=run_env,
|
|
146
|
+
check=False,
|
|
147
|
+
) # nosec
|
|
148
|
+
return result.stdout, result.stderr, result.returncode, False
|
|
149
|
+
if cancel_event.is_set():
|
|
150
|
+
return "", "", -1, True
|
|
151
|
+
popen_kwargs: dict[str, Any] = {}
|
|
152
|
+
if sys.platform != "win32":
|
|
153
|
+
# New session/process group so os.killpg reaches the shell's children.
|
|
154
|
+
popen_kwargs["start_new_session"] = True
|
|
155
|
+
proc = subprocess.Popen(
|
|
156
|
+
command,
|
|
157
|
+
stdout=subprocess.PIPE,
|
|
158
|
+
stderr=subprocess.PIPE,
|
|
159
|
+
text=True,
|
|
160
|
+
shell=True,
|
|
161
|
+
encoding="utf-8",
|
|
162
|
+
errors="replace",
|
|
163
|
+
env=run_env,
|
|
164
|
+
**popen_kwargs,
|
|
165
|
+
) # nosec
|
|
166
|
+
# Poll so a cancel signal can terminate a still-running command.
|
|
167
|
+
while True:
|
|
168
|
+
try:
|
|
169
|
+
stdout, stderr = proc.communicate(timeout=0.1)
|
|
170
|
+
return stdout, stderr, proc.returncode, False
|
|
171
|
+
except subprocess.TimeoutExpired:
|
|
172
|
+
if cancel_event.is_set():
|
|
173
|
+
_terminate_process_tree(proc)
|
|
174
|
+
try:
|
|
175
|
+
proc.communicate(timeout=5)
|
|
176
|
+
except subprocess.TimeoutExpired: # pragma: no cover
|
|
177
|
+
proc.kill()
|
|
178
|
+
return "", "", proc.returncode or -1, True
|
|
179
|
+
|
|
180
|
+
|
|
105
181
|
def run_tool(
|
|
106
182
|
tool_config: ToolConfig,
|
|
107
183
|
context_path: str,
|
|
@@ -109,6 +185,7 @@ def run_tool(
|
|
|
109
185
|
*,
|
|
110
186
|
precomputed_hash: str | None = None,
|
|
111
187
|
project_tag: str | None = None,
|
|
188
|
+
cancel_event: threading.Event | None = None,
|
|
112
189
|
) -> RawResult:
|
|
113
190
|
"""Runs a tool defined by its configuration and returns the execution result.
|
|
114
191
|
|
|
@@ -195,11 +272,12 @@ def run_tool(
|
|
|
195
272
|
python=python,
|
|
196
273
|
exclude=exclude,
|
|
197
274
|
)
|
|
198
|
-
|
|
199
|
-
precomputed_hash
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
275
|
+
if precomputed_hash is not None:
|
|
276
|
+
context_hash = precomputed_hash
|
|
277
|
+
elif tool_config.cache_invariant == "ast":
|
|
278
|
+
context_hash = get_context_hash(context_path, normalize=True)
|
|
279
|
+
else:
|
|
280
|
+
context_hash = get_context_hash(context_path)
|
|
203
281
|
cache_key = f"{command}:{context_hash}"
|
|
204
282
|
|
|
205
283
|
t_cache0 = time.perf_counter()
|
|
@@ -223,18 +301,20 @@ def run_tool(
|
|
|
223
301
|
run_env["COVERAGE_FILE"] = coverage_tmp
|
|
224
302
|
t_sub0 = time.perf_counter()
|
|
225
303
|
try:
|
|
226
|
-
|
|
227
|
-
command,
|
|
228
|
-
|
|
229
|
-
text=True,
|
|
230
|
-
shell=True,
|
|
231
|
-
encoding="utf-8",
|
|
232
|
-
errors="replace",
|
|
233
|
-
env=run_env,
|
|
234
|
-
) # nosec
|
|
304
|
+
stdout, stderr, return_code, cancelled = _run_command(
|
|
305
|
+
command, run_env, cancel_event
|
|
306
|
+
)
|
|
235
307
|
finally:
|
|
236
308
|
Path(coverage_tmp).unlink(missing_ok=True)
|
|
237
309
|
t_sub = time.perf_counter() - t_sub0
|
|
310
|
+
if cancelled:
|
|
311
|
+
# A dependency (e.g. pytest) already failed, so this run is moot. Return
|
|
312
|
+
# an empty result - uncached, since it was terminated, not completed -
|
|
313
|
+
# and let the parser produce its skip/zero output.
|
|
314
|
+
log.debug(
|
|
315
|
+
f"{tool_config.name}: [CANCELLED] {tool_config.skip_if} failed; terminated after {t_sub * 1000:.0f}ms"
|
|
316
|
+
)
|
|
317
|
+
return RawResult(tool_name=tool_config.name)
|
|
238
318
|
log.debug(
|
|
239
319
|
f"{tool_config.name}: [MISS] cache={t_cache * 1000:.1f}ms tool={t_sub * 1000:.0f}ms: {command}"
|
|
240
320
|
)
|
|
@@ -242,9 +322,9 @@ def run_tool(
|
|
|
242
322
|
raw_result = RawResult(
|
|
243
323
|
tool_name=tool_config.name,
|
|
244
324
|
command=command,
|
|
245
|
-
stdout=
|
|
246
|
-
stderr=
|
|
247
|
-
return_code=
|
|
325
|
+
stdout=stdout,
|
|
326
|
+
stderr=stderr,
|
|
327
|
+
return_code=return_code,
|
|
248
328
|
timestamp=timestamp,
|
|
249
329
|
project_path=project_dir,
|
|
250
330
|
)
|
|
@@ -308,16 +388,29 @@ def run_tools(
|
|
|
308
388
|
shared_hash = get_context_hash(root)
|
|
309
389
|
log.debug(f"context_hash: {(time.perf_counter() - t_hash0) * 1000:.1f}ms {shared_hash}")
|
|
310
390
|
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
391
|
+
# Cache keys are content-based, so a changed project naturally produces new
|
|
392
|
+
# keys; stale entries age out via TTL + size_limit (no blanket eviction).
|
|
393
|
+
# Tools flagged cache_invariant="ast" key on the docstring-stripped AST so
|
|
394
|
+
# docstring/comment/format-only edits stay cache hits. Compute it once here
|
|
395
|
+
# (not lazily in the worker threads, where pytest and coverage would race
|
|
396
|
+
# and each re-parse the whole tree) and only when some tool needs it.
|
|
397
|
+
norm_hash = shared_hash
|
|
398
|
+
if any(tc.cache_invariant == "ast" for tc in tool_configs):
|
|
399
|
+
t_norm0 = time.perf_counter()
|
|
400
|
+
norm_hash = get_context_hash(root, normalize=True)
|
|
401
|
+
log.debug(f"context_hash(ast): {(time.perf_counter() - t_norm0) * 1000:.1f}ms {norm_hash}")
|
|
402
|
+
|
|
403
|
+
def _hash_for(tool_config: ToolConfig) -> str:
|
|
404
|
+
return norm_hash if tool_config.cache_invariant == "ast" else shared_hash
|
|
405
|
+
|
|
406
|
+
def _run_and_parse(
|
|
407
|
+
tool_config: ToolConfig, cancel_event: threading.Event | None = None
|
|
408
|
+
) -> tuple[int, ToolResult]:
|
|
319
409
|
t0 = time.perf_counter()
|
|
320
|
-
|
|
410
|
+
# Only pass cancel_event when present so patched run_tool stubs in tests
|
|
411
|
+
# (which don't declare the kwarg) keep working for non-cancellable tools.
|
|
412
|
+
extra = {"cancel_event": cancel_event} if cancel_event is not None else {}
|
|
413
|
+
raw_result = run_tool(tool_config, path, excludes, precomputed_hash=_hash_for(tool_config), project_tag=root, **extra)
|
|
321
414
|
tr = tool_config.parser_class(tool_config.parser_config).parse(raw_result)
|
|
322
415
|
tr.duration_s = time.perf_counter() - t0
|
|
323
416
|
return tool_config.order, tr
|
|
@@ -329,7 +422,7 @@ def run_tools(
|
|
|
329
422
|
for i, tool_config in enumerate(sorted_configs):
|
|
330
423
|
try:
|
|
331
424
|
prioritized.append(_run_and_parse(tool_config))
|
|
332
|
-
except Exception as exc:
|
|
425
|
+
except Exception as exc: # noqa: BLE001 - a crashing tool must not abort the run
|
|
333
426
|
log.error(f"{tool_config.name} generated an exception: {exc} {exc.__traceback__}")
|
|
334
427
|
n_skipped = n_total - i - 1
|
|
335
428
|
if n_skipped:
|
|
@@ -357,52 +450,52 @@ def run_tools(
|
|
|
357
450
|
return [tr for _, tr in sorted(prioritized)]
|
|
358
451
|
configs_by_name = {tc.name: tc for tc in tool_configs}
|
|
359
452
|
|
|
360
|
-
def _run_if_dep_passed(dep_future, tool_config: ToolConfig, dep_config: ToolConfig) -> tuple[int, ToolResult]:
|
|
361
|
-
# Wait for the dependency within the same thread pool - no phase split,
|
|
362
|
-
# so all other tools keep running in parallel while we block here.
|
|
363
|
-
try:
|
|
364
|
-
_, dep_tr = dep_future.result()
|
|
365
|
-
except Exception:
|
|
366
|
-
return _run_and_parse(tool_config)
|
|
367
|
-
if dep_tr.metrics and min(dep_tr.metrics.values()) < dep_config.error_threshold:
|
|
368
|
-
# Dependency failed its error threshold: skip this tool and return a
|
|
369
|
-
# synthetic empty result. Coverage declares skip_if = "pytest" so that
|
|
370
|
-
# it never re-invokes pytest (which the coverage command does internally)
|
|
371
|
-
# when we already know pytest is broken.
|
|
372
|
-
log.debug(f"{tool_config.name}: skipping because {tool_config.skip_if} failed")
|
|
373
|
-
t0 = time.perf_counter()
|
|
374
|
-
tr = tool_config.parser_class(tool_config.parser_config).parse(
|
|
375
|
-
RawResult(tool_name=tool_config.name)
|
|
376
|
-
)
|
|
377
|
-
tr.duration_s = time.perf_counter() - t0
|
|
378
|
-
return tool_config.order, tr
|
|
379
|
-
return _run_and_parse(tool_config)
|
|
380
|
-
|
|
381
453
|
timings: list[tuple[int, str, float]] = []
|
|
382
|
-
# Sort by order so that when we build name_to_future below, a dependency's
|
|
383
|
-
# future is always registered before the tool that declares skip_if on it.
|
|
384
454
|
sorted_configs = sorted(tool_configs, key=lambda tc: tc.order)
|
|
455
|
+
# A tool with skip_if gets a cancel event: it runs in parallel with its
|
|
456
|
+
# dependency, and is terminated mid-run only if that dependency *fails* its
|
|
457
|
+
# error threshold. Coverage declares skip_if = "pytest" so a broken pytest
|
|
458
|
+
# kills the coverage run (which re-invokes pytest internally) instead of
|
|
459
|
+
# letting it finish a pointless second suite execution.
|
|
460
|
+
cancel_events = {
|
|
461
|
+
tc.name: threading.Event()
|
|
462
|
+
for tc in sorted_configs
|
|
463
|
+
if tc.skip_if and tc.skip_if in configs_by_name
|
|
464
|
+
}
|
|
385
465
|
name_to_future: dict[str, Any] = {}
|
|
386
466
|
with ThreadPoolExecutor(max_workers=max_workers or len(tool_configs)) as executor:
|
|
387
467
|
future_to_tool = {}
|
|
388
468
|
for tc in sorted_configs:
|
|
389
|
-
|
|
390
|
-
dep_future = name_to_future.get(dep_name) if dep_name else None
|
|
391
|
-
dep_config = configs_by_name.get(dep_name) if dep_name else None
|
|
392
|
-
if dep_future and dep_config:
|
|
393
|
-
# Submit a wrapper that waits on the dependency future first.
|
|
394
|
-
f = executor.submit(_run_if_dep_passed, dep_future, tc, dep_config)
|
|
395
|
-
else:
|
|
396
|
-
f = executor.submit(_run_and_parse, tc)
|
|
469
|
+
f = executor.submit(_run_and_parse, tc, cancel_events.get(tc.name))
|
|
397
470
|
future_to_tool[f] = tc
|
|
398
471
|
name_to_future[tc.name] = f
|
|
472
|
+
|
|
473
|
+
def _cancel_on_dep_failure(
|
|
474
|
+
dep_future, ev: threading.Event, dep_config: ToolConfig, name: str
|
|
475
|
+
) -> None:
|
|
476
|
+
try:
|
|
477
|
+
_, dep_tr = dep_future.result()
|
|
478
|
+
except Exception: # noqa: BLE001 - a crashed dep cancels nothing
|
|
479
|
+
return
|
|
480
|
+
if dep_tr.metrics and min(dep_tr.metrics.values()) < dep_config.error_threshold:
|
|
481
|
+
log.debug(f"{name}: cancelling because {dep_config.name} failed")
|
|
482
|
+
ev.set()
|
|
483
|
+
|
|
484
|
+
for tc in sorted_configs:
|
|
485
|
+
if tc.name in cancel_events:
|
|
486
|
+
dep_config = configs_by_name[tc.skip_if]
|
|
487
|
+
name_to_future[tc.skip_if].add_done_callback(
|
|
488
|
+
lambda fut, ev=cancel_events[tc.name], dc=dep_config, nm=tc.name: _cancel_on_dep_failure(
|
|
489
|
+
fut, ev, dc, nm
|
|
490
|
+
)
|
|
491
|
+
)
|
|
399
492
|
for future in as_completed(future_to_tool):
|
|
400
493
|
tool_config = future_to_tool[future]
|
|
401
494
|
try:
|
|
402
495
|
tc_order, tr = future.result()
|
|
403
496
|
prioritized.append((tc_order, tr))
|
|
404
497
|
timings.append((tc_order, tool_config.name, tr.duration_s))
|
|
405
|
-
except Exception as exc:
|
|
498
|
+
except Exception as exc: # noqa: BLE001 - a crashing tool must not abort the run
|
|
406
499
|
log.error(f"{tool_config.name} generated an exception: {exc}")
|
|
407
500
|
per_tool = ", ".join(f"{name}={dur:.2f}s" for _, name, dur in sorted(timings))
|
|
408
501
|
log.debug(f"run_tools elapsed: {time.perf_counter() - t_start:.2f}s [{per_tool}]")
|
py_cq/localtypes.py
CHANGED
|
@@ -62,6 +62,9 @@ class ToolConfig:
|
|
|
62
62
|
skip_for_file: bool = False # If True, skip when context_path is a single file
|
|
63
63
|
gate_strict: bool = True # If False, small regressions above warning_threshold are accepted
|
|
64
64
|
skip_if: str = "" # Tool name; if that tool's score is below its error_threshold, skip this tool
|
|
65
|
+
cache_invariant: str = (
|
|
66
|
+
"" # "ast": key the cache on the docstring-stripped AST, ignoring docstrings/comments/format
|
|
67
|
+
)
|
|
65
68
|
|
|
66
69
|
|
|
67
70
|
@dataclass
|
|
@@ -172,7 +175,6 @@ class AbstractParser(ABC):
|
|
|
172
175
|
@abstractmethod
|
|
173
176
|
def parse(self, raw_result: RawResult) -> ToolResult:
|
|
174
177
|
"""Converts raw tool output into a structured ToolResult."""
|
|
175
|
-
pass
|
|
176
178
|
|
|
177
179
|
def format_llm_message(
|
|
178
180
|
self, tr: ToolResult, *, context_lines: int = 15, limit: int = 1
|
py_cq/parsers/common.py
CHANGED
|
@@ -48,9 +48,7 @@ def format_source_context(
|
|
|
48
48
|
collected = []
|
|
49
49
|
for i, rline in enumerate(raw_lines):
|
|
50
50
|
if i > error_offset and (
|
|
51
|
-
rline.startswith("def ")
|
|
52
|
-
or rline.startswith("async def ")
|
|
53
|
-
or rline.startswith("class ")
|
|
51
|
+
rline.startswith(("def ", "async def ", "class "))
|
|
54
52
|
):
|
|
55
53
|
break
|
|
56
54
|
collected.append(f"{context_start + i}: {rline}")
|
|
@@ -382,8 +380,7 @@ def score_logistic_variant(
|
|
|
382
380
|
1.0
|
|
383
381
|
>>> score_logistic_variant(10, scale_factor=0)
|
|
384
382
|
0.0"""
|
|
385
|
-
|
|
386
|
-
errors = 0
|
|
383
|
+
errors = max(errors, 0)
|
|
387
384
|
if scale_factor == 0:
|
|
388
385
|
return 1.0 if errors == 0 else 0.0
|
|
389
386
|
try:
|
py_cq/parsers/coverageparser.py
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
"""Parses raw coverage tool output into structured ToolResult instances with per-function granularity."""
|
|
2
2
|
|
|
3
|
-
import ast
|
|
4
3
|
import logging
|
|
5
4
|
from pathlib import Path
|
|
6
5
|
|
|
7
6
|
from py_cq.localtypes import AbstractParser, RawResult, ToolResult
|
|
8
7
|
from py_cq.parsers.common import find_function_source, resolve_path
|
|
8
|
+
from py_cq.source_file import get_source
|
|
9
9
|
|
|
10
10
|
log = logging.getLogger("cq")
|
|
11
11
|
|
|
@@ -29,30 +29,19 @@ def _parse_line_ranges(s: str) -> set[int]:
|
|
|
29
29
|
return result
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
def _get_signature(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str:
|
|
33
|
-
"""Return the signature of the function definition as a string."""
|
|
34
|
-
prefix = "async def" if isinstance(node, ast.AsyncFunctionDef) else "def"
|
|
35
|
-
args = ast.unparse(node.args)
|
|
36
|
-
returns = f" -> {ast.unparse(node.returns)}" if node.returns else ""
|
|
37
|
-
return f"{prefix} {node.name}({args}){returns}"
|
|
38
|
-
|
|
39
|
-
|
|
40
32
|
def _extract_functions(file: str, missing_lines_str: str) -> list[tuple[str, int, str]]:
|
|
41
33
|
"""Return (name, lineno, signature) for functions whose bodies overlap with the missing line ranges."""
|
|
42
34
|
try:
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
except (OSError, SyntaxError, ValueError):
|
|
35
|
+
funcs = get_source(file).functions
|
|
36
|
+
except OSError:
|
|
46
37
|
return []
|
|
47
38
|
missing = _parse_line_ranges(missing_lines_str)
|
|
48
39
|
seen: set[str] = set()
|
|
49
40
|
result: list[tuple[str, int, str]] = []
|
|
50
|
-
for
|
|
51
|
-
if
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
seen.add(node.name)
|
|
55
|
-
result.append((node.name, node.lineno, _get_signature(node)))
|
|
41
|
+
for fn in funcs:
|
|
42
|
+
if missing & set(range(fn.lineno, fn.end_lineno + 1)) and fn.name not in seen:
|
|
43
|
+
seen.add(fn.name)
|
|
44
|
+
result.append((fn.name, fn.lineno, fn.signature))
|
|
56
45
|
return result
|
|
57
46
|
|
|
58
47
|
|
|
@@ -21,6 +21,7 @@ from py_cq.parsers.common import (
|
|
|
21
21
|
format_issue_header,
|
|
22
22
|
format_source_context,
|
|
23
23
|
)
|
|
24
|
+
from py_cq.source_file import get_source
|
|
24
25
|
|
|
25
26
|
_SKIP_PARAMS = {"self", "cls"}
|
|
26
27
|
|
|
@@ -34,9 +35,10 @@ def _format_missing_docstring(file_str: str, line: int, code: str, message: str)
|
|
|
34
35
|
return base + "\n\nInsert a module-level docstring as the very first statement in the file."
|
|
35
36
|
|
|
36
37
|
try:
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
38
|
+
tree = get_source(file_str).tree
|
|
39
|
+
except OSError:
|
|
40
|
+
tree = None
|
|
41
|
+
if tree is None:
|
|
40
42
|
return base + "\n\nInsert a docstring as the first statement in the body."
|
|
41
43
|
|
|
42
44
|
for node in ast.walk(tree):
|
|
@@ -88,7 +90,7 @@ def _load_interrogate_cfg(context_path: str) -> dict:
|
|
|
88
90
|
try:
|
|
89
91
|
data = tomllib.loads(pyproject.read_text(encoding="utf-8"))
|
|
90
92
|
return data.get("tool", {}).get("interrogate", {})
|
|
91
|
-
except
|
|
93
|
+
except (tomllib.TOMLDecodeError, OSError, UnicodeDecodeError):
|
|
92
94
|
return {}
|
|
93
95
|
return {}
|
|
94
96
|
|
|
@@ -104,9 +106,7 @@ def _skip_node(name: str, cfg: dict) -> bool:
|
|
|
104
106
|
return True
|
|
105
107
|
if is_private and cfg.get("ignore-private"):
|
|
106
108
|
return True
|
|
107
|
-
|
|
108
|
-
return True
|
|
109
|
-
return False
|
|
109
|
+
return bool(is_semiprivate and cfg.get("ignore-semiprivate"))
|
|
110
110
|
|
|
111
111
|
|
|
112
112
|
def _missing_docstrings(file_path: Path, cfg: dict | None = None) -> list[tuple[int, str, str]]:
|
|
@@ -117,11 +117,13 @@ def _missing_docstrings(file_path: Path, cfg: dict | None = None) -> list[tuple[
|
|
|
117
117
|
the module.
|
|
118
118
|
"""
|
|
119
119
|
try:
|
|
120
|
-
|
|
121
|
-
tree =
|
|
122
|
-
except
|
|
120
|
+
sf = get_source(str(file_path))
|
|
121
|
+
tree = sf.tree
|
|
122
|
+
except OSError:
|
|
123
123
|
return []
|
|
124
|
-
|
|
124
|
+
if tree is None:
|
|
125
|
+
return []
|
|
126
|
+
src_lines = sf.text.splitlines()
|
|
125
127
|
|
|
126
128
|
def src_line(lineno: int) -> str:
|
|
127
129
|
"""Return the stripped content of the source line at the given line number."""
|
|
@@ -137,16 +139,13 @@ def _missing_docstrings(file_path: Path, cfg: dict | None = None) -> list[tuple[
|
|
|
137
139
|
|
|
138
140
|
effective_cfg = cfg or {}
|
|
139
141
|
results = []
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
elif isinstance(node, ast.ClassDef):
|
|
148
|
-
if not ast.get_docstring(node) and not _skip_node(node.name, effective_cfg):
|
|
149
|
-
results.append((node.lineno, "class", src_line(node.lineno)))
|
|
142
|
+
if not sf.module_has_docstring:
|
|
143
|
+
results.append((0, "module", first_code_line()))
|
|
144
|
+
for d in sf.definitions:
|
|
145
|
+
if d.has_docstring or _skip_node(d.name, effective_cfg):
|
|
146
|
+
continue
|
|
147
|
+
kind = "def" if d.kind == "function" else "class"
|
|
148
|
+
results.append((d.lineno, kind, src_line(d.lineno)))
|
|
150
149
|
results.sort(key=lambda x: x[0])
|
|
151
150
|
return results
|
|
152
151
|
|
py_cq/parsers/pytestparser.py
CHANGED
py_cq/source_file.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""Process-scoped cache of parsed source files, shared across one ``cq check``.
|
|
2
|
+
|
|
3
|
+
Within a single run the same ``.py`` file is otherwise read and parsed several
|
|
4
|
+
times: by the normalized context hash, the coverage parser, the interrogate
|
|
5
|
+
parser, and the LLM formatters. :class:`SourceFile` parses each file at most
|
|
6
|
+
once and exposes lazy projections (functions, classes, docstring presence) that
|
|
7
|
+
downstream consumers walk instead of re-parsing.
|
|
8
|
+
|
|
9
|
+
The cache is keyed on ``(path, size, mtime)`` so a content change (which bumps
|
|
10
|
+
``mtime``) misses and forces a fresh parse; correctness never depends on a hit.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import ast
|
|
14
|
+
import hashlib
|
|
15
|
+
import os
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class FuncInfo:
|
|
22
|
+
"""A function or class definition discovered in a source file.
|
|
23
|
+
|
|
24
|
+
``signature`` is the rendered ``def`` line for functions and ``""`` for
|
|
25
|
+
classes (no consumer needs a class signature). ``kind`` is ``"function"``
|
|
26
|
+
or ``"class"``.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
name: str
|
|
30
|
+
kind: str
|
|
31
|
+
lineno: int
|
|
32
|
+
end_lineno: int
|
|
33
|
+
has_docstring: bool
|
|
34
|
+
signature: str
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _docstring_invariant_digest(tree: ast.AST) -> str:
|
|
38
|
+
"""Return an md5 of ``tree`` with docstrings removed, leaving ``tree`` intact.
|
|
39
|
+
|
|
40
|
+
Docstring statements are temporarily detached, the tree is dumped and
|
|
41
|
+
hashed, then they are reattached - so the digest is docstring-invariant
|
|
42
|
+
without parsing a second time or mutating the tree that other consumers
|
|
43
|
+
share. Safe because the hash pass runs single-threaded before the tool pool.
|
|
44
|
+
"""
|
|
45
|
+
detached = []
|
|
46
|
+
for node in ast.walk(tree):
|
|
47
|
+
if isinstance(
|
|
48
|
+
node, (ast.Module, ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)
|
|
49
|
+
):
|
|
50
|
+
body = node.body
|
|
51
|
+
if (
|
|
52
|
+
body
|
|
53
|
+
and isinstance(body[0], ast.Expr)
|
|
54
|
+
and isinstance(body[0].value, ast.Constant)
|
|
55
|
+
and isinstance(body[0].value.value, str)
|
|
56
|
+
):
|
|
57
|
+
detached.append((node, body[0]))
|
|
58
|
+
node.body = body[1:]
|
|
59
|
+
digest = hashlib.md5(ast.dump(tree).encode()).hexdigest() # nosec
|
|
60
|
+
for node, doc in detached:
|
|
61
|
+
node.body = [doc, *node.body]
|
|
62
|
+
return digest
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _signature(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str:
|
|
66
|
+
"""Return the signature of the function definition as a string."""
|
|
67
|
+
prefix = "async def" if isinstance(node, ast.AsyncFunctionDef) else "def"
|
|
68
|
+
args = ast.unparse(node.args)
|
|
69
|
+
returns = f" -> {ast.unparse(node.returns)}" if node.returns else ""
|
|
70
|
+
return f"{prefix} {node.name}({args}){returns}"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class SourceFile:
|
|
74
|
+
"""Lazily-realized view of one source file: text, AST, and projections.
|
|
75
|
+
|
|
76
|
+
Fields are computed on first access and cached. ``.tree`` stays pristine
|
|
77
|
+
(docstrings intact) so multiple consumers can share it; ``.norm_digest``
|
|
78
|
+
derives its docstring-stripped digest from a throwaway reparse rather than
|
|
79
|
+
mutating ``.tree``.
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
def __init__(self, path: str) -> None:
|
|
83
|
+
self.path = path
|
|
84
|
+
self._raw: bytes | None = None
|
|
85
|
+
self._text: str | None = None
|
|
86
|
+
self._tree: ast.AST | None = None
|
|
87
|
+
self._parsed = False
|
|
88
|
+
self._norm_digest: str | None = None
|
|
89
|
+
self._definitions: list[FuncInfo] | None = None
|
|
90
|
+
|
|
91
|
+
def _read_bytes(self) -> bytes:
|
|
92
|
+
if self._raw is None:
|
|
93
|
+
self._raw = Path(self.path).read_bytes()
|
|
94
|
+
return self._raw
|
|
95
|
+
|
|
96
|
+
@property
|
|
97
|
+
def text(self) -> str:
|
|
98
|
+
"""Return the file contents decoded as UTF-8 (undecodable bytes replaced)."""
|
|
99
|
+
if self._text is None:
|
|
100
|
+
self._text = self._read_bytes().decode("utf-8", "replace")
|
|
101
|
+
return self._text
|
|
102
|
+
|
|
103
|
+
@property
|
|
104
|
+
def tree(self) -> ast.AST | None:
|
|
105
|
+
"""Return the parsed AST, or ``None`` if the file can't be parsed."""
|
|
106
|
+
if not self._parsed:
|
|
107
|
+
self._parsed = True
|
|
108
|
+
try:
|
|
109
|
+
self._tree = ast.parse(self.text)
|
|
110
|
+
except (SyntaxError, ValueError):
|
|
111
|
+
self._tree = None
|
|
112
|
+
return self._tree
|
|
113
|
+
|
|
114
|
+
@property
|
|
115
|
+
def norm_digest(self) -> str:
|
|
116
|
+
"""Return a docstring/comment/format-invariant digest of the file.
|
|
117
|
+
|
|
118
|
+
Falls back to a byte digest when the file can't be parsed (e.g. a
|
|
119
|
+
mid-edit syntax error), so callers always get a usable digest.
|
|
120
|
+
"""
|
|
121
|
+
if self._norm_digest is None:
|
|
122
|
+
tree = self.tree
|
|
123
|
+
if tree is None:
|
|
124
|
+
self._norm_digest = hashlib.md5(self._read_bytes()).hexdigest() # nosec
|
|
125
|
+
else:
|
|
126
|
+
self._norm_digest = _docstring_invariant_digest(tree)
|
|
127
|
+
return self._norm_digest
|
|
128
|
+
|
|
129
|
+
@property
|
|
130
|
+
def definitions(self) -> list[FuncInfo]:
|
|
131
|
+
"""Return every function and class definition, in source order."""
|
|
132
|
+
if self._definitions is None:
|
|
133
|
+
defs: list[FuncInfo] = []
|
|
134
|
+
tree = self.tree
|
|
135
|
+
if tree is not None:
|
|
136
|
+
for node in ast.walk(tree):
|
|
137
|
+
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
138
|
+
defs.append(
|
|
139
|
+
FuncInfo(
|
|
140
|
+
node.name,
|
|
141
|
+
"function",
|
|
142
|
+
node.lineno,
|
|
143
|
+
getattr(node, "end_lineno", node.lineno),
|
|
144
|
+
ast.get_docstring(node) is not None,
|
|
145
|
+
_signature(node),
|
|
146
|
+
)
|
|
147
|
+
)
|
|
148
|
+
elif isinstance(node, ast.ClassDef):
|
|
149
|
+
defs.append(
|
|
150
|
+
FuncInfo(
|
|
151
|
+
node.name,
|
|
152
|
+
"class",
|
|
153
|
+
node.lineno,
|
|
154
|
+
getattr(node, "end_lineno", node.lineno),
|
|
155
|
+
ast.get_docstring(node) is not None,
|
|
156
|
+
"",
|
|
157
|
+
)
|
|
158
|
+
)
|
|
159
|
+
defs.sort(key=lambda d: d.lineno)
|
|
160
|
+
self._definitions = defs
|
|
161
|
+
return self._definitions
|
|
162
|
+
|
|
163
|
+
@property
|
|
164
|
+
def functions(self) -> list[FuncInfo]:
|
|
165
|
+
"""Return the function definitions only (no classes), in source order."""
|
|
166
|
+
return [d for d in self.definitions if d.kind == "function"]
|
|
167
|
+
|
|
168
|
+
@property
|
|
169
|
+
def module_has_docstring(self) -> bool:
|
|
170
|
+
"""Return True if the module has a docstring."""
|
|
171
|
+
tree = self.tree
|
|
172
|
+
return isinstance(tree, ast.Module) and ast.get_docstring(tree) is not None
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
# Process-scoped memo: (path, size, mtime) -> SourceFile. A stale mtime only
|
|
176
|
+
# forces a re-parse that reproduces the same projections, so correctness never
|
|
177
|
+
# depends on this cache (matches the lock-free design: a duplicate parse is
|
|
178
|
+
# harmless and GIL-serialized).
|
|
179
|
+
_source_cache: dict[tuple[str, int, float], SourceFile] = {}
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def get_source(path: str) -> SourceFile:
|
|
183
|
+
"""Return the cached :class:`SourceFile` for ``path``, creating it on miss.
|
|
184
|
+
|
|
185
|
+
Keyed on ``(canonical_path, size, mtime)``; a content change misses and
|
|
186
|
+
re-parses. The path is canonicalized (absolute + OS case folding) so the
|
|
187
|
+
hash pass and the parsers - which spell the same file differently (relative
|
|
188
|
+
vs. resolved/posix) - share one cache entry.
|
|
189
|
+
|
|
190
|
+
Raises:
|
|
191
|
+
OSError: If ``path`` cannot be stat'd.
|
|
192
|
+
"""
|
|
193
|
+
s = os.stat(path)
|
|
194
|
+
key = (os.path.normcase(os.path.abspath(path)), s.st_size, s.st_mtime)
|
|
195
|
+
sf = _source_cache.get(key)
|
|
196
|
+
if sf is None:
|
|
197
|
+
sf = SourceFile(path)
|
|
198
|
+
_source_cache[key] = sf
|
|
199
|
+
return sf
|
py_cq/table_formatter.py
CHANGED
|
@@ -21,7 +21,7 @@ def format_as_table(
|
|
|
21
21
|
table.add_column("Status")
|
|
22
22
|
for tr in data.tool_results:
|
|
23
23
|
tool_name = tr.raw.tool_name
|
|
24
|
-
config = next(
|
|
24
|
+
config = next(t for t in registry.values() if t.name == tool_name)
|
|
25
25
|
for i, (name, value) in enumerate(tr.metrics.items()):
|
|
26
26
|
if value < config.error_threshold:
|
|
27
27
|
status = "[bold red]Error[/]"
|
py_cq/tool_registry.py
CHANGED
|
@@ -34,6 +34,7 @@ def load_tool_configs() -> dict[str, ToolConfig]:
|
|
|
34
34
|
skip_for_file=tool_data.get("skip_for_file", False),
|
|
35
35
|
gate_strict=tool_data.get("gate_strict", True),
|
|
36
36
|
skip_if=tool_data.get("skip_if", ""),
|
|
37
|
+
cache_invariant=tool_data.get("cache_invariant", ""),
|
|
37
38
|
)
|
|
38
39
|
return registry
|
|
39
40
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: python-code-quality
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Python Code Quality Analysis Tool - feed the results from 11 CQ tools straight into an LLM. Minimal tokens.
|
|
5
5
|
Author: Chris Kilner
|
|
6
6
|
Author-email: Chris Kilner <chris@rhiza.fr>
|
|
@@ -17,7 +17,7 @@ Requires-Dist: pytest-cov>=6.1.1
|
|
|
17
17
|
Requires-Dist: pytest-json-report>=1.5.0
|
|
18
18
|
Requires-Dist: radon>=6.0.1
|
|
19
19
|
Requires-Dist: rich>=14.0.0
|
|
20
|
-
Requires-Dist: ruff>=0.14.1
|
|
20
|
+
Requires-Dist: ruff>=0.14.1,<0.17
|
|
21
21
|
Requires-Dist: tomlkit>=0.13.0
|
|
22
22
|
Requires-Dist: ty>=0.0.17
|
|
23
23
|
Requires-Dist: typer>=0.16.0
|
|
@@ -65,6 +65,8 @@ uv tool install .
|
|
|
65
65
|
|
|
66
66
|
These tools are run in **parallel** except:
|
|
67
67
|
When running '-o llm', we run sequentially and exit early at the first error.
|
|
68
|
+
`coverage` is a special case: it starts alongside `pytest` and is cancelled if
|
|
69
|
+
`pytest` fails, scoring `0%` without a second test run.
|
|
68
70
|
|
|
69
71
|
| Order | Tool | Measures |
|
|
70
72
|
|----------|------|----------|
|
|
@@ -74,11 +76,11 @@ When running '-o llm', we run sequentially and exit early at the first error.
|
|
|
74
76
|
| 4 | bandit | Security vulnerabilities |
|
|
75
77
|
| 5 | pytest | Test pass rate |
|
|
76
78
|
| 6 | coverage | Test coverage |
|
|
77
|
-
| 7 |
|
|
78
|
-
| 8 |
|
|
79
|
-
| 9 | radon
|
|
80
|
-
| 10 |
|
|
81
|
-
| 11 |
|
|
79
|
+
| 7 | vulture | Dead code |
|
|
80
|
+
| 8 | interrogate | Docstring coverage |
|
|
81
|
+
| 9 | radon cc | Cyclomatic complexity |
|
|
82
|
+
| 10 | radon mi | Maintainability index |
|
|
83
|
+
| 11 | radon hal | Halstead volume / bug estimate |
|
|
82
84
|
|
|
83
85
|
Diskcache is used to cache tool output for lightning fast re-runs. Sane defaults: <100 Mb, <5 days, No pickle risk.
|
|
84
86
|
|
|
@@ -337,106 +339,117 @@ warning = 0.9
|
|
|
337
339
|
error = 0.7
|
|
338
340
|
```
|
|
339
341
|
|
|
340
|
-
Tool IDs match the keys in `config/config.
|
|
342
|
+
Tool IDs match the keys in `src/py_cq/config/config.toml`: `compile`, `ruff`, `ty`, `bandit`, `pytest`, `coverage`, `radon-cc`, `radon-mi`, `radon-hal`, `vulture`, `interrogate`.
|
|
341
343
|
|
|
342
344
|
|
|
343
345
|
### Default config
|
|
344
346
|
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
347
|
+
Defined in `src/py_cq/config/config.toml` (sections are namespaced by language):
|
|
348
|
+
|
|
349
|
+
```toml
|
|
350
|
+
[python.compile]
|
|
351
|
+
command = "{python} -m compileall -r 10 -j 8 \"{context_path}\" -x .*venv|\\.claude"
|
|
352
|
+
parser = "CompileParser"
|
|
353
|
+
order = 1
|
|
354
|
+
warning_threshold = 0.9999
|
|
355
|
+
error_threshold = 0.9999
|
|
356
|
+
cache_invariant = "ast"
|
|
357
|
+
|
|
358
|
+
[python.ruff]
|
|
359
|
+
command = "{python} -m ruff check --output-format concise --no-cache \"{context_path}\" --exclude .claude{exclude}"
|
|
360
|
+
exclude_format = " --exclude {path}"
|
|
361
|
+
parser = "RuffParser"
|
|
362
|
+
order = 2
|
|
363
|
+
warning_threshold = 0.9999
|
|
364
|
+
error_threshold = 0.9
|
|
365
|
+
|
|
366
|
+
[python.ty]
|
|
367
|
+
command = "{python} -m ty check --output-format concise --color never \"{context_path}\" --exclude .claude{exclude}"
|
|
368
|
+
exclude_format = " --exclude {path}"
|
|
369
|
+
parser = "TyParser"
|
|
370
|
+
order = 3
|
|
371
|
+
warning_threshold = 0.9999
|
|
372
|
+
error_threshold = 0.8
|
|
373
|
+
run_in_target_env = true
|
|
374
|
+
extra_deps = ["ty"]
|
|
375
|
+
|
|
376
|
+
[python.bandit]
|
|
377
|
+
command = "{python} -m bandit -r {scan_targets} -f json -q -s B101 --severity-level medium{exclude}"
|
|
378
|
+
scan_exclude_names = [".venv", ".claude", "tests"]
|
|
379
|
+
exclude_format = " --exclude {abs_native_path}"
|
|
380
|
+
parser = "BanditParser"
|
|
381
|
+
order = 4
|
|
382
|
+
warning_threshold = 0.9999
|
|
383
|
+
error_threshold = 0.8
|
|
384
|
+
|
|
385
|
+
[python.pytest]
|
|
386
|
+
command = "{python} -m pytest -vv \"{context_path}\" --ignore .claude{exclude}"
|
|
387
|
+
exclude_format = " --ignore {path}"
|
|
388
|
+
parser = "PytestParser"
|
|
389
|
+
order = 5
|
|
390
|
+
warning_threshold = 1.0
|
|
391
|
+
error_threshold = 1.0
|
|
392
|
+
run_in_target_env = true
|
|
393
|
+
extra_deps = ["pytest"]
|
|
394
|
+
skip_for_file = true
|
|
395
|
+
cache_invariant = "ast"
|
|
396
|
+
|
|
397
|
+
[python.coverage]
|
|
398
|
+
command = "{python} -m coverage run --omit=*/tests/*,*/test_*.py -m pytest \"{context_path}\" --ignore .claude{exclude} && {python} -m coverage report --show-missing --omit=*/tests/*,*/test_*.py"
|
|
399
|
+
exclude_format = " --ignore {path}"
|
|
400
|
+
parser = "CoverageParser"
|
|
401
|
+
order = 6
|
|
402
|
+
warning_threshold = 0.9
|
|
403
|
+
error_threshold = 0.5
|
|
404
|
+
run_in_target_env = true
|
|
405
|
+
extra_deps = ["coverage", "pytest"]
|
|
406
|
+
skip_for_file = true
|
|
407
|
+
skip_if = "pytest"
|
|
408
|
+
cache_invariant = "ast"
|
|
409
|
+
|
|
410
|
+
[python.vulture]
|
|
411
|
+
command = "{python} -m vulture \"{context_path}\" --min-confidence 80 --exclude .venv,dist,.*_cache,docs,.git,.claude{exclude}"
|
|
412
|
+
exclude_format = ",{path}"
|
|
413
|
+
parser = "VultureParser"
|
|
414
|
+
order = 7
|
|
415
|
+
warning_threshold = 0.9999
|
|
416
|
+
error_threshold = 0.8
|
|
417
|
+
cache_invariant = "ast"
|
|
418
|
+
|
|
419
|
+
[python.interrogate]
|
|
420
|
+
command = "{python} -m interrogate \"{context_path}\" -e .claude{exclude} -v --fail-under 0"
|
|
421
|
+
exclude_format = " -e {path}"
|
|
422
|
+
parser = "InterrogateParser"
|
|
423
|
+
order = 8
|
|
424
|
+
warning_threshold = 0.8
|
|
425
|
+
error_threshold = 0.5
|
|
426
|
+
|
|
427
|
+
[python.interrogate.parser_config]
|
|
428
|
+
skip_empty_init = true
|
|
429
|
+
|
|
430
|
+
[python.radon-cc]
|
|
431
|
+
command = "{python} -m radon cc --json --exclude '.claude/**' \"{context_path}\""
|
|
432
|
+
parser = "ComplexityParser"
|
|
433
|
+
order = 9
|
|
434
|
+
gate_strict = false
|
|
435
|
+
warning_threshold = 0.6
|
|
436
|
+
error_threshold = 0.4
|
|
437
|
+
|
|
438
|
+
[python.radon-mi]
|
|
439
|
+
command = "{python} -m radon mi -s --json --exclude '.claude/**' \"{context_path}\""
|
|
440
|
+
parser = "MaintainabilityParser"
|
|
441
|
+
order = 10
|
|
442
|
+
gate_strict = false
|
|
443
|
+
warning_threshold = 0.6
|
|
444
|
+
error_threshold = 0.4
|
|
445
|
+
|
|
446
|
+
[python.radon-hal]
|
|
447
|
+
command = "{python} -m radon hal -f --json --exclude '.claude/**' \"{context_path}\""
|
|
448
|
+
parser = "HalsteadParser"
|
|
449
|
+
order = 11
|
|
450
|
+
gate_strict = false
|
|
451
|
+
warning_threshold = 0.35
|
|
452
|
+
error_threshold = 0.3
|
|
440
453
|
```
|
|
441
454
|
|
|
442
455
|
## Respect
|
|
@@ -1,35 +1,36 @@
|
|
|
1
1
|
py_cq/__init__.py,sha256=-C6_QzQeydxft3ndlExTgWnvlqH8fJNPJxfySXULQmU,56
|
|
2
2
|
py_cq/api.py,sha256=AxhTQVX4ACZmgCpdfJgB2_1VXtClxc531NkRoLK1vA0,10118
|
|
3
|
-
py_cq/cli.py,sha256=
|
|
3
|
+
py_cq/cli.py,sha256=aaLnBXF81fkUtCvO7DGzmogcq50X5bF5dC4XFC4sbqY,13814
|
|
4
4
|
py_cq/config/__init__.py,sha256=f0wc51O_3kGDTZUnCbGv8_zWnC5yYGl4NWcf2buSImQ,670
|
|
5
|
-
py_cq/config/config.toml,sha256=
|
|
6
|
-
py_cq/context_hash.py,sha256=
|
|
7
|
-
py_cq/execution_engine.py,sha256=
|
|
5
|
+
py_cq/config/config.toml,sha256=fco0qKWfIg84e3ex2pmAcSPYBmcHdbiqL-Geo__EIkM,3017
|
|
6
|
+
py_cq/context_hash.py,sha256=uKMkN394v6QYMukZj9usbxnq8px3i1khyIRPKwe09DE,5614
|
|
7
|
+
py_cq/execution_engine.py,sha256=vAbNDCHP5c-2ADWs9SM0eCEpZBiabhpl-WZp6Er3qb4,20644
|
|
8
8
|
py_cq/language_detector.py,sha256=rh5bAA3LyCQR3RODmx_EalqRNYvSYK2SJzh2hIkxOUo,1139
|
|
9
9
|
py_cq/llm_formatter.py,sha256=nCV3nFdxJAi2CnH9FKb5tB-0w-yjLWmjywQI5GJ2cSU,8412
|
|
10
|
-
py_cq/localtypes.py,sha256=
|
|
10
|
+
py_cq/localtypes.py,sha256=mt4sqUg1bz2BenirGA18X4CdtyZahTAuKyOfwFnwfSE,7907
|
|
11
11
|
py_cq/main.py,sha256=5dgr8Zk1LNK5M-0wncYVksBp0qE-jWZYqmOXfb1eOhk,400
|
|
12
12
|
py_cq/metric_aggregator.py,sha256=M2ymo62S7p7qPUqjjoiPg4IVyXQhLMuTr9-jxLiFjCY,853
|
|
13
13
|
py_cq/parsers/__init__.py,sha256=uCy7R0cnvDPNvB5aWMLgM3FB9nDcu1jwUmznFMRROSY,28
|
|
14
14
|
py_cq/parsers/banditparser.py,sha256=YqpWljU5D4IhLdtzat7XVGBm9NXrbOQbr9LI9NKybK8,3298
|
|
15
|
-
py_cq/parsers/common.py,sha256=
|
|
15
|
+
py_cq/parsers/common.py,sha256=r8UD7pch-txlJSTsJrOff2o3_0WHRlZhgKbJ-C58L7g,13818
|
|
16
16
|
py_cq/parsers/compileparser.py,sha256=jl0DloyXTKqVHRIUft-t5lvJOndY-PdLPrXB38v9zuA,6927
|
|
17
17
|
py_cq/parsers/complexityparser.py,sha256=mX1WHPRgBBc6kOqHfubowRbYXY0E1k4JVwr908Cj10U,5582
|
|
18
|
-
py_cq/parsers/coverageparser.py,sha256=
|
|
18
|
+
py_cq/parsers/coverageparser.py,sha256=QGjZ8JJF0G-0uiZ07BzxSnblGiysHtOwq0SYvWrtOhA,7441
|
|
19
19
|
py_cq/parsers/exitcodeparser.py,sha256=b4LWIwwklPwCX3MnqNsGQC1slQU56DKE05vJ0OdWVoA,957
|
|
20
20
|
py_cq/parsers/halsteadparser.py,sha256=789c_WWfomskM4i7SUEbKvB67-F2y5nfqt48aFXOusY,10192
|
|
21
|
-
py_cq/parsers/interrogateparser.py,sha256=
|
|
21
|
+
py_cq/parsers/interrogateparser.py,sha256=55HZ2j1Rz34eremwGx-W9oIMI21DIqwxEG524UgsAhk,11249
|
|
22
22
|
py_cq/parsers/linecountparser.py,sha256=-hqebwG4j7_QugzGLUUQRcPukGHicV2nbIp1pR8ZcAw,1310
|
|
23
23
|
py_cq/parsers/maintainabilityparser.py,sha256=3a4m93XNtUurw3IxEI1Iae9idgJY6qiEt7HMYl1wTu0,4793
|
|
24
|
-
py_cq/parsers/pytestparser.py,sha256=
|
|
24
|
+
py_cq/parsers/pytestparser.py,sha256=cIaTs2XxgeJlVragLEVTZcrbB-bzfNYYMdUrakRa-rQ,11545
|
|
25
25
|
py_cq/parsers/regexcountparser.py,sha256=zGzHEYDIori6aiWnVkRFixdE7fWt66npAAWjzvpoBF4,1592
|
|
26
26
|
py_cq/parsers/ruffparser.py,sha256=uE0QEaZ4Ch7IsWY_2ZkfHXPU0TYPGsGPfv66hGe5G34,7634
|
|
27
27
|
py_cq/parsers/typarser.py,sha256=x835OrapG1aUteW2NhC5U0bhj1w1pMtbfmbvb74-1AI,8077
|
|
28
28
|
py_cq/parsers/vultureparser.py,sha256=-yGAXzRY4_fzLRTk7dfwDrrRZ-n97j1Ph5_xo3c6TfU,2229
|
|
29
29
|
py_cq/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
30
|
-
py_cq/
|
|
31
|
-
py_cq/
|
|
32
|
-
|
|
33
|
-
python_code_quality-0.
|
|
34
|
-
python_code_quality-0.
|
|
35
|
-
python_code_quality-0.
|
|
30
|
+
py_cq/source_file.py,sha256=Uyv-CSVeliFnE6TpRB4Y15ykFQs4n0r9-amzxXXlqN0,7562
|
|
31
|
+
py_cq/table_formatter.py,sha256=-22nzVIOaDe3QOBmZ2iBPH7ewIEIPEn5WYSjCoB4Hj4,1608
|
|
32
|
+
py_cq/tool_registry.py,sha256=0s0sD9_iQW-HU0SIKSS8CFDlrzW0c-vlYrAFODEHiSc,1894
|
|
33
|
+
python_code_quality-0.3.0.dist-info/WHEEL,sha256=ZFFp7t7R4RYQ5KYZkmiFWoQvHay7SrTmn-6ZYfoFZ3U,80
|
|
34
|
+
python_code_quality-0.3.0.dist-info/entry_points.txt,sha256=cfWbTw7eYO6Trv1-Z_odL6Zta9CqYU6Vk9lAHdfB60Q,40
|
|
35
|
+
python_code_quality-0.3.0.dist-info/METADATA,sha256=hQBSoblR8TKmTPRriXRt5Ed-uO9jexstNj0nPU4Li4A,16213
|
|
36
|
+
python_code_quality-0.3.0.dist-info/RECORD,,
|
{python_code_quality-0.2.3.dist-info → python_code_quality-0.3.0.dist-info}/entry_points.txt
RENAMED
|
File without changes
|