python-code-quality 0.2.3__py3-none-any.whl → 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- py_cq/api.py +9 -3
- py_cq/cli.py +12 -14
- py_cq/config/config.toml +13 -0
- py_cq/context_hash.py +58 -15
- py_cq/execution_engine.py +172 -61
- py_cq/localtypes.py +4 -2
- py_cq/parsers/common.py +2 -7
- py_cq/parsers/coverageparser.py +7 -18
- py_cq/parsers/interrogateparser.py +31 -24
- py_cq/parsers/pytestparser.py +1 -1
- py_cq/parsers/ruffformatparser.py +62 -0
- py_cq/source_file.py +199 -0
- py_cq/table_formatter.py +1 -1
- py_cq/tool_registry.py +1 -0
- {python_code_quality-0.2.3.dist-info → python_code_quality-0.3.1.dist-info}/METADATA +136 -112
- {python_code_quality-0.2.3.dist-info → python_code_quality-0.3.1.dist-info}/RECORD +18 -16
- {python_code_quality-0.2.3.dist-info → python_code_quality-0.3.1.dist-info}/WHEEL +1 -1
- {python_code_quality-0.2.3.dist-info → python_code_quality-0.3.1.dist-info}/entry_points.txt +0 -0
py_cq/api.py
CHANGED
|
@@ -148,7 +148,9 @@ class CQ:
|
|
|
148
148
|
if clear_cache:
|
|
149
149
|
_cache.clear()
|
|
150
150
|
|
|
151
|
-
def raw(
|
|
151
|
+
def raw(
|
|
152
|
+
self, *, early_exit: bool = False, order: str = "severity"
|
|
153
|
+
) -> list[ToolResult]:
|
|
152
154
|
"""Run all tools and return parsed results before aggregation."""
|
|
153
155
|
return run_tools(
|
|
154
156
|
self._registry.values(),
|
|
@@ -218,7 +220,7 @@ class CQ:
|
|
|
218
220
|
)
|
|
219
221
|
if fp.tool not in tool_registry:
|
|
220
222
|
raise ValueError(f"Unknown tool: {fp.tool!r}")
|
|
221
|
-
|
|
223
|
+
|
|
222
224
|
if fp.code and fp.path:
|
|
223
225
|
# Specific issue: target only the affected file for a fast re-check
|
|
224
226
|
file_path = Path(fp.path)
|
|
@@ -242,7 +244,11 @@ class CQ:
|
|
|
242
244
|
|
|
243
245
|
only_registry = {fp.tool: tool_registry[fp.tool]}
|
|
244
246
|
tool_results = run_tools(
|
|
245
|
-
only_registry.values(),
|
|
247
|
+
only_registry.values(),
|
|
248
|
+
target,
|
|
249
|
+
max_workers=1,
|
|
250
|
+
early_exit=False,
|
|
251
|
+
excludes=[],
|
|
246
252
|
project_root=str(self.path),
|
|
247
253
|
)
|
|
248
254
|
|
py_cq/cli.py
CHANGED
|
@@ -17,7 +17,7 @@ import logging
|
|
|
17
17
|
import time
|
|
18
18
|
import tomllib
|
|
19
19
|
from enum import Enum
|
|
20
|
-
from importlib.metadata import requires, version
|
|
20
|
+
from importlib.metadata import PackageNotFoundError, requires, version
|
|
21
21
|
from pathlib import Path
|
|
22
22
|
|
|
23
23
|
import tomlkit
|
|
@@ -88,7 +88,7 @@ def _version_callback(value: bool) -> None:
|
|
|
88
88
|
dep_name = re.split(r"[>=<!;\s\[]", req)[0]
|
|
89
89
|
try:
|
|
90
90
|
dep_versions.append((dep_name, version(dep_name)))
|
|
91
|
-
except
|
|
91
|
+
except PackageNotFoundError:
|
|
92
92
|
pass
|
|
93
93
|
typer.echo(f"{pkg} v{pkg_version}")
|
|
94
94
|
for dep_name, dep_ver in sorted(dep_versions):
|
|
@@ -107,7 +107,7 @@ def callback(
|
|
|
107
107
|
help="Show version and dependencies",
|
|
108
108
|
),
|
|
109
109
|
) -> None:
|
|
110
|
-
"""Feed the results from
|
|
110
|
+
"""Feed the results from 12 code quality tools to an LLM. Try: cq check . -o llm"""
|
|
111
111
|
import sys
|
|
112
112
|
|
|
113
113
|
if isinstance(sys.stdout, io.TextIOWrapper):
|
|
@@ -120,11 +120,11 @@ console = Console()
|
|
|
120
120
|
@app.command()
|
|
121
121
|
def check(
|
|
122
122
|
path: str = typer.Argument(".", help="Path to Python file or project directory"),
|
|
123
|
-
output: OutputMode = typer.Option(
|
|
123
|
+
output: OutputMode = typer.Option( # noqa: B008
|
|
124
124
|
OutputMode.TABLE,
|
|
125
125
|
"--output",
|
|
126
126
|
"-o",
|
|
127
|
-
help="Output mode: table (default), score, json, llm",
|
|
127
|
+
help="Output mode: table (default), score, json, raw, llm, llm-json",
|
|
128
128
|
),
|
|
129
129
|
log_level: str = typer.Option(
|
|
130
130
|
"CRITICAL",
|
|
@@ -160,19 +160,19 @@ def check(
|
|
|
160
160
|
limit: int = typer.Option(
|
|
161
161
|
1, "--limit", help="Number of issues to show with -o llm (default: 1)"
|
|
162
162
|
),
|
|
163
|
-
silence: list[str] = typer.Option(
|
|
163
|
+
silence: list[str] = typer.Option( # noqa: B008
|
|
164
164
|
[],
|
|
165
165
|
"--silence",
|
|
166
166
|
"-s",
|
|
167
167
|
help="Silence issues from -o llm output (e.g. -s src/foo.py or -s src/foo.py:42:E501)",
|
|
168
168
|
),
|
|
169
|
-
order: OrderMode = typer.Option(
|
|
169
|
+
order: OrderMode = typer.Option( # noqa: B008
|
|
170
170
|
OrderMode.SEVERITY,
|
|
171
171
|
"--order",
|
|
172
172
|
help="Issue selection order for -o llm/llm-json: severity (most severe first) or phase (first non-clean phase, in dependency order)",
|
|
173
173
|
),
|
|
174
174
|
):
|
|
175
|
-
"""Feed the results from
|
|
175
|
+
"""Feed the results from 12 code quality tools to an LLM. Try: cq check . -o llm""" # --help
|
|
176
176
|
path_obj = Path(path)
|
|
177
177
|
if not path_obj.exists():
|
|
178
178
|
raise typer.BadParameter(f"Path does not exist: {path}")
|
|
@@ -186,12 +186,10 @@ def check(
|
|
|
186
186
|
)
|
|
187
187
|
raise typer.Exit(0)
|
|
188
188
|
|
|
189
|
-
if path_obj.is_file():
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
if not (path_obj / "pyproject.toml").exists():
|
|
194
|
-
raise typer.BadParameter(f"Directory must contain pyproject.toml: {path}")
|
|
189
|
+
if path_obj.is_file() and path_obj.suffix != ".py":
|
|
190
|
+
raise typer.BadParameter(f"File must be a Python file (.py): {path}")
|
|
191
|
+
elif path_obj.is_dir() and not (path_obj / "pyproject.toml").exists():
|
|
192
|
+
raise typer.BadParameter(f"Directory must contain pyproject.toml: {path}")
|
|
195
193
|
log.setLevel(log_level)
|
|
196
194
|
|
|
197
195
|
only_list = [t.strip() for t in only.split(",")] if only else None
|
py_cq/config/config.toml
CHANGED
|
@@ -4,6 +4,7 @@ parser = "CompileParser"
|
|
|
4
4
|
order = 1
|
|
5
5
|
warning_threshold = 0.9999
|
|
6
6
|
error_threshold = 0.9999
|
|
7
|
+
cache_invariant = "ast"
|
|
7
8
|
|
|
8
9
|
[python.ruff]
|
|
9
10
|
command = "{python} -m ruff check --output-format concise --no-cache \"{context_path}\" --exclude .claude{exclude}"
|
|
@@ -42,6 +43,7 @@ error_threshold = 1.0
|
|
|
42
43
|
run_in_target_env = true
|
|
43
44
|
extra_deps = ["pytest"]
|
|
44
45
|
skip_for_file = true
|
|
46
|
+
cache_invariant = "ast"
|
|
45
47
|
|
|
46
48
|
[python.coverage]
|
|
47
49
|
command = "{python} -m coverage run --omit=*/tests/*,*/test_*.py -m pytest \"{context_path}\" --ignore .claude{exclude} && {python} -m coverage report --show-missing --omit=*/tests/*,*/test_*.py"
|
|
@@ -54,6 +56,7 @@ run_in_target_env = true
|
|
|
54
56
|
extra_deps = ["coverage", "pytest"]
|
|
55
57
|
skip_for_file = true
|
|
56
58
|
skip_if = "pytest"
|
|
59
|
+
cache_invariant = "ast"
|
|
57
60
|
|
|
58
61
|
[python.vulture]
|
|
59
62
|
command = "{python} -m vulture \"{context_path}\" --min-confidence 80 --exclude .venv,dist,.*_cache,docs,.git,.claude{exclude}"
|
|
@@ -62,6 +65,7 @@ parser = "VultureParser"
|
|
|
62
65
|
order = 7
|
|
63
66
|
warning_threshold = 0.9999
|
|
64
67
|
error_threshold = 0.8
|
|
68
|
+
cache_invariant = "ast"
|
|
65
69
|
|
|
66
70
|
[python.interrogate]
|
|
67
71
|
command = "{python} -m interrogate \"{context_path}\" -e .claude{exclude} -v --fail-under 0"
|
|
@@ -97,3 +101,12 @@ order = 11
|
|
|
97
101
|
gate_strict = false
|
|
98
102
|
warning_threshold = 0.35
|
|
99
103
|
error_threshold = 0.3
|
|
104
|
+
|
|
105
|
+
[python.ruff-format]
|
|
106
|
+
command = "{python} -m ruff format --check --output-format concise --no-cache \"{context_path}\" --exclude .claude --config \"include=['*.py','*.pyi','*.ipynb']\"{exclude}"
|
|
107
|
+
exclude_format = " --exclude {path}"
|
|
108
|
+
parser = "RuffFormatParser"
|
|
109
|
+
order = 12
|
|
110
|
+
gate_strict = false
|
|
111
|
+
warning_threshold = 0.9999
|
|
112
|
+
error_threshold = 0.5
|
py_cq/context_hash.py
CHANGED
|
@@ -12,23 +12,56 @@ change-detection in larger systems:
|
|
|
12
12
|
a file or directory. For a file it hashes its path, size, and modification
|
|
13
13
|
time; for a directory it aggregates the signatures of all contained files.
|
|
14
14
|
|
|
15
|
+
When called with ``normalize=True`` the hash is derived from the AST of each
|
|
16
|
+
``.py`` file with docstrings stripped, so changes that cannot affect runtime
|
|
17
|
+
behaviour (docstrings, comments, formatting, mtime-only touches such as a
|
|
18
|
+
``git checkout``) leave the hash unchanged. This is used as the cache key for
|
|
19
|
+
tools whose results only depend on executable code (pytest, coverage).
|
|
20
|
+
|
|
15
21
|
These functions provide deterministic fingerprints that can be used for
|
|
16
22
|
file integrity verification, caching, and change-detection logic.
|
|
17
23
|
"""
|
|
18
24
|
|
|
19
25
|
import hashlib
|
|
20
26
|
import os
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
from py_cq.source_file import get_source
|
|
30
|
+
|
|
31
|
+
_ENV_FILES = {".python-version", "pyproject.toml", "uv.lock"}
|
|
32
|
+
_SKIP_DIRS = {".venv", "venv", "__pycache__", ".git"}
|
|
21
33
|
|
|
22
34
|
|
|
23
|
-
def
|
|
35
|
+
def _normalized_sig(path: str) -> str:
|
|
36
|
+
"""Return a docstring/comment/format-invariant signature for a ``.py`` file.
|
|
37
|
+
|
|
38
|
+
Backed by the shared :class:`~py_cq.source_file.SourceFile` cache, so the
|
|
39
|
+
AST parsed here is reused by the coverage and interrogate parsers. Falls
|
|
40
|
+
back to a byte digest when the file can't be parsed (e.g. a mid-edit syntax
|
|
41
|
+
error), so callers always get a usable signature.
|
|
42
|
+
"""
|
|
43
|
+
return f"{path}:{get_source(path).norm_digest}"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _byte_sig(path: str) -> str:
|
|
47
|
+
"""Return a path + content digest, independent of mtime."""
|
|
48
|
+
return f"{path}:{hashlib.md5(Path(path).read_bytes()).hexdigest()}" # nosec
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def get_sigs(path: str, *, normalize: bool = False):
|
|
24
52
|
"""Recursively scans a directory tree and returns a list of signatures for all Python files.
|
|
25
53
|
|
|
26
54
|
The signature format is ``<file_path>:<size_bytes>:<mtime>`` where ``mtime`` is the
|
|
27
55
|
last modification timestamp retrieved from ``os.stat``. The traversal skips
|
|
28
56
|
directories named ``.venv``, ``venv`` and ``__pycache__``.
|
|
29
57
|
|
|
58
|
+
When ``normalize`` is ``True``, ``.py`` files are fingerprinted by their
|
|
59
|
+
docstring-stripped AST and env files by their byte content, so the result is
|
|
60
|
+
invariant to docstrings, comments, formatting, and mtime-only changes.
|
|
61
|
+
|
|
30
62
|
Args:
|
|
31
63
|
path (str): The root directory to scan.
|
|
64
|
+
normalize (bool): Use AST/content fingerprints instead of size+mtime.
|
|
32
65
|
|
|
33
66
|
Returns:
|
|
34
67
|
list[str]: A signature string for each ``.py`` file found.
|
|
@@ -41,7 +74,6 @@ def get_sigs(path: str):
|
|
|
41
74
|
>>> get_sigs('/tmp/project')
|
|
42
75
|
['/tmp/project/main.py:1024:1680000000.0', ...]
|
|
43
76
|
"""
|
|
44
|
-
_ENV_FILES = {".python-version", "pyproject.toml", "uv.lock"}
|
|
45
77
|
items = []
|
|
46
78
|
with os.scandir(path) as entries:
|
|
47
79
|
for entry in entries:
|
|
@@ -50,27 +82,35 @@ def get_sigs(path: str):
|
|
|
50
82
|
if entry.is_file(follow_symlinks=False) and (
|
|
51
83
|
entry.name.endswith(".py") or entry.name in _ENV_FILES
|
|
52
84
|
):
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
85
|
+
if normalize:
|
|
86
|
+
if entry.name.endswith(".py"):
|
|
87
|
+
items.append(_normalized_sig(entry.path))
|
|
88
|
+
else:
|
|
89
|
+
items.append(_byte_sig(entry.path))
|
|
90
|
+
else:
|
|
91
|
+
stat_info = entry.stat(follow_symlinks=False)
|
|
92
|
+
items.append(
|
|
93
|
+
f"{entry.path}:{stat_info.st_size}:{stat_info.st_mtime}"
|
|
94
|
+
)
|
|
95
|
+
if entry.is_dir(follow_symlinks=False) and entry.name not in _SKIP_DIRS:
|
|
96
|
+
items.extend(get_sigs(entry.path, normalize=normalize))
|
|
62
97
|
return items
|
|
63
98
|
|
|
64
99
|
|
|
65
|
-
def get_context_hash(path: str) -> str:
|
|
100
|
+
def get_context_hash(path: str, *, normalize: bool = False) -> str:
|
|
66
101
|
"""Compute an MD5 hash that uniquely identifies a file or directory.
|
|
67
102
|
|
|
68
103
|
The hash is derived from a signature string. For a file, the signature consists of
|
|
69
104
|
its path, size, and modification time. For a directory, the signature is the
|
|
70
105
|
concatenation of the signatures of all files within it (recursively).
|
|
71
106
|
|
|
107
|
+
When ``normalize`` is ``True`` the signatures are derived from each ``.py``
|
|
108
|
+
file's docstring-stripped AST, making the hash invariant to docstrings,
|
|
109
|
+
comments, formatting, and mtime-only changes.
|
|
110
|
+
|
|
72
111
|
Args:
|
|
73
112
|
path (str): The filesystem path to hash.
|
|
113
|
+
normalize (bool): Use AST/content fingerprints instead of size+mtime.
|
|
74
114
|
|
|
75
115
|
Returns:
|
|
76
116
|
str: The hexadecimal MD5 digest.
|
|
@@ -84,10 +124,13 @@ def get_context_hash(path: str) -> str:
|
|
|
84
124
|
"""
|
|
85
125
|
h = hashlib.md5() # nosec
|
|
86
126
|
if os.path.isfile(path):
|
|
87
|
-
|
|
88
|
-
|
|
127
|
+
if normalize and path.endswith(".py"):
|
|
128
|
+
h.update(_normalized_sig(path).encode())
|
|
129
|
+
else:
|
|
130
|
+
s = os.stat(path)
|
|
131
|
+
h.update(f"{path}:{s.st_size}:{s.st_mtime}".encode())
|
|
89
132
|
elif os.path.isdir(path):
|
|
90
|
-
for sig in sorted(get_sigs(path)):
|
|
133
|
+
for sig in sorted(get_sigs(path, normalize=normalize)):
|
|
91
134
|
h.update(sig.encode())
|
|
92
135
|
else:
|
|
93
136
|
h.update(b"empty")
|
py_cq/execution_engine.py
CHANGED
|
@@ -16,9 +16,11 @@ import logging
|
|
|
16
16
|
import os
|
|
17
17
|
import shlex
|
|
18
18
|
import shutil
|
|
19
|
+
import signal
|
|
19
20
|
import subprocess
|
|
20
21
|
import sys
|
|
21
22
|
import tempfile
|
|
23
|
+
import threading
|
|
22
24
|
import time
|
|
23
25
|
from collections.abc import Collection
|
|
24
26
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
@@ -102,6 +104,80 @@ def _build_exclude_str(
|
|
|
102
104
|
return "".join(parts)
|
|
103
105
|
|
|
104
106
|
|
|
107
|
+
def _terminate_process_tree(proc: "subprocess.Popen") -> None:
|
|
108
|
+
"""Best-effort kill of a shell subprocess and all its children, cross-platform.
|
|
109
|
+
|
|
110
|
+
``shell=True`` spawns a shell that spawns ``uv``/``python``, so killing the
|
|
111
|
+
immediate child is not enough - the whole tree must go.
|
|
112
|
+
"""
|
|
113
|
+
if proc.poll() is not None:
|
|
114
|
+
return
|
|
115
|
+
if sys.platform == "win32":
|
|
116
|
+
subprocess.run(
|
|
117
|
+
["taskkill", "/F", "/T", "/PID", str(proc.pid)],
|
|
118
|
+
capture_output=True,
|
|
119
|
+
check=False,
|
|
120
|
+
) # nosec
|
|
121
|
+
else:
|
|
122
|
+
try:
|
|
123
|
+
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
|
124
|
+
except (ProcessLookupError, PermissionError):
|
|
125
|
+
proc.kill()
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _run_command(
|
|
129
|
+
command: str, run_env: dict, cancel_event: threading.Event | None
|
|
130
|
+
) -> tuple[str, str, int, bool]:
|
|
131
|
+
"""Run *command* in a shell and return ``(stdout, stderr, returncode, cancelled)``.
|
|
132
|
+
|
|
133
|
+
When *cancel_event* fires mid-run the process tree is terminated and
|
|
134
|
+
``cancelled`` is True (captured output is discarded). With no event this is
|
|
135
|
+
a plain blocking ``subprocess.run`` (no poll overhead for the common path).
|
|
136
|
+
"""
|
|
137
|
+
if cancel_event is None:
|
|
138
|
+
result = subprocess.run(
|
|
139
|
+
command,
|
|
140
|
+
capture_output=True,
|
|
141
|
+
text=True,
|
|
142
|
+
shell=True,
|
|
143
|
+
encoding="utf-8",
|
|
144
|
+
errors="replace",
|
|
145
|
+
env=run_env,
|
|
146
|
+
check=False,
|
|
147
|
+
) # nosec
|
|
148
|
+
return result.stdout, result.stderr, result.returncode, False
|
|
149
|
+
if cancel_event.is_set():
|
|
150
|
+
return "", "", -1, True
|
|
151
|
+
popen_kwargs: dict[str, Any] = {}
|
|
152
|
+
if sys.platform != "win32":
|
|
153
|
+
# New session/process group so os.killpg reaches the shell's children.
|
|
154
|
+
popen_kwargs["start_new_session"] = True
|
|
155
|
+
proc = subprocess.Popen(
|
|
156
|
+
command,
|
|
157
|
+
stdout=subprocess.PIPE,
|
|
158
|
+
stderr=subprocess.PIPE,
|
|
159
|
+
text=True,
|
|
160
|
+
shell=True,
|
|
161
|
+
encoding="utf-8",
|
|
162
|
+
errors="replace",
|
|
163
|
+
env=run_env,
|
|
164
|
+
**popen_kwargs,
|
|
165
|
+
) # nosec
|
|
166
|
+
# Poll so a cancel signal can terminate a still-running command.
|
|
167
|
+
while True:
|
|
168
|
+
try:
|
|
169
|
+
stdout, stderr = proc.communicate(timeout=0.1)
|
|
170
|
+
return stdout, stderr, proc.returncode, False
|
|
171
|
+
except subprocess.TimeoutExpired:
|
|
172
|
+
if cancel_event.is_set():
|
|
173
|
+
_terminate_process_tree(proc)
|
|
174
|
+
try:
|
|
175
|
+
proc.communicate(timeout=5)
|
|
176
|
+
except subprocess.TimeoutExpired: # pragma: no cover
|
|
177
|
+
proc.kill()
|
|
178
|
+
return "", "", proc.returncode or -1, True
|
|
179
|
+
|
|
180
|
+
|
|
105
181
|
def run_tool(
|
|
106
182
|
tool_config: ToolConfig,
|
|
107
183
|
context_path: str,
|
|
@@ -109,6 +185,7 @@ def run_tool(
|
|
|
109
185
|
*,
|
|
110
186
|
precomputed_hash: str | None = None,
|
|
111
187
|
project_tag: str | None = None,
|
|
188
|
+
cancel_event: threading.Event | None = None,
|
|
112
189
|
) -> RawResult:
|
|
113
190
|
"""Runs a tool defined by its configuration and returns the execution result.
|
|
114
191
|
|
|
@@ -195,11 +272,12 @@ def run_tool(
|
|
|
195
272
|
python=python,
|
|
196
273
|
exclude=exclude,
|
|
197
274
|
)
|
|
198
|
-
|
|
199
|
-
precomputed_hash
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
275
|
+
if precomputed_hash is not None:
|
|
276
|
+
context_hash = precomputed_hash
|
|
277
|
+
elif tool_config.cache_invariant == "ast":
|
|
278
|
+
context_hash = get_context_hash(context_path, normalize=True)
|
|
279
|
+
else:
|
|
280
|
+
context_hash = get_context_hash(context_path)
|
|
203
281
|
cache_key = f"{command}:{context_hash}"
|
|
204
282
|
|
|
205
283
|
t_cache0 = time.perf_counter()
|
|
@@ -223,18 +301,20 @@ def run_tool(
|
|
|
223
301
|
run_env["COVERAGE_FILE"] = coverage_tmp
|
|
224
302
|
t_sub0 = time.perf_counter()
|
|
225
303
|
try:
|
|
226
|
-
|
|
227
|
-
command,
|
|
228
|
-
|
|
229
|
-
text=True,
|
|
230
|
-
shell=True,
|
|
231
|
-
encoding="utf-8",
|
|
232
|
-
errors="replace",
|
|
233
|
-
env=run_env,
|
|
234
|
-
) # nosec
|
|
304
|
+
stdout, stderr, return_code, cancelled = _run_command(
|
|
305
|
+
command, run_env, cancel_event
|
|
306
|
+
)
|
|
235
307
|
finally:
|
|
236
308
|
Path(coverage_tmp).unlink(missing_ok=True)
|
|
237
309
|
t_sub = time.perf_counter() - t_sub0
|
|
310
|
+
if cancelled:
|
|
311
|
+
# A dependency (e.g. pytest) already failed, so this run is moot. Return
|
|
312
|
+
# an empty result - uncached, since it was terminated, not completed -
|
|
313
|
+
# and let the parser produce its skip/zero output.
|
|
314
|
+
log.debug(
|
|
315
|
+
f"{tool_config.name}: [CANCELLED] {tool_config.skip_if} failed; terminated after {t_sub * 1000:.0f}ms"
|
|
316
|
+
)
|
|
317
|
+
return RawResult(tool_name=tool_config.name)
|
|
238
318
|
log.debug(
|
|
239
319
|
f"{tool_config.name}: [MISS] cache={t_cache * 1000:.1f}ms tool={t_sub * 1000:.0f}ms: {command}"
|
|
240
320
|
)
|
|
@@ -242,13 +322,15 @@ def run_tool(
|
|
|
242
322
|
raw_result = RawResult(
|
|
243
323
|
tool_name=tool_config.name,
|
|
244
324
|
command=command,
|
|
245
|
-
stdout=
|
|
246
|
-
stderr=
|
|
247
|
-
return_code=
|
|
325
|
+
stdout=stdout,
|
|
326
|
+
stderr=stderr,
|
|
327
|
+
return_code=return_code,
|
|
248
328
|
timestamp=timestamp,
|
|
249
329
|
project_path=project_dir,
|
|
250
330
|
)
|
|
251
|
-
_cache.set(
|
|
331
|
+
_cache.set(
|
|
332
|
+
cache_key, raw_result.to_dict(), expire=5 * 24 * 60 * 60, tag=project_tag
|
|
333
|
+
)
|
|
252
334
|
return raw_result
|
|
253
335
|
|
|
254
336
|
|
|
@@ -306,18 +388,42 @@ def run_tools(
|
|
|
306
388
|
t_hash0 = time.perf_counter()
|
|
307
389
|
root = project_root or str(Path(path).resolve())
|
|
308
390
|
shared_hash = get_context_hash(root)
|
|
309
|
-
log.debug(
|
|
391
|
+
log.debug(
|
|
392
|
+
f"context_hash: {(time.perf_counter() - t_hash0) * 1000:.1f}ms {shared_hash}"
|
|
393
|
+
)
|
|
310
394
|
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
395
|
+
# Cache keys are content-based, so a changed project naturally produces new
|
|
396
|
+
# keys; stale entries age out via TTL + size_limit (no blanket eviction).
|
|
397
|
+
# Tools flagged cache_invariant="ast" key on the docstring-stripped AST so
|
|
398
|
+
# docstring/comment/format-only edits stay cache hits. Compute it once here
|
|
399
|
+
# (not lazily in the worker threads, where pytest and coverage would race
|
|
400
|
+
# and each re-parse the whole tree) and only when some tool needs it.
|
|
401
|
+
norm_hash = shared_hash
|
|
402
|
+
if any(tc.cache_invariant == "ast" for tc in tool_configs):
|
|
403
|
+
t_norm0 = time.perf_counter()
|
|
404
|
+
norm_hash = get_context_hash(root, normalize=True)
|
|
405
|
+
log.debug(
|
|
406
|
+
f"context_hash(ast): {(time.perf_counter() - t_norm0) * 1000:.1f}ms {norm_hash}"
|
|
407
|
+
)
|
|
408
|
+
|
|
409
|
+
def _hash_for(tool_config: ToolConfig) -> str:
|
|
410
|
+
return norm_hash if tool_config.cache_invariant == "ast" else shared_hash
|
|
317
411
|
|
|
318
|
-
def _run_and_parse(
|
|
412
|
+
def _run_and_parse(
|
|
413
|
+
tool_config: ToolConfig, cancel_event: threading.Event | None = None
|
|
414
|
+
) -> tuple[int, ToolResult]:
|
|
319
415
|
t0 = time.perf_counter()
|
|
320
|
-
|
|
416
|
+
# Only pass cancel_event when present so patched run_tool stubs in tests
|
|
417
|
+
# (which don't declare the kwarg) keep working for non-cancellable tools.
|
|
418
|
+
extra = {"cancel_event": cancel_event} if cancel_event is not None else {}
|
|
419
|
+
raw_result = run_tool(
|
|
420
|
+
tool_config,
|
|
421
|
+
path,
|
|
422
|
+
excludes,
|
|
423
|
+
precomputed_hash=_hash_for(tool_config),
|
|
424
|
+
project_tag=root,
|
|
425
|
+
**extra,
|
|
426
|
+
)
|
|
321
427
|
tr = tool_config.parser_class(tool_config.parser_config).parse(raw_result)
|
|
322
428
|
tr.duration_s = time.perf_counter() - t0
|
|
323
429
|
return tool_config.order, tr
|
|
@@ -329,8 +435,10 @@ def run_tools(
|
|
|
329
435
|
for i, tool_config in enumerate(sorted_configs):
|
|
330
436
|
try:
|
|
331
437
|
prioritized.append(_run_and_parse(tool_config))
|
|
332
|
-
except Exception as exc:
|
|
333
|
-
log.error(
|
|
438
|
+
except Exception as exc: # noqa: BLE001 - a crashing tool must not abort the run
|
|
439
|
+
log.error(
|
|
440
|
+
f"{tool_config.name} generated an exception: {exc} {exc.__traceback__}"
|
|
441
|
+
)
|
|
334
442
|
n_skipped = n_total - i - 1
|
|
335
443
|
if n_skipped:
|
|
336
444
|
remaining = ", ".join(tc.name for tc in sorted_configs[i + 1 :])
|
|
@@ -357,52 +465,55 @@ def run_tools(
|
|
|
357
465
|
return [tr for _, tr in sorted(prioritized)]
|
|
358
466
|
configs_by_name = {tc.name: tc for tc in tool_configs}
|
|
359
467
|
|
|
360
|
-
def _run_if_dep_passed(dep_future, tool_config: ToolConfig, dep_config: ToolConfig) -> tuple[int, ToolResult]:
|
|
361
|
-
# Wait for the dependency within the same thread pool - no phase split,
|
|
362
|
-
# so all other tools keep running in parallel while we block here.
|
|
363
|
-
try:
|
|
364
|
-
_, dep_tr = dep_future.result()
|
|
365
|
-
except Exception:
|
|
366
|
-
return _run_and_parse(tool_config)
|
|
367
|
-
if dep_tr.metrics and min(dep_tr.metrics.values()) < dep_config.error_threshold:
|
|
368
|
-
# Dependency failed its error threshold: skip this tool and return a
|
|
369
|
-
# synthetic empty result. Coverage declares skip_if = "pytest" so that
|
|
370
|
-
# it never re-invokes pytest (which the coverage command does internally)
|
|
371
|
-
# when we already know pytest is broken.
|
|
372
|
-
log.debug(f"{tool_config.name}: skipping because {tool_config.skip_if} failed")
|
|
373
|
-
t0 = time.perf_counter()
|
|
374
|
-
tr = tool_config.parser_class(tool_config.parser_config).parse(
|
|
375
|
-
RawResult(tool_name=tool_config.name)
|
|
376
|
-
)
|
|
377
|
-
tr.duration_s = time.perf_counter() - t0
|
|
378
|
-
return tool_config.order, tr
|
|
379
|
-
return _run_and_parse(tool_config)
|
|
380
|
-
|
|
381
468
|
timings: list[tuple[int, str, float]] = []
|
|
382
|
-
# Sort by order so that when we build name_to_future below, a dependency's
|
|
383
|
-
# future is always registered before the tool that declares skip_if on it.
|
|
384
469
|
sorted_configs = sorted(tool_configs, key=lambda tc: tc.order)
|
|
470
|
+
# A tool with skip_if gets a cancel event: it runs in parallel with its
|
|
471
|
+
# dependency, and is terminated mid-run only if that dependency *fails* its
|
|
472
|
+
# error threshold. Coverage declares skip_if = "pytest" so a broken pytest
|
|
473
|
+
# kills the coverage run (which re-invokes pytest internally) instead of
|
|
474
|
+
# letting it finish a pointless second suite execution.
|
|
475
|
+
cancel_events = {
|
|
476
|
+
tc.name: threading.Event()
|
|
477
|
+
for tc in sorted_configs
|
|
478
|
+
if tc.skip_if and tc.skip_if in configs_by_name
|
|
479
|
+
}
|
|
385
480
|
name_to_future: dict[str, Any] = {}
|
|
386
481
|
with ThreadPoolExecutor(max_workers=max_workers or len(tool_configs)) as executor:
|
|
387
482
|
future_to_tool = {}
|
|
388
483
|
for tc in sorted_configs:
|
|
389
|
-
|
|
390
|
-
dep_future = name_to_future.get(dep_name) if dep_name else None
|
|
391
|
-
dep_config = configs_by_name.get(dep_name) if dep_name else None
|
|
392
|
-
if dep_future and dep_config:
|
|
393
|
-
# Submit a wrapper that waits on the dependency future first.
|
|
394
|
-
f = executor.submit(_run_if_dep_passed, dep_future, tc, dep_config)
|
|
395
|
-
else:
|
|
396
|
-
f = executor.submit(_run_and_parse, tc)
|
|
484
|
+
f = executor.submit(_run_and_parse, tc, cancel_events.get(tc.name))
|
|
397
485
|
future_to_tool[f] = tc
|
|
398
486
|
name_to_future[tc.name] = f
|
|
487
|
+
|
|
488
|
+
def _cancel_on_dep_failure(
|
|
489
|
+
dep_future, ev: threading.Event, dep_config: ToolConfig, name: str
|
|
490
|
+
) -> None:
|
|
491
|
+
try:
|
|
492
|
+
_, dep_tr = dep_future.result()
|
|
493
|
+
except Exception: # noqa: BLE001 - a crashed dep cancels nothing
|
|
494
|
+
return
|
|
495
|
+
if (
|
|
496
|
+
dep_tr.metrics
|
|
497
|
+
and min(dep_tr.metrics.values()) < dep_config.error_threshold
|
|
498
|
+
):
|
|
499
|
+
log.debug(f"{name}: cancelling because {dep_config.name} failed")
|
|
500
|
+
ev.set()
|
|
501
|
+
|
|
502
|
+
for tc in sorted_configs:
|
|
503
|
+
if tc.name in cancel_events:
|
|
504
|
+
dep_config = configs_by_name[tc.skip_if]
|
|
505
|
+
name_to_future[tc.skip_if].add_done_callback(
|
|
506
|
+
lambda fut, ev=cancel_events[tc.name], dc=dep_config, nm=tc.name: (
|
|
507
|
+
_cancel_on_dep_failure(fut, ev, dc, nm)
|
|
508
|
+
)
|
|
509
|
+
)
|
|
399
510
|
for future in as_completed(future_to_tool):
|
|
400
511
|
tool_config = future_to_tool[future]
|
|
401
512
|
try:
|
|
402
513
|
tc_order, tr = future.result()
|
|
403
514
|
prioritized.append((tc_order, tr))
|
|
404
515
|
timings.append((tc_order, tool_config.name, tr.duration_s))
|
|
405
|
-
except Exception as exc:
|
|
516
|
+
except Exception as exc: # noqa: BLE001 - a crashing tool must not abort the run
|
|
406
517
|
log.error(f"{tool_config.name} generated an exception: {exc}")
|
|
407
518
|
per_tool = ", ".join(f"{name}={dur:.2f}s" for _, name, dur in sorted(timings))
|
|
408
519
|
log.debug(f"run_tools elapsed: {time.perf_counter() - t_start:.2f}s [{per_tool}]")
|
py_cq/localtypes.py
CHANGED
|
@@ -60,8 +60,11 @@ class ToolConfig:
|
|
|
60
60
|
default_factory=list
|
|
61
61
|
) # Top-level dir/file names to omit from {scan_targets}
|
|
62
62
|
skip_for_file: bool = False # If True, skip when context_path is a single file
|
|
63
|
-
gate_strict: bool =
|
|
63
|
+
gate_strict: bool = (
|
|
64
|
+
True # If False, small regressions above warning_threshold are accepted
|
|
65
|
+
)
|
|
64
66
|
skip_if: str = "" # Tool name; if that tool's score is below its error_threshold, skip this tool
|
|
67
|
+
cache_invariant: str = "" # "ast": key the cache on the docstring-stripped AST, ignoring docstrings/comments/format
|
|
65
68
|
|
|
66
69
|
|
|
67
70
|
@dataclass
|
|
@@ -172,7 +175,6 @@ class AbstractParser(ABC):
|
|
|
172
175
|
@abstractmethod
|
|
173
176
|
def parse(self, raw_result: RawResult) -> ToolResult:
|
|
174
177
|
"""Converts raw tool output into a structured ToolResult."""
|
|
175
|
-
pass
|
|
176
178
|
|
|
177
179
|
def format_llm_message(
|
|
178
180
|
self, tr: ToolResult, *, context_lines: int = 15, limit: int = 1
|
py_cq/parsers/common.py
CHANGED
|
@@ -47,11 +47,7 @@ def format_source_context(
|
|
|
47
47
|
error_offset = line - context_start # 0-based index of the error line in raw_lines
|
|
48
48
|
collected = []
|
|
49
49
|
for i, rline in enumerate(raw_lines):
|
|
50
|
-
if i > error_offset and (
|
|
51
|
-
rline.startswith("def ")
|
|
52
|
-
or rline.startswith("async def ")
|
|
53
|
-
or rline.startswith("class ")
|
|
54
|
-
):
|
|
50
|
+
if i > error_offset and (rline.startswith(("def ", "async def ", "class "))):
|
|
55
51
|
break
|
|
56
52
|
collected.append(f"{context_start + i}: {rline}")
|
|
57
53
|
src = "\n".join(collected)
|
|
@@ -382,8 +378,7 @@ def score_logistic_variant(
|
|
|
382
378
|
1.0
|
|
383
379
|
>>> score_logistic_variant(10, scale_factor=0)
|
|
384
380
|
0.0"""
|
|
385
|
-
|
|
386
|
-
errors = 0
|
|
381
|
+
errors = max(errors, 0)
|
|
387
382
|
if scale_factor == 0:
|
|
388
383
|
return 1.0 if errors == 0 else 0.0
|
|
389
384
|
try:
|