python-code-quality 0.2.2__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- py_cq/api.py +26 -5
- py_cq/cli.py +21 -11
- py_cq/config/config.toml +30 -22
- py_cq/context_hash.py +61 -15
- py_cq/execution_engine.py +175 -38
- py_cq/localtypes.py +10 -7
- py_cq/parsers/common.py +17 -18
- py_cq/parsers/complexityparser.py +2 -0
- py_cq/parsers/coverageparser.py +11 -18
- py_cq/parsers/halsteadparser.py +4 -3
- py_cq/parsers/interrogateparser.py +20 -21
- py_cq/parsers/pytestparser.py +1 -1
- py_cq/parsers/ruffparser.py +1 -1
- py_cq/source_file.py +199 -0
- py_cq/table_formatter.py +1 -1
- py_cq/tool_registry.py +3 -0
- {python_code_quality-0.2.2.dist-info → python_code_quality-0.3.0.dist-info}/METADATA +116 -103
- python_code_quality-0.3.0.dist-info/RECORD +36 -0
- {python_code_quality-0.2.2.dist-info → python_code_quality-0.3.0.dist-info}/WHEEL +1 -1
- python_code_quality-0.2.2.dist-info/RECORD +0 -35
- {python_code_quality-0.2.2.dist-info → python_code_quality-0.3.0.dist-info}/entry_points.txt +0 -0
py_cq/api.py
CHANGED
|
@@ -148,13 +148,14 @@ class CQ:
|
|
|
148
148
|
if clear_cache:
|
|
149
149
|
_cache.clear()
|
|
150
150
|
|
|
151
|
-
def raw(self, *, early_exit: bool = False) -> list[ToolResult]:
|
|
151
|
+
def raw(self, *, early_exit: bool = False, order: str = "severity") -> list[ToolResult]:
|
|
152
152
|
"""Run all tools and return parsed results before aggregation."""
|
|
153
153
|
return run_tools(
|
|
154
154
|
self._registry.values(),
|
|
155
155
|
str(self.path),
|
|
156
156
|
self._workers,
|
|
157
157
|
early_exit=early_exit,
|
|
158
|
+
order=order,
|
|
158
159
|
excludes=self._excludes,
|
|
159
160
|
)
|
|
160
161
|
|
|
@@ -168,22 +169,42 @@ class CQ:
|
|
|
168
169
|
limit: int = 1,
|
|
169
170
|
silence: list[str] | None = None,
|
|
170
171
|
hint: bool = False,
|
|
172
|
+
order: str = "severity",
|
|
171
173
|
) -> dict:
|
|
172
174
|
"""Return the top defect as a dict with keys: id, file, project, message.
|
|
173
175
|
|
|
174
|
-
Stops running tools after the first
|
|
176
|
+
Stops running tools after the first issue (early_exit) for speed.
|
|
177
|
+
``order="phase"`` selects the first issue of the earliest non-clean
|
|
178
|
+
phase (dependency order); ``"severity"`` (default) selects the most
|
|
179
|
+
severe issue first.
|
|
175
180
|
"""
|
|
176
|
-
|
|
181
|
+
_silence = silence or []
|
|
182
|
+
results = self.raw(early_exit=True, order=order)
|
|
177
183
|
combined = aggregate_metrics(str(self.path), results)
|
|
178
|
-
|
|
184
|
+
issue = format_for_llm_json(
|
|
179
185
|
self._registry,
|
|
180
186
|
combined,
|
|
181
187
|
context_lines=self._context_lines,
|
|
182
188
|
hint=hint,
|
|
183
189
|
limit=limit,
|
|
184
|
-
silence=
|
|
190
|
+
silence=_silence,
|
|
185
191
|
project_root=self._project_root,
|
|
186
192
|
)
|
|
193
|
+
if issue.get("id") is None and _silence:
|
|
194
|
+
# Early-exit stopped at a phase whose only issues are silenced.
|
|
195
|
+
# Re-run all tools to surface issues in later phases.
|
|
196
|
+
results = self.raw(early_exit=False, order=order)
|
|
197
|
+
combined = aggregate_metrics(str(self.path), results)
|
|
198
|
+
issue = format_for_llm_json(
|
|
199
|
+
self._registry,
|
|
200
|
+
combined,
|
|
201
|
+
context_lines=self._context_lines,
|
|
202
|
+
hint=hint,
|
|
203
|
+
limit=limit,
|
|
204
|
+
silence=_silence,
|
|
205
|
+
project_root=self._project_root,
|
|
206
|
+
)
|
|
207
|
+
return issue
|
|
187
208
|
|
|
188
209
|
def is_fixed(self, fingerprint: str) -> bool:
|
|
189
210
|
"""Return True if the fingerprinted issue is no longer present.
|
py_cq/cli.py
CHANGED
|
@@ -17,7 +17,7 @@ import logging
|
|
|
17
17
|
import time
|
|
18
18
|
import tomllib
|
|
19
19
|
from enum import Enum
|
|
20
|
-
from importlib.metadata import requires, version
|
|
20
|
+
from importlib.metadata import PackageNotFoundError, requires, version
|
|
21
21
|
from pathlib import Path
|
|
22
22
|
|
|
23
23
|
import tomlkit
|
|
@@ -67,6 +67,13 @@ class OutputMode(str, Enum):
|
|
|
67
67
|
RAW = "raw"
|
|
68
68
|
|
|
69
69
|
|
|
70
|
+
class OrderMode(str, Enum):
|
|
71
|
+
"""Selection order for the top issue in -o llm/llm-json."""
|
|
72
|
+
|
|
73
|
+
SEVERITY = "severity"
|
|
74
|
+
PHASE = "phase"
|
|
75
|
+
|
|
76
|
+
|
|
70
77
|
def _version_callback(value: bool) -> None:
|
|
71
78
|
if not value:
|
|
72
79
|
return
|
|
@@ -81,7 +88,7 @@ def _version_callback(value: bool) -> None:
|
|
|
81
88
|
dep_name = re.split(r"[>=<!;\s\[]", req)[0]
|
|
82
89
|
try:
|
|
83
90
|
dep_versions.append((dep_name, version(dep_name)))
|
|
84
|
-
except
|
|
91
|
+
except PackageNotFoundError:
|
|
85
92
|
pass
|
|
86
93
|
typer.echo(f"{pkg} v{pkg_version}")
|
|
87
94
|
for dep_name, dep_ver in sorted(dep_versions):
|
|
@@ -113,7 +120,7 @@ console = Console()
|
|
|
113
120
|
@app.command()
|
|
114
121
|
def check(
|
|
115
122
|
path: str = typer.Argument(".", help="Path to Python file or project directory"),
|
|
116
|
-
output: OutputMode = typer.Option(
|
|
123
|
+
output: OutputMode = typer.Option( # noqa: B008
|
|
117
124
|
OutputMode.TABLE,
|
|
118
125
|
"--output",
|
|
119
126
|
"-o",
|
|
@@ -153,12 +160,17 @@ def check(
|
|
|
153
160
|
limit: int = typer.Option(
|
|
154
161
|
1, "--limit", help="Number of issues to show with -o llm (default: 1)"
|
|
155
162
|
),
|
|
156
|
-
silence: list[str] = typer.Option(
|
|
163
|
+
silence: list[str] = typer.Option( # noqa: B008
|
|
157
164
|
[],
|
|
158
165
|
"--silence",
|
|
159
166
|
"-s",
|
|
160
167
|
help="Silence issues from -o llm output (e.g. -s src/foo.py or -s src/foo.py:42:E501)",
|
|
161
168
|
),
|
|
169
|
+
order: OrderMode = typer.Option( # noqa: B008
|
|
170
|
+
OrderMode.SEVERITY,
|
|
171
|
+
"--order",
|
|
172
|
+
help="Issue selection order for -o llm/llm-json: severity (most severe first) or phase (first non-clean phase, in dependency order)",
|
|
173
|
+
),
|
|
162
174
|
):
|
|
163
175
|
"""Feed the results from 11+ code quality tools to an LLM. Try: cq check . -o llm""" # --help
|
|
164
176
|
path_obj = Path(path)
|
|
@@ -174,12 +186,10 @@ def check(
|
|
|
174
186
|
)
|
|
175
187
|
raise typer.Exit(0)
|
|
176
188
|
|
|
177
|
-
if path_obj.is_file():
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
if not (path_obj / "pyproject.toml").exists():
|
|
182
|
-
raise typer.BadParameter(f"Directory must contain pyproject.toml: {path}")
|
|
189
|
+
if path_obj.is_file() and path_obj.suffix != ".py":
|
|
190
|
+
raise typer.BadParameter(f"File must be a Python file (.py): {path}")
|
|
191
|
+
elif path_obj.is_dir() and not (path_obj / "pyproject.toml").exists():
|
|
192
|
+
raise typer.BadParameter(f"Directory must contain pyproject.toml: {path}")
|
|
183
193
|
log.setLevel(log_level)
|
|
184
194
|
|
|
185
195
|
only_list = [t.strip() for t in only.split(",")] if only else None
|
|
@@ -200,7 +210,7 @@ def check(
|
|
|
200
210
|
|
|
201
211
|
is_llm = output in (OutputMode.LLM, OutputMode.LLM_JSON)
|
|
202
212
|
t0 = time.perf_counter()
|
|
203
|
-
tool_results = cq.raw(early_exit=is_llm)
|
|
213
|
+
tool_results = cq.raw(early_exit=is_llm, order=order.value)
|
|
204
214
|
total_s = time.perf_counter() - t0
|
|
205
215
|
combined = aggregate_metrics(path, tool_results)
|
|
206
216
|
|
py_cq/config/config.toml
CHANGED
|
@@ -4,6 +4,7 @@ parser = "CompileParser"
|
|
|
4
4
|
order = 1
|
|
5
5
|
warning_threshold = 0.9999
|
|
6
6
|
error_threshold = 0.9999
|
|
7
|
+
cache_invariant = "ast"
|
|
7
8
|
|
|
8
9
|
[python.ruff]
|
|
9
10
|
command = "{python} -m ruff check --output-format concise --no-cache \"{context_path}\" --exclude .claude{exclude}"
|
|
@@ -42,6 +43,7 @@ error_threshold = 1.0
|
|
|
42
43
|
run_in_target_env = true
|
|
43
44
|
extra_deps = ["pytest"]
|
|
44
45
|
skip_for_file = true
|
|
46
|
+
cache_invariant = "ast"
|
|
45
47
|
|
|
46
48
|
[python.coverage]
|
|
47
49
|
command = "{python} -m coverage run --omit=*/tests/*,*/test_*.py -m pytest \"{context_path}\" --ignore .claude{exclude} && {python} -m coverage report --show-missing --omit=*/tests/*,*/test_*.py"
|
|
@@ -53,43 +55,49 @@ error_threshold = 0.5
|
|
|
53
55
|
run_in_target_env = true
|
|
54
56
|
extra_deps = ["coverage", "pytest"]
|
|
55
57
|
skip_for_file = true
|
|
58
|
+
skip_if = "pytest"
|
|
59
|
+
cache_invariant = "ast"
|
|
60
|
+
|
|
61
|
+
[python.vulture]
|
|
62
|
+
command = "{python} -m vulture \"{context_path}\" --min-confidence 80 --exclude .venv,dist,.*_cache,docs,.git,.claude{exclude}"
|
|
63
|
+
exclude_format = ",{path}"
|
|
64
|
+
parser = "VultureParser"
|
|
65
|
+
order = 7
|
|
66
|
+
warning_threshold = 0.9999
|
|
67
|
+
error_threshold = 0.8
|
|
68
|
+
cache_invariant = "ast"
|
|
69
|
+
|
|
70
|
+
[python.interrogate]
|
|
71
|
+
command = "{python} -m interrogate \"{context_path}\" -e .claude{exclude} -v --fail-under 0"
|
|
72
|
+
exclude_format = " -e {path}"
|
|
73
|
+
parser = "InterrogateParser"
|
|
74
|
+
order = 8
|
|
75
|
+
warning_threshold = 0.8
|
|
76
|
+
error_threshold = 0.5
|
|
77
|
+
|
|
78
|
+
[python.interrogate.parser_config]
|
|
79
|
+
skip_empty_init = true
|
|
56
80
|
|
|
57
81
|
[python.radon-cc]
|
|
58
82
|
command = "{python} -m radon cc --json --exclude '.claude/**' \"{context_path}\""
|
|
59
83
|
parser = "ComplexityParser"
|
|
60
|
-
order =
|
|
84
|
+
order = 9
|
|
85
|
+
gate_strict = false
|
|
61
86
|
warning_threshold = 0.6
|
|
62
87
|
error_threshold = 0.4
|
|
63
88
|
|
|
64
89
|
[python.radon-mi]
|
|
65
90
|
command = "{python} -m radon mi -s --json --exclude '.claude/**' \"{context_path}\""
|
|
66
91
|
parser = "MaintainabilityParser"
|
|
67
|
-
order =
|
|
92
|
+
order = 10
|
|
93
|
+
gate_strict = false
|
|
68
94
|
warning_threshold = 0.6
|
|
69
95
|
error_threshold = 0.4
|
|
70
96
|
|
|
71
97
|
[python.radon-hal]
|
|
72
98
|
command = "{python} -m radon hal -f --json --exclude '.claude/**' \"{context_path}\""
|
|
73
99
|
parser = "HalsteadParser"
|
|
74
|
-
order = 9
|
|
75
|
-
warning_threshold = 0.5
|
|
76
|
-
error_threshold = 0.3
|
|
77
|
-
|
|
78
|
-
[python.vulture]
|
|
79
|
-
command = "{python} -m vulture \"{context_path}\" --min-confidence 80 --exclude .venv,dist,.*_cache,docs,.git,.claude{exclude}"
|
|
80
|
-
exclude_format = ",{path}"
|
|
81
|
-
parser = "VultureParser"
|
|
82
|
-
order = 10
|
|
83
|
-
warning_threshold = 0.9999
|
|
84
|
-
error_threshold = 0.8
|
|
85
|
-
|
|
86
|
-
[python.interrogate]
|
|
87
|
-
command = "{python} -m interrogate \"{context_path}\" -e .claude{exclude} -v --fail-under 0"
|
|
88
|
-
exclude_format = " -e {path}"
|
|
89
|
-
parser = "InterrogateParser"
|
|
90
100
|
order = 11
|
|
91
|
-
|
|
101
|
+
gate_strict = false
|
|
102
|
+
warning_threshold = 0.35
|
|
92
103
|
error_threshold = 0.3
|
|
93
|
-
|
|
94
|
-
[python.interrogate.parser_config]
|
|
95
|
-
skip_empty_init = true
|
py_cq/context_hash.py
CHANGED
|
@@ -12,23 +12,56 @@ change-detection in larger systems:
|
|
|
12
12
|
a file or directory. For a file it hashes its path, size, and modification
|
|
13
13
|
time; for a directory it aggregates the signatures of all contained files.
|
|
14
14
|
|
|
15
|
+
When called with ``normalize=True`` the hash is derived from the AST of each
|
|
16
|
+
``.py`` file with docstrings stripped, so changes that cannot affect runtime
|
|
17
|
+
behaviour (docstrings, comments, formatting, mtime-only touches such as a
|
|
18
|
+
``git checkout``) leave the hash unchanged. This is used as the cache key for
|
|
19
|
+
tools whose results only depend on executable code (pytest, coverage).
|
|
20
|
+
|
|
15
21
|
These functions provide deterministic fingerprints that can be used for
|
|
16
22
|
file integrity verification, caching, and change-detection logic.
|
|
17
23
|
"""
|
|
18
24
|
|
|
19
25
|
import hashlib
|
|
20
26
|
import os
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
from py_cq.source_file import get_source
|
|
30
|
+
|
|
31
|
+
_ENV_FILES = {".python-version", "pyproject.toml", "uv.lock"}
|
|
32
|
+
_SKIP_DIRS = {".venv", "venv", "__pycache__", ".git"}
|
|
21
33
|
|
|
22
34
|
|
|
23
|
-
def
|
|
35
|
+
def _normalized_sig(path: str) -> str:
|
|
36
|
+
"""Return a docstring/comment/format-invariant signature for a ``.py`` file.
|
|
37
|
+
|
|
38
|
+
Backed by the shared :class:`~py_cq.source_file.SourceFile` cache, so the
|
|
39
|
+
AST parsed here is reused by the coverage and interrogate parsers. Falls
|
|
40
|
+
back to a byte digest when the file can't be parsed (e.g. a mid-edit syntax
|
|
41
|
+
error), so callers always get a usable signature.
|
|
42
|
+
"""
|
|
43
|
+
return f"{path}:{get_source(path).norm_digest}"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _byte_sig(path: str) -> str:
|
|
47
|
+
"""Return a path + content digest, independent of mtime."""
|
|
48
|
+
return f"{path}:{hashlib.md5(Path(path).read_bytes()).hexdigest()}" # nosec
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def get_sigs(path: str, *, normalize: bool = False):
|
|
24
52
|
"""Recursively scans a directory tree and returns a list of signatures for all Python files.
|
|
25
53
|
|
|
26
54
|
The signature format is ``<file_path>:<size_bytes>:<mtime>`` where ``mtime`` is the
|
|
27
55
|
last modification timestamp retrieved from ``os.stat``. The traversal skips
|
|
28
56
|
directories named ``.venv``, ``venv`` and ``__pycache__``.
|
|
29
57
|
|
|
58
|
+
When ``normalize`` is ``True``, ``.py`` files are fingerprinted by their
|
|
59
|
+
docstring-stripped AST and env files by their byte content, so the result is
|
|
60
|
+
invariant to docstrings, comments, formatting, and mtime-only changes.
|
|
61
|
+
|
|
30
62
|
Args:
|
|
31
63
|
path (str): The root directory to scan.
|
|
64
|
+
normalize (bool): Use AST/content fingerprints instead of size+mtime.
|
|
32
65
|
|
|
33
66
|
Returns:
|
|
34
67
|
list[str]: A signature string for each ``.py`` file found.
|
|
@@ -46,28 +79,38 @@ def get_sigs(path: str):
|
|
|
46
79
|
for entry in entries:
|
|
47
80
|
# Use follow_symlinks=False to prevent cache poisoning from
|
|
48
81
|
# symlinks pointing outside the project tree (M-2)
|
|
49
|
-
if entry.is_file(follow_symlinks=False) and
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
82
|
+
if entry.is_file(follow_symlinks=False) and (
|
|
83
|
+
entry.name.endswith(".py") or entry.name in _ENV_FILES
|
|
84
|
+
):
|
|
85
|
+
if normalize:
|
|
86
|
+
if entry.name.endswith(".py"):
|
|
87
|
+
items.append(_normalized_sig(entry.path))
|
|
88
|
+
else:
|
|
89
|
+
items.append(_byte_sig(entry.path))
|
|
90
|
+
else:
|
|
91
|
+
stat_info = entry.stat(follow_symlinks=False)
|
|
92
|
+
items.append(
|
|
93
|
+
f"{entry.path}:{stat_info.st_size}:{stat_info.st_mtime}"
|
|
94
|
+
)
|
|
95
|
+
if entry.is_dir(follow_symlinks=False) and entry.name not in _SKIP_DIRS:
|
|
96
|
+
items.extend(get_sigs(entry.path, normalize=normalize))
|
|
59
97
|
return items
|
|
60
98
|
|
|
61
99
|
|
|
62
|
-
def get_context_hash(path: str) -> str:
|
|
100
|
+
def get_context_hash(path: str, *, normalize: bool = False) -> str:
|
|
63
101
|
"""Compute an MD5 hash that uniquely identifies a file or directory.
|
|
64
102
|
|
|
65
103
|
The hash is derived from a signature string. For a file, the signature consists of
|
|
66
104
|
its path, size, and modification time. For a directory, the signature is the
|
|
67
105
|
concatenation of the signatures of all files within it (recursively).
|
|
68
106
|
|
|
107
|
+
When ``normalize`` is ``True`` the signatures are derived from each ``.py``
|
|
108
|
+
file's docstring-stripped AST, making the hash invariant to docstrings,
|
|
109
|
+
comments, formatting, and mtime-only changes.
|
|
110
|
+
|
|
69
111
|
Args:
|
|
70
112
|
path (str): The filesystem path to hash.
|
|
113
|
+
normalize (bool): Use AST/content fingerprints instead of size+mtime.
|
|
71
114
|
|
|
72
115
|
Returns:
|
|
73
116
|
str: The hexadecimal MD5 digest.
|
|
@@ -81,10 +124,13 @@ def get_context_hash(path: str) -> str:
|
|
|
81
124
|
"""
|
|
82
125
|
h = hashlib.md5() # nosec
|
|
83
126
|
if os.path.isfile(path):
|
|
84
|
-
|
|
85
|
-
|
|
127
|
+
if normalize and path.endswith(".py"):
|
|
128
|
+
h.update(_normalized_sig(path).encode())
|
|
129
|
+
else:
|
|
130
|
+
s = os.stat(path)
|
|
131
|
+
h.update(f"{path}:{s.st_size}:{s.st_mtime}".encode())
|
|
86
132
|
elif os.path.isdir(path):
|
|
87
|
-
for sig in sorted(get_sigs(path)):
|
|
133
|
+
for sig in sorted(get_sigs(path, normalize=normalize)):
|
|
88
134
|
h.update(sig.encode())
|
|
89
135
|
else:
|
|
90
136
|
h.update(b"empty")
|
py_cq/execution_engine.py
CHANGED
|
@@ -16,9 +16,11 @@ import logging
|
|
|
16
16
|
import os
|
|
17
17
|
import shlex
|
|
18
18
|
import shutil
|
|
19
|
+
import signal
|
|
19
20
|
import subprocess
|
|
20
21
|
import sys
|
|
21
22
|
import tempfile
|
|
23
|
+
import threading
|
|
22
24
|
import time
|
|
23
25
|
from collections.abc import Collection
|
|
24
26
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
@@ -102,6 +104,80 @@ def _build_exclude_str(
|
|
|
102
104
|
return "".join(parts)
|
|
103
105
|
|
|
104
106
|
|
|
107
|
+
def _terminate_process_tree(proc: "subprocess.Popen") -> None:
|
|
108
|
+
"""Best-effort kill of a shell subprocess and all its children, cross-platform.
|
|
109
|
+
|
|
110
|
+
``shell=True`` spawns a shell that spawns ``uv``/``python``, so killing the
|
|
111
|
+
immediate child is not enough - the whole tree must go.
|
|
112
|
+
"""
|
|
113
|
+
if proc.poll() is not None:
|
|
114
|
+
return
|
|
115
|
+
if sys.platform == "win32":
|
|
116
|
+
subprocess.run(
|
|
117
|
+
["taskkill", "/F", "/T", "/PID", str(proc.pid)],
|
|
118
|
+
capture_output=True,
|
|
119
|
+
check=False,
|
|
120
|
+
) # nosec
|
|
121
|
+
else:
|
|
122
|
+
try:
|
|
123
|
+
os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
|
|
124
|
+
except (ProcessLookupError, PermissionError):
|
|
125
|
+
proc.kill()
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _run_command(
|
|
129
|
+
command: str, run_env: dict, cancel_event: threading.Event | None
|
|
130
|
+
) -> tuple[str, str, int, bool]:
|
|
131
|
+
"""Run *command* in a shell and return ``(stdout, stderr, returncode, cancelled)``.
|
|
132
|
+
|
|
133
|
+
When *cancel_event* fires mid-run the process tree is terminated and
|
|
134
|
+
``cancelled`` is True (captured output is discarded). With no event this is
|
|
135
|
+
a plain blocking ``subprocess.run`` (no poll overhead for the common path).
|
|
136
|
+
"""
|
|
137
|
+
if cancel_event is None:
|
|
138
|
+
result = subprocess.run(
|
|
139
|
+
command,
|
|
140
|
+
capture_output=True,
|
|
141
|
+
text=True,
|
|
142
|
+
shell=True,
|
|
143
|
+
encoding="utf-8",
|
|
144
|
+
errors="replace",
|
|
145
|
+
env=run_env,
|
|
146
|
+
check=False,
|
|
147
|
+
) # nosec
|
|
148
|
+
return result.stdout, result.stderr, result.returncode, False
|
|
149
|
+
if cancel_event.is_set():
|
|
150
|
+
return "", "", -1, True
|
|
151
|
+
popen_kwargs: dict[str, Any] = {}
|
|
152
|
+
if sys.platform != "win32":
|
|
153
|
+
# New session/process group so os.killpg reaches the shell's children.
|
|
154
|
+
popen_kwargs["start_new_session"] = True
|
|
155
|
+
proc = subprocess.Popen(
|
|
156
|
+
command,
|
|
157
|
+
stdout=subprocess.PIPE,
|
|
158
|
+
stderr=subprocess.PIPE,
|
|
159
|
+
text=True,
|
|
160
|
+
shell=True,
|
|
161
|
+
encoding="utf-8",
|
|
162
|
+
errors="replace",
|
|
163
|
+
env=run_env,
|
|
164
|
+
**popen_kwargs,
|
|
165
|
+
) # nosec
|
|
166
|
+
# Poll so a cancel signal can terminate a still-running command.
|
|
167
|
+
while True:
|
|
168
|
+
try:
|
|
169
|
+
stdout, stderr = proc.communicate(timeout=0.1)
|
|
170
|
+
return stdout, stderr, proc.returncode, False
|
|
171
|
+
except subprocess.TimeoutExpired:
|
|
172
|
+
if cancel_event.is_set():
|
|
173
|
+
_terminate_process_tree(proc)
|
|
174
|
+
try:
|
|
175
|
+
proc.communicate(timeout=5)
|
|
176
|
+
except subprocess.TimeoutExpired: # pragma: no cover
|
|
177
|
+
proc.kill()
|
|
178
|
+
return "", "", proc.returncode or -1, True
|
|
179
|
+
|
|
180
|
+
|
|
105
181
|
def run_tool(
|
|
106
182
|
tool_config: ToolConfig,
|
|
107
183
|
context_path: str,
|
|
@@ -109,6 +185,7 @@ def run_tool(
|
|
|
109
185
|
*,
|
|
110
186
|
precomputed_hash: str | None = None,
|
|
111
187
|
project_tag: str | None = None,
|
|
188
|
+
cancel_event: threading.Event | None = None,
|
|
112
189
|
) -> RawResult:
|
|
113
190
|
"""Runs a tool defined by its configuration and returns the execution result.
|
|
114
191
|
|
|
@@ -195,11 +272,12 @@ def run_tool(
|
|
|
195
272
|
python=python,
|
|
196
273
|
exclude=exclude,
|
|
197
274
|
)
|
|
198
|
-
|
|
199
|
-
precomputed_hash
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
275
|
+
if precomputed_hash is not None:
|
|
276
|
+
context_hash = precomputed_hash
|
|
277
|
+
elif tool_config.cache_invariant == "ast":
|
|
278
|
+
context_hash = get_context_hash(context_path, normalize=True)
|
|
279
|
+
else:
|
|
280
|
+
context_hash = get_context_hash(context_path)
|
|
203
281
|
cache_key = f"{command}:{context_hash}"
|
|
204
282
|
|
|
205
283
|
t_cache0 = time.perf_counter()
|
|
@@ -223,18 +301,20 @@ def run_tool(
|
|
|
223
301
|
run_env["COVERAGE_FILE"] = coverage_tmp
|
|
224
302
|
t_sub0 = time.perf_counter()
|
|
225
303
|
try:
|
|
226
|
-
|
|
227
|
-
command,
|
|
228
|
-
|
|
229
|
-
text=True,
|
|
230
|
-
shell=True,
|
|
231
|
-
encoding="utf-8",
|
|
232
|
-
errors="replace",
|
|
233
|
-
env=run_env,
|
|
234
|
-
) # nosec
|
|
304
|
+
stdout, stderr, return_code, cancelled = _run_command(
|
|
305
|
+
command, run_env, cancel_event
|
|
306
|
+
)
|
|
235
307
|
finally:
|
|
236
308
|
Path(coverage_tmp).unlink(missing_ok=True)
|
|
237
309
|
t_sub = time.perf_counter() - t_sub0
|
|
310
|
+
if cancelled:
|
|
311
|
+
# A dependency (e.g. pytest) already failed, so this run is moot. Return
|
|
312
|
+
# an empty result - uncached, since it was terminated, not completed -
|
|
313
|
+
# and let the parser produce its skip/zero output.
|
|
314
|
+
log.debug(
|
|
315
|
+
f"{tool_config.name}: [CANCELLED] {tool_config.skip_if} failed; terminated after {t_sub * 1000:.0f}ms"
|
|
316
|
+
)
|
|
317
|
+
return RawResult(tool_name=tool_config.name)
|
|
238
318
|
log.debug(
|
|
239
319
|
f"{tool_config.name}: [MISS] cache={t_cache * 1000:.1f}ms tool={t_sub * 1000:.0f}ms: {command}"
|
|
240
320
|
)
|
|
@@ -242,9 +322,9 @@ def run_tool(
|
|
|
242
322
|
raw_result = RawResult(
|
|
243
323
|
tool_name=tool_config.name,
|
|
244
324
|
command=command,
|
|
245
|
-
stdout=
|
|
246
|
-
stderr=
|
|
247
|
-
return_code=
|
|
325
|
+
stdout=stdout,
|
|
326
|
+
stderr=stderr,
|
|
327
|
+
return_code=return_code,
|
|
248
328
|
timestamp=timestamp,
|
|
249
329
|
project_path=project_dir,
|
|
250
330
|
)
|
|
@@ -257,6 +337,7 @@ def run_tools(
|
|
|
257
337
|
path: str,
|
|
258
338
|
max_workers: int = 0,
|
|
259
339
|
early_exit: bool = False,
|
|
340
|
+
order: str = "severity",
|
|
260
341
|
excludes: list[str] | None = None,
|
|
261
342
|
project_root: str | None = None,
|
|
262
343
|
) -> list[ToolResult]:
|
|
@@ -307,16 +388,29 @@ def run_tools(
|
|
|
307
388
|
shared_hash = get_context_hash(root)
|
|
308
389
|
log.debug(f"context_hash: {(time.perf_counter() - t_hash0) * 1000:.1f}ms {shared_hash}")
|
|
309
390
|
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
391
|
+
# Cache keys are content-based, so a changed project naturally produces new
|
|
392
|
+
# keys; stale entries age out via TTL + size_limit (no blanket eviction).
|
|
393
|
+
# Tools flagged cache_invariant="ast" key on the docstring-stripped AST so
|
|
394
|
+
# docstring/comment/format-only edits stay cache hits. Compute it once here
|
|
395
|
+
# (not lazily in the worker threads, where pytest and coverage would race
|
|
396
|
+
# and each re-parse the whole tree) and only when some tool needs it.
|
|
397
|
+
norm_hash = shared_hash
|
|
398
|
+
if any(tc.cache_invariant == "ast" for tc in tool_configs):
|
|
399
|
+
t_norm0 = time.perf_counter()
|
|
400
|
+
norm_hash = get_context_hash(root, normalize=True)
|
|
401
|
+
log.debug(f"context_hash(ast): {(time.perf_counter() - t_norm0) * 1000:.1f}ms {norm_hash}")
|
|
402
|
+
|
|
403
|
+
def _hash_for(tool_config: ToolConfig) -> str:
|
|
404
|
+
return norm_hash if tool_config.cache_invariant == "ast" else shared_hash
|
|
405
|
+
|
|
406
|
+
def _run_and_parse(
|
|
407
|
+
tool_config: ToolConfig, cancel_event: threading.Event | None = None
|
|
408
|
+
) -> tuple[int, ToolResult]:
|
|
318
409
|
t0 = time.perf_counter()
|
|
319
|
-
|
|
410
|
+
# Only pass cancel_event when present so patched run_tool stubs in tests
|
|
411
|
+
# (which don't declare the kwarg) keep working for non-cancellable tools.
|
|
412
|
+
extra = {"cancel_event": cancel_event} if cancel_event is not None else {}
|
|
413
|
+
raw_result = run_tool(tool_config, path, excludes, precomputed_hash=_hash_for(tool_config), project_tag=root, **extra)
|
|
320
414
|
tr = tool_config.parser_class(tool_config.parser_config).parse(raw_result)
|
|
321
415
|
tr.duration_s = time.perf_counter() - t0
|
|
322
416
|
return tool_config.order, tr
|
|
@@ -328,7 +422,7 @@ def run_tools(
|
|
|
328
422
|
for i, tool_config in enumerate(sorted_configs):
|
|
329
423
|
try:
|
|
330
424
|
prioritized.append(_run_and_parse(tool_config))
|
|
331
|
-
except Exception as exc:
|
|
425
|
+
except Exception as exc: # noqa: BLE001 - a crashing tool must not abort the run
|
|
332
426
|
log.error(f"{tool_config.name} generated an exception: {exc} {exc.__traceback__}")
|
|
333
427
|
n_skipped = n_total - i - 1
|
|
334
428
|
if n_skipped:
|
|
@@ -336,29 +430,72 @@ def run_tools(
|
|
|
336
430
|
log.warning(f"Early exit: skipped {n_skipped} tool(s): {remaining}")
|
|
337
431
|
break
|
|
338
432
|
_, tr = prioritized[-1]
|
|
339
|
-
|
|
433
|
+
# order="phase" stops at the first phase with *any* finding (not
|
|
434
|
+
# clean), so phase order is absolute and independent of error/warning
|
|
435
|
+
# threshold tuning; "severity" stops at the first error.
|
|
436
|
+
threshold = (
|
|
437
|
+
tool_config.warning_threshold
|
|
438
|
+
if order == "phase"
|
|
439
|
+
else tool_config.error_threshold
|
|
440
|
+
)
|
|
441
|
+
if tr.metrics and min(tr.metrics.values()) < threshold:
|
|
340
442
|
n_skipped = n_total - i - 1
|
|
341
443
|
if n_skipped:
|
|
342
444
|
remaining = ", ".join(tc.name for tc in sorted_configs[i + 1 :])
|
|
343
445
|
log.debug(
|
|
344
|
-
f"Error threshold hit at {tool_config.name}: skipped {n_skipped} tool(s): {remaining}"
|
|
446
|
+
f"{'Not clean' if order == 'phase' else 'Error threshold hit'} at {tool_config.name}: skipped {n_skipped} tool(s): {remaining}"
|
|
345
447
|
)
|
|
346
448
|
break
|
|
347
449
|
log.info(f"cq run_tools elapsed: {time.perf_counter() - t_start:.2f}s")
|
|
348
450
|
return [tr for _, tr in sorted(prioritized)]
|
|
451
|
+
configs_by_name = {tc.name: tc for tc in tool_configs}
|
|
452
|
+
|
|
453
|
+
timings: list[tuple[int, str, float]] = []
|
|
454
|
+
sorted_configs = sorted(tool_configs, key=lambda tc: tc.order)
|
|
455
|
+
# A tool with skip_if gets a cancel event: it runs in parallel with its
|
|
456
|
+
# dependency, and is terminated mid-run only if that dependency *fails* its
|
|
457
|
+
# error threshold. Coverage declares skip_if = "pytest" so a broken pytest
|
|
458
|
+
# kills the coverage run (which re-invokes pytest internally) instead of
|
|
459
|
+
# letting it finish a pointless second suite execution.
|
|
460
|
+
cancel_events = {
|
|
461
|
+
tc.name: threading.Event()
|
|
462
|
+
for tc in sorted_configs
|
|
463
|
+
if tc.skip_if and tc.skip_if in configs_by_name
|
|
464
|
+
}
|
|
465
|
+
name_to_future: dict[str, Any] = {}
|
|
349
466
|
with ThreadPoolExecutor(max_workers=max_workers or len(tool_configs)) as executor:
|
|
350
|
-
future_to_tool = {
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
467
|
+
future_to_tool = {}
|
|
468
|
+
for tc in sorted_configs:
|
|
469
|
+
f = executor.submit(_run_and_parse, tc, cancel_events.get(tc.name))
|
|
470
|
+
future_to_tool[f] = tc
|
|
471
|
+
name_to_future[tc.name] = f
|
|
472
|
+
|
|
473
|
+
def _cancel_on_dep_failure(
|
|
474
|
+
dep_future, ev: threading.Event, dep_config: ToolConfig, name: str
|
|
475
|
+
) -> None:
|
|
476
|
+
try:
|
|
477
|
+
_, dep_tr = dep_future.result()
|
|
478
|
+
except Exception: # noqa: BLE001 - a crashed dep cancels nothing
|
|
479
|
+
return
|
|
480
|
+
if dep_tr.metrics and min(dep_tr.metrics.values()) < dep_config.error_threshold:
|
|
481
|
+
log.debug(f"{name}: cancelling because {dep_config.name} failed")
|
|
482
|
+
ev.set()
|
|
483
|
+
|
|
484
|
+
for tc in sorted_configs:
|
|
485
|
+
if tc.name in cancel_events:
|
|
486
|
+
dep_config = configs_by_name[tc.skip_if]
|
|
487
|
+
name_to_future[tc.skip_if].add_done_callback(
|
|
488
|
+
lambda fut, ev=cancel_events[tc.name], dc=dep_config, nm=tc.name: _cancel_on_dep_failure(
|
|
489
|
+
fut, ev, dc, nm
|
|
490
|
+
)
|
|
491
|
+
)
|
|
355
492
|
for future in as_completed(future_to_tool):
|
|
356
493
|
tool_config = future_to_tool[future]
|
|
357
494
|
try:
|
|
358
|
-
|
|
359
|
-
prioritized.append((
|
|
360
|
-
timings.append((
|
|
361
|
-
except Exception as exc:
|
|
495
|
+
tc_order, tr = future.result()
|
|
496
|
+
prioritized.append((tc_order, tr))
|
|
497
|
+
timings.append((tc_order, tool_config.name, tr.duration_s))
|
|
498
|
+
except Exception as exc: # noqa: BLE001 - a crashing tool must not abort the run
|
|
362
499
|
log.error(f"{tool_config.name} generated an exception: {exc}")
|
|
363
500
|
per_tool = ", ".join(f"{name}={dur:.2f}s" for _, name, dur in sorted(timings))
|
|
364
501
|
log.debug(f"run_tools elapsed: {time.perf_counter() - t_start:.2f}s [{per_tool}]")
|