errgrade 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- errgrade/__init__.py +3 -0
- errgrade/analyze.py +269 -0
- errgrade/cli.py +108 -0
- errgrade/grade.py +134 -0
- errgrade-0.1.0.dist-info/METADATA +160 -0
- errgrade-0.1.0.dist-info/RECORD +10 -0
- errgrade-0.1.0.dist-info/WHEEL +5 -0
- errgrade-0.1.0.dist-info/entry_points.txt +2 -0
- errgrade-0.1.0.dist-info/licenses/LICENSE +21 -0
- errgrade-0.1.0.dist-info/top_level.txt +1 -0
errgrade/__init__.py
ADDED
errgrade/analyze.py
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
"""Find the messages a Python program would actually show a person.
|
|
2
|
+
|
|
3
|
+
Only ``raise`` sites count. A string handed to ``logging.error`` is for operators who
|
|
4
|
+
have the code open; a string in ``raise`` is what a user or a caller reads when
|
|
5
|
+
something is already going wrong, and that is the one worth grading.
|
|
6
|
+
|
|
7
|
+
Extraction is AST-based on purpose. Grepping for ``raise`` finds it inside comments,
|
|
8
|
+
docstrings and string literals, which is how most "quick linter" scripts end up with
|
|
9
|
+
noise instead of findings.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import ast
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import List, Optional, Sequence, Tuple
|
|
18
|
+
|
|
19
|
+
from .grade import PLACEHOLDER_RE, Grade, grade_message
|
|
20
|
+
|
|
21
|
+
SEVERITY_ORDER = {"high": 0, "medium": 1, "low": 2}
|
|
22
|
+
|
|
23
|
+
#: Frameworks put the human-facing text in a keyword, not a positional argument:
|
|
24
|
+
#: ``HTTPException(status_code=401, detail="invalid signature")``.
|
|
25
|
+
MESSAGE_KWARGS = ("detail", "message", "msg", "reason")
|
|
26
|
+
|
|
27
|
+
#: Inline suppression, so a deliberate message does not need a config file.
|
|
28
|
+
PRAGMA = "errgrade: ignore"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class Finding:
|
|
33
|
+
"""One suspicious message, located well enough to fix without searching."""
|
|
34
|
+
|
|
35
|
+
path: str
|
|
36
|
+
line: int
|
|
37
|
+
rule: str
|
|
38
|
+
severity: str
|
|
39
|
+
message: str
|
|
40
|
+
detail: str
|
|
41
|
+
grade: Grade
|
|
42
|
+
|
|
43
|
+
def to_dict(self) -> dict:
|
|
44
|
+
return {
|
|
45
|
+
"path": self.path,
|
|
46
|
+
"line": self.line,
|
|
47
|
+
"rule": self.rule,
|
|
48
|
+
"severity": self.severity,
|
|
49
|
+
"message": self.message,
|
|
50
|
+
"detail": self.detail,
|
|
51
|
+
**self.grade.to_dict(),
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass
|
|
56
|
+
class Report:
|
|
57
|
+
findings: List[Finding] = field(default_factory=list)
|
|
58
|
+
graded: int = 0
|
|
59
|
+
files: int = 0
|
|
60
|
+
below_c: int = 0
|
|
61
|
+
|
|
62
|
+
def sorted_findings(self) -> List[Finding]:
|
|
63
|
+
return sorted(self.findings, key=lambda f: (SEVERITY_ORDER[f.severity], f.path, f.line))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _call_name(node: ast.AST) -> str:
|
|
67
|
+
if isinstance(node, ast.Call):
|
|
68
|
+
func = node.func
|
|
69
|
+
if isinstance(func, ast.Name):
|
|
70
|
+
return func.id
|
|
71
|
+
if isinstance(func, ast.Attribute):
|
|
72
|
+
return func.attr
|
|
73
|
+
if isinstance(node, ast.Name):
|
|
74
|
+
return node.id
|
|
75
|
+
return ""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _string_parts(node: ast.AST) -> Optional[Tuple[str, bool]]:
|
|
79
|
+
"""Return (text, interpolated) for a message expression, or None if unreadable.
|
|
80
|
+
|
|
81
|
+
``interpolated`` marks a value flowing into the text. That is the only honest
|
|
82
|
+
signal that a message says *where* the failure happened.
|
|
83
|
+
"""
|
|
84
|
+
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
|
85
|
+
return node.value, False
|
|
86
|
+
if isinstance(node, ast.JoinedStr):
|
|
87
|
+
text = "".join(
|
|
88
|
+
part.value if isinstance(part, ast.Constant) and isinstance(part.value, str) else "<value>"
|
|
89
|
+
for part in node.values
|
|
90
|
+
)
|
|
91
|
+
has_value = any(isinstance(part, ast.FormattedValue) for part in node.values)
|
|
92
|
+
return text, has_value
|
|
93
|
+
if isinstance(node, ast.BinOp):
|
|
94
|
+
# "x" % name or "x" + name
|
|
95
|
+
if isinstance(node.op, (ast.Mod, ast.Add)):
|
|
96
|
+
left = _string_parts(node.left)
|
|
97
|
+
if left:
|
|
98
|
+
text, _ = left
|
|
99
|
+
return text, True
|
|
100
|
+
right = _string_parts(node.right)
|
|
101
|
+
if right and isinstance(node.op, ast.Add):
|
|
102
|
+
text, _ = right
|
|
103
|
+
return text, True
|
|
104
|
+
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
|
105
|
+
# "x {}".format(name) / f-string style helpers
|
|
106
|
+
if node.func.attr == "format":
|
|
107
|
+
base = _string_parts(node.func.value)
|
|
108
|
+
if base:
|
|
109
|
+
text, _ = base
|
|
110
|
+
# "{}".format(x) interpolates; "{a}" without args does not.
|
|
111
|
+
return text, bool(node.args or node.keywords)
|
|
112
|
+
return None
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _message_node(exc: ast.AST) -> Optional[ast.AST]:
|
|
116
|
+
"""The expression holding the message, wherever this framework keeps it."""
|
|
117
|
+
if not isinstance(exc, ast.Call):
|
|
118
|
+
return None
|
|
119
|
+
if exc.args:
|
|
120
|
+
return exc.args[0]
|
|
121
|
+
for kw in exc.keywords:
|
|
122
|
+
if kw.arg in MESSAGE_KWARGS:
|
|
123
|
+
return kw.value
|
|
124
|
+
return None
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _docstring_lines(tree: ast.AST) -> set:
|
|
128
|
+
"""Lines occupied by docstrings, so a message quoted in prose is not graded."""
|
|
129
|
+
lines = set()
|
|
130
|
+
for node in ast.walk(tree):
|
|
131
|
+
if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
132
|
+
body = getattr(node, "body", None)
|
|
133
|
+
if body and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant) \
|
|
134
|
+
and isinstance(body[0].value.value, str):
|
|
135
|
+
first, last = body[0].lineno, getattr(body[0], "end_lineno", body[0].lineno)
|
|
136
|
+
lines.update(range(first, last + 1))
|
|
137
|
+
return lines
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def analyze_source(source: str, path: str = "<src>") -> List[Finding]:
|
|
141
|
+
"""Every finding in one file."""
|
|
142
|
+
try:
|
|
143
|
+
tree = ast.parse(source)
|
|
144
|
+
except SyntaxError:
|
|
145
|
+
return []
|
|
146
|
+
|
|
147
|
+
ignored = {i + 1 for i, line in enumerate(source.splitlines()) if PRAGMA in line}
|
|
148
|
+
doc_lines = _docstring_lines(tree)
|
|
149
|
+
findings: List[Finding] = []
|
|
150
|
+
|
|
151
|
+
for node in ast.walk(tree):
|
|
152
|
+
if not isinstance(node, ast.Raise) or node.exc is None:
|
|
153
|
+
continue
|
|
154
|
+
exc = node.exc
|
|
155
|
+
name = _call_name(exc)
|
|
156
|
+
if not name:
|
|
157
|
+
continue
|
|
158
|
+
line = node.lineno
|
|
159
|
+
if line in ignored:
|
|
160
|
+
continue
|
|
161
|
+
|
|
162
|
+
node_for_message = _message_node(exc)
|
|
163
|
+
if node_for_message is None:
|
|
164
|
+
# A raise with framing keywords but no readable text (e.g. status_code=500)
|
|
165
|
+
# cannot be graded honestly, so it is skipped rather than guessed at.
|
|
166
|
+
if isinstance(exc, ast.Call) and (exc.args or exc.keywords):
|
|
167
|
+
continue
|
|
168
|
+
findings.append(Finding(
|
|
169
|
+
path, line, "bare-exception", "medium", name + "()",
|
|
170
|
+
"raised with no message, so the caller learns nothing beyond the type",
|
|
171
|
+
grade_message(""),
|
|
172
|
+
))
|
|
173
|
+
continue
|
|
174
|
+
|
|
175
|
+
parts = _string_parts(node_for_message)
|
|
176
|
+
if parts is None:
|
|
177
|
+
continue
|
|
178
|
+
text, interpolated = parts
|
|
179
|
+
|
|
180
|
+
if not text.strip():
|
|
181
|
+
continue
|
|
182
|
+
grade = grade_message(text, interpolated=interpolated)
|
|
183
|
+
|
|
184
|
+
# A brace placeholder in a string that is not interpolated reaches the user
|
|
185
|
+
# literally, which is a bug rather than a style problem.
|
|
186
|
+
if not interpolated and PLACEHOLDER_RE.search(text):
|
|
187
|
+
findings.append(Finding(
|
|
188
|
+
path, line, "literal-placeholder", "high", text,
|
|
189
|
+
"braces are shown to the reader verbatim: this string is never formatted",
|
|
190
|
+
grade,
|
|
191
|
+
))
|
|
192
|
+
continue
|
|
193
|
+
|
|
194
|
+
if not grade.what:
|
|
195
|
+
findings.append(Finding(
|
|
196
|
+
path, line, "vague-error", "high", text,
|
|
197
|
+
"the message does not say what went wrong",
|
|
198
|
+
grade,
|
|
199
|
+
))
|
|
200
|
+
continue # one primary verdict per message; the low rules below add noise
|
|
201
|
+
elif not grade.where:
|
|
202
|
+
findings.append(Finding(
|
|
203
|
+
path, line, "no-context", "medium", text,
|
|
204
|
+
"no value or location: the reader cannot tell which case this is",
|
|
205
|
+
grade,
|
|
206
|
+
))
|
|
207
|
+
continue
|
|
208
|
+
|
|
209
|
+
if not grade.next:
|
|
210
|
+
findings.append(Finding(
|
|
211
|
+
path, line, "no-next-step", "low", text,
|
|
212
|
+
"nothing tells the reader what to do next",
|
|
213
|
+
grade,
|
|
214
|
+
))
|
|
215
|
+
|
|
216
|
+
return findings
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def analyze_files(files: Sequence[Path], *, include_tests: bool = False) -> Report:
|
|
220
|
+
"""Analyse a list of paths, expanding directories."""
|
|
221
|
+
report = Report()
|
|
222
|
+
paths: List[Path] = []
|
|
223
|
+
for p in files:
|
|
224
|
+
if p.is_dir():
|
|
225
|
+
paths.extend(sorted(p.rglob("*.py")))
|
|
226
|
+
elif p.suffix == ".py":
|
|
227
|
+
paths.append(p)
|
|
228
|
+
|
|
229
|
+
for path in paths:
|
|
230
|
+
if _looks_like_artifact(path):
|
|
231
|
+
continue
|
|
232
|
+
if not include_tests and _looks_like_test(path):
|
|
233
|
+
continue
|
|
234
|
+
try:
|
|
235
|
+
source = path.read_text(encoding="utf-8")
|
|
236
|
+
except (OSError, UnicodeDecodeError):
|
|
237
|
+
continue
|
|
238
|
+
report.files += 1
|
|
239
|
+
file_findings = analyze_source(source, str(path))
|
|
240
|
+
report.findings.extend(file_findings)
|
|
241
|
+
report.graded += _count_raises(source)
|
|
242
|
+
report.below_c += sum(1 for f in file_findings if f.grade.letter in ("C", "D"))
|
|
243
|
+
return report
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
SKIP_DIRS = frozenset({"build", "dist", "node_modules", ".git", ".venv", "venv",
|
|
247
|
+
"site-packages", "__pycache__", ".tox", ".mypy_cache",
|
|
248
|
+
"egg-info", ".eggs"})
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _looks_like_artifact(path: Path) -> bool:
|
|
252
|
+
"""Build outputs and vendored trees are not your code to fix."""
|
|
253
|
+
return any(p.lower() in SKIP_DIRS or p.lower().endswith(".egg-info") for p in path.parts)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _looks_like_test(path: Path) -> bool:
|
|
257
|
+
parts = [p.lower() for p in path.parts]
|
|
258
|
+
if "tests" in parts or "test" in parts:
|
|
259
|
+
return True
|
|
260
|
+
name = path.name.lower()
|
|
261
|
+
return name.startswith("test_") or name.endswith("_test.py") or name == "conftest.py"
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _count_raises(source: str) -> int:
|
|
265
|
+
try:
|
|
266
|
+
tree = ast.parse(source)
|
|
267
|
+
except SyntaxError:
|
|
268
|
+
return 0
|
|
269
|
+
return sum(1 for node in ast.walk(tree) if isinstance(node, ast.Raise) and node.exc is not None)
|
errgrade/cli.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""CLI. Reads paths, prints a report, exits with a documented code.
|
|
2
|
+
|
|
3
|
+
Exit codes: 0 nothing at or above the threshold, 1 findings at or above it,
|
|
4
|
+
2 usage or configuration problem. A linter that exits 1 for its own errors makes CI
|
|
5
|
+
lie, which is why 2 is distinct.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import sys
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import List, Optional, Sequence
|
|
15
|
+
|
|
16
|
+
from . import __version__
|
|
17
|
+
from .analyze import SEVERITY_ORDER, Report, analyze_files
|
|
18
|
+
|
|
19
|
+
EXIT_OK = 0
|
|
20
|
+
EXIT_FINDINGS = 1
|
|
21
|
+
EXIT_USAGE = 2
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _report(rep: Report, args: argparse.Namespace) -> str:
|
|
25
|
+
findings = rep.sorted_findings()
|
|
26
|
+
if args.top:
|
|
27
|
+
findings = findings[: args.top]
|
|
28
|
+
|
|
29
|
+
if not findings:
|
|
30
|
+
return (
|
|
31
|
+
"No issues found by errgrade v%s across %d message(s) in %d file(s).\n"
|
|
32
|
+
"That means no known weak pattern matched; it is not a guarantee."
|
|
33
|
+
% (__version__, rep.graded, rep.files)
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
worst = SEVERITY_ORDER[findings[0].severity]
|
|
37
|
+
worst_name = {0: "high", 1: "medium", 2: "low"}[worst]
|
|
38
|
+
lines = [
|
|
39
|
+
"errgrade: %d finding(s) across %d message(s) in %d file(s), worst %s"
|
|
40
|
+
% (len(findings), rep.graded, rep.files, worst_name),
|
|
41
|
+
"%d of %d graded message(s) fall below C" % (rep.below_c, rep.graded),
|
|
42
|
+
"",
|
|
43
|
+
]
|
|
44
|
+
for f in findings:
|
|
45
|
+
lines.append("[%s] %s:%d (%s)" % (f.severity.upper(), f.path, f.line, f.rule))
|
|
46
|
+
lines.append(" message: %r" % f.message)
|
|
47
|
+
lines.append(" why: %s" % f.detail)
|
|
48
|
+
if f.grade.missing:
|
|
49
|
+
lines.append(" missing: %s" % ", ".join(f.grade.missing))
|
|
50
|
+
lines.append(" grade: %s (%d/100)" % (f.grade.letter, f.grade.score))
|
|
51
|
+
lines.append("")
|
|
52
|
+
return "\n".join(lines).rstrip()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
56
|
+
parser = argparse.ArgumentParser(
|
|
57
|
+
prog="errgrade",
|
|
58
|
+
description="Grade the error messages your code raises: what broke, where, what to do next.",
|
|
59
|
+
)
|
|
60
|
+
parser.add_argument("--version", action="version", version="errgrade %s" % __version__)
|
|
61
|
+
parser.add_argument("paths", nargs="*", help="files or directories (default: current directory)")
|
|
62
|
+
parser.add_argument("--json", action="store_true", help="machine-readable output")
|
|
63
|
+
parser.add_argument("--fail-on", choices=["high", "medium", "low", "none"], default="medium",
|
|
64
|
+
help="lowest severity that fails the run (default: medium)")
|
|
65
|
+
parser.add_argument("--include-tests", action="store_true",
|
|
66
|
+
help="also grade messages under tests/ (they are not user-facing)")
|
|
67
|
+
parser.add_argument("--top", type=int, default=0, help="show only the N worst findings")
|
|
68
|
+
args = parser.parse_args(argv)
|
|
69
|
+
|
|
70
|
+
raw: List[str] = list(args.paths)
|
|
71
|
+
if raw and raw[0] == "check": # optional verb, so `errgrade check .` works
|
|
72
|
+
raw = raw[1:]
|
|
73
|
+
if not raw:
|
|
74
|
+
raw = ["."]
|
|
75
|
+
|
|
76
|
+
targets = [Path(p) for p in raw]
|
|
77
|
+
missing = [str(p) for p in targets if not p.exists()]
|
|
78
|
+
if missing:
|
|
79
|
+
print("errgrade: no such path: %s" % ", ".join(missing), file=sys.stderr)
|
|
80
|
+
return EXIT_USAGE
|
|
81
|
+
|
|
82
|
+
rep = analyze_files(targets, include_tests=args.include_tests)
|
|
83
|
+
|
|
84
|
+
if args.json:
|
|
85
|
+
payload = {
|
|
86
|
+
"version": __version__,
|
|
87
|
+
"summary": {
|
|
88
|
+
"graded": rep.graded,
|
|
89
|
+
"files": rep.files,
|
|
90
|
+
"findings": len(rep.findings),
|
|
91
|
+
"below_c": rep.below_c,
|
|
92
|
+
},
|
|
93
|
+
"findings": [f.to_dict() for f in rep.sorted_findings()],
|
|
94
|
+
}
|
|
95
|
+
print(json.dumps(payload, indent=2))
|
|
96
|
+
else:
|
|
97
|
+
print(_report(rep, args))
|
|
98
|
+
|
|
99
|
+
if args.fail_on == "none":
|
|
100
|
+
return EXIT_OK
|
|
101
|
+
limit = SEVERITY_ORDER[args.fail_on]
|
|
102
|
+
if any(SEVERITY_ORDER[f.severity] <= limit for f in rep.findings):
|
|
103
|
+
return EXIT_FINDINGS
|
|
104
|
+
return EXIT_OK
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
if __name__ == "__main__": # so `python -m errgrade.cli` works too
|
|
108
|
+
raise SystemExit(main())
|
errgrade/grade.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Grade one error message on the three questions a stuck person actually asks.
|
|
2
|
+
|
|
3
|
+
The vocabulary here is deliberately small and boring. A linter that claims to judge
|
|
4
|
+
prose is lying; a linter that can reliably tell "failed" from "failed to open
|
|
5
|
+
config.yaml: no such file" is not.
|
|
6
|
+
|
|
7
|
+
Nothing in this module parses Python. It takes a message string and returns a grade,
|
|
8
|
+
which is why the same grading can later be pointed at other languages.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import re
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from typing import Tuple
|
|
16
|
+
|
|
17
|
+
#: Messages that communicate nothing on their own. Compared case-folded with
|
|
18
|
+
#: surrounding punctuation and whitespace removed.
|
|
19
|
+
VAGUE = frozenset({
|
|
20
|
+
"error", "errors", "an error", "an error occurred", "error occurred",
|
|
21
|
+
"failed", "failure", "failed.", "it failed", "something failed",
|
|
22
|
+
"invalid", "invalid input", "invalid request", "bad request",
|
|
23
|
+
"unknown error", "unexpected error", "unexpected", "oops", "whoops",
|
|
24
|
+
"problem", "a problem occurred", "not allowed", "not permitted",
|
|
25
|
+
"something went wrong", "went wrong", "cannot proceed", "aborted",
|
|
26
|
+
"unknown", "exception", "unable", "unable to proceed",
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
#: Words that place a failure somewhere: a file, a field, a table, a key.
|
|
30
|
+
LOCATORS = (
|
|
31
|
+
"file", "path", "line", "field", "column", "key", "variable", "env",
|
|
32
|
+
"environment", "table", "endpoint", "route", "header", "argument",
|
|
33
|
+
"parameter", "config", "value", "record", "row", "migration", "schema",
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
#: Openers that tell the reader what to do next. Imperatives and pointers only.
|
|
37
|
+
ACTION_HINTS = (
|
|
38
|
+
"use ", "try ", "check ", "set ", "run ", "see ", "install ", "pass ", "give ",
|
|
39
|
+
"provide ", "ensure ", "add ", "remove ", "configure ", "specify ",
|
|
40
|
+
"retry", "call ", "import ", "regenerate", "upgrade", "downgrade",
|
|
41
|
+
"verify ", "read ", "instead", "prefix", "suffix", "must ", "should ",
|
|
42
|
+
"requires ", "expected ", "supported ", "allowed", "consider ",
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
#: Naming the knob that is wrong ("run_id is required") locates the failure even with
|
|
46
|
+
#: no interpolated value, so the "where" axis has to credit it.
|
|
47
|
+
#: Naming the thing and saying something about it locates the failure: "tenant is
|
|
48
|
+
#: archived" tells the reader far more than "invalid input", with no value in it.
|
|
49
|
+
PARAM_HINT_RE = re.compile(
|
|
50
|
+
r"\b[A-Za-z_][A-Za-z0-9_.]*\s+(?:is|are|was|were|must|cannot|can't|should"
|
|
51
|
+
r"|has been|have been|does not|did not|expected)\b")
|
|
52
|
+
|
|
53
|
+
#: Listing the acceptable values doubles as a next step: the reader now knows what to pass.
|
|
54
|
+
VALUES_HINTS = ("one of", "allowed", "supported", "valid values", "choose from",
|
|
55
|
+
"expected one of", "instead of")
|
|
56
|
+
|
|
57
|
+
#: A brace placeholder in a string that was never formatted reaches the user verbatim.
|
|
58
|
+
PLACEHOLDER_RE = re.compile(r"\{[A-Za-z_][A-Za-z0-9_.\[\]'\"]*\}")
|
|
59
|
+
|
|
60
|
+
WORD_RE = re.compile(r"[A-Za-z][A-Za-z'_-]*")
|
|
61
|
+
|
|
62
|
+
WEIGHTS = {"what": 40, "where": 35, "next": 25}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _normalize(text: str) -> str:
|
|
66
|
+
"""Case-fold and strip punctuation so "Error!" and "error" compare equal."""
|
|
67
|
+
return " ".join(re.sub(r"[^a-z0-9 ]+", " ", text.casefold()).split())
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass(frozen=True)
|
|
71
|
+
class Grade:
|
|
72
|
+
"""The verdict on one message, with the axes kept separate.
|
|
73
|
+
|
|
74
|
+
``missing`` names the axes that failed, so a report can say "no next step" rather
|
|
75
|
+
than repeating the whole sentence.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
message: str
|
|
79
|
+
score: int
|
|
80
|
+
letter: str
|
|
81
|
+
what: bool
|
|
82
|
+
where: bool
|
|
83
|
+
next: bool
|
|
84
|
+
missing: Tuple[str, ...] = field(default_factory=tuple)
|
|
85
|
+
|
|
86
|
+
def to_dict(self) -> dict:
|
|
87
|
+
return {
|
|
88
|
+
"score": self.score,
|
|
89
|
+
"letter": self.letter,
|
|
90
|
+
"has_what": self.what,
|
|
91
|
+
"has_where": self.where,
|
|
92
|
+
"has_next": self.next,
|
|
93
|
+
"missing": list(self.missing),
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _letter(score: int) -> str:
|
|
98
|
+
if score >= 85:
|
|
99
|
+
return "A"
|
|
100
|
+
if score >= 70:
|
|
101
|
+
return "B"
|
|
102
|
+
if score >= 50:
|
|
103
|
+
return "C"
|
|
104
|
+
return "D"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def grade_message(message: str, *, interpolated: bool = False) -> Grade:
|
|
108
|
+
"""Grade a message.
|
|
109
|
+
|
|
110
|
+
``interpolated`` is True when the source interpolates a value into the string
|
|
111
|
+
(f-string, ``.format``, ``%``). Plain prose cannot show us a value, so that signal
|
|
112
|
+
has to come from the syntax tree rather than the text.
|
|
113
|
+
"""
|
|
114
|
+
text = (message or "").strip()
|
|
115
|
+
norm = _normalize(text)
|
|
116
|
+
words = WORD_RE.findall(text)
|
|
117
|
+
|
|
118
|
+
has_what = bool(norm) and norm not in VAGUE and len(words) >= 2
|
|
119
|
+
names_a_knob = bool(PARAM_HINT_RE.search(text))
|
|
120
|
+
lists_values = any(v in norm for v in VALUES_HINTS)
|
|
121
|
+
|
|
122
|
+
has_where = interpolated or names_a_knob or lists_values or any(loc in norm for loc in LOCATORS)
|
|
123
|
+
has_next = lists_values or any(hint in (" " + norm + " ") for hint in ACTION_HINTS)
|
|
124
|
+
|
|
125
|
+
score = (
|
|
126
|
+
WEIGHTS["what"] * int(has_what)
|
|
127
|
+
+ WEIGHTS["where"] * int(has_where)
|
|
128
|
+
+ WEIGHTS["next"] * int(has_next)
|
|
129
|
+
)
|
|
130
|
+
missing = tuple(
|
|
131
|
+
axis for axis, ok in (("what", has_what), ("where", has_where), ("next", has_next))
|
|
132
|
+
if not ok
|
|
133
|
+
)
|
|
134
|
+
return Grade(text, score, _letter(score), has_what, has_where, has_next, missing)
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: errgrade
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Grade the error messages your code raises: what broke, where, and what to do next.
|
|
5
|
+
Author: mgkvibez
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: linter,errors,dx,code-review,developer-tools,static-analysis
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Environment :: Console
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
14
|
+
Requires-Python: >=3.9
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Provides-Extra: dev
|
|
18
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# errgrade
|
|
22
|
+
|
|
23
|
+
Grade the error messages your code raises: does it say **what** broke, **where**, and **what to do next**.
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
$ errgrade examples/before.py
|
|
27
|
+
errgrade: 4 finding(s) across 6 message(s) in 1 file(s), worst high
|
|
28
|
+
1 of 6 graded message(s) fall below C
|
|
29
|
+
|
|
30
|
+
[HIGH] examples/before.py:6 (vague-error)
|
|
31
|
+
message: 'failed'
|
|
32
|
+
why: the message does not say what went wrong
|
|
33
|
+
missing: what, where, next
|
|
34
|
+
grade: D (0/100)
|
|
35
|
+
|
|
36
|
+
[HIGH] examples/before.py:8 (literal-placeholder)
|
|
37
|
+
message: 'missing {path}'
|
|
38
|
+
why: braces are shown to the reader verbatim: this string is never formatted
|
|
39
|
+
missing: next
|
|
40
|
+
grade: B (75/100)
|
|
41
|
+
|
|
42
|
+
[LOW] examples/before.py:10 (no-next-step)
|
|
43
|
+
message: 'config file is empty'
|
|
44
|
+
why: nothing tells the reader what to do next
|
|
45
|
+
missing: next
|
|
46
|
+
grade: B (75/100)
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Note what it does **not** flag: `amount must be positive, got {amount}` and
|
|
50
|
+
`currency must be one of NGN, USD` pass, because they answer all three questions.
|
|
51
|
+
|
|
52
|
+
## Install
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pipx install errgrade
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Python 3.9+, **zero runtime dependencies**. The analysis is a standard-library AST pass.
|
|
59
|
+
|
|
60
|
+
## Usage
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
# grade the current directory (tests excluded: their messages are not user-facing)
|
|
64
|
+
errgrade
|
|
65
|
+
|
|
66
|
+
# one file or directory, and see the worst offenders first
|
|
67
|
+
errgrade src --top 5
|
|
68
|
+
|
|
69
|
+
# machine-readable, for a bot or a CI annotation step
|
|
70
|
+
errgrade --json
|
|
71
|
+
|
|
72
|
+
# fail CI only on the messages a user would actually see going wrong
|
|
73
|
+
errgrade src --fail-on high
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Exit codes: `0` nothing at or above the threshold, `1` findings at or above it,
|
|
77
|
+
`2` usage error. A linter that exits 1 for its own mistakes makes CI lie, which is why
|
|
78
|
+
those are separate.
|
|
79
|
+
|
|
80
|
+
`--fail-on` defaults to `medium`. `low` findings never fail a run unless you ask.
|
|
81
|
+
|
|
82
|
+
## Why it exists
|
|
83
|
+
|
|
84
|
+
Error messages are the only part of your code that runs at the worst moment of someone's
|
|
85
|
+
day. They are also the one thing nobody reviews: a pull request diff shows a new
|
|
86
|
+
`raise`, and reviewers look at the condition, not the sentence.
|
|
87
|
+
|
|
88
|
+
The cost is invisible and real. `ValueError("failed")` reaches a user as a dead end, and
|
|
89
|
+
reaches you as a support ticket that says "it doesn't work". The difference between that
|
|
90
|
+
and `amount must be positive, got -5` is four words and one value.
|
|
91
|
+
|
|
92
|
+
## How the grading works
|
|
93
|
+
|
|
94
|
+
Every message is scored on three axes, because those are the three questions a stuck
|
|
95
|
+
person asks:
|
|
96
|
+
|
|
97
|
+
| Axis | Weight | Satisfied by |
|
|
98
|
+
|---|---|---|
|
|
99
|
+
| what | 40 | not a known filler phrase, and at least two words |
|
|
100
|
+
| where | 35 | an interpolated value, a named subject ("tenant is archived"), a location noun, or a list of valid values |
|
|
101
|
+
| next | 25 | an imperative ("set", "pass", "check", "give"), a constraint ("must be", "expected"), or the allowed values |
|
|
102
|
+
|
|
103
|
+
100 becomes `A`, 85+ `A`, 70+ `B`, 50+ `C`, below that `D`. A message with no
|
|
104
|
+
interpolated value and no named subject cannot tell one failure from another, which is
|
|
105
|
+
what pushes it below `C`.
|
|
106
|
+
|
|
107
|
+
The values come from the syntax tree, not the text: an f-string, `.format()` or `%`
|
|
108
|
+
counts as interpolation because a value genuinely flows in. `"{name}"` written as a plain
|
|
109
|
+
string counts as a bug, because the reader sees the braces.
|
|
110
|
+
|
|
111
|
+
## Rules
|
|
112
|
+
|
|
113
|
+
| Severity | Rule | What it means |
|
|
114
|
+
|---|---|---|
|
|
115
|
+
| high | `vague-error` | the message does not say what went wrong (`"failed"`, `"invalid input"`) |
|
|
116
|
+
| high | `literal-placeholder` | braces reach the reader verbatim because the string is never formatted |
|
|
117
|
+
| medium | `no-context` | no value and no subject: the reader cannot tell which case this is |
|
|
118
|
+
| medium | `bare-exception` | `raise ValueError()` with no message at all |
|
|
119
|
+
| low | `no-next-step` | nothing tells the reader what to do next |
|
|
120
|
+
|
|
121
|
+
One primary finding per message: a vague message is not also reported for context and
|
|
122
|
+
next steps, because stacking three findings on one word is noise, not help.
|
|
123
|
+
|
|
124
|
+
## Suppressing a message
|
|
125
|
+
|
|
126
|
+
Some messages are deliberately terse. Say so on the line:
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
raise ValueError("failed") # errgrade: ignore
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Only that line is suppressed. There is no config file to fall out of sync.
|
|
133
|
+
|
|
134
|
+
## What this deliberately does not do
|
|
135
|
+
|
|
136
|
+
- **Python only, for now.** The grading itself takes a string and does not care where it
|
|
137
|
+
came from; only the AST extraction is Python-specific.
|
|
138
|
+
- **It grades literal messages.** `raise ValueError(msg)` is skipped, because grading a
|
|
139
|
+
variable would mean inventing its contents.
|
|
140
|
+
- **It is a heuristic, not a review.** A high grade means the message answers the three
|
|
141
|
+
questions as far as a static tool can tell, not that it is good writing. "No issues
|
|
142
|
+
found" means no known weak pattern matched, and is not a guarantee.
|
|
143
|
+
- **It does not touch logging.** Log lines are for people with the code open; `raise`
|
|
144
|
+
strings are what users and callers read.
|
|
145
|
+
- **It does not rewrite your messages.** It tells you which ones are a dead end and why.
|
|
146
|
+
|
|
147
|
+
## Tests
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
python -m pytest -q # 52 tests
|
|
151
|
+
bash scripts/verify.sh # the release gate, 6 checks
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
Contributions: the rules live in `errgrade/analyze.py`, the scoring in
|
|
155
|
+
`errgrade/grade.py`, and every rule has a test in `tests/test_analyze.py`. A new
|
|
156
|
+
heuristic needs a case it must flag and a case it must not.
|
|
157
|
+
|
|
158
|
+
## License
|
|
159
|
+
|
|
160
|
+
MIT.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
errgrade/__init__.py,sha256=Z_ST0cii8ECNmUme1HkHtXCoviwMNnmaCnGAIcydqtM,82
|
|
2
|
+
errgrade/analyze.py,sha256=t0AMmqZLbWGrMgS9bGwQyT9xjEJ99bVihrRnFyweJow,9427
|
|
3
|
+
errgrade/cli.py,sha256=kIxSGz2fH4qautdw-FEcwXgTju0-voguUiaIasMJtnc,3984
|
|
4
|
+
errgrade/grade.py,sha256=TrmR5kt37Pr0sbDZyGut3KdiQ4jExJDcJTmyIcKdOds,5114
|
|
5
|
+
errgrade-0.1.0.dist-info/licenses/LICENSE,sha256=e-0Bks6hOWW2m30Xgh8w6CENzPJoV246vIq2lz_9Ing,1065
|
|
6
|
+
errgrade-0.1.0.dist-info/METADATA,sha256=WE1YITpsPs3MdeJ8NXD6Vk7FUqANXfPv__riWBB0bf0,5858
|
|
7
|
+
errgrade-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
8
|
+
errgrade-0.1.0.dist-info/entry_points.txt,sha256=Djl3oZQ_UVt1P3ll1UmsdrFsGJZz8l4YFqMZJVpfaGs,47
|
|
9
|
+
errgrade-0.1.0.dist-info/top_level.txt,sha256=l9EYDCCi3CNO7NThXq6xJwYhIS_5JRbNesEtnTleD_8,9
|
|
10
|
+
errgrade-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 mgkvibez
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
errgrade
|