wouldcatch 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wouldcatch/__init__.py +1 -0
- wouldcatch/__main__.py +5 -0
- wouldcatch/cli.py +150 -0
- wouldcatch/diff.py +58 -0
- wouldcatch/mutate.py +127 -0
- wouldcatch-0.1.0.dist-info/METADATA +170 -0
- wouldcatch-0.1.0.dist-info/RECORD +11 -0
- wouldcatch-0.1.0.dist-info/WHEEL +5 -0
- wouldcatch-0.1.0.dist-info/entry_points.txt +2 -0
- wouldcatch-0.1.0.dist-info/licenses/LICENSE +21 -0
- wouldcatch-0.1.0.dist-info/top_level.txt +1 -0
wouldcatch/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
wouldcatch/__main__.py
ADDED
wouldcatch/cli.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""wouldcatch: would your tests catch a bug in the lines you just changed?"""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import signal
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import tempfile
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
from .diff import changed_lines, pick_base, repo_root
|
|
14
|
+
from .mutate import find_mutants
|
|
15
|
+
|
|
16
|
+
BACKUP = ".wouldcatch-backup.json"
|
|
17
|
+
COLOR = sys.stdout.isatty() and not os.environ.get("NO_COLOR")
|
|
18
|
+
if COLOR and os.name == "nt":
|
|
19
|
+
os.system("") # turns on ANSI colors in the classic Windows console
|
|
20
|
+
DEFAULT_CMD = f'"{sys.executable}" -m pytest -x -q -p no:cacheprovider'
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def read(path):
|
|
24
|
+
with open(path, encoding="utf-8", newline="") as f: # keep CRLF as-is
|
|
25
|
+
return f.read()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def write(path, text):
|
|
29
|
+
with open(path, "w", encoding="utf-8", newline="") as f:
|
|
30
|
+
f.write(text)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def paint(text, code):
|
|
34
|
+
return f"\033[{code}m{text}\033[0m" if COLOR else text
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def run_tests(cmd, root, timeout):
|
|
38
|
+
env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1")
|
|
39
|
+
# A fresh bytecode cache per run, so a stale .pyc can never hide a mutant.
|
|
40
|
+
with tempfile.TemporaryDirectory() as cache:
|
|
41
|
+
env["PYTHONPYCACHEPREFIX"] = cache
|
|
42
|
+
try:
|
|
43
|
+
p = subprocess.run(cmd, shell=True, cwd=root, env=env, timeout=timeout,
|
|
44
|
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
45
|
+
except subprocess.TimeoutExpired:
|
|
46
|
+
return "timeout"
|
|
47
|
+
return "passed" if p.returncode == 0 else "failed"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def restore_stale_backup(root):
|
|
51
|
+
"""If a previous run was killed mid-mutation, put the original file back."""
|
|
52
|
+
backup = root / BACKUP
|
|
53
|
+
if backup.exists():
|
|
54
|
+
for path, text in json.loads(backup.read_text(encoding="utf-8")).items():
|
|
55
|
+
write(root / path, text)
|
|
56
|
+
print(f"wouldcatch: restored {path} from an interrupted run", file=sys.stderr)
|
|
57
|
+
backup.unlink()
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_mutant(m, root, cmd, timeout):
|
|
61
|
+
target, backup = root / m.path, root / BACKUP
|
|
62
|
+
original = read(target)
|
|
63
|
+
backup.write_text(json.dumps({m.path: original}), encoding="utf-8")
|
|
64
|
+
try:
|
|
65
|
+
write(target, m.apply(original))
|
|
66
|
+
result = run_tests(cmd, root, timeout)
|
|
67
|
+
finally:
|
|
68
|
+
write(target, original)
|
|
69
|
+
backup.unlink()
|
|
70
|
+
return {"passed": "missed", "failed": "caught", "timeout": "caught"}[result]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def main(argv=None):
|
|
74
|
+
ap = argparse.ArgumentParser(prog="wouldcatch", description=__doc__)
|
|
75
|
+
ap.add_argument("--base", help="git ref to diff against (default: uncommitted changes, else last commit)")
|
|
76
|
+
ap.add_argument("--cmd", default=DEFAULT_CMD, help="test command (default: pytest)")
|
|
77
|
+
ap.add_argument("--max", type=int, default=50, help="max mutants to try (default: 50)")
|
|
78
|
+
ap.add_argument("--json", action="store_true", help="print results as JSON (for scripts and AI agents)")
|
|
79
|
+
ap.add_argument("--all", action="store_true", help="list caught mutants too")
|
|
80
|
+
ap.add_argument("--version", action="version", version=f"wouldcatch {__version__}")
|
|
81
|
+
args = ap.parse_args(argv)
|
|
82
|
+
signal.signal(signal.SIGTERM, lambda *_: sys.exit(143)) # still runs the restore in `finally`
|
|
83
|
+
log = (lambda *a, **k: None) if args.json else (lambda *a, **k: print(*a, file=sys.stderr, **k))
|
|
84
|
+
|
|
85
|
+
try:
|
|
86
|
+
root = repo_root()
|
|
87
|
+
except RuntimeError as e:
|
|
88
|
+
print(f"wouldcatch: {e}", file=sys.stderr)
|
|
89
|
+
return 2
|
|
90
|
+
restore_stale_backup(root)
|
|
91
|
+
base = args.base or pick_base(root)
|
|
92
|
+
try:
|
|
93
|
+
changes = changed_lines(root, base)
|
|
94
|
+
except RuntimeError as e:
|
|
95
|
+
print(f"wouldcatch: {e}", file=sys.stderr)
|
|
96
|
+
return 2
|
|
97
|
+
|
|
98
|
+
mutants = []
|
|
99
|
+
for path, lines in sorted(changes.items()):
|
|
100
|
+
text = read(root / path)
|
|
101
|
+
mutants += find_mutants(path, text, lines)
|
|
102
|
+
skipped = max(0, len(mutants) - args.max)
|
|
103
|
+
mutants = mutants[: args.max]
|
|
104
|
+
|
|
105
|
+
if not mutants:
|
|
106
|
+
msg = f"no mutable Python lines changed since {base}"
|
|
107
|
+
print(json.dumps({"base": base, "mutants": [], "message": msg}) if args.json else f"wouldcatch: {msg}")
|
|
108
|
+
return 0
|
|
109
|
+
|
|
110
|
+
log(f"wouldcatch: {len(mutants)} mutants on {sum(map(len, changes.values()))} changed lines "
|
|
111
|
+
f"in {len(changes)} file(s), base {base}")
|
|
112
|
+
t0 = time.time()
|
|
113
|
+
if run_tests(args.cmd, root, None) != "passed":
|
|
114
|
+
print("wouldcatch: your tests fail before any change. Fix them first.\n"
|
|
115
|
+
f" command: {args.cmd}", file=sys.stderr)
|
|
116
|
+
return 2
|
|
117
|
+
timeout = max(10.0, 3 * (time.time() - t0))
|
|
118
|
+
|
|
119
|
+
for i, m in enumerate(mutants, 1):
|
|
120
|
+
log(f" [{i}/{len(mutants)}] {m.path}:{m.line} {m.describe()}", end="\r" if sys.stderr.isatty() else "\n")
|
|
121
|
+
m.status = test_mutant(m, root, args.cmd, timeout)
|
|
122
|
+
if sys.stderr.isatty() and not args.json:
|
|
123
|
+
print(" " * 100, end="\r", file=sys.stderr)
|
|
124
|
+
|
|
125
|
+
missed = [m for m in mutants if m.status == "missed"]
|
|
126
|
+
caught = len(mutants) - len(missed)
|
|
127
|
+
if args.json:
|
|
128
|
+
print(json.dumps({
|
|
129
|
+
"base": base, "caught": caught, "missed": len(missed), "skipped": skipped,
|
|
130
|
+
"mutants": [{"file": m.path, "line": m.line, "original": m.original,
|
|
131
|
+
"mutated": m.replacement, "status": m.status} for m in mutants],
|
|
132
|
+
}, indent=2))
|
|
133
|
+
else:
|
|
134
|
+
for m in mutants:
|
|
135
|
+
if m.status == "missed" or args.all:
|
|
136
|
+
label = paint("MISSED", "1;31") if m.status == "missed" else paint("caught", "32")
|
|
137
|
+
print(f" {label} {paint(f'{m.path}:{m.line}', '36')} {m.describe()}")
|
|
138
|
+
print(paint(f"\n{caught}/{len(mutants)} caught.", "1;32" if not missed else "1"), end=" ")
|
|
139
|
+
if missed:
|
|
140
|
+
print(paint(f"{len(missed)} bug(s) your tests would not notice.", "31"),
|
|
141
|
+
"Add a test that fails for each MISSED line.")
|
|
142
|
+
else:
|
|
143
|
+
print("Your tests would notice every one of these bugs.")
|
|
144
|
+
if skipped:
|
|
145
|
+
print(f"({skipped} more mutants not tried; raise --max to try them.)")
|
|
146
|
+
return 1 if missed else 0
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
if __name__ == "__main__":
|
|
150
|
+
sys.exit(main())
|
wouldcatch/diff.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Find which Python lines changed, using git."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import subprocess
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@")
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def git(*args, cwd="."):
|
|
11
|
+
out = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True)
|
|
12
|
+
if out.returncode != 0:
|
|
13
|
+
raise RuntimeError(out.stderr.strip() or f"git {' '.join(args)} failed")
|
|
14
|
+
return out.stdout
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def repo_root(cwd="."):
|
|
18
|
+
return Path(git("rev-parse", "--show-toplevel", cwd=cwd).strip())
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def is_test_file(path):
|
|
22
|
+
p = Path(path)
|
|
23
|
+
return (
|
|
24
|
+
p.name.startswith("test_")
|
|
25
|
+
or p.name.endswith("_test.py")
|
|
26
|
+
or p.name == "conftest.py"
|
|
27
|
+
or any(part in ("tests", "test") for part in p.parts[:-1])
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def untracked(root):
|
|
32
|
+
out = git("ls-files", "--others", "--exclude-standard", "--", "*.py", cwd=root)
|
|
33
|
+
return [line for line in out.splitlines() if line]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def pick_base(root):
|
|
37
|
+
"""Uncommitted changes if there are any, otherwise the last commit."""
|
|
38
|
+
dirty = git("diff", "--name-only", "HEAD", "--", "*.py", cwd=root).strip()
|
|
39
|
+
if dirty or untracked(root):
|
|
40
|
+
return "HEAD"
|
|
41
|
+
return "HEAD~1"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def changed_lines(root, base):
|
|
45
|
+
"""Return {path: set of changed line numbers} for non-test .py files."""
|
|
46
|
+
out = git("diff", "-U0", "--no-color", "--no-ext-diff", base, "--", "*.py", cwd=root)
|
|
47
|
+
changes, path = {}, None
|
|
48
|
+
for line in out.splitlines():
|
|
49
|
+
if line.startswith("+++ "):
|
|
50
|
+
path = None if line == "+++ /dev/null" else line[6:]
|
|
51
|
+
elif path and (m := HUNK.match(line)):
|
|
52
|
+
start, count = int(m.group(1)), int(m.group(2) or 1)
|
|
53
|
+
changes.setdefault(path, set()).update(range(start, start + count))
|
|
54
|
+
if base == "HEAD":
|
|
55
|
+
for path in untracked(root):
|
|
56
|
+
n = len((root / path).read_text(encoding="utf-8", errors="replace").splitlines())
|
|
57
|
+
changes[path] = set(range(1, n + 1))
|
|
58
|
+
return {p: ls for p, ls in changes.items() if ls and not is_test_file(p)}
|
wouldcatch/mutate.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Plant small bugs in Python source, only on the lines you choose.
|
|
2
|
+
|
|
3
|
+
Each mutant is a text edit (start, end, replacement) so the rest of the
|
|
4
|
+
file, comments and formatting included, stays exactly as written.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import ast
|
|
8
|
+
import re
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
|
|
11
|
+
IGNORE = "wouldcatch: ignore"
|
|
12
|
+
|
|
13
|
+
COMPARE = {
|
|
14
|
+
ast.Eq: ("==", "!="), ast.NotEq: ("!=", "=="),
|
|
15
|
+
ast.Lt: ("<", "<="), ast.LtE: ("<=", "<"),
|
|
16
|
+
ast.Gt: (">", ">="), ast.GtE: (">=", ">"),
|
|
17
|
+
ast.Is: (r"\bis\b", "is not"), ast.IsNot: (r"\bis\s+not\b", "is"),
|
|
18
|
+
ast.In: (r"\bin\b", "not in"), ast.NotIn: (r"\bnot\s+in\b", "in"),
|
|
19
|
+
}
|
|
20
|
+
BINARY = {ast.Add: ("+", "-"), ast.Sub: ("-", "+"), ast.Mult: ("*", "/"), ast.Div: ("/", "*")}
|
|
21
|
+
BOOL = {ast.And: (r"\band\b", "or"), ast.Or: (r"\bor\b", "and")}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class Mutant:
|
|
26
|
+
path: str
|
|
27
|
+
line: int
|
|
28
|
+
start: int
|
|
29
|
+
end: int
|
|
30
|
+
original: str
|
|
31
|
+
replacement: str
|
|
32
|
+
status: str = "pending"
|
|
33
|
+
|
|
34
|
+
def apply(self, text):
|
|
35
|
+
return text[: self.start] + self.replacement + text[self.end :]
|
|
36
|
+
|
|
37
|
+
def describe(self, width=40):
|
|
38
|
+
short = lambda s: s if len(s) <= width else s[: width - 3] + "..."
|
|
39
|
+
return f"`{short(' '.join(self.original.split()))}` -> `{short(' '.join(self.replacement.split()))}`"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class _Offsets:
|
|
43
|
+
"""ast columns are UTF-8 byte offsets; convert them to str indexes."""
|
|
44
|
+
|
|
45
|
+
def __init__(self, text):
|
|
46
|
+
# Only \r\n, \r and \n end a line for ast; str.splitlines also splits on \f etc.
|
|
47
|
+
self.lines = re.findall(r"[^\r\n]*(?:\r\n|\r|\n|$)", text)
|
|
48
|
+
self.starts = [0]
|
|
49
|
+
for line in self.lines:
|
|
50
|
+
self.starts.append(self.starts[-1] + len(line))
|
|
51
|
+
|
|
52
|
+
def __call__(self, lineno, col):
|
|
53
|
+
line = self.lines[lineno - 1].encode("utf-8")
|
|
54
|
+
return self.starts[lineno - 1] + len(line[:col].decode("utf-8", errors="replace"))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _span(off, node):
|
|
58
|
+
return off(node.lineno, node.col_offset), off(node.end_lineno, node.end_col_offset)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def find_mutants(path, text, lines):
|
|
62
|
+
"""All mutants whose node starts on one of `lines` (a set of 1-based ints)."""
|
|
63
|
+
try:
|
|
64
|
+
tree = ast.parse(text)
|
|
65
|
+
except SyntaxError:
|
|
66
|
+
return []
|
|
67
|
+
off = _Offsets(text)
|
|
68
|
+
src = [line.rstrip("\r\n") for line in off.lines]
|
|
69
|
+
out = []
|
|
70
|
+
|
|
71
|
+
def add(line, start, end, replacement):
|
|
72
|
+
if line in lines and IGNORE not in src[line - 1]:
|
|
73
|
+
out.append(Mutant(path, line, start, end, text[start:end], replacement))
|
|
74
|
+
|
|
75
|
+
def swap_in_gap(line, left, right, pattern, replacement):
|
|
76
|
+
"""Replace the operator that sits between two nodes."""
|
|
77
|
+
a, b = _span(off, left)[1], _span(off, right)[0]
|
|
78
|
+
gap = text[a:b]
|
|
79
|
+
m = re.search(pattern if pattern.startswith("\\") else re.escape(pattern), gap)
|
|
80
|
+
if m and gap[m.end() : m.end() + 1] != "=": # "<" must not match inside "<="
|
|
81
|
+
add(line, a + m.start(), a + m.end(), replacement)
|
|
82
|
+
|
|
83
|
+
for node in ast.walk(tree):
|
|
84
|
+
line = getattr(node, "lineno", None)
|
|
85
|
+
if line is None:
|
|
86
|
+
continue
|
|
87
|
+
if isinstance(node, ast.Compare):
|
|
88
|
+
left = node.left
|
|
89
|
+
for op, right in zip(node.ops, node.comparators):
|
|
90
|
+
if type(op) in COMPARE:
|
|
91
|
+
swap_in_gap(line, left, right, *COMPARE[type(op)])
|
|
92
|
+
left = right
|
|
93
|
+
elif isinstance(node, ast.BinOp) and type(node.op) in BINARY:
|
|
94
|
+
swap_in_gap(line, node.left, node.right, *BINARY[type(node.op)])
|
|
95
|
+
elif isinstance(node, ast.AugAssign) and type(node.op) in BINARY:
|
|
96
|
+
sym, rep = BINARY[type(node.op)]
|
|
97
|
+
swap_in_gap(line, node.target, node.value, sym + "=", rep + "=")
|
|
98
|
+
elif isinstance(node, ast.BoolOp):
|
|
99
|
+
for left, right in zip(node.values, node.values[1:]):
|
|
100
|
+
swap_in_gap(line, left, right, *BOOL[type(node.op)])
|
|
101
|
+
elif isinstance(node, ast.Constant):
|
|
102
|
+
if node.value is True or node.value is False:
|
|
103
|
+
add(line, *_span(off, node), str(not node.value))
|
|
104
|
+
elif type(node.value) is int:
|
|
105
|
+
add(line, *_span(off, node), str(node.value + 1))
|
|
106
|
+
elif isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not):
|
|
107
|
+
a, b = _span(off, node.operand)
|
|
108
|
+
add(line, *_span(off, node), text[a:b])
|
|
109
|
+
elif isinstance(node, ast.Return) and node.value is not None:
|
|
110
|
+
if not (isinstance(node.value, ast.Constant) and node.value.value is None):
|
|
111
|
+
add(line, *_span(off, node.value), "None")
|
|
112
|
+
elif isinstance(node, (ast.If, ast.While)):
|
|
113
|
+
a, b = _span(off, node.test)
|
|
114
|
+
add(line, a, b, f"not ({text[a:b]})")
|
|
115
|
+
|
|
116
|
+
seen, result = set(), []
|
|
117
|
+
for m in sorted(out, key=lambda m: (m.line, m.start)):
|
|
118
|
+
key = (m.start, m.end, m.replacement)
|
|
119
|
+
if key in seen:
|
|
120
|
+
continue
|
|
121
|
+
seen.add(key)
|
|
122
|
+
try:
|
|
123
|
+
compile(m.apply(text), path, "exec")
|
|
124
|
+
except (SyntaxError, ValueError):
|
|
125
|
+
continue
|
|
126
|
+
result.append(m)
|
|
127
|
+
return result
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: wouldcatch
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Would your tests catch a bug in the lines you just changed? Mutation testing for your git diff.
|
|
5
|
+
Author: Sahil Lenka
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Sahil-u07/wouldcatch
|
|
8
|
+
Project-URL: Issues, https://github.com/Sahil-u07/wouldcatch/issues
|
|
9
|
+
Keywords: mutation-testing,testing,pytest,git,code-review,ai-agents
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Topic :: Software Development :: Testing
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
<h1 align="center">wouldcatch</h1>
|
|
22
|
+
|
|
23
|
+
<p align="center">
|
|
24
|
+
<b>Your tests pass. Would they catch a bug?</b><br>
|
|
25
|
+
wouldcatch plants small bugs in the lines you just changed, runs your tests, and shows the ones nobody noticed.
|
|
26
|
+
</p>
|
|
27
|
+
|
|
28
|
+
<p align="center">
|
|
29
|
+
<a href="https://github.com/Sahil-u07/wouldcatch/actions/workflows/ci.yml"><img src="https://github.com/Sahil-u07/wouldcatch/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
30
|
+
<img src="https://img.shields.io/badge/python-3.10%2B-blue" alt="Python 3.10+">
|
|
31
|
+
<img src="https://img.shields.io/badge/dependencies-0-brightgreen" alt="Zero dependencies">
|
|
32
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-green" alt="MIT license"></a>
|
|
33
|
+
</p>
|
|
34
|
+
|
|
35
|
+
<p align="center">
|
|
36
|
+
<img src="docs/demo.gif" alt="wouldcatch finds 3 bugs the tests miss, then 0 after one more test" width="800">
|
|
37
|
+
</p>
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pipx install git+https://github.com/Sahil-u07/wouldcatch
|
|
41
|
+
cd your-repo && wouldcatch
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
On Windows without pipx: `py -m pip install git+https://github.com/Sahil-u07/wouldcatch`, then run `wouldcatch` from PowerShell or Windows Terminal.
|
|
45
|
+
|
|
46
|
+
No config. No dependencies. Works on any git repo with pytest (or any test command).
|
|
47
|
+
|
|
48
|
+
## Green tests can still miss bugs
|
|
49
|
+
|
|
50
|
+
You add a member discount and write two tests. Every line runs. Everything passes.
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
def discount(price, is_member):
|
|
54
|
+
if is_member and price >= 100:
|
|
55
|
+
return price - 10
|
|
56
|
+
return price
|
|
57
|
+
|
|
58
|
+
def test_member():
|
|
59
|
+
assert discount(200, True) == 190
|
|
60
|
+
|
|
61
|
+
def test_guest():
|
|
62
|
+
assert discount(50, False) == 50
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
```
|
|
66
|
+
$ wouldcatch
|
|
67
|
+
MISSED shop.py:2 `and` -> `or`
|
|
68
|
+
MISSED shop.py:2 `>=` -> `>`
|
|
69
|
+
MISSED shop.py:2 `100` -> `101`
|
|
70
|
+
|
|
71
|
+
4/7 caught. 3 bug(s) your tests would not notice.
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Each MISSED line is a bug someone could ship and your suite would stay green:
|
|
75
|
+
|
|
76
|
+
- `>=` becoming `>` means a member spending exactly 100 gets no discount. No test checks 100.
|
|
77
|
+
- `and` becoming `or` gives every guest over 100 the discount. No test has a guest over 100.
|
|
78
|
+
|
|
79
|
+
Add `assert discount(100, True) == 90` and `assert discount(200, False) == 200`, run it again, and you get `7/7 caught`.
|
|
80
|
+
|
|
81
|
+
Coverage can't tell you this. It counts which lines ran, not whether a test would fail if they were wrong.
|
|
82
|
+
|
|
83
|
+
## Why only the diff
|
|
84
|
+
|
|
85
|
+
Mutation testing is the honest way to grade a test suite, but running it on a whole project can take hours, so almost nobody does. wouldcatch only mutates the lines in your `git diff`. A normal change gives 5 to 30 mutants, so it finishes while you're still writing the commit message.
|
|
86
|
+
|
|
87
|
+
| | Line coverage | Whole-project mutation testing | **wouldcatch** |
|
|
88
|
+
|---|---|---|---|
|
|
89
|
+
| Tells you a test would fail on a bug | no | yes | **yes** |
|
|
90
|
+
| Time on a typical change | seconds | minutes to hours | **seconds to a few minutes** |
|
|
91
|
+
| Setup | plugin | config files | **none** |
|
|
92
|
+
| Output | a percentage | a report on everything | **the exact lines to test** |
|
|
93
|
+
|
|
94
|
+
## How it works
|
|
95
|
+
|
|
96
|
+
1. Reads `git diff` to find the Python lines you changed. Test files are skipped.
|
|
97
|
+
2. Makes one small change at a time on those lines: `<` to `<=`, `==` to `!=`, `and` to `or`, `+` to `-`, `True` to `False`, `100` to `101`, a return value to `None`, an `if` flipped.
|
|
98
|
+
3. Runs your tests after each change. Tests still green means that bug would have gotten through.
|
|
99
|
+
4. Puts the file back byte for byte (CRLF included) after every run, even on Ctrl+C or a killed process.
|
|
100
|
+
|
|
101
|
+
## Usage
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
wouldcatch # uncommitted changes, or the last commit if the tree is clean
|
|
105
|
+
wouldcatch --base main # everything on this branch since main
|
|
106
|
+
wouldcatch --cmd "pytest tests/unit -x -q" # run only the tests that matter
|
|
107
|
+
wouldcatch --json # machine-readable output for scripts and agents
|
|
108
|
+
wouldcatch --all # list caught mutants too
|
|
109
|
+
wouldcatch --max 100 # try more mutants (default 50)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Exit codes: `0` all caught, `1` something missed, `2` setup problem (not a git repo, tests already failing).
|
|
113
|
+
|
|
114
|
+
Skip a line on purpose:
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
RETRY_DELAY = 5 # wouldcatch: ignore
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
## Use it everywhere
|
|
121
|
+
|
|
122
|
+
**GitHub Actions**, on every pull request:
|
|
123
|
+
|
|
124
|
+
```yaml
|
|
125
|
+
- uses: actions/checkout@v5
|
|
126
|
+
with:
|
|
127
|
+
fetch-depth: 0
|
|
128
|
+
- run: pip install git+https://github.com/Sahil-u07/wouldcatch pytest
|
|
129
|
+
- run: wouldcatch --base origin/${{ github.base_ref }}
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
**pre-commit**, before every commit (with wouldcatch installed in your project environment):
|
|
133
|
+
|
|
134
|
+
```yaml
|
|
135
|
+
- repo: https://github.com/Sahil-u07/wouldcatch
|
|
136
|
+
rev: main
|
|
137
|
+
hooks:
|
|
138
|
+
- id: wouldcatch
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
**AI coding agents.** Agents are good at writing tests that pass and bad at writing tests that check anything. wouldcatch gives them a concrete target: make every MISSED line fail.
|
|
142
|
+
|
|
143
|
+
- Claude Code: copy [`skills/wouldcatch`](skills/wouldcatch) into `.claude/skills/` in your project.
|
|
144
|
+
- Any other agent: add this to `AGENTS.md`:
|
|
145
|
+
|
|
146
|
+
```
|
|
147
|
+
After changing Python code, run `wouldcatch --json`. For every mutant with
|
|
148
|
+
"status": "missed", add a test that fails for that change. Re-run until none are missed.
|
|
149
|
+
Never change the source code to make a mutant go away.
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
## FAQ
|
|
153
|
+
|
|
154
|
+
**Will it mess up my files?** It edits one file at a time and restores it right after. If the process dies mid-run, the next run restores the file from `.wouldcatch-backup.json` before doing anything else.
|
|
155
|
+
|
|
156
|
+
**My test suite is slow.** Each mutant is one run of your test command. Point `--cmd` at the relevant tests and use `-x` so pytest stops at the first failure.
|
|
157
|
+
|
|
158
|
+
**A mutant can't be caught, the code really doesn't care.** That happens (a log interval, a cache size). Mark the line `# wouldcatch: ignore`.
|
|
159
|
+
|
|
160
|
+
**JavaScript, TypeScript, Go?** Python today. The git part is language-neutral; JS/TS is next. Join [the discussion](https://github.com/Sahil-u07/wouldcatch/issues/4).
|
|
161
|
+
|
|
162
|
+
## Contributing
|
|
163
|
+
|
|
164
|
+
New mutations are small, self-contained PRs. Start with a [good first issue](https://github.com/Sahil-u07/wouldcatch/labels/good%20first%20issue) and see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
165
|
+
|
|
166
|
+
If wouldcatch found a bug your tests missed, a star helps other people find it.
|
|
167
|
+
|
|
168
|
+
## License
|
|
169
|
+
|
|
170
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
wouldcatch/__init__.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
|
|
2
|
+
wouldcatch/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
|
|
3
|
+
wouldcatch/cli.py,sha256=ZMt-YmYutZeQbEh_6bHepjnm9kATqUJpqv43bKfnAeg,5963
|
|
4
|
+
wouldcatch/diff.py,sha256=UeGaJEk6L8TF2_DyZF2PoaBO1U5hrZkr0POTyiPkrRU,1994
|
|
5
|
+
wouldcatch/mutate.py,sha256=2XR4wh2epYlJSOZwFFYPBawVOVDVeUZbmeyFidIvV6o,4950
|
|
6
|
+
wouldcatch-0.1.0.dist-info/licenses/LICENSE,sha256=x68zHYUKYVZLcEHCvBoPEp-fCDtz-Z55DcXIaPNIxhE,1068
|
|
7
|
+
wouldcatch-0.1.0.dist-info/METADATA,sha256=p3NwffBW89iWT5PmWbOWwR4BwSToasrF0mpByiLvANM,6882
|
|
8
|
+
wouldcatch-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
9
|
+
wouldcatch-0.1.0.dist-info/entry_points.txt,sha256=FCuDWDV1_FdyBqLmstR7RBdJqwpWSlmTgJQokVvUfZs,51
|
|
10
|
+
wouldcatch-0.1.0.dist-info/top_level.txt,sha256=GJPfR2T2fzNEmYnystHh-alS5D0aZeBnT7Hw7IMqbmU,11
|
|
11
|
+
wouldcatch-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Sahil Lenka
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
wouldcatch
|