mutgate 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mutgate-0.1.0/.gitignore +15 -0
- mutgate-0.1.0/CHANGELOG.md +38 -0
- mutgate-0.1.0/CITATION.cff +29 -0
- mutgate-0.1.0/LICENSE +21 -0
- mutgate-0.1.0/PKG-INFO +157 -0
- mutgate-0.1.0/README.md +128 -0
- mutgate-0.1.0/docs/RELEASING.md +36 -0
- mutgate-0.1.0/pyproject.toml +71 -0
- mutgate-0.1.0/src/mutgate/__init__.py +22 -0
- mutgate-0.1.0/src/mutgate/__main__.py +3 -0
- mutgate-0.1.0/src/mutgate/cli.py +82 -0
- mutgate-0.1.0/src/mutgate/core.py +453 -0
- mutgate-0.1.0/tests/mutations.py +67 -0
- mutgate-0.1.0/tests/test_mutgate.py +411 -0
mutgate-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## Unreleased
|
|
4
|
+
|
|
5
|
+
Planned: a `control=` field naming an invariance's firing sibling, with a warning on an
|
|
6
|
+
unpaired invariance, once a real declaration carries more than a handful of them.
|
|
7
|
+
|
|
8
|
+
## 0.1.0 — 2026-09-07
|
|
9
|
+
|
|
10
|
+
First release. `Mutation` with `fires` / `may_fire` / `invisible` / `count`; the sandbox
|
|
11
|
+
copy via `git ls-files`; verdicts OK, DECORATION, OVERREACH, VISIBLE, NOT_APPLIED, ERROR;
|
|
12
|
+
a red baseline aborts; `mutgate run` and `mutgate list`, with `--markdown` for build
|
|
13
|
+
records. Extracted from three hand-rolled mutation harnesses written in one week across
|
|
14
|
+
the Morphospace and abiogenesis repositories, after the third one found a duplicated
|
|
15
|
+
convention by firing on the wrong kind of assertion.
|
|
16
|
+
|
|
17
|
+
### Fixed before release, on a second session's review of the first commit
|
|
18
|
+
|
|
19
|
+
- **A symlinked TMPDIR crashed every mutation** ("escapes the sandbox"): the sandbox
|
|
20
|
+
directory is now resolved before the containment check. macOS keeps `/var` behind a
|
|
21
|
+
symlink, so this was every macOS user.
|
|
22
|
+
- **A suppressed pytest summary was a silent false pass.** `pytest` exit 1 with no
|
|
23
|
+
failure readable is now `ERROR`, never `OK`; the same at baseline.
|
|
24
|
+
- **`fires="Name"` (a bare string) was a silent false pass**: it became one-letter
|
|
25
|
+
fragments matching every node id. Strings and empty fragments are refused.
|
|
26
|
+
- **A typo in `--only`, or an emptied `MUTATIONS`, exited 0.** Both are usage errors (2).
|
|
27
|
+
- **A project whose config stops at the first failure (`-x`, `--maxfail=N`) truncated the
|
|
28
|
+
fired set with a consistent-looking exit**, so an OVERREACH read OK. mutgate now owns
|
|
29
|
+
`--maxfail=0` after every user argument, and pytest's own "stopping after N failures"
|
|
30
|
+
line (a conftest can still force it) makes the run `ERROR`. Second review round.
|
|
31
|
+
- **An interrupted run with a partial fired set read as a verdict** (a conftest setting
|
|
32
|
+
`session.shouldstop` after the first failure: exit 2, one failure parsed). Only exit codes
|
|
33
|
+
0 and 1 now count as a completed run; anything else is `ERROR` whatever parsed. Third
|
|
34
|
+
review round.
|
|
35
|
+
- Node ids containing " - " inside a parametrisation are no longer truncated; `--timeout`
|
|
36
|
+
makes a hanging suite an `ERROR`; a missing `--python` is a usage error; `ROOT` is
|
|
37
|
+
relative to the declaration file; the `git ls-files` sandbox and real PYTHONPATH
|
|
38
|
+
shadowing are now tested; the `Framework :: Pytest` classifier (for plugins) is dropped.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
title: "mutgate: named mutations as contracts on a test suite"
|
|
3
|
+
message: "If you use this software, please cite it using these metadata."
|
|
4
|
+
type: software
|
|
5
|
+
authors:
|
|
6
|
+
- given-names: James P.
|
|
7
|
+
family-names: Galasyn
|
|
8
|
+
- given-names: Claude
|
|
9
|
+
family-names: Théodore
|
|
10
|
+
repository-code: "https://github.com/JimGalasyn/mutgate"
|
|
11
|
+
url: "https://github.com/JimGalasyn/mutgate"
|
|
12
|
+
abstract: >-
|
|
13
|
+
A dependency-free tool that turns "break the code on purpose and see whether the
|
|
14
|
+
test notices" into a checked artefact. Each mutation is a named, exact-string change
|
|
15
|
+
to one file of the project under test, applied in a sandbox copy, with a contract
|
|
16
|
+
naming the tests it must turn red or the statement that it must turn nothing red.
|
|
17
|
+
Verdicts distinguish a guard that does not fire (a decoration) from a mutation that
|
|
18
|
+
fires beyond its contract (the tell for a convention duplicated in two places) and
|
|
19
|
+
from an invariance that breaks; a mutation whose text is not found exactly once is
|
|
20
|
+
reported rather than silently skipped.
|
|
21
|
+
keywords:
|
|
22
|
+
- mutation-testing
|
|
23
|
+
- pytest
|
|
24
|
+
- test-quality
|
|
25
|
+
- scientific-software
|
|
26
|
+
- reproducibility
|
|
27
|
+
license: MIT
|
|
28
|
+
version: 0.1.0
|
|
29
|
+
date-released: "2026-09-07"
|
mutgate-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 James P. Galasyn
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
mutgate-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: mutgate
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Named mutations as contracts on a test suite: each mutation must turn exactly the tests it names red, or nothing at all.
|
|
5
|
+
Project-URL: Homepage, https://github.com/JimGalasyn/mutgate
|
|
6
|
+
Project-URL: Issues, https://github.com/JimGalasyn/mutgate/issues
|
|
7
|
+
Project-URL: Changelog, https://github.com/JimGalasyn/mutgate/blob/main/CHANGELOG.md
|
|
8
|
+
Author: James P. Galasyn, Claude Théodore
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: contracts,mutation-testing,pytest,reproducibility,scientific-software,test-quality,testing
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Topic :: Software Development :: Testing
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Provides-Extra: test
|
|
26
|
+
Requires-Dist: pytest-cov>=4; extra == 'test'
|
|
27
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# mutgate
|
|
31
|
+
|
|
32
|
+
**Named mutations as contracts on a test suite.** Each mutation is a deliberate, named
|
|
33
|
+
change to the code under test, together with the list of tests it must turn red — or the
|
|
34
|
+
statement that it must turn nothing red. `mutgate` applies each one in a sandbox copy, runs
|
|
35
|
+
the suite, and judges the contract.
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
# tests/mutations.py
|
|
39
|
+
from mutgate import Mutation
|
|
40
|
+
|
|
41
|
+
MUTATIONS = [
|
|
42
|
+
Mutation("keep-larger", file="pkg/engine.py",
|
|
43
|
+
old="if (birth[a], -a) >= (birth[b], -b):",
|
|
44
|
+
new="if size[a] >= size[b]:",
|
|
45
|
+
fires=("TestElderRule",)),
|
|
46
|
+
Mutation("skip-absolute-floor", file="pkg/engine.py",
|
|
47
|
+
old="if u.birth < abs_floor:", new="if False:",
|
|
48
|
+
fires=("TestAbsoluteFloor",)),
|
|
49
|
+
Mutation("tie-to-higher-index", file="pkg/engine.py",
|
|
50
|
+
old="(birth[a], -a) >= (birth[b], -b)",
|
|
51
|
+
new="(birth[a], a) >= (birth[b], b)",
|
|
52
|
+
invisible=True), # an invariance: the measurement must not depend on it
|
|
53
|
+
]
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
$ mutgate run tests/mutations.py
|
|
58
|
+
mutation verdict fired expected
|
|
59
|
+
keep-larger OK 3 TestElderRule
|
|
60
|
+
skip-absolute-floor DECORATION 0 TestAbsoluteFloor
|
|
61
|
+
did not fire: TestAbsoluteFloor
|
|
62
|
+
tie-to-higher-index OK 0 nothing
|
|
63
|
+
|
|
64
|
+
2 OK, 1 DECORATION
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Why
|
|
68
|
+
|
|
69
|
+
A green test suite guards only the defects that shaped it. The usual way to find out
|
|
70
|
+
whether a test is load-bearing is to break the code on purpose and see whether the test
|
|
71
|
+
notices — and in practice that is done by hand, in a scratch copy, with `sed`, once, and then
|
|
72
|
+
the result is written into a design note as prose. `mutgate` makes that a checked artefact:
|
|
73
|
+
|
|
74
|
+
- **the mutation is named and exact.** `old` must occur exactly `count` times in the file,
|
|
75
|
+
or the verdict is `NOT_APPLIED`. A mutation that silently does not land looks identical
|
|
76
|
+
to a working guard; this is the trap that motivated the tool.
|
|
77
|
+
- **the contract says which tests must fire.** A named test that does not fire is a
|
|
78
|
+
`DECORATION` — a guard that would not go red.
|
|
79
|
+
- **a test outside the contract firing is a finding, not noise.** `OVERREACH` means the
|
|
80
|
+
mutation reaches further than its author believed. When a convention (a tie-breaking rule,
|
|
81
|
+
a sign, an ordering) is changed and tests that should only *relabel* things fail on
|
|
82
|
+
*values*, the convention exists in two places that have drifted apart.
|
|
83
|
+
- **an invariance is a mutation that must fire nothing.** `invisible=True` pins that the
|
|
84
|
+
suite passes unchanged under the alternative convention; if it does not, the verdict is
|
|
85
|
+
`VISIBLE`. ⚠ An `OK` on an invariance is a *negative* result, and it is only as strong as
|
|
86
|
+
the evidence that the mutated line ran: a dead function, a site the named `TESTS` never
|
|
87
|
+
reach, or a suite that imports an installed copy instead of the sandbox all read `OK`
|
|
88
|
+
honestly. **Pair every invariance with a firing mutation at the same site in the same
|
|
89
|
+
declaration** — the known-positive control — so the file itself proves the site is
|
|
90
|
+
exercised by these tests. An unpaired `OK` on an invariance is unevidenced.
|
|
91
|
+
- **the working tree is never touched.** Each mutation is applied and restored in a
|
|
92
|
+
temporary copy (`git ls-files`, so uncommitted work is included and the venv is not), and
|
|
93
|
+
the copy is put first on `PYTHONPATH` so an editable install elsewhere cannot shadow it.
|
|
94
|
+
- **a red baseline aborts.** Mutations mean nothing on a suite that already fails.
|
|
95
|
+
- **a run whose failures cannot be read is an error, never a verdict.** If the project's
|
|
96
|
+
pytest configuration suppresses the short summary (`--no-summary`), pytest's exit code
|
|
97
|
+
says "failed" while nothing parses; that is reported as `ERROR`, not silently as `OK`.
|
|
98
|
+
And a run cut short is never a verdict either: `mutgate` owns `--maxfail=0` after every
|
|
99
|
+
user argument (a project's `-x` would truncate the fired set to one test), and a
|
|
100
|
+
`conftest` that forces `maxfail` anyway trips pytest's own "stopping after N failures"
|
|
101
|
+
line, which reads as `ERROR`.
|
|
102
|
+
|
|
103
|
+
The sandbox holds the project's files but not its `.git`; a suite that shells out to git
|
|
104
|
+
(a `setuptools_scm`-style version check, say) goes red at baseline and says so.
|
|
105
|
+
|
|
106
|
+
## What it is not
|
|
107
|
+
|
|
108
|
+
It is not automatic mutation testing. Tools like `mutmut` and `cosmic-ray` generate operator
|
|
109
|
+
mutations at random and report a kill rate; that answers "how thorough is this suite?".
|
|
110
|
+
`mutgate` answers a narrower question that random mutation cannot: *does this specific guard
|
|
111
|
+
fire under the specific defect it was written for, and only that?* The two are complementary.
|
|
112
|
+
|
|
113
|
+
## Declaration file
|
|
114
|
+
|
|
115
|
+
A Python file (conventionally `tests/mutations.py`) defining:
|
|
116
|
+
|
|
117
|
+
| name | required | meaning |
|
|
118
|
+
|---|---|---|
|
|
119
|
+
| `MUTATIONS` | yes | a sequence of `Mutation` |
|
|
120
|
+
| `TESTS` | no | pytest targets; default: the declaration file's own directory |
|
|
121
|
+
| `PATHS` | no | `PYTHONPATH` entries relative to the root; default `("src", ".")` |
|
|
122
|
+
| `ROOT` | no | project root, relative to the declaration file; default: the nearest ancestor with `pyproject.toml` or `.git` |
|
|
123
|
+
| `PYTHON` | no | interpreter to run pytest with; default: the one running `mutgate` |
|
|
124
|
+
|
|
125
|
+
`Mutation(name, file, old, new, fires=(), may_fire=(), invisible=False, count=1, note="")`.
|
|
126
|
+
Entries in `fires` and `may_fire` are substrings of pytest node ids, so `"TestElderRule"`,
|
|
127
|
+
`"test_engine.py::TestElderRule"` and `"test_tie_goes_to_lower_index"` all work.
|
|
128
|
+
|
|
129
|
+
## CLI
|
|
130
|
+
|
|
131
|
+
```
|
|
132
|
+
mutgate run tests/mutations.py [--tests T ...] [--only NAME ...] [--python PY]
|
|
133
|
+
[--pytest-arg ARG] [--markdown] [--keep] [-x] [-v]
|
|
134
|
+
mutgate list tests/mutations.py
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
`--markdown` prints a table meant to be pasted into a design note's build record. Exit code
|
|
138
|
+
0 when every contract holds, 1 when a verdict is not `OK`, 2 when the baseline is red or the
|
|
139
|
+
file cannot be loaded.
|
|
140
|
+
|
|
141
|
+
## In CI
|
|
142
|
+
|
|
143
|
+
```yaml
|
|
144
|
+
- run: pip install mutgate
|
|
145
|
+
- run: mutgate run tests/mutations.py
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Each mutation runs the named tests once, so the cost is the suite's cost times the number of
|
|
149
|
+
mutations; point `TESTS` at the module the mutations concern.
|
|
150
|
+
|
|
151
|
+
## Install
|
|
152
|
+
|
|
153
|
+
```
|
|
154
|
+
pip install mutgate
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
No runtime dependencies. Python 3.10+. MIT.
|
mutgate-0.1.0/README.md
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# mutgate
|
|
2
|
+
|
|
3
|
+
**Named mutations as contracts on a test suite.** Each mutation is a deliberate, named
|
|
4
|
+
change to the code under test, together with the list of tests it must turn red — or the
|
|
5
|
+
statement that it must turn nothing red. `mutgate` applies each one in a sandbox copy, runs
|
|
6
|
+
the suite, and judges the contract.
|
|
7
|
+
|
|
8
|
+
```python
|
|
9
|
+
# tests/mutations.py
|
|
10
|
+
from mutgate import Mutation
|
|
11
|
+
|
|
12
|
+
MUTATIONS = [
|
|
13
|
+
Mutation("keep-larger", file="pkg/engine.py",
|
|
14
|
+
old="if (birth[a], -a) >= (birth[b], -b):",
|
|
15
|
+
new="if size[a] >= size[b]:",
|
|
16
|
+
fires=("TestElderRule",)),
|
|
17
|
+
Mutation("skip-absolute-floor", file="pkg/engine.py",
|
|
18
|
+
old="if u.birth < abs_floor:", new="if False:",
|
|
19
|
+
fires=("TestAbsoluteFloor",)),
|
|
20
|
+
Mutation("tie-to-higher-index", file="pkg/engine.py",
|
|
21
|
+
old="(birth[a], -a) >= (birth[b], -b)",
|
|
22
|
+
new="(birth[a], a) >= (birth[b], b)",
|
|
23
|
+
invisible=True), # an invariance: the measurement must not depend on it
|
|
24
|
+
]
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
```
|
|
28
|
+
$ mutgate run tests/mutations.py
|
|
29
|
+
mutation verdict fired expected
|
|
30
|
+
keep-larger OK 3 TestElderRule
|
|
31
|
+
skip-absolute-floor DECORATION 0 TestAbsoluteFloor
|
|
32
|
+
did not fire: TestAbsoluteFloor
|
|
33
|
+
tie-to-higher-index OK 0 nothing
|
|
34
|
+
|
|
35
|
+
2 OK, 1 DECORATION
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Why
|
|
39
|
+
|
|
40
|
+
A green test suite guards only the defects that shaped it. The usual way to find out
|
|
41
|
+
whether a test is load-bearing is to break the code on purpose and see whether the test
|
|
42
|
+
notices — and in practice that is done by hand, in a scratch copy, with `sed`, once, and then
|
|
43
|
+
the result is written into a design note as prose. `mutgate` makes that a checked artefact:
|
|
44
|
+
|
|
45
|
+
- **the mutation is named and exact.** `old` must occur exactly `count` times in the file,
|
|
46
|
+
or the verdict is `NOT_APPLIED`. A mutation that silently does not land looks identical
|
|
47
|
+
to a working guard; this is the trap that motivated the tool.
|
|
48
|
+
- **the contract says which tests must fire.** A named test that does not fire is a
|
|
49
|
+
`DECORATION` — a guard that would not go red.
|
|
50
|
+
- **a test outside the contract firing is a finding, not noise.** `OVERREACH` means the
|
|
51
|
+
mutation reaches further than its author believed. When a convention (a tie-breaking rule,
|
|
52
|
+
a sign, an ordering) is changed and tests that should only *relabel* things fail on
|
|
53
|
+
*values*, the convention exists in two places that have drifted apart.
|
|
54
|
+
- **an invariance is a mutation that must fire nothing.** `invisible=True` pins that the
|
|
55
|
+
suite passes unchanged under the alternative convention; if it does not, the verdict is
|
|
56
|
+
`VISIBLE`. ⚠ An `OK` on an invariance is a *negative* result, and it is only as strong as
|
|
57
|
+
the evidence that the mutated line ran: a dead function, a site the named `TESTS` never
|
|
58
|
+
reach, or a suite that imports an installed copy instead of the sandbox all read `OK`
|
|
59
|
+
honestly. **Pair every invariance with a firing mutation at the same site in the same
|
|
60
|
+
declaration** — the known-positive control — so the file itself proves the site is
|
|
61
|
+
exercised by these tests. An unpaired `OK` on an invariance is unevidenced.
|
|
62
|
+
- **the working tree is never touched.** Each mutation is applied and restored in a
|
|
63
|
+
temporary copy (`git ls-files`, so uncommitted work is included and the venv is not), and
|
|
64
|
+
the copy is put first on `PYTHONPATH` so an editable install elsewhere cannot shadow it.
|
|
65
|
+
- **a red baseline aborts.** Mutations mean nothing on a suite that already fails.
|
|
66
|
+
- **a run whose failures cannot be read is an error, never a verdict.** If the project's
|
|
67
|
+
pytest configuration suppresses the short summary (`--no-summary`), pytest's exit code
|
|
68
|
+
says "failed" while nothing parses; that is reported as `ERROR`, not silently as `OK`.
|
|
69
|
+
And a run cut short is never a verdict either: `mutgate` owns `--maxfail=0` after every
|
|
70
|
+
user argument (a project's `-x` would truncate the fired set to one test), and a
|
|
71
|
+
`conftest` that forces `maxfail` anyway trips pytest's own "stopping after N failures"
|
|
72
|
+
line, which reads as `ERROR`.
|
|
73
|
+
|
|
74
|
+
The sandbox holds the project's files but not its `.git`; a suite that shells out to git
|
|
75
|
+
(a `setuptools_scm`-style version check, say) goes red at baseline and says so.
|
|
76
|
+
|
|
77
|
+
## What it is not
|
|
78
|
+
|
|
79
|
+
It is not automatic mutation testing. Tools like `mutmut` and `cosmic-ray` generate operator
|
|
80
|
+
mutations at random and report a kill rate; that answers "how thorough is this suite?".
|
|
81
|
+
`mutgate` answers a narrower question that random mutation cannot: *does this specific guard
|
|
82
|
+
fire under the specific defect it was written for, and only that?* The two are complementary.
|
|
83
|
+
|
|
84
|
+
## Declaration file
|
|
85
|
+
|
|
86
|
+
A Python file (conventionally `tests/mutations.py`) defining:
|
|
87
|
+
|
|
88
|
+
| name | required | meaning |
|
|
89
|
+
|---|---|---|
|
|
90
|
+
| `MUTATIONS` | yes | a sequence of `Mutation` |
|
|
91
|
+
| `TESTS` | no | pytest targets; default: the declaration file's own directory |
|
|
92
|
+
| `PATHS` | no | `PYTHONPATH` entries relative to the root; default `("src", ".")` |
|
|
93
|
+
| `ROOT` | no | project root, relative to the declaration file; default: the nearest ancestor with `pyproject.toml` or `.git` |
|
|
94
|
+
| `PYTHON` | no | interpreter to run pytest with; default: the one running `mutgate` |
|
|
95
|
+
|
|
96
|
+
`Mutation(name, file, old, new, fires=(), may_fire=(), invisible=False, count=1, note="")`.
|
|
97
|
+
Entries in `fires` and `may_fire` are substrings of pytest node ids, so `"TestElderRule"`,
|
|
98
|
+
`"test_engine.py::TestElderRule"` and `"test_tie_goes_to_lower_index"` all work.
|
|
99
|
+
|
|
100
|
+
## CLI
|
|
101
|
+
|
|
102
|
+
```
|
|
103
|
+
mutgate run tests/mutations.py [--tests T ...] [--only NAME ...] [--python PY]
|
|
104
|
+
[--pytest-arg ARG] [--markdown] [--keep] [-x] [-v]
|
|
105
|
+
mutgate list tests/mutations.py
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
`--markdown` prints a table meant to be pasted into a design note's build record. Exit code
|
|
109
|
+
0 when every contract holds, 1 when a verdict is not `OK`, 2 when the baseline is red or the
|
|
110
|
+
file cannot be loaded.
|
|
111
|
+
|
|
112
|
+
## In CI
|
|
113
|
+
|
|
114
|
+
```yaml
|
|
115
|
+
- run: pip install mutgate
|
|
116
|
+
- run: mutgate run tests/mutations.py
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Each mutation runs the named tests once, so the cost is the suite's cost times the number of
|
|
120
|
+
mutations; point `TESTS` at the module the mutations concern.
|
|
121
|
+
|
|
122
|
+
## Install
|
|
123
|
+
|
|
124
|
+
```
|
|
125
|
+
pip install mutgate
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
No runtime dependencies. Python 3.10+. MIT.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Releasing
|
|
2
|
+
|
|
3
|
+
The version is static in three files and they must agree:
|
|
4
|
+
|
|
5
|
+
- `pyproject.toml` → `[project] version`
|
|
6
|
+
- `src/mutgate/__init__.py` → `__version__`
|
|
7
|
+
- `CITATION.cff` → `version` (and `date-released`)
|
|
8
|
+
|
|
9
|
+
## Cutting a release
|
|
10
|
+
|
|
11
|
+
1. Bump those three, and move the `CHANGELOG.md` `Unreleased` section under the new
|
|
12
|
+
version heading.
|
|
13
|
+
2. Commit, then tag and push:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
git tag vX.Y.Z && git push origin vX.Y.Z
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
3. Publish a GitHub Release for the tag. That triggers `.github/workflows/publish-pypi.yml`,
|
|
20
|
+
which refuses to build unless the tag matches the `pyproject.toml` version, runs
|
|
21
|
+
`twine check`, and uploads to PyPI by trusted publishing.
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
gh release create vX.Y.Z --title "vX.Y.Z" --notes-file <(sed -n '/^## X.Y.Z/,/^## /p' CHANGELOG.md)
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Before the first publish (one-time, web console)
|
|
28
|
+
|
|
29
|
+
- pypi.org → Publishing → add a **pending** trusted publisher: project `mutgate`, owner
|
|
30
|
+
`JimGalasyn`, repository `mutgate`, workflow `publish-pypi.yml`, environment `pypi`.
|
|
31
|
+
- GitHub → Settings → Environments → create `pypi`.
|
|
32
|
+
- Zenodo → GitHub integration → enable `JimGalasyn/mutgate` before the first release, so
|
|
33
|
+
the first release mints the concept DOI; then record it in `CITATION.cff`.
|
|
34
|
+
|
|
35
|
+
The `publish-release` skill in the Claude Code setup walks these steps for the sibling
|
|
36
|
+
repositories (rafkit, run-farm, jax-morpho, jax-solitons, proc-warden); this one is the same.
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "mutgate"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Named mutations as contracts on a test suite: each mutation must turn exactly the tests it names red, or nothing at all."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "James P. Galasyn" },
|
|
14
|
+
{ name = "Claude Théodore" },
|
|
15
|
+
]
|
|
16
|
+
keywords = [
|
|
17
|
+
"mutation-testing", "pytest", "testing", "test-quality", "contracts",
|
|
18
|
+
"reproducibility", "scientific-software",
|
|
19
|
+
]
|
|
20
|
+
classifiers = [
|
|
21
|
+
"Development Status :: 3 - Alpha",
|
|
22
|
+
"Environment :: Console",
|
|
23
|
+
"Intended Audience :: Developers",
|
|
24
|
+
"Intended Audience :: Science/Research",
|
|
25
|
+
"License :: OSI Approved :: MIT License",
|
|
26
|
+
"Operating System :: OS Independent",
|
|
27
|
+
"Programming Language :: Python :: 3",
|
|
28
|
+
"Programming Language :: Python :: 3.10",
|
|
29
|
+
"Programming Language :: Python :: 3.11",
|
|
30
|
+
"Programming Language :: Python :: 3.12",
|
|
31
|
+
"Programming Language :: Python :: 3.13",
|
|
32
|
+
"Topic :: Software Development :: Testing",
|
|
33
|
+
]
|
|
34
|
+
# No runtime dependencies, deliberately: pytest is invoked as a subprocess of whichever
|
|
35
|
+
# interpreter owns the project under test, so mutgate never has to agree with it on versions.
|
|
36
|
+
dependencies = []
|
|
37
|
+
|
|
38
|
+
[project.optional-dependencies]
|
|
39
|
+
test = [
|
|
40
|
+
"pytest>=8",
|
|
41
|
+
"pytest-cov>=4",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[project.urls]
|
|
45
|
+
Homepage = "https://github.com/JimGalasyn/mutgate"
|
|
46
|
+
Issues = "https://github.com/JimGalasyn/mutgate/issues"
|
|
47
|
+
Changelog = "https://github.com/JimGalasyn/mutgate/blob/main/CHANGELOG.md"
|
|
48
|
+
|
|
49
|
+
[project.scripts]
|
|
50
|
+
mutgate = "mutgate.cli:run_cli"
|
|
51
|
+
|
|
52
|
+
[tool.hatch.build.targets.wheel]
|
|
53
|
+
packages = ["src/mutgate"]
|
|
54
|
+
|
|
55
|
+
[tool.hatch.build.targets.sdist]
|
|
56
|
+
include = [
|
|
57
|
+
"src/", "tests/", "docs/",
|
|
58
|
+
"README.md", "CHANGELOG.md", "LICENSE", "CITATION.cff",
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
[tool.pytest.ini_options]
|
|
62
|
+
testpaths = ["tests"]
|
|
63
|
+
|
|
64
|
+
[tool.coverage.run]
|
|
65
|
+
source = ["mutgate"]
|
|
66
|
+
|
|
67
|
+
[tool.coverage.report]
|
|
68
|
+
exclude_also = [
|
|
69
|
+
"if __name__ == .__main__.:",
|
|
70
|
+
"if TYPE_CHECKING:",
|
|
71
|
+
]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""mutgate — named mutations as contracts on a test suite.
|
|
2
|
+
|
|
3
|
+
from mutgate import Mutation
|
|
4
|
+
MUTATIONS = [
|
|
5
|
+
Mutation("keep-larger", file="pkg/engine.py",
|
|
6
|
+
old="if birth[a] >= birth[b]:", new="if size[a] >= size[b]:",
|
|
7
|
+
fires=("TestElderRule",)),
|
|
8
|
+
Mutation("tie-to-higher-index", file="pkg/engine.py",
|
|
9
|
+
old="(birth[a], -a) >= (birth[b], -b)", new="(birth[a], a) >= (birth[b], b)",
|
|
10
|
+
invisible=True), # an invariance: must fire nothing
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
Then `mutgate run tests/mutations.py`. Each mutation is applied in a sandbox copy, the
|
|
14
|
+
tests run, and the contract judged: OK, DECORATION (a named guard did not fire),
|
|
15
|
+
OVERREACH (something outside the contract fired — two copies of one convention?),
|
|
16
|
+
VISIBLE (an invariance was broken), NOT_APPLIED (the text was not found), ERROR.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from .core import Mutation, Report, Verdict, load, run
|
|
20
|
+
|
|
21
|
+
__version__ = "0.1.0"
|
|
22
|
+
__all__ = ["Mutation", "Report", "Verdict", "load", "run", "__version__"]
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""`mutgate run tests/mutations.py` — check every named mutation's contract.
|
|
2
|
+
|
|
3
|
+
Exit codes: 0 every contract holds; 1 a verdict is not OK; 2 the baseline is red or the
|
|
4
|
+
declaration cannot be loaded.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import shutil
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from . import __version__
|
|
15
|
+
from .core import load, run
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _parser() -> argparse.ArgumentParser:
|
|
19
|
+
ap = argparse.ArgumentParser(prog="mutgate",
|
|
20
|
+
description="Named mutations as contracts on a test suite.")
|
|
21
|
+
ap.add_argument("--version", action="version", version=f"mutgate {__version__}")
|
|
22
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
23
|
+
|
|
24
|
+
r = sub.add_parser("run", help="apply each mutation in a sandbox and judge its contract")
|
|
25
|
+
r.add_argument("file", help="the mutations file (defines MUTATIONS, optionally TESTS/PATHS/ROOT/PYTHON)")
|
|
26
|
+
r.add_argument("--tests", nargs="+", help="pytest targets (override the file's TESTS)")
|
|
27
|
+
r.add_argument("--only", nargs="+", metavar="NAME", help="run only these mutations")
|
|
28
|
+
r.add_argument("--python", help="interpreter to run pytest with (default: this one)")
|
|
29
|
+
r.add_argument("--timeout", type=float, default=None, metavar="SECONDS",
|
|
30
|
+
help="abort a pytest run that takes longer (verdict ERROR)")
|
|
31
|
+
r.add_argument("--pytest-arg", action="append", default=[], metavar="ARG",
|
|
32
|
+
help="extra argument passed to pytest (repeatable)")
|
|
33
|
+
r.add_argument("--markdown", action="store_true", help="print the build-record table instead of the plain one")
|
|
34
|
+
r.add_argument("--keep", action="store_true", help="keep the sandbox directory")
|
|
35
|
+
r.add_argument("-x", "--stop", action="store_true", help="stop at the first verdict that is not OK")
|
|
36
|
+
r.add_argument("-v", "--verbose", action="store_true")
|
|
37
|
+
|
|
38
|
+
ls = sub.add_parser("list", help="list the mutations a file declares")
|
|
39
|
+
ls.add_argument("file")
|
|
40
|
+
return ap
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def main(argv=None) -> int:
|
|
44
|
+
args = _parser().parse_args(argv)
|
|
45
|
+
try:
|
|
46
|
+
decl = load(Path(args.file))
|
|
47
|
+
except Exception as exc: # noqa: BLE001 — every load failure is a usage error here
|
|
48
|
+
print(f"mutgate: cannot load {args.file}: {exc}", file=sys.stderr)
|
|
49
|
+
return 2
|
|
50
|
+
if args.cmd == "list":
|
|
51
|
+
w = max(len(m.name) for m in decl.mutations) if decl.mutations else 8
|
|
52
|
+
for m in decl.mutations:
|
|
53
|
+
exp = "invisible" if m.invisible else ", ".join(m.fires)
|
|
54
|
+
print(f"{m.name:<{w}} {m.file} -> {exp}")
|
|
55
|
+
print(f"\nroot {decl.root}\ntests {' '.join(decl.tests)}\npaths {' '.join(decl.paths)}")
|
|
56
|
+
return 0
|
|
57
|
+
log = (lambda s: print(f"mutgate: {s}", file=sys.stderr, flush=True)) if args.verbose else None
|
|
58
|
+
python = args.python or decl.python or sys.executable
|
|
59
|
+
if not (Path(python).is_file() or shutil.which(python)):
|
|
60
|
+
print(f"mutgate: interpreter not found: {python}", file=sys.stderr)
|
|
61
|
+
return 2
|
|
62
|
+
try:
|
|
63
|
+
report = run(decl.mutations, decl.root, args.tests or decl.tests,
|
|
64
|
+
python=python, paths=decl.paths, only=args.only, keep=args.keep,
|
|
65
|
+
stop_early=args.stop, extra_pytest_args=args.pytest_arg, log=log,
|
|
66
|
+
timeout=args.timeout)
|
|
67
|
+
except ValueError as exc:
|
|
68
|
+
print(f"mutgate: {exc}", file=sys.stderr)
|
|
69
|
+
return 2
|
|
70
|
+
if report.baseline_failed or report.baseline_error:
|
|
71
|
+
print(report.table())
|
|
72
|
+
return 2
|
|
73
|
+
print(report.markdown() if args.markdown else report.table())
|
|
74
|
+
return 0 if report.ok else 1
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def run_cli() -> None:
|
|
78
|
+
sys.exit(main())
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
if __name__ == "__main__":
|
|
82
|
+
run_cli()
|