temper-python 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,26 @@
1
+ name: ci
2
+ on:
3
+ push:
4
+ branches: [main]
5
+ pull_request:
6
+
7
+ jobs:
8
+ test:
9
+ runs-on: ubuntu-latest
10
+ strategy:
11
+ fail-fast: false
12
+ matrix:
13
+ python: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+ - uses: actions/setup-go@v5
17
+ with:
18
+ go-version: stable
19
+ - run: go install github.com/turbolytics/temper/cmd/temper@v0.1.0
20
+ - uses: actions/setup-python@v5
21
+ with:
22
+ python-version: ${{ matrix.python }}
23
+ - run: pip install -e '.[dev]'
24
+ - run: temper check
25
+ - run: temper run --level unit -- python -m pytest -q
26
+ - run: temper matrix --check
@@ -0,0 +1,40 @@
1
+ name: release
2
+ on:
3
+ push:
4
+ tags: ["v*"]
5
+
6
+ jobs:
7
+ build:
8
+ runs-on: ubuntu-latest
9
+ steps:
10
+ - uses: actions/checkout@v4
11
+ - uses: actions/setup-python@v5
12
+ with:
13
+ python-version: "3.13"
14
+ - name: The tag is the package version
15
+ run: |
16
+ version=$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml","rb"))["project"]["version"])')
17
+ if [ "v$version" != "$GITHUB_REF_NAME" ]; then
18
+ echo "tag $GITHUB_REF_NAME does not match pyproject.toml version $version" >&2
19
+ exit 1
20
+ fi
21
+ - run: pip install -e '.[dev]' build
22
+ - run: python -m pytest -q
23
+ - run: python -m build
24
+ - uses: actions/upload-artifact@v4
25
+ with:
26
+ name: dist
27
+ path: dist/
28
+
29
+ publish:
30
+ needs: build
31
+ runs-on: ubuntu-latest
32
+ environment: pypi
33
+ permissions:
34
+ id-token: write # PyPI trusted publishing; no stored token
35
+ steps:
36
+ - uses: actions/download-artifact@v4
37
+ with:
38
+ name: dist
39
+ path: dist/
40
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,6 @@
1
+ .venv/
2
+ __pycache__/
3
+ *.egg-info/
4
+ /.temper/
5
+ /dist/
6
+ .pytest_cache/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 turbolytics
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,51 @@
1
+ <!-- Generated by temper. Do not edit by hand. -->
2
+ # temper-python matrix
3
+
4
+ Every row is a claim this project declares in `temper/`. A cell is covered only by a
5
+ passing evidence record from a test that ran at that level; a skip proves nothing.
6
+ Exempt cells are excused by a passing proof. Regenerate with `temper matrix`.
7
+
8
+ ✅ covered · ☑️ exempt · ❌ failing · ⬜ missing · ⏳ unenforced · — not required
9
+
10
+ ## Summary
11
+
12
+ | covered | exempt | failing | missing | unenforced |
13
+ | ---: | ---: | ---: | ---: | ---: |
14
+ | 10 | 0 | 0 | 0 | 0 |
15
+
16
+ ## Features
17
+
18
+ | Feature | What it does | unit |
19
+ | --- | --- | --- |
20
+ | `adapter.python` | Records what each pytest test proves and how it ended, as Temper evidence. | ✅ |
21
+
22
+ ## Invariants
23
+
24
+ ### adapter
25
+
26
+ | Invariant | Subject | unit |
27
+ | --- | --- | --- |
28
+ | `adapter.failure_never_passes` | `adapter.python` | ✅ |
29
+ | `adapter.level_from_runner` | `adapter.python` | ✅ |
30
+ | `adapter.never_truncates` | `adapter.python` | ✅ |
31
+ | `adapter.silent_without_runner` | `adapter.python` | ✅ |
32
+ | `adapter.skip_records_skip` | `adapter.python` | ✅ |
33
+
34
+ - `adapter.failure_never_passes` — A test that fails, errors or panics never records pass.
35
+ - `adapter.level_from_runner` — The level a record carries comes only from TEMPER_LEVEL; no test can set it.
36
+ - `adapter.never_truncates` — The adapter only appends, to a file unique to its process; it never truncates or removes evidence.
37
+ - `adapter.silent_without_runner` — Without TEMPER_LEVEL and TEMPER_EVIDENCE the adapter records nothing and changes nothing.
38
+ - `adapter.skip_records_skip` — A skipped test records skip.
39
+
40
+ ## Lattices
41
+
42
+ ### `outcomes`
43
+
44
+ Every outcome an evidence record can carry, from the spec's record format. An adapter answers for each one it can produce.
45
+
46
+ | Member | `adapter.python` |
47
+ | --- | --- |
48
+ | `pass` | ✅ |
49
+ | `fail` | ✅ |
50
+ | `skip` | ✅ |
51
+ | `error` | ✅ |
@@ -0,0 +1,19 @@
1
+ PYTHON ?= python
2
+ TEMPER ?= temper
3
+
4
+ .PHONY: test matrix check-matrix
5
+
6
+ test:
7
+ $(PYTHON) -m pytest -q
8
+
9
+ matrix:
10
+ $(TEMPER) check
11
+ $(TEMPER) clean
12
+ $(TEMPER) run --level unit -- $(PYTHON) -m pytest -q
13
+ $(TEMPER) matrix --all-levels
14
+
15
+ check-matrix:
16
+ $(TEMPER) check
17
+ $(TEMPER) clean
18
+ $(TEMPER) run --level unit -- $(PYTHON) -m pytest -q
19
+ $(TEMPER) matrix --check
@@ -0,0 +1,135 @@
1
+ Metadata-Version: 2.5
2
+ Name: temper-python
3
+ Version: 0.1.0
4
+ Summary: pytest adapter for Temper: tests state what they prove, and write evidence.
5
+ Project-URL: Homepage, https://github.com/turbolytics/temper-python
6
+ Project-URL: Temper, https://github.com/turbolytics/temper
7
+ License: MIT
8
+ License-File: LICENSE
9
+ Classifier: Framework :: Pytest
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Topic :: Software Development :: Testing
13
+ Requires-Python: >=3.10
14
+ Requires-Dist: pytest>=7
15
+ Provides-Extra: dev
16
+ Requires-Dist: pytest-xdist; extra == 'dev'
17
+ Description-Content-Type: text/markdown
18
+
19
+ # temper-python
20
+
21
+ The [Temper](https://github.com/turbolytics/temper) adapter for pytest. A test
22
+ states what it proves; the adapter records how it ended; `temper matrix` turns
23
+ those records into a matrix of what is proven and what is not.
24
+
25
+ ## Getting started
26
+
27
+ You need the `temper` CLI ([install](https://github.com/turbolytics/temper#install))
28
+ and a project with `temper.yml` and a `temper/` directory of declarations.
29
+ `temper guide` explains both.
30
+
31
+ **1. Install the adapter.**
32
+
33
+ ```sh
34
+ pip install temper-python
35
+ ```
36
+
37
+ It is a pytest plugin: nothing to import or enable.
38
+
39
+ **2. Mark each test with what it proves.**
40
+
41
+ ```python
42
+ import pytest
43
+
44
+ @pytest.mark.covers("store.reserve.never_negative", subject="store.file")
45
+ def test_reserve_never_goes_negative(): ...
46
+
47
+ @pytest.mark.exempts("store.persists_across_restart", subject="store.memory")
48
+ def test_memory_store_forgets_on_restart(): ...
49
+ ```
50
+
51
+ **3. Run the tests through `temper run`, then build the matrix.**
52
+
53
+ ```sh
54
+ temper clean
55
+ temper run --level unit -- pytest
56
+ temper matrix --all-levels
57
+ ```
58
+
59
+ That is the whole integration. Plain `pytest` behaves exactly as before: without
60
+ the variables `temper run` sets, the plugin records nothing.
61
+
62
+ ## What a test can say
63
+
64
+ | Mark | Meaning |
65
+ |---|---|
66
+ | `@pytest.mark.covers(ref)` | The test proves a feature or lattice member (`lattice/key`). |
67
+ | `@pytest.mark.covers(ref, subject=id)` | The test proves an invariant or lattice member for that subject. |
68
+ | `@pytest.mark.exempts(ref, subject=id)` | The test proves the subject is excused. Use it only in the test a declaration names in `proven_by`. |
69
+ | `scenario=id` (on either) | The claim came from that declared scenario. |
70
+
71
+ Marks on a class or a module apply to every test inside. A test may carry many
72
+ marks; each is one record with the same outcome.
73
+
74
+ A scenario runner that parametrizes one test over many scenarios claims from
75
+ inside the test, where the scenario is known:
76
+
77
+ ```python
78
+ import temper
79
+
80
+ @pytest.mark.parametrize("sid", SCENARIOS)
81
+ def test_scenario(request, sid):
82
+ temper.covers(request, "alerts.low_stock", scenario=sid)
83
+ temper.covers(request, sid, scenario=sid)
84
+ ...
85
+ ```
86
+
87
+ `temper.exempts(request, ref, subject=...)` is the same for exemptions.
88
+
89
+ There is no way to set a level. The runner does (`temper run --level`), and a
90
+ test that changes `TEMPER_LEVEL` in its own environment changes nothing: the
91
+ environment is read once, when pytest starts.
92
+
93
+ A malformed mark (no reference, a stray keyword such as `level=`, a non-string)
94
+ stops the run with a message naming the test, when the runner is active.
95
+
96
+ ## How outcomes are recorded
97
+
98
+ A record is written once the test is final, after fixtures have torn down.
99
+
100
+ | Outcome | When |
101
+ |---|---|
102
+ | `pass` | The test, its setup and its teardown all passed. The only outcome that counts. |
103
+ | `fail` | An assertion failed, `pytest.fail` was called, or the test was `xfail` (whether it failed or passed). |
104
+ | `skip` | The test was skipped, in any phase. |
105
+ | `error` | An exception other than a failed assertion, or any failure in setup or teardown. |
106
+
107
+ A test that fails and then skips records `fail`. Evidence goes to one new file
108
+ per process, `<level>-<pid>-<8 hex>.jsonl`, opened for append, one write per
109
+ line. Existing evidence is never truncated, so `pytest-xdist` works without
110
+ configuration. The contract every adapter follows is
111
+ [docs/adapters.md](https://github.com/turbolytics/temper/blob/main/docs/adapters.md).
112
+
113
+ Not verified: retry plugins such as `pytest-rerunfailures`.
114
+
115
+ ## Development
116
+
117
+ ```sh
118
+ python -m venv .venv && .venv/bin/pip install -e '.[dev]'
119
+ make matrix # clean, run the tests through temper, rebuild MATRIX.md
120
+ make check-matrix # fail if the committed matrix is stale
121
+ ```
122
+
123
+ This project is declared with Temper in `temper/`, and its own tests prove the
124
+ adapter claims with the adapter: see [MATRIX.md](MATRIX.md).
125
+
126
+ ## Releasing
127
+
128
+ Set `version` in `pyproject.toml`, merge, then push the matching tag:
129
+
130
+ ```sh
131
+ git tag v0.1.0 && git push origin v0.1.0
132
+ ```
133
+
134
+ The release workflow checks the tag matches the version, runs the tests, builds,
135
+ and publishes to PyPI through trusted publishing.
@@ -0,0 +1,117 @@
1
+ # temper-python
2
+
3
+ The [Temper](https://github.com/turbolytics/temper) adapter for pytest. A test
4
+ states what it proves; the adapter records how it ended; `temper matrix` turns
5
+ those records into a matrix of what is proven and what is not.
6
+
7
+ ## Getting started
8
+
9
+ You need the `temper` CLI ([install](https://github.com/turbolytics/temper#install))
10
+ and a project with `temper.yml` and a `temper/` directory of declarations.
11
+ `temper guide` explains both.
12
+
13
+ **1. Install the adapter.**
14
+
15
+ ```sh
16
+ pip install temper-python
17
+ ```
18
+
19
+ It is a pytest plugin: nothing to import or enable.
20
+
21
+ **2. Mark each test with what it proves.**
22
+
23
+ ```python
24
+ import pytest
25
+
26
+ @pytest.mark.covers("store.reserve.never_negative", subject="store.file")
27
+ def test_reserve_never_goes_negative(): ...
28
+
29
+ @pytest.mark.exempts("store.persists_across_restart", subject="store.memory")
30
+ def test_memory_store_forgets_on_restart(): ...
31
+ ```
32
+
33
+ **3. Run the tests through `temper run`, then build the matrix.**
34
+
35
+ ```sh
36
+ temper clean
37
+ temper run --level unit -- pytest
38
+ temper matrix --all-levels
39
+ ```
40
+
41
+ That is the whole integration. Plain `pytest` behaves exactly as before: without
42
+ the variables `temper run` sets, the plugin records nothing.
43
+
44
+ ## What a test can say
45
+
46
+ | Mark | Meaning |
47
+ |---|---|
48
+ | `@pytest.mark.covers(ref)` | The test proves a feature or lattice member (`lattice/key`). |
49
+ | `@pytest.mark.covers(ref, subject=id)` | The test proves an invariant or lattice member for that subject. |
50
+ | `@pytest.mark.exempts(ref, subject=id)` | The test proves the subject is excused. Use it only in the test a declaration names in `proven_by`. |
51
+ | `scenario=id` (on either) | The claim came from that declared scenario. |
52
+
53
+ Marks on a class or a module apply to every test inside. A test may carry many
54
+ marks; each is one record with the same outcome.
55
+
56
+ A scenario runner that parametrizes one test over many scenarios claims from
57
+ inside the test, where the scenario is known:
58
+
59
+ ```python
60
+ import temper
61
+
62
+ @pytest.mark.parametrize("sid", SCENARIOS)
63
+ def test_scenario(request, sid):
64
+ temper.covers(request, "alerts.low_stock", scenario=sid)
65
+ temper.covers(request, sid, scenario=sid)
66
+ ...
67
+ ```
68
+
69
+ `temper.exempts(request, ref, subject=...)` is the same for exemptions.
70
+
71
+ There is no way to set a level. The runner does (`temper run --level`), and a
72
+ test that changes `TEMPER_LEVEL` in its own environment changes nothing: the
73
+ environment is read once, when pytest starts.
74
+
75
+ A malformed mark (no reference, a stray keyword such as `level=`, a non-string)
76
+ stops the run with a message naming the test, when the runner is active.
77
+
78
+ ## How outcomes are recorded
79
+
80
+ A record is written once the test is final, after fixtures have torn down.
81
+
82
+ | Outcome | When |
83
+ |---|---|
84
+ | `pass` | The test, its setup and its teardown all passed. The only outcome that counts. |
85
+ | `fail` | An assertion failed, `pytest.fail` was called, or the test was `xfail` (whether it failed or passed). |
86
+ | `skip` | The test was skipped, in any phase. |
87
+ | `error` | An exception other than a failed assertion, or any failure in setup or teardown. |
88
+
89
+ A test that fails and then skips records `fail`. Evidence goes to one new file
90
+ per process, `<level>-<pid>-<8 hex>.jsonl`, opened for append, one write per
91
+ line. Existing evidence is never truncated, so `pytest-xdist` works without
92
+ configuration. The contract every adapter follows is
93
+ [docs/adapters.md](https://github.com/turbolytics/temper/blob/main/docs/adapters.md).
94
+
95
+ Not verified: retry plugins such as `pytest-rerunfailures`.
96
+
97
+ ## Development
98
+
99
+ ```sh
100
+ python -m venv .venv && .venv/bin/pip install -e '.[dev]'
101
+ make matrix # clean, run the tests through temper, rebuild MATRIX.md
102
+ make check-matrix # fail if the committed matrix is stale
103
+ ```
104
+
105
+ This project is declared with Temper in `temper/`, and its own tests prove the
106
+ adapter claims with the adapter: see [MATRIX.md](MATRIX.md).
107
+
108
+ ## Releasing
109
+
110
+ Set `version` in `pyproject.toml`, merge, then push the matching tag:
111
+
112
+ ```sh
113
+ git tag v0.1.0 && git push origin v0.1.0
114
+ ```
115
+
116
+ The release workflow checks the tag matches the version, runs the tests, builds,
117
+ and publishes to PyPI through trusted publishing.
@@ -0,0 +1,34 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "temper-python"
7
+ version = "0.1.0"
8
+ description = "pytest adapter for Temper: tests state what they prove, and write evidence."
9
+ readme = "README.md"
10
+ license = { text = "MIT" }
11
+ requires-python = ">=3.10"
12
+ dependencies = ["pytest>=7"]
13
+ classifiers = [
14
+ "Framework :: Pytest",
15
+ "License :: OSI Approved :: MIT License",
16
+ "Programming Language :: Python :: 3",
17
+ "Topic :: Software Development :: Testing",
18
+ ]
19
+
20
+ [project.urls]
21
+ Homepage = "https://github.com/turbolytics/temper-python"
22
+ Temper = "https://github.com/turbolytics/temper"
23
+
24
+ [project.entry-points.pytest11]
25
+ temper = "temper.plugin"
26
+
27
+ [project.optional-dependencies]
28
+ dev = ["pytest-xdist"]
29
+
30
+ [tool.hatch.build.targets.wheel]
31
+ packages = ["src/temper"]
32
+
33
+ [tool.pytest.ini_options]
34
+ testpaths = ["tests"]
@@ -0,0 +1,26 @@
1
+ """Temper adapter for pytest.
2
+
3
+ Mark a test with what it proves::
4
+
5
+ @pytest.mark.covers("store.reserve.never_negative", subject="store.file")
6
+ def test_reserve(): ...
7
+
8
+ or claim from inside it, which is how a parametrized scenario runner names its
9
+ scenario::
10
+
11
+ temper.covers(request, "alerts.low_stock", scenario="alerts.sold_out")
12
+ """
13
+
14
+ from .plugin import add_claim
15
+
16
+ __all__ = ["covers", "exempts"]
17
+
18
+
19
+ def covers(request, ref, *, subject=None, scenario=None):
20
+ """The running test proves ``ref`` (for ``subject``, when it applies per subject)."""
21
+ add_claim(request, "covers", ref, subject, scenario)
22
+
23
+
24
+ def exempts(request, ref, *, subject=None, scenario=None):
25
+ """The running test proves ``subject`` is excused from ``ref``."""
26
+ add_claim(request, "exempts", ref, subject, scenario)
@@ -0,0 +1,60 @@
1
+ """The evidence file: append-only, one file per process, one write per line."""
2
+
3
+ import json
4
+ import os
5
+ import re
6
+ import secrets
7
+ from pathlib import Path
8
+
9
+ _SEPARATORS = (",", ":")
10
+
11
+
12
+ def encode(record):
13
+ """One record as one UTF-8 line, keys in the order given, compact."""
14
+ try:
15
+ return (json.dumps(record, separators=_SEPARATORS, ensure_ascii=False) + "\n").encode("utf-8")
16
+ except UnicodeEncodeError:
17
+ # A lone surrogate (an odd test id) cannot be UTF-8; JSON can still escape it.
18
+ return (json.dumps(record, separators=_SEPARATORS, ensure_ascii=True) + "\n").encode("utf-8")
19
+
20
+
21
+ class Writer:
22
+ """Appends records to ``<level>-<pid>-<8 hex>.jsonl`` in a directory.
23
+
24
+ The file is created on the first record, never opened for truncation, and
25
+ never shared: a forked child opens its own.
26
+ """
27
+
28
+ def __init__(self, directory, level):
29
+ self._dir = Path(directory)
30
+ self._level = level
31
+ self._fd = None
32
+ self._pid = None
33
+
34
+ def ensure_directory(self):
35
+ self._dir.mkdir(parents=True, exist_ok=True)
36
+
37
+ def write(self, record):
38
+ data = encode(record)
39
+ fd = self._open()
40
+ written = os.write(fd, data)
41
+ while written < len(data): # a short write is rare; finish the line
42
+ written += os.write(fd, data[written:])
43
+
44
+ def close(self):
45
+ if self._fd is not None and self._pid == os.getpid():
46
+ os.close(self._fd)
47
+ self._fd = None
48
+
49
+ def _open(self):
50
+ pid = os.getpid()
51
+ if self._fd is not None and self._pid == pid:
52
+ return self._fd
53
+ self._fd = None # inherited across a fork: leave the parent's file alone
54
+ self.ensure_directory()
55
+ safe = re.sub(r"[^A-Za-z0-9._-]", "_", self._level)
56
+ name = f"{safe}-{pid}-{secrets.token_hex(4)}.jsonl"
57
+ flags = os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_EXCL
58
+ self._fd = os.open(self._dir / name, flags, 0o644)
59
+ self._pid = pid
60
+ return self._fd
@@ -0,0 +1,140 @@
1
+ """The pytest plugin: reads what each test claims, records how it ended.
2
+
3
+ The level and the evidence directory come from ``TEMPER_LEVEL`` and
4
+ ``TEMPER_EVIDENCE``, read once when pytest starts. Without both, the plugin
5
+ does nothing.
6
+ """
7
+
8
+ import os
9
+ from datetime import datetime, timezone
10
+
11
+ import pytest
12
+
13
+ from ._evidence import Writer
14
+
15
+ CLAIM_KINDS = ("covers", "exempts")
16
+ _KEYWORDS = ("subject", "scenario")
17
+
18
+ _CLAIMS = pytest.StashKey() # runtime claims made through temper.covers / temper.exempts
19
+ _OUTCOMES = pytest.StashKey() # one outcome per phase report
20
+ _STATE = pytest.StashKey()
21
+
22
+ _PRECEDENCE = ("error", "fail", "skip", "pass") # first one present wins
23
+
24
+
25
+ class _State:
26
+ def __init__(self, level, directory, run):
27
+ self.level = level
28
+ self.run = run
29
+ self.writer = Writer(directory, level)
30
+
31
+
32
+ def _claim(kind, args, kwargs, where):
33
+ """Validate one claim and return (kind, ref, subject, scenario)."""
34
+ if len(args) != 1 or not isinstance(args[0], str) or not args[0]:
35
+ raise ValueError(f"{where}: {kind} takes one non-empty reference, as a string")
36
+ for key in kwargs:
37
+ if key not in _KEYWORDS:
38
+ hint = " (the level comes from the runner, never the test)" if key == "level" else ""
39
+ raise ValueError(f"{where}: {kind} has no keyword {key!r}{hint}; use subject or scenario")
40
+ for key in _KEYWORDS:
41
+ value = kwargs.get(key)
42
+ if value is not None and (not isinstance(value, str) or not value):
43
+ raise ValueError(f"{where}: {kind} {key} must be a non-empty string")
44
+ return kind, args[0], kwargs.get("subject"), kwargs.get("scenario")
45
+
46
+
47
+ def add_claim(request, kind, ref, subject=None, scenario=None):
48
+ """Claim from inside a test; the claim follows the test's outcome."""
49
+ node = getattr(request, "node", None)
50
+ if not isinstance(node, pytest.Item):
51
+ raise TypeError(f"temper.{kind} needs the `request` fixture of a test")
52
+ kwargs = {k: v for k, v in (("subject", subject), ("scenario", scenario)) if v is not None}
53
+ claim = _claim(kind, (ref,), kwargs, node.nodeid)
54
+ node.stash.setdefault(_CLAIMS, []).append(claim)
55
+
56
+
57
+ def _claims(item):
58
+ seen, out = set(), []
59
+ marked = [_claim(m.name, m.args, m.kwargs, item.nodeid) for m in item.iter_markers() if m.name in CLAIM_KINDS]
60
+ for claim in marked + item.stash.get(_CLAIMS, []):
61
+ if claim not in seen:
62
+ seen.add(claim)
63
+ out.append(claim)
64
+ return out
65
+
66
+
67
+ def pytest_configure(config):
68
+ config.addinivalue_line("markers", "covers(ref, subject=None, scenario=None): the test proves this Temper claim")
69
+ config.addinivalue_line("markers", "exempts(ref, subject=None, scenario=None): the test proves this exemption")
70
+ level = os.environ.get("TEMPER_LEVEL", "")
71
+ directory = os.environ.get("TEMPER_EVIDENCE", "")
72
+ if not level or not directory:
73
+ return
74
+ state = _State(level, directory, os.environ.get("TEMPER_RUN", ""))
75
+ try:
76
+ state.writer.ensure_directory()
77
+ except OSError as err:
78
+ raise pytest.UsageError(f"temper: cannot write evidence to TEMPER_EVIDENCE={directory!r}: {err}")
79
+ config.stash[_STATE] = state
80
+
81
+
82
+ def pytest_unconfigure(config):
83
+ state = config.stash.get(_STATE, None)
84
+ if state is not None:
85
+ state.writer.close()
86
+
87
+
88
+ def pytest_collection_modifyitems(config, items):
89
+ if config.stash.get(_STATE, None) is None:
90
+ return
91
+ for item in items:
92
+ try:
93
+ _claims(item)
94
+ except ValueError as err:
95
+ raise pytest.UsageError(f"temper: {err}")
96
+
97
+
98
+ def _outcome(report, call):
99
+ if hasattr(report, "wasxfail"):
100
+ return "fail" # an expected failure never proves anything, whichever way it went
101
+ if report.skipped:
102
+ return "skip"
103
+ if report.passed:
104
+ return "pass"
105
+ if report.when != "call":
106
+ return "error" # setup or teardown broke
107
+ excinfo = call.excinfo
108
+ if excinfo is None or isinstance(excinfo.value, (AssertionError, pytest.fail.Exception)):
109
+ return "fail"
110
+ return "error"
111
+
112
+
113
+ @pytest.hookimpl(hookwrapper=True, tryfirst=True)
114
+ def pytest_runtest_makereport(item, call):
115
+ outcome = yield
116
+ if item.config.stash.get(_STATE, None) is None:
117
+ return
118
+ item.stash.setdefault(_OUTCOMES, []).append(_outcome(outcome.get_result(), call))
119
+
120
+
121
+ @pytest.hookimpl(hookwrapper=True)
122
+ def pytest_runtest_protocol(item, nextitem):
123
+ yield
124
+ state = item.config.stash.get(_STATE, None)
125
+ outcomes = item.stash.get(_OUTCOMES, None)
126
+ if state is None or not outcomes:
127
+ return
128
+ final = next(o for o in _PRECEDENCE if o in outcomes)
129
+ at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
130
+ for kind, ref, subject, scenario in _claims(item):
131
+ record = {kind: ref}
132
+ if subject:
133
+ record["subject"] = subject
134
+ if scenario:
135
+ record["scenario"] = scenario
136
+ record.update(test=item.nodeid, level=state.level, outcome=final)
137
+ if state.run:
138
+ record["run"] = state.run
139
+ record["at"] = at
140
+ state.writer.write(record)
@@ -0,0 +1,4 @@
1
+ features:
2
+ - id: adapter.python
3
+ description: Records what each pytest test proves and how it ended, as Temper evidence.
4
+ requires: [unit]
@@ -0,0 +1,35 @@
1
+ invariants:
2
+ - id: adapter.failure_never_passes
3
+ claim: A test that fails, errors or panics never records pass.
4
+ applies_to: adapter
5
+ class: safety
6
+ family: adapter
7
+ requires: [unit]
8
+
9
+ - id: adapter.skip_records_skip
10
+ claim: A skipped test records skip.
11
+ applies_to: adapter
12
+ class: safety
13
+ family: adapter
14
+ requires: [unit]
15
+
16
+ - id: adapter.level_from_runner
17
+ claim: The level a record carries comes only from TEMPER_LEVEL; no test can set it.
18
+ applies_to: adapter
19
+ class: safety
20
+ family: adapter
21
+ requires: [unit]
22
+
23
+ - id: adapter.silent_without_runner
24
+ claim: Without TEMPER_LEVEL and TEMPER_EVIDENCE the adapter records nothing and changes nothing.
25
+ applies_to: adapter
26
+ class: safety
27
+ family: adapter
28
+ requires: [unit]
29
+
30
+ - id: adapter.never_truncates
31
+ claim: The adapter only appends, to a file unique to its process; it never truncates or removes evidence.
32
+ applies_to: adapter
33
+ class: safety
34
+ family: adapter
35
+ requires: [unit]
@@ -0,0 +1,11 @@
1
+ id: outcomes
2
+ description: >
3
+ Every outcome an evidence record can carry, from the spec's record format.
4
+ An adapter answers for each one it can produce.
5
+ applies_to: adapter
6
+ requires: [unit]
7
+ members:
8
+ - key: pass
9
+ - key: fail
10
+ - key: skip
11
+ - key: error
@@ -0,0 +1,3 @@
1
+ # Generated by temper. Do not edit by hand.
2
+ # One line per feature, sorted by id: the status at each level.
3
+ adapter.python: {unit: covered}
@@ -0,0 +1,11 @@
1
+ # Generated by temper. Do not edit by hand.
2
+ # One line per invariant or lattice member that applies to adapter.python, sorted by id: the status at each level.
3
+ adapter.failure_never_passes: {unit: covered}
4
+ adapter.level_from_runner: {unit: covered}
5
+ adapter.never_truncates: {unit: covered}
6
+ adapter.silent_without_runner: {unit: covered}
7
+ adapter.skip_records_skip: {unit: covered}
8
+ outcomes/error: {unit: covered}
9
+ outcomes/fail: {unit: covered}
10
+ outcomes/pass: {unit: covered}
11
+ outcomes/skip: {unit: covered}
@@ -0,0 +1,9 @@
1
+ id: adapter.python
2
+ kind: adapter
3
+ feature: adapter.python
4
+ answers:
5
+ outcomes:
6
+ pass: supported
7
+ fail: supported
8
+ skip: supported
9
+ error: supported
@@ -0,0 +1,6 @@
1
+ temper: "0.1"
2
+ project: temper-python
3
+
4
+ levels:
5
+ - name: unit
6
+ description: pytest; every test runs the plugin in a fresh pytest process
@@ -0,0 +1,51 @@
1
+ """Helpers for running an inner pytest, as a runner would, and reading what it wrote."""
2
+
3
+ import json
4
+ from pathlib import Path
5
+
6
+ import pytest
7
+
8
+ pytest_plugins = ["pytester"]
9
+
10
+ VARS = ("TEMPER_LEVEL", "TEMPER_EVIDENCE", "TEMPER_RUN")
11
+
12
+
13
+ class Inner:
14
+ """Runs a test file in a fresh pytest process with an explicit Temper environment."""
15
+
16
+ def __init__(self, pytester, monkeypatch, tmp_path):
17
+ self.pytester = pytester
18
+ self.monkeypatch = monkeypatch
19
+ self.dir = tmp_path / "evidence"
20
+
21
+ def run(self, source, *args, level="unit", run=None, runner=True, evidence=None):
22
+ for name in VARS:
23
+ self.monkeypatch.delenv(name, raising=False)
24
+ if runner:
25
+ self.monkeypatch.setenv("TEMPER_LEVEL", level)
26
+ self.monkeypatch.setenv("TEMPER_EVIDENCE", str(evidence or self.dir))
27
+ if run:
28
+ self.monkeypatch.setenv("TEMPER_RUN", run)
29
+ self.pytester.makepyfile(test_inner=source)
30
+ return self.pytester.runpytest_subprocess(*args)
31
+
32
+ def files(self):
33
+ return sorted(self.dir.glob("*.jsonl")) if self.dir.exists() else []
34
+
35
+ def lines(self):
36
+ out = []
37
+ for f in self.files():
38
+ out.extend(f.read_bytes().decode("utf-8").splitlines(keepends=True))
39
+ return out
40
+
41
+ def records(self):
42
+ return [json.loads(line) for line in self.lines()]
43
+
44
+ def outcomes(self):
45
+ """test name -> outcome, for inner files with one claim per test."""
46
+ return {r["test"].split("::", 1)[1]: r["outcome"] for r in self.records()}
47
+
48
+
49
+ @pytest.fixture
50
+ def inner(pytester, monkeypatch, tmp_path):
51
+ return Inner(pytester, monkeypatch, tmp_path)
@@ -0,0 +1,423 @@
1
+ """The adapter contract (docs/adapters.md in turbolytics/temper), proven through real pytest runs."""
2
+
3
+ import json
4
+ import re
5
+ from pathlib import Path
6
+
7
+ import pytest
8
+
9
+ SUBJECT = "adapter.python"
10
+ CLAIM = '@pytest.mark.covers("demo.claim")\n'
11
+
12
+
13
+ def inner_test(body, mark=CLAIM, name="test_it"):
14
+ return "import pytest\n\n" + mark + f"def {name}():\n" + "".join(
15
+ f" {line}\n" for line in body.strip().splitlines()
16
+ )
17
+
18
+
19
+ @pytest.mark.covers("adapter.python")
20
+ @pytest.mark.covers("outcomes/pass", subject=SUBJECT)
21
+ def test_pass_writes_one_exact_record(inner):
22
+ inner.run(
23
+ "import pytest\n\n"
24
+ '@pytest.mark.covers("sink.flush.keeps_batch", subject="sink.kafka")\n'
25
+ "def test_flush():\n assert True\n",
26
+ run="ci-1",
27
+ )
28
+ (line,) = inner.lines()
29
+ assert line.endswith("\n") and not line.endswith("\n\n")
30
+ rec = json.loads(line)
31
+ assert list(rec) == ["covers", "subject", "test", "level", "outcome", "run", "at"]
32
+ assert rec["covers"] == "sink.flush.keeps_batch"
33
+ assert rec["subject"] == "sink.kafka"
34
+ assert rec["test"] == "test_inner.py::test_flush"
35
+ assert (rec["level"], rec["outcome"], rec["run"]) == ("unit", "pass", "ci-1")
36
+ assert re.fullmatch(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ", rec["at"])
37
+ assert line == json.dumps(rec, separators=(",", ":"), ensure_ascii=False) + "\n"
38
+
39
+
40
+ @pytest.mark.covers("adapter.python")
41
+ def test_empty_optional_keys_are_left_out(inner):
42
+ inner.run(inner_test("pass"))
43
+ (rec,) = inner.records()
44
+ assert list(rec) == ["covers", "test", "level", "outcome", "at"]
45
+
46
+
47
+ @pytest.mark.covers("adapter.python")
48
+ def test_unmarked_tests_record_nothing(inner):
49
+ inner.run("def test_plain():\n assert True\n")
50
+ assert inner.files() == []
51
+
52
+
53
+ # --- adapter.failure_never_passes ------------------------------------------------
54
+
55
+ FAILURES = {
56
+ "assert": ("assert 1 == 2", "fail"),
57
+ "pytest_fail": ("pytest.fail('no')", "fail"),
58
+ "exception": ("raise ValueError('boom')", "error"),
59
+ "xfail_fails": ("pytest.xfail('expected')", "fail"),
60
+ }
61
+
62
+
63
+ @pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
64
+ @pytest.mark.covers("outcomes/fail", subject=SUBJECT)
65
+ @pytest.mark.covers("outcomes/error", subject=SUBJECT)
66
+ @pytest.mark.parametrize("name", sorted(FAILURES))
67
+ def test_a_failing_body_never_records_pass(inner, name):
68
+ body, want = FAILURES[name]
69
+ inner.run(inner_test(body))
70
+ (rec,) = inner.records()
71
+ assert rec["outcome"] == want
72
+
73
+
74
+ @pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
75
+ @pytest.mark.covers("outcomes/error", subject=SUBJECT)
76
+ def test_a_setup_error_records_error(inner):
77
+ inner.run(
78
+ "import pytest\n\n"
79
+ "@pytest.fixture\ndef broken():\n raise RuntimeError('setup')\n\n"
80
+ '@pytest.mark.covers("demo.claim")\n'
81
+ "def test_it(broken):\n assert True\n"
82
+ )
83
+ (rec,) = inner.records()
84
+ assert rec["outcome"] == "error"
85
+
86
+
87
+ @pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
88
+ @pytest.mark.covers("outcomes/error", subject=SUBJECT)
89
+ def test_a_teardown_error_after_a_passing_body_records_error_not_pass(inner):
90
+ inner.run(
91
+ "import pytest\n\n"
92
+ "@pytest.fixture\ndef leaky():\n yield\n raise RuntimeError('teardown')\n\n"
93
+ '@pytest.mark.covers("demo.claim")\n'
94
+ "def test_it(leaky):\n assert True\n"
95
+ )
96
+ (rec,) = inner.records() # exactly one record, written after teardown
97
+ assert rec["outcome"] == "error"
98
+
99
+
100
+ @pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
101
+ @pytest.mark.covers("outcomes/fail", subject=SUBJECT)
102
+ @pytest.mark.parametrize(
103
+ "mark",
104
+ ["@pytest.mark.xfail", "@pytest.mark.xfail(strict=True)"],
105
+ ids=["xfail", "strict"],
106
+ )
107
+ def test_xfail_never_records_pass(inner, mark):
108
+ # fails as expected -> fail; passes unexpectedly -> fail
109
+ for body in ("assert False", "assert True"):
110
+ inner.run(inner_test(body, mark=mark + "\n" + CLAIM), "-p", "no:cacheprovider")
111
+ assert [r["outcome"] for r in inner.records()] == ["fail", "fail"]
112
+
113
+
114
+ @pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
115
+ def test_fail_then_skip_records_fail(inner):
116
+ inner.run(
117
+ "import pytest\n\n"
118
+ "@pytest.fixture\ndef skipper():\n yield\n pytest.skip('late')\n\n"
119
+ '@pytest.mark.covers("demo.claim")\n'
120
+ "def test_it(skipper):\n assert False\n"
121
+ )
122
+ (rec,) = inner.records()
123
+ assert rec["outcome"] == "fail"
124
+
125
+
126
+ @pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
127
+ def test_a_crashing_run_is_never_recorded_as_pass(inner):
128
+ # the process dies mid-test: nothing is written for it (no record, so no pass)
129
+ inner.run(
130
+ "import os\nimport pytest\n\n" + CLAIM + "def test_it():\n os._exit(7)\n"
131
+ )
132
+ assert inner.records() == []
133
+
134
+
135
+ @pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
136
+ @pytest.mark.covers("outcomes/error", subject=SUBJECT)
137
+ @pytest.mark.parametrize("phase", ["setup", "teardown"])
138
+ def test_an_assertion_in_setup_or_teardown_is_an_error_not_a_test_failure(inner, phase):
139
+ fixture = (
140
+ " assert False\n yield\n" if phase == "setup" else " yield\n assert False\n"
141
+ )
142
+ inner.run(
143
+ "import pytest\n\n@pytest.fixture\ndef fx():\n" + fixture + "\n"
144
+ '@pytest.mark.covers("demo.claim")\n'
145
+ "def test_it(fx):\n assert True\n"
146
+ )
147
+ (rec,) = inner.records()
148
+ assert rec["outcome"] == "error"
149
+
150
+
151
+ # --- adapter.skip_records_skip ---------------------------------------------------
152
+
153
+ SKIPS = {
154
+ "runtime": inner_test("pytest.skip('later')"),
155
+ "skipif": inner_test("assert True", mark='@pytest.mark.skipif(True, reason="x")\n' + CLAIM),
156
+ "skip_mark": inner_test("assert True", mark='@pytest.mark.skip\n' + CLAIM),
157
+ "importorskip": inner_test("pytest.importorskip('no_such_module_xyz')"),
158
+ }
159
+
160
+
161
+ @pytest.mark.covers("adapter.skip_records_skip", subject=SUBJECT)
162
+ @pytest.mark.covers("outcomes/skip", subject=SUBJECT)
163
+ @pytest.mark.parametrize("name", sorted(SKIPS))
164
+ def test_a_skipped_test_records_skip(inner, name):
165
+ inner.run(SKIPS[name])
166
+ (rec,) = inner.records()
167
+ assert rec["outcome"] == "skip"
168
+
169
+
170
+ @pytest.mark.covers("adapter.skip_records_skip", subject=SUBJECT)
171
+ def test_a_skip_in_setup_records_skip(inner):
172
+ inner.run(
173
+ "import pytest\n\n"
174
+ "@pytest.fixture\ndef need():\n pytest.skip('no service')\n\n"
175
+ '@pytest.mark.covers("demo.claim")\n'
176
+ "def test_it(need):\n assert True\n"
177
+ )
178
+ (rec,) = inner.records()
179
+ assert rec["outcome"] == "skip"
180
+
181
+
182
+ # --- adapter.level_from_runner ---------------------------------------------------
183
+
184
+
185
+ @pytest.mark.covers("adapter.level_from_runner", subject=SUBJECT)
186
+ def test_the_level_is_the_runners_verbatim(inner):
187
+ inner.run(inner_test("pass"), level="integration")
188
+ (rec,) = inner.records()
189
+ assert rec["level"] == "integration"
190
+
191
+
192
+ @pytest.mark.covers("adapter.level_from_runner", subject=SUBJECT)
193
+ def test_a_test_that_changes_its_environment_cannot_change_level_or_place(inner, tmp_path):
194
+ elsewhere = tmp_path / "elsewhere"
195
+ inner.run(
196
+ inner_test(
197
+ "import os\n"
198
+ "os.environ['TEMPER_LEVEL'] = 'release'\n"
199
+ f"os.environ['TEMPER_EVIDENCE'] = {str(elsewhere)!r}\n"
200
+ "os.environ['TEMPER_RUN'] = 'forged'"
201
+ ),
202
+ run="real",
203
+ )
204
+ (rec,) = inner.records()
205
+ assert (rec["level"], rec["run"]) == ("unit", "real")
206
+ assert not elsewhere.exists()
207
+
208
+
209
+ @pytest.mark.covers("adapter.level_from_runner", subject=SUBJECT)
210
+ def test_a_mark_cannot_set_a_level(inner):
211
+ result = inner.run(
212
+ "import pytest\n\n"
213
+ '@pytest.mark.covers("demo.claim", level="release")\n'
214
+ "def test_it():\n assert True\n"
215
+ )
216
+ assert result.ret != 0
217
+ result.stderr.fnmatch_lines(["*level*"])
218
+ assert inner.records() == []
219
+
220
+
221
+ # --- adapter.silent_without_runner -----------------------------------------------
222
+
223
+
224
+ @pytest.mark.covers("adapter.silent_without_runner", subject=SUBJECT)
225
+ @pytest.mark.parametrize("missing", ["TEMPER_LEVEL", "TEMPER_EVIDENCE", "both", "empty"])
226
+ def test_without_the_runner_environment_nothing_is_recorded(inner, tmp_path, monkeypatch, missing):
227
+ src = inner_test("assert True") + "\n" + inner_test("assert False", name="test_bad")
228
+ inner.pytester.makepyfile(test_inner=src)
229
+ for name in ("TEMPER_LEVEL", "TEMPER_EVIDENCE", "TEMPER_RUN"):
230
+ monkeypatch.delenv(name, raising=False)
231
+ if missing in ("TEMPER_EVIDENCE",):
232
+ monkeypatch.setenv("TEMPER_LEVEL", "unit")
233
+ if missing == "TEMPER_LEVEL":
234
+ monkeypatch.setenv("TEMPER_EVIDENCE", str(inner.dir))
235
+ if missing == "empty":
236
+ monkeypatch.setenv("TEMPER_LEVEL", "")
237
+ monkeypatch.setenv("TEMPER_EVIDENCE", "")
238
+ with_plugin = inner.pytester.runpytest_subprocess()
239
+ without = inner.pytester.runpytest_subprocess("-p", "no:temper")
240
+ assert not inner.dir.exists()
241
+ assert with_plugin.ret == without.ret
242
+ # Without the plugin the marks are unknown and pytest warns; the results are the same.
243
+ def results(r):
244
+ return {k: v for k, v in r.parseoutcomes().items() if k != "warnings"}
245
+
246
+ assert results(with_plugin) == results(without)
247
+
248
+
249
+ @pytest.mark.covers("adapter.silent_without_runner", subject=SUBJECT)
250
+ def test_a_malformed_mark_is_ignored_without_the_runner(inner):
251
+ result = inner.run(
252
+ "import pytest\n\n"
253
+ '@pytest.mark.covers(42)\n'
254
+ "def test_it():\n assert True\n",
255
+ runner=False,
256
+ )
257
+ assert result.ret == 0
258
+
259
+
260
+ # --- adapter.never_truncates -----------------------------------------------------
261
+
262
+
263
+ @pytest.mark.covers("adapter.never_truncates", subject=SUBJECT)
264
+ def test_existing_evidence_is_untouched_and_each_process_has_its_own_file(inner):
265
+ inner.dir.mkdir()
266
+ old = inner.dir / "unit-1-deadbeef.jsonl"
267
+ old.write_bytes(b'{"covers":"old.claim","test":"t","level":"unit","outcome":"pass"}\n')
268
+ stray = inner.dir / "notes.txt"
269
+ stray.write_bytes(b"keep me")
270
+ inner.run(inner_test("pass"))
271
+ inner.run(inner_test("pass"))
272
+ assert old.read_bytes().startswith(b'{"covers":"old.claim"')
273
+ assert old.read_bytes().count(b"\n") == 1
274
+ assert stray.read_bytes() == b"keep me"
275
+ new = [f for f in inner.files() if f != old]
276
+ assert len(new) == 2
277
+ for f in new:
278
+ assert re.fullmatch(r"unit-\d+-[0-9a-f]{8}\.jsonl", f.name)
279
+
280
+
281
+ @pytest.mark.covers("adapter.never_truncates", subject=SUBJECT)
282
+ def test_the_evidence_directory_is_created_when_missing(inner, tmp_path):
283
+ deep = tmp_path / "a" / "b" / "evidence"
284
+ inner.run(inner_test("pass"), evidence=deep)
285
+ assert len(list(deep.glob("*.jsonl"))) == 1
286
+
287
+
288
+ # --- claims ----------------------------------------------------------------------
289
+
290
+
291
+ @pytest.mark.covers("adapter.python")
292
+ def test_each_claim_is_its_own_record_with_the_same_outcome(inner):
293
+ inner.run(
294
+ "import pytest\n\n"
295
+ '@pytest.mark.covers("a.one")\n'
296
+ '@pytest.mark.covers("b.two", subject="s.x")\n'
297
+ '@pytest.mark.exempts("c.three", subject="s.y")\n'
298
+ "def test_it():\n assert False\n"
299
+ )
300
+ recs = inner.records()
301
+ assert len(recs) == 3
302
+ assert {r["outcome"] for r in recs} == {"fail"}
303
+ by = {r.get("covers") or r["exempts"]: r for r in recs}
304
+ assert "exempts" in by["c.three"] and "covers" not in by["c.three"]
305
+ assert by["c.three"]["subject"] == "s.y"
306
+ assert "subject" not in by["a.one"]
307
+
308
+
309
+ @pytest.mark.covers("adapter.python")
310
+ def test_class_and_module_marks_apply_to_each_test_inside(inner):
311
+ inner.run(
312
+ "import pytest\n\n"
313
+ 'pytestmark = pytest.mark.covers("mod.claim")\n\n'
314
+ '@pytest.mark.covers("cls.claim", subject="s")\n'
315
+ "class TestGroup:\n"
316
+ " def test_a(self):\n assert True\n"
317
+ " def test_b(self):\n assert True\n"
318
+ )
319
+ seen = sorted((r["test"].split("::")[-1], r["covers"]) for r in inner.records())
320
+ assert seen == [
321
+ ("test_a", "cls.claim"),
322
+ ("test_a", "mod.claim"),
323
+ ("test_b", "cls.claim"),
324
+ ("test_b", "mod.claim"),
325
+ ]
326
+
327
+
328
+ @pytest.mark.covers("adapter.python")
329
+ def test_the_same_claim_twice_is_one_record(inner):
330
+ inner.run(
331
+ "import pytest\n\n"
332
+ 'pytestmark = pytest.mark.covers("same.claim", subject="s")\n\n'
333
+ '@pytest.mark.covers("same.claim", subject="s")\n'
334
+ "def test_it():\n assert True\n"
335
+ )
336
+ assert len(inner.records()) == 1
337
+
338
+
339
+ @pytest.mark.covers("adapter.python")
340
+ def test_a_scenario_is_recorded_for_parametrized_scenario_runners(inner):
341
+ inner.run(
342
+ "import pytest\nimport temper\n\n"
343
+ '@pytest.mark.parametrize("sid", ["alerts.sold_out", "alerts.low"])\n'
344
+ "def test_scenario(request, sid):\n"
345
+ ' temper.covers(request, "alerts.low_stock", scenario=sid)\n'
346
+ ' temper.covers(request, sid, scenario=sid)\n'
347
+ )
348
+ recs = inner.records()
349
+ assert len(recs) == 4
350
+ assert {(r["covers"], r["scenario"]) for r in recs} == {
351
+ ("alerts.low_stock", "alerts.sold_out"),
352
+ ("alerts.sold_out", "alerts.sold_out"),
353
+ ("alerts.low_stock", "alerts.low"),
354
+ ("alerts.low", "alerts.low"),
355
+ }
356
+ assert list(recs[0]) == ["covers", "scenario", "test", "level", "outcome", "at"]
357
+
358
+
359
+ @pytest.mark.covers("adapter.python")
360
+ def test_a_runtime_claim_follows_the_tests_outcome(inner):
361
+ inner.run(
362
+ "import temper\n\n"
363
+ "def test_it(request):\n"
364
+ ' temper.covers(request, "late.claim")\n'
365
+ " assert False\n"
366
+ )
367
+ (rec,) = inner.records()
368
+ assert (rec["covers"], rec["outcome"]) == ("late.claim", "fail")
369
+
370
+
371
+ @pytest.mark.covers("adapter.python")
372
+ def test_exempts_has_a_runtime_form_too(inner):
373
+ inner.run(
374
+ "import temper\n\n"
375
+ "def test_it(request):\n"
376
+ ' temper.exempts(request, "inv.x", subject="s")\n'
377
+ )
378
+ (rec,) = inner.records()
379
+ assert (rec["exempts"], rec["subject"], rec["outcome"]) == ("inv.x", "s", "pass")
380
+
381
+
382
+ MALFORMED = {
383
+ "no reference": "@pytest.mark.covers()",
384
+ "empty reference": '@pytest.mark.covers("")',
385
+ "non-string reference": "@pytest.mark.covers(42)",
386
+ "second positional": '@pytest.mark.covers("a.b", "c")',
387
+ "unknown keyword": '@pytest.mark.covers("a.b", subjcet="s")',
388
+ "non-string subject": '@pytest.mark.covers("a.b", subject=3)',
389
+ "empty subject": '@pytest.mark.covers("a.b", subject="")',
390
+ }
391
+
392
+
393
+ @pytest.mark.covers("adapter.python")
394
+ @pytest.mark.parametrize("name", sorted(MALFORMED))
395
+ def test_a_malformed_mark_stops_the_run_loudly(inner, name):
396
+ result = inner.run(
397
+ "import pytest\n\n" + MALFORMED[name] + "\ndef test_it():\n assert True\n"
398
+ )
399
+ assert result.ret != 0
400
+ result.stderr.fnmatch_lines(["*test_inner.py::test_it*"])
401
+ assert inner.records() == []
402
+
403
+
404
+ @pytest.mark.covers("adapter.python")
405
+ def test_an_unwritable_evidence_directory_stops_the_run_loudly(inner, tmp_path):
406
+ blocker = tmp_path / "file"
407
+ blocker.write_text("x")
408
+ result = inner.run(inner_test("pass"), evidence=blocker / "evidence")
409
+ assert result.ret != 0
410
+
411
+
412
+ @pytest.mark.covers("adapter.python")
413
+ def test_parallel_workers_write_separate_whole_lines(inner):
414
+ body = "".join(
415
+ f'@pytest.mark.covers("demo.claim")\ndef test_{i}():\n assert True\n\n'
416
+ for i in range(40)
417
+ )
418
+ result = inner.run("import pytest\n\n" + body, "-n", "3")
419
+ assert result.ret == 0
420
+ assert len(inner.files()) >= 2
421
+ assert len(inner.records()) == 40
422
+ for line in inner.lines():
423
+ assert line.endswith("\n")
@@ -0,0 +1,91 @@
1
+ """The evidence file writer, directly."""
2
+
3
+ import json
4
+ import os
5
+ import re
6
+
7
+ import pytest
8
+
9
+ from temper import _evidence
10
+
11
+
12
+ @pytest.mark.covers("adapter.never_truncates", subject="adapter.python")
13
+ def test_each_record_is_one_write_in_append_mode(tmp_path, monkeypatch):
14
+ writes = []
15
+ real = os.write
16
+ monkeypatch.setattr(os, "write", lambda fd, data: (writes.append(data), real(fd, data))[1])
17
+ w = _evidence.Writer(tmp_path, "unit")
18
+ for i in range(5):
19
+ w.write({"covers": f"c.{i}", "test": "t", "level": "unit", "outcome": "pass"})
20
+ w.close()
21
+ assert len(writes) == 5
22
+ assert all(d.endswith(b"\n") and d.count(b"\n") == 1 for d in writes)
23
+ (f,) = tmp_path.glob("*.jsonl")
24
+ assert re.fullmatch(r"unit-%d-[0-9a-f]{8}\.jsonl" % os.getpid(), f.name)
25
+ assert len(f.read_bytes().splitlines()) == 5
26
+
27
+
28
+ @pytest.mark.covers("adapter.never_truncates", subject="adapter.python")
29
+ def test_two_writers_never_share_a_file(tmp_path):
30
+ a = _evidence.Writer(tmp_path, "unit")
31
+ b = _evidence.Writer(tmp_path, "unit")
32
+ a.write({"test": "t", "level": "unit", "outcome": "pass"})
33
+ b.write({"test": "t", "level": "unit", "outcome": "pass"})
34
+ a.close()
35
+ b.close()
36
+ assert len(list(tmp_path.glob("*.jsonl"))) == 2
37
+
38
+
39
+ @pytest.mark.covers("adapter.never_truncates", subject="adapter.python")
40
+ def test_nothing_is_created_until_the_first_record(tmp_path):
41
+ target = tmp_path / "evidence"
42
+ w = _evidence.Writer(target, "unit")
43
+ w.close()
44
+ assert not target.exists()
45
+
46
+
47
+ @pytest.mark.covers("adapter.python")
48
+ def test_a_forked_child_writes_to_its_own_file(tmp_path):
49
+ if not hasattr(os, "fork"):
50
+ pytest.skip("no fork on this platform")
51
+ w = _evidence.Writer(tmp_path, "unit")
52
+ w.write({"test": "parent", "level": "unit", "outcome": "pass"})
53
+ pid = os.fork()
54
+ if pid == 0:
55
+ try:
56
+ w.write({"test": "child", "level": "unit", "outcome": "pass"})
57
+ finally:
58
+ os._exit(0)
59
+ os.waitpid(pid, 0)
60
+ w.close()
61
+ files = sorted(tmp_path.glob("*.jsonl"))
62
+ assert len(files) == 2
63
+ for f in files:
64
+ assert len(f.read_bytes().splitlines()) == 1
65
+
66
+
67
+ @pytest.mark.covers("adapter.python")
68
+ def test_a_lone_surrogate_in_a_test_id_is_escaped_not_fatal(tmp_path):
69
+ w = _evidence.Writer(tmp_path, "unit")
70
+ w.write({"test": "bad\ud800id", "level": "unit", "outcome": "pass"})
71
+ w.close()
72
+ (f,) = tmp_path.glob("*.jsonl")
73
+ assert json.loads(f.read_bytes())["test"] == "bad\ud800id"
74
+
75
+
76
+ @pytest.mark.covers("adapter.python")
77
+ def test_non_ascii_is_written_as_utf8_not_escaped(tmp_path):
78
+ w = _evidence.Writer(tmp_path, "unit")
79
+ w.write({"test": "héllo日本", "level": "unit", "outcome": "pass"})
80
+ w.close()
81
+ (f,) = tmp_path.glob("*.jsonl")
82
+ assert "héllo日本".encode("utf-8") in f.read_bytes()
83
+
84
+
85
+ @pytest.mark.covers("adapter.python")
86
+ def test_a_level_cannot_escape_the_directory_through_the_file_name(tmp_path):
87
+ w = _evidence.Writer(tmp_path / "ev", "../../x/y")
88
+ w.write({"test": "t", "level": "../../x/y", "outcome": "pass"})
89
+ w.close()
90
+ (f,) = (tmp_path / "ev").glob("*.jsonl")
91
+ assert "/" not in f.name