temper-python 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- temper_python-0.1.0/.github/workflows/ci.yml +26 -0
- temper_python-0.1.0/.github/workflows/release.yml +40 -0
- temper_python-0.1.0/.gitignore +6 -0
- temper_python-0.1.0/LICENSE +21 -0
- temper_python-0.1.0/MATRIX.md +51 -0
- temper_python-0.1.0/Makefile +19 -0
- temper_python-0.1.0/PKG-INFO +135 -0
- temper_python-0.1.0/README.md +117 -0
- temper_python-0.1.0/pyproject.toml +34 -0
- temper_python-0.1.0/src/temper/__init__.py +26 -0
- temper_python-0.1.0/src/temper/_evidence.py +60 -0
- temper_python-0.1.0/src/temper/plugin.py +140 -0
- temper_python-0.1.0/temper/features.yml +4 -0
- temper_python-0.1.0/temper/invariants.yml +35 -0
- temper_python-0.1.0/temper/lattices/outcomes.yml +11 -0
- temper_python-0.1.0/temper/status/features.yml +3 -0
- temper_python-0.1.0/temper/status/subjects/adapter.python.yml +11 -0
- temper_python-0.1.0/temper/subjects/adapter.python.yml +9 -0
- temper_python-0.1.0/temper.yml +6 -0
- temper_python-0.1.0/tests/conftest.py +51 -0
- temper_python-0.1.0/tests/test_adapter.py +423 -0
- temper_python-0.1.0/tests/test_writer.py +91 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches: [main]
|
|
5
|
+
pull_request:
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
test:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
strategy:
|
|
11
|
+
fail-fast: false
|
|
12
|
+
matrix:
|
|
13
|
+
python: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: actions/setup-go@v5
|
|
17
|
+
with:
|
|
18
|
+
go-version: stable
|
|
19
|
+
- run: go install github.com/turbolytics/temper/cmd/temper@v0.1.0
|
|
20
|
+
- uses: actions/setup-python@v5
|
|
21
|
+
with:
|
|
22
|
+
python-version: ${{ matrix.python }}
|
|
23
|
+
- run: pip install -e '.[dev]'
|
|
24
|
+
- run: temper check
|
|
25
|
+
- run: temper run --level unit -- python -m pytest -q
|
|
26
|
+
- run: temper matrix --check
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
tags: ["v*"]
|
|
5
|
+
|
|
6
|
+
jobs:
|
|
7
|
+
build:
|
|
8
|
+
runs-on: ubuntu-latest
|
|
9
|
+
steps:
|
|
10
|
+
- uses: actions/checkout@v4
|
|
11
|
+
- uses: actions/setup-python@v5
|
|
12
|
+
with:
|
|
13
|
+
python-version: "3.13"
|
|
14
|
+
- name: The tag is the package version
|
|
15
|
+
run: |
|
|
16
|
+
version=$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml","rb"))["project"]["version"])')
|
|
17
|
+
if [ "v$version" != "$GITHUB_REF_NAME" ]; then
|
|
18
|
+
echo "tag $GITHUB_REF_NAME does not match pyproject.toml version $version" >&2
|
|
19
|
+
exit 1
|
|
20
|
+
fi
|
|
21
|
+
- run: pip install -e '.[dev]' build
|
|
22
|
+
- run: python -m pytest -q
|
|
23
|
+
- run: python -m build
|
|
24
|
+
- uses: actions/upload-artifact@v4
|
|
25
|
+
with:
|
|
26
|
+
name: dist
|
|
27
|
+
path: dist/
|
|
28
|
+
|
|
29
|
+
publish:
|
|
30
|
+
needs: build
|
|
31
|
+
runs-on: ubuntu-latest
|
|
32
|
+
environment: pypi
|
|
33
|
+
permissions:
|
|
34
|
+
id-token: write # PyPI trusted publishing; no stored token
|
|
35
|
+
steps:
|
|
36
|
+
- uses: actions/download-artifact@v4
|
|
37
|
+
with:
|
|
38
|
+
name: dist
|
|
39
|
+
path: dist/
|
|
40
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 turbolytics
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
<!-- Generated by temper. Do not edit by hand. -->
|
|
2
|
+
# temper-python matrix
|
|
3
|
+
|
|
4
|
+
Every row is a claim this project declares in `temper/`. A cell is covered only by a
|
|
5
|
+
passing evidence record from a test that ran at that level; a skip proves nothing.
|
|
6
|
+
Exempt cells are excused by a passing proof. Regenerate with `temper matrix`.
|
|
7
|
+
|
|
8
|
+
✅ covered · ☑️ exempt · ❌ failing · ⬜ missing · ⏳ unenforced · — not required
|
|
9
|
+
|
|
10
|
+
## Summary
|
|
11
|
+
|
|
12
|
+
| covered | exempt | failing | missing | unenforced |
|
|
13
|
+
| ---: | ---: | ---: | ---: | ---: |
|
|
14
|
+
| 10 | 0 | 0 | 0 | 0 |
|
|
15
|
+
|
|
16
|
+
## Features
|
|
17
|
+
|
|
18
|
+
| Feature | What it does | unit |
|
|
19
|
+
| --- | --- | --- |
|
|
20
|
+
| `adapter.python` | Records what each pytest test proves and how it ended, as Temper evidence. | ✅ |
|
|
21
|
+
|
|
22
|
+
## Invariants
|
|
23
|
+
|
|
24
|
+
### adapter
|
|
25
|
+
|
|
26
|
+
| Invariant | Subject | unit |
|
|
27
|
+
| --- | --- | --- |
|
|
28
|
+
| `adapter.failure_never_passes` | `adapter.python` | ✅ |
|
|
29
|
+
| `adapter.level_from_runner` | `adapter.python` | ✅ |
|
|
30
|
+
| `adapter.never_truncates` | `adapter.python` | ✅ |
|
|
31
|
+
| `adapter.silent_without_runner` | `adapter.python` | ✅ |
|
|
32
|
+
| `adapter.skip_records_skip` | `adapter.python` | ✅ |
|
|
33
|
+
|
|
34
|
+
- `adapter.failure_never_passes` — A test that fails, errors or panics never records pass.
|
|
35
|
+
- `adapter.level_from_runner` — The level a record carries comes only from TEMPER_LEVEL; no test can set it.
|
|
36
|
+
- `adapter.never_truncates` — The adapter only appends, to a file unique to its process; it never truncates or removes evidence.
|
|
37
|
+
- `adapter.silent_without_runner` — Without TEMPER_LEVEL and TEMPER_EVIDENCE the adapter records nothing and changes nothing.
|
|
38
|
+
- `adapter.skip_records_skip` — A skipped test records skip.
|
|
39
|
+
|
|
40
|
+
## Lattices
|
|
41
|
+
|
|
42
|
+
### `outcomes`
|
|
43
|
+
|
|
44
|
+
Every outcome an evidence record can carry, from the spec's record format. An adapter answers for each one it can produce.
|
|
45
|
+
|
|
46
|
+
| Member | `adapter.python` |
|
|
47
|
+
| --- | --- |
|
|
48
|
+
| `pass` | ✅ |
|
|
49
|
+
| `fail` | ✅ |
|
|
50
|
+
| `skip` | ✅ |
|
|
51
|
+
| `error` | ✅ |
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
PYTHON ?= python
|
|
2
|
+
TEMPER ?= temper
|
|
3
|
+
|
|
4
|
+
.PHONY: test matrix check-matrix
|
|
5
|
+
|
|
6
|
+
test:
|
|
7
|
+
$(PYTHON) -m pytest -q
|
|
8
|
+
|
|
9
|
+
matrix:
|
|
10
|
+
$(TEMPER) check
|
|
11
|
+
$(TEMPER) clean
|
|
12
|
+
$(TEMPER) run --level unit -- $(PYTHON) -m pytest -q
|
|
13
|
+
$(TEMPER) matrix --all-levels
|
|
14
|
+
|
|
15
|
+
check-matrix:
|
|
16
|
+
$(TEMPER) check
|
|
17
|
+
$(TEMPER) clean
|
|
18
|
+
$(TEMPER) run --level unit -- $(PYTHON) -m pytest -q
|
|
19
|
+
$(TEMPER) matrix --check
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: temper-python
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: pytest adapter for Temper: tests state what they prove, and write evidence.
|
|
5
|
+
Project-URL: Homepage, https://github.com/turbolytics/temper-python
|
|
6
|
+
Project-URL: Temper, https://github.com/turbolytics/temper
|
|
7
|
+
License: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Framework :: Pytest
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Topic :: Software Development :: Testing
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Requires-Dist: pytest>=7
|
|
15
|
+
Provides-Extra: dev
|
|
16
|
+
Requires-Dist: pytest-xdist; extra == 'dev'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# temper-python
|
|
20
|
+
|
|
21
|
+
The [Temper](https://github.com/turbolytics/temper) adapter for pytest. A test
|
|
22
|
+
states what it proves; the adapter records how it ended; `temper matrix` turns
|
|
23
|
+
those records into a matrix of what is proven and what is not.
|
|
24
|
+
|
|
25
|
+
## Getting started
|
|
26
|
+
|
|
27
|
+
You need the `temper` CLI ([install](https://github.com/turbolytics/temper#install))
|
|
28
|
+
and a project with `temper.yml` and a `temper/` directory of declarations.
|
|
29
|
+
`temper guide` explains both.
|
|
30
|
+
|
|
31
|
+
**1. Install the adapter.**
|
|
32
|
+
|
|
33
|
+
```sh
|
|
34
|
+
pip install temper-python
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
It is a pytest plugin: nothing to import or enable.
|
|
38
|
+
|
|
39
|
+
**2. Mark each test with what it proves.**
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
import pytest
|
|
43
|
+
|
|
44
|
+
@pytest.mark.covers("store.reserve.never_negative", subject="store.file")
|
|
45
|
+
def test_reserve_never_goes_negative(): ...
|
|
46
|
+
|
|
47
|
+
@pytest.mark.exempts("store.persists_across_restart", subject="store.memory")
|
|
48
|
+
def test_memory_store_forgets_on_restart(): ...
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
**3. Run the tests through `temper run`, then build the matrix.**
|
|
52
|
+
|
|
53
|
+
```sh
|
|
54
|
+
temper clean
|
|
55
|
+
temper run --level unit -- pytest
|
|
56
|
+
temper matrix --all-levels
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
That is the whole integration. Plain `pytest` behaves exactly as before: without
|
|
60
|
+
the variables `temper run` sets, the plugin records nothing.
|
|
61
|
+
|
|
62
|
+
## What a test can say
|
|
63
|
+
|
|
64
|
+
| Mark | Meaning |
|
|
65
|
+
|---|---|
|
|
66
|
+
| `@pytest.mark.covers(ref)` | The test proves a feature or lattice member (`lattice/key`). |
|
|
67
|
+
| `@pytest.mark.covers(ref, subject=id)` | The test proves an invariant or lattice member for that subject. |
|
|
68
|
+
| `@pytest.mark.exempts(ref, subject=id)` | The test proves the subject is excused. Use it only in the test a declaration names in `proven_by`. |
|
|
69
|
+
| `scenario=id` (on either) | The claim came from that declared scenario. |
|
|
70
|
+
|
|
71
|
+
Marks on a class or a module apply to every test inside. A test may carry many
|
|
72
|
+
marks; each is one record with the same outcome.
|
|
73
|
+
|
|
74
|
+
A scenario runner that parametrizes one test over many scenarios claims from
|
|
75
|
+
inside the test, where the scenario is known:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
import temper
|
|
79
|
+
|
|
80
|
+
@pytest.mark.parametrize("sid", SCENARIOS)
|
|
81
|
+
def test_scenario(request, sid):
|
|
82
|
+
temper.covers(request, "alerts.low_stock", scenario=sid)
|
|
83
|
+
temper.covers(request, sid, scenario=sid)
|
|
84
|
+
...
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
`temper.exempts(request, ref, subject=...)` is the same for exemptions.
|
|
88
|
+
|
|
89
|
+
There is no way to set a level. The runner does (`temper run --level`), and a
|
|
90
|
+
test that changes `TEMPER_LEVEL` in its own environment changes nothing: the
|
|
91
|
+
environment is read once, when pytest starts.
|
|
92
|
+
|
|
93
|
+
A malformed mark (no reference, a stray keyword such as `level=`, a non-string)
|
|
94
|
+
stops the run with a message naming the test, when the runner is active.
|
|
95
|
+
|
|
96
|
+
## How outcomes are recorded
|
|
97
|
+
|
|
98
|
+
A record is written once the test is final, after fixtures have torn down.
|
|
99
|
+
|
|
100
|
+
| Outcome | When |
|
|
101
|
+
|---|---|
|
|
102
|
+
| `pass` | The test, its setup and its teardown all passed. The only outcome that counts. |
|
|
103
|
+
| `fail` | An assertion failed, `pytest.fail` was called, or the test was `xfail` (whether it failed or passed). |
|
|
104
|
+
| `skip` | The test was skipped, in any phase. |
|
|
105
|
+
| `error` | An exception other than a failed assertion, or any failure in setup or teardown. |
|
|
106
|
+
|
|
107
|
+
A test that fails and then skips records `fail`. Evidence goes to one new file
|
|
108
|
+
per process, `<level>-<pid>-<8 hex>.jsonl`, opened for append, one write per
|
|
109
|
+
line. Existing evidence is never truncated, so `pytest-xdist` works without
|
|
110
|
+
configuration. The contract every adapter follows is
|
|
111
|
+
[docs/adapters.md](https://github.com/turbolytics/temper/blob/main/docs/adapters.md).
|
|
112
|
+
|
|
113
|
+
Not verified: retry plugins such as `pytest-rerunfailures`.
|
|
114
|
+
|
|
115
|
+
## Development
|
|
116
|
+
|
|
117
|
+
```sh
|
|
118
|
+
python -m venv .venv && .venv/bin/pip install -e '.[dev]'
|
|
119
|
+
make matrix # clean, run the tests through temper, rebuild MATRIX.md
|
|
120
|
+
make check-matrix # fail if the committed matrix is stale
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
This project is declared with Temper in `temper/`, and its own tests prove the
|
|
124
|
+
adapter claims with the adapter: see [MATRIX.md](MATRIX.md).
|
|
125
|
+
|
|
126
|
+
## Releasing
|
|
127
|
+
|
|
128
|
+
Set `version` in `pyproject.toml`, merge, then push the matching tag:
|
|
129
|
+
|
|
130
|
+
```sh
|
|
131
|
+
git tag v0.1.0 && git push origin v0.1.0
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
The release workflow checks the tag matches the version, runs the tests, builds,
|
|
135
|
+
and publishes to PyPI through trusted publishing.
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# temper-python
|
|
2
|
+
|
|
3
|
+
The [Temper](https://github.com/turbolytics/temper) adapter for pytest. A test
|
|
4
|
+
states what it proves; the adapter records how it ended; `temper matrix` turns
|
|
5
|
+
those records into a matrix of what is proven and what is not.
|
|
6
|
+
|
|
7
|
+
## Getting started
|
|
8
|
+
|
|
9
|
+
You need the `temper` CLI ([install](https://github.com/turbolytics/temper#install))
|
|
10
|
+
and a project with `temper.yml` and a `temper/` directory of declarations.
|
|
11
|
+
`temper guide` explains both.
|
|
12
|
+
|
|
13
|
+
**1. Install the adapter.**
|
|
14
|
+
|
|
15
|
+
```sh
|
|
16
|
+
pip install temper-python
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
It is a pytest plugin: nothing to import or enable.
|
|
20
|
+
|
|
21
|
+
**2. Mark each test with what it proves.**
|
|
22
|
+
|
|
23
|
+
```python
|
|
24
|
+
import pytest
|
|
25
|
+
|
|
26
|
+
@pytest.mark.covers("store.reserve.never_negative", subject="store.file")
|
|
27
|
+
def test_reserve_never_goes_negative(): ...
|
|
28
|
+
|
|
29
|
+
@pytest.mark.exempts("store.persists_across_restart", subject="store.memory")
|
|
30
|
+
def test_memory_store_forgets_on_restart(): ...
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
**3. Run the tests through `temper run`, then build the matrix.**
|
|
34
|
+
|
|
35
|
+
```sh
|
|
36
|
+
temper clean
|
|
37
|
+
temper run --level unit -- pytest
|
|
38
|
+
temper matrix --all-levels
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
That is the whole integration. Plain `pytest` behaves exactly as before: without
|
|
42
|
+
the variables `temper run` sets, the plugin records nothing.
|
|
43
|
+
|
|
44
|
+
## What a test can say
|
|
45
|
+
|
|
46
|
+
| Mark | Meaning |
|
|
47
|
+
|---|---|
|
|
48
|
+
| `@pytest.mark.covers(ref)` | The test proves a feature or lattice member (`lattice/key`). |
|
|
49
|
+
| `@pytest.mark.covers(ref, subject=id)` | The test proves an invariant or lattice member for that subject. |
|
|
50
|
+
| `@pytest.mark.exempts(ref, subject=id)` | The test proves the subject is excused. Use it only in the test a declaration names in `proven_by`. |
|
|
51
|
+
| `scenario=id` (on either) | The claim came from that declared scenario. |
|
|
52
|
+
|
|
53
|
+
Marks on a class or a module apply to every test inside. A test may carry many
|
|
54
|
+
marks; each is one record with the same outcome.
|
|
55
|
+
|
|
56
|
+
A scenario runner that parametrizes one test over many scenarios claims from
|
|
57
|
+
inside the test, where the scenario is known:
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
import temper
|
|
61
|
+
|
|
62
|
+
@pytest.mark.parametrize("sid", SCENARIOS)
|
|
63
|
+
def test_scenario(request, sid):
|
|
64
|
+
temper.covers(request, "alerts.low_stock", scenario=sid)
|
|
65
|
+
temper.covers(request, sid, scenario=sid)
|
|
66
|
+
...
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
`temper.exempts(request, ref, subject=...)` is the same for exemptions.
|
|
70
|
+
|
|
71
|
+
There is no way to set a level. The runner does (`temper run --level`), and a
|
|
72
|
+
test that changes `TEMPER_LEVEL` in its own environment changes nothing: the
|
|
73
|
+
environment is read once, when pytest starts.
|
|
74
|
+
|
|
75
|
+
A malformed mark (no reference, a stray keyword such as `level=`, a non-string)
|
|
76
|
+
stops the run with a message naming the test, when the runner is active.
|
|
77
|
+
|
|
78
|
+
## How outcomes are recorded
|
|
79
|
+
|
|
80
|
+
A record is written once the test is final, after fixtures have torn down.
|
|
81
|
+
|
|
82
|
+
| Outcome | When |
|
|
83
|
+
|---|---|
|
|
84
|
+
| `pass` | The test, its setup and its teardown all passed. The only outcome that counts. |
|
|
85
|
+
| `fail` | An assertion failed, `pytest.fail` was called, or the test was `xfail` (whether it failed or passed). |
|
|
86
|
+
| `skip` | The test was skipped, in any phase. |
|
|
87
|
+
| `error` | An exception other than a failed assertion, or any failure in setup or teardown. |
|
|
88
|
+
|
|
89
|
+
A test that fails and then skips records `fail`. Evidence goes to one new file
|
|
90
|
+
per process, `<level>-<pid>-<8 hex>.jsonl`, opened for append, one write per
|
|
91
|
+
line. Existing evidence is never truncated, so `pytest-xdist` works without
|
|
92
|
+
configuration. The contract every adapter follows is
|
|
93
|
+
[docs/adapters.md](https://github.com/turbolytics/temper/blob/main/docs/adapters.md).
|
|
94
|
+
|
|
95
|
+
Not verified: retry plugins such as `pytest-rerunfailures`.
|
|
96
|
+
|
|
97
|
+
## Development
|
|
98
|
+
|
|
99
|
+
```sh
|
|
100
|
+
python -m venv .venv && .venv/bin/pip install -e '.[dev]'
|
|
101
|
+
make matrix # clean, run the tests through temper, rebuild MATRIX.md
|
|
102
|
+
make check-matrix # fail if the committed matrix is stale
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
This project is declared with Temper in `temper/`, and its own tests prove the
|
|
106
|
+
adapter claims with the adapter: see [MATRIX.md](MATRIX.md).
|
|
107
|
+
|
|
108
|
+
## Releasing
|
|
109
|
+
|
|
110
|
+
Set `version` in `pyproject.toml`, merge, then push the matching tag:
|
|
111
|
+
|
|
112
|
+
```sh
|
|
113
|
+
git tag v0.1.0 && git push origin v0.1.0
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The release workflow checks the tag matches the version, runs the tests, builds,
|
|
117
|
+
and publishes to PyPI through trusted publishing.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "temper-python"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "pytest adapter for Temper: tests state what they prove, and write evidence."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
dependencies = ["pytest>=7"]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Framework :: Pytest",
|
|
15
|
+
"License :: OSI Approved :: MIT License",
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Topic :: Software Development :: Testing",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[project.urls]
|
|
21
|
+
Homepage = "https://github.com/turbolytics/temper-python"
|
|
22
|
+
Temper = "https://github.com/turbolytics/temper"
|
|
23
|
+
|
|
24
|
+
[project.entry-points.pytest11]
|
|
25
|
+
temper = "temper.plugin"
|
|
26
|
+
|
|
27
|
+
[project.optional-dependencies]
|
|
28
|
+
dev = ["pytest-xdist"]
|
|
29
|
+
|
|
30
|
+
[tool.hatch.build.targets.wheel]
|
|
31
|
+
packages = ["src/temper"]
|
|
32
|
+
|
|
33
|
+
[tool.pytest.ini_options]
|
|
34
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Temper adapter for pytest.
|
|
2
|
+
|
|
3
|
+
Mark a test with what it proves::
|
|
4
|
+
|
|
5
|
+
@pytest.mark.covers("store.reserve.never_negative", subject="store.file")
|
|
6
|
+
def test_reserve(): ...
|
|
7
|
+
|
|
8
|
+
or claim from inside it, which is how a parametrized scenario runner names its
|
|
9
|
+
scenario::
|
|
10
|
+
|
|
11
|
+
temper.covers(request, "alerts.low_stock", scenario="alerts.sold_out")
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from .plugin import add_claim
|
|
15
|
+
|
|
16
|
+
__all__ = ["covers", "exempts"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def covers(request, ref, *, subject=None, scenario=None):
|
|
20
|
+
"""The running test proves ``ref`` (for ``subject``, when it applies per subject)."""
|
|
21
|
+
add_claim(request, "covers", ref, subject, scenario)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def exempts(request, ref, *, subject=None, scenario=None):
|
|
25
|
+
"""The running test proves ``subject`` is excused from ``ref``."""
|
|
26
|
+
add_claim(request, "exempts", ref, subject, scenario)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""The evidence file: append-only, one file per process, one write per line."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import secrets
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
_SEPARATORS = (",", ":")
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def encode(record):
|
|
13
|
+
"""One record as one UTF-8 line, keys in the order given, compact."""
|
|
14
|
+
try:
|
|
15
|
+
return (json.dumps(record, separators=_SEPARATORS, ensure_ascii=False) + "\n").encode("utf-8")
|
|
16
|
+
except UnicodeEncodeError:
|
|
17
|
+
# A lone surrogate (an odd test id) cannot be UTF-8; JSON can still escape it.
|
|
18
|
+
return (json.dumps(record, separators=_SEPARATORS, ensure_ascii=True) + "\n").encode("utf-8")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Writer:
|
|
22
|
+
"""Appends records to ``<level>-<pid>-<8 hex>.jsonl`` in a directory.
|
|
23
|
+
|
|
24
|
+
The file is created on the first record, never opened for truncation, and
|
|
25
|
+
never shared: a forked child opens its own.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
def __init__(self, directory, level):
|
|
29
|
+
self._dir = Path(directory)
|
|
30
|
+
self._level = level
|
|
31
|
+
self._fd = None
|
|
32
|
+
self._pid = None
|
|
33
|
+
|
|
34
|
+
def ensure_directory(self):
|
|
35
|
+
self._dir.mkdir(parents=True, exist_ok=True)
|
|
36
|
+
|
|
37
|
+
def write(self, record):
|
|
38
|
+
data = encode(record)
|
|
39
|
+
fd = self._open()
|
|
40
|
+
written = os.write(fd, data)
|
|
41
|
+
while written < len(data): # a short write is rare; finish the line
|
|
42
|
+
written += os.write(fd, data[written:])
|
|
43
|
+
|
|
44
|
+
def close(self):
|
|
45
|
+
if self._fd is not None and self._pid == os.getpid():
|
|
46
|
+
os.close(self._fd)
|
|
47
|
+
self._fd = None
|
|
48
|
+
|
|
49
|
+
def _open(self):
|
|
50
|
+
pid = os.getpid()
|
|
51
|
+
if self._fd is not None and self._pid == pid:
|
|
52
|
+
return self._fd
|
|
53
|
+
self._fd = None # inherited across a fork: leave the parent's file alone
|
|
54
|
+
self.ensure_directory()
|
|
55
|
+
safe = re.sub(r"[^A-Za-z0-9._-]", "_", self._level)
|
|
56
|
+
name = f"{safe}-{pid}-{secrets.token_hex(4)}.jsonl"
|
|
57
|
+
flags = os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_EXCL
|
|
58
|
+
self._fd = os.open(self._dir / name, flags, 0o644)
|
|
59
|
+
self._pid = pid
|
|
60
|
+
return self._fd
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""The pytest plugin: reads what each test claims, records how it ended.
|
|
2
|
+
|
|
3
|
+
The level and the evidence directory come from ``TEMPER_LEVEL`` and
|
|
4
|
+
``TEMPER_EVIDENCE``, read once when pytest starts. Without both, the plugin
|
|
5
|
+
does nothing.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
from datetime import datetime, timezone
|
|
10
|
+
|
|
11
|
+
import pytest
|
|
12
|
+
|
|
13
|
+
from ._evidence import Writer
|
|
14
|
+
|
|
15
|
+
CLAIM_KINDS = ("covers", "exempts")
|
|
16
|
+
_KEYWORDS = ("subject", "scenario")
|
|
17
|
+
|
|
18
|
+
_CLAIMS = pytest.StashKey() # runtime claims made through temper.covers / temper.exempts
|
|
19
|
+
_OUTCOMES = pytest.StashKey() # one outcome per phase report
|
|
20
|
+
_STATE = pytest.StashKey()
|
|
21
|
+
|
|
22
|
+
_PRECEDENCE = ("error", "fail", "skip", "pass") # first one present wins
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class _State:
|
|
26
|
+
def __init__(self, level, directory, run):
|
|
27
|
+
self.level = level
|
|
28
|
+
self.run = run
|
|
29
|
+
self.writer = Writer(directory, level)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _claim(kind, args, kwargs, where):
|
|
33
|
+
"""Validate one claim and return (kind, ref, subject, scenario)."""
|
|
34
|
+
if len(args) != 1 or not isinstance(args[0], str) or not args[0]:
|
|
35
|
+
raise ValueError(f"{where}: {kind} takes one non-empty reference, as a string")
|
|
36
|
+
for key in kwargs:
|
|
37
|
+
if key not in _KEYWORDS:
|
|
38
|
+
hint = " (the level comes from the runner, never the test)" if key == "level" else ""
|
|
39
|
+
raise ValueError(f"{where}: {kind} has no keyword {key!r}{hint}; use subject or scenario")
|
|
40
|
+
for key in _KEYWORDS:
|
|
41
|
+
value = kwargs.get(key)
|
|
42
|
+
if value is not None and (not isinstance(value, str) or not value):
|
|
43
|
+
raise ValueError(f"{where}: {kind} {key} must be a non-empty string")
|
|
44
|
+
return kind, args[0], kwargs.get("subject"), kwargs.get("scenario")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def add_claim(request, kind, ref, subject=None, scenario=None):
|
|
48
|
+
"""Claim from inside a test; the claim follows the test's outcome."""
|
|
49
|
+
node = getattr(request, "node", None)
|
|
50
|
+
if not isinstance(node, pytest.Item):
|
|
51
|
+
raise TypeError(f"temper.{kind} needs the `request` fixture of a test")
|
|
52
|
+
kwargs = {k: v for k, v in (("subject", subject), ("scenario", scenario)) if v is not None}
|
|
53
|
+
claim = _claim(kind, (ref,), kwargs, node.nodeid)
|
|
54
|
+
node.stash.setdefault(_CLAIMS, []).append(claim)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _claims(item):
|
|
58
|
+
seen, out = set(), []
|
|
59
|
+
marked = [_claim(m.name, m.args, m.kwargs, item.nodeid) for m in item.iter_markers() if m.name in CLAIM_KINDS]
|
|
60
|
+
for claim in marked + item.stash.get(_CLAIMS, []):
|
|
61
|
+
if claim not in seen:
|
|
62
|
+
seen.add(claim)
|
|
63
|
+
out.append(claim)
|
|
64
|
+
return out
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def pytest_configure(config):
|
|
68
|
+
config.addinivalue_line("markers", "covers(ref, subject=None, scenario=None): the test proves this Temper claim")
|
|
69
|
+
config.addinivalue_line("markers", "exempts(ref, subject=None, scenario=None): the test proves this exemption")
|
|
70
|
+
level = os.environ.get("TEMPER_LEVEL", "")
|
|
71
|
+
directory = os.environ.get("TEMPER_EVIDENCE", "")
|
|
72
|
+
if not level or not directory:
|
|
73
|
+
return
|
|
74
|
+
state = _State(level, directory, os.environ.get("TEMPER_RUN", ""))
|
|
75
|
+
try:
|
|
76
|
+
state.writer.ensure_directory()
|
|
77
|
+
except OSError as err:
|
|
78
|
+
raise pytest.UsageError(f"temper: cannot write evidence to TEMPER_EVIDENCE={directory!r}: {err}")
|
|
79
|
+
config.stash[_STATE] = state
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def pytest_unconfigure(config):
|
|
83
|
+
state = config.stash.get(_STATE, None)
|
|
84
|
+
if state is not None:
|
|
85
|
+
state.writer.close()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def pytest_collection_modifyitems(config, items):
|
|
89
|
+
if config.stash.get(_STATE, None) is None:
|
|
90
|
+
return
|
|
91
|
+
for item in items:
|
|
92
|
+
try:
|
|
93
|
+
_claims(item)
|
|
94
|
+
except ValueError as err:
|
|
95
|
+
raise pytest.UsageError(f"temper: {err}")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _outcome(report, call):
|
|
99
|
+
if hasattr(report, "wasxfail"):
|
|
100
|
+
return "fail" # an expected failure never proves anything, whichever way it went
|
|
101
|
+
if report.skipped:
|
|
102
|
+
return "skip"
|
|
103
|
+
if report.passed:
|
|
104
|
+
return "pass"
|
|
105
|
+
if report.when != "call":
|
|
106
|
+
return "error" # setup or teardown broke
|
|
107
|
+
excinfo = call.excinfo
|
|
108
|
+
if excinfo is None or isinstance(excinfo.value, (AssertionError, pytest.fail.Exception)):
|
|
109
|
+
return "fail"
|
|
110
|
+
return "error"
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@pytest.hookimpl(hookwrapper=True, tryfirst=True)
|
|
114
|
+
def pytest_runtest_makereport(item, call):
|
|
115
|
+
outcome = yield
|
|
116
|
+
if item.config.stash.get(_STATE, None) is None:
|
|
117
|
+
return
|
|
118
|
+
item.stash.setdefault(_OUTCOMES, []).append(_outcome(outcome.get_result(), call))
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@pytest.hookimpl(hookwrapper=True)
|
|
122
|
+
def pytest_runtest_protocol(item, nextitem):
|
|
123
|
+
yield
|
|
124
|
+
state = item.config.stash.get(_STATE, None)
|
|
125
|
+
outcomes = item.stash.get(_OUTCOMES, None)
|
|
126
|
+
if state is None or not outcomes:
|
|
127
|
+
return
|
|
128
|
+
final = next(o for o in _PRECEDENCE if o in outcomes)
|
|
129
|
+
at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
130
|
+
for kind, ref, subject, scenario in _claims(item):
|
|
131
|
+
record = {kind: ref}
|
|
132
|
+
if subject:
|
|
133
|
+
record["subject"] = subject
|
|
134
|
+
if scenario:
|
|
135
|
+
record["scenario"] = scenario
|
|
136
|
+
record.update(test=item.nodeid, level=state.level, outcome=final)
|
|
137
|
+
if state.run:
|
|
138
|
+
record["run"] = state.run
|
|
139
|
+
record["at"] = at
|
|
140
|
+
state.writer.write(record)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
invariants:
|
|
2
|
+
- id: adapter.failure_never_passes
|
|
3
|
+
claim: A test that fails, errors or panics never records pass.
|
|
4
|
+
applies_to: adapter
|
|
5
|
+
class: safety
|
|
6
|
+
family: adapter
|
|
7
|
+
requires: [unit]
|
|
8
|
+
|
|
9
|
+
- id: adapter.skip_records_skip
|
|
10
|
+
claim: A skipped test records skip.
|
|
11
|
+
applies_to: adapter
|
|
12
|
+
class: safety
|
|
13
|
+
family: adapter
|
|
14
|
+
requires: [unit]
|
|
15
|
+
|
|
16
|
+
- id: adapter.level_from_runner
|
|
17
|
+
claim: The level a record carries comes only from TEMPER_LEVEL; no test can set it.
|
|
18
|
+
applies_to: adapter
|
|
19
|
+
class: safety
|
|
20
|
+
family: adapter
|
|
21
|
+
requires: [unit]
|
|
22
|
+
|
|
23
|
+
- id: adapter.silent_without_runner
|
|
24
|
+
claim: Without TEMPER_LEVEL and TEMPER_EVIDENCE the adapter records nothing and changes nothing.
|
|
25
|
+
applies_to: adapter
|
|
26
|
+
class: safety
|
|
27
|
+
family: adapter
|
|
28
|
+
requires: [unit]
|
|
29
|
+
|
|
30
|
+
- id: adapter.never_truncates
|
|
31
|
+
claim: The adapter only appends, to a file unique to its process; it never truncates or removes evidence.
|
|
32
|
+
applies_to: adapter
|
|
33
|
+
class: safety
|
|
34
|
+
family: adapter
|
|
35
|
+
requires: [unit]
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
id: outcomes
|
|
2
|
+
description: >
|
|
3
|
+
Every outcome an evidence record can carry, from the spec's record format.
|
|
4
|
+
An adapter answers for each one it can produce.
|
|
5
|
+
applies_to: adapter
|
|
6
|
+
requires: [unit]
|
|
7
|
+
members:
|
|
8
|
+
- key: pass
|
|
9
|
+
- key: fail
|
|
10
|
+
- key: skip
|
|
11
|
+
- key: error
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Generated by temper. Do not edit by hand.
|
|
2
|
+
# One line per invariant or lattice member that applies to adapter.python, sorted by id: the status at each level.
|
|
3
|
+
adapter.failure_never_passes: {unit: covered}
|
|
4
|
+
adapter.level_from_runner: {unit: covered}
|
|
5
|
+
adapter.never_truncates: {unit: covered}
|
|
6
|
+
adapter.silent_without_runner: {unit: covered}
|
|
7
|
+
adapter.skip_records_skip: {unit: covered}
|
|
8
|
+
outcomes/error: {unit: covered}
|
|
9
|
+
outcomes/fail: {unit: covered}
|
|
10
|
+
outcomes/pass: {unit: covered}
|
|
11
|
+
outcomes/skip: {unit: covered}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Helpers for running an inner pytest, as a runner would, and reading what it wrote."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
pytest_plugins = ["pytester"]
|
|
9
|
+
|
|
10
|
+
VARS = ("TEMPER_LEVEL", "TEMPER_EVIDENCE", "TEMPER_RUN")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Inner:
|
|
14
|
+
"""Runs a test file in a fresh pytest process with an explicit Temper environment."""
|
|
15
|
+
|
|
16
|
+
def __init__(self, pytester, monkeypatch, tmp_path):
|
|
17
|
+
self.pytester = pytester
|
|
18
|
+
self.monkeypatch = monkeypatch
|
|
19
|
+
self.dir = tmp_path / "evidence"
|
|
20
|
+
|
|
21
|
+
def run(self, source, *args, level="unit", run=None, runner=True, evidence=None):
|
|
22
|
+
for name in VARS:
|
|
23
|
+
self.monkeypatch.delenv(name, raising=False)
|
|
24
|
+
if runner:
|
|
25
|
+
self.monkeypatch.setenv("TEMPER_LEVEL", level)
|
|
26
|
+
self.monkeypatch.setenv("TEMPER_EVIDENCE", str(evidence or self.dir))
|
|
27
|
+
if run:
|
|
28
|
+
self.monkeypatch.setenv("TEMPER_RUN", run)
|
|
29
|
+
self.pytester.makepyfile(test_inner=source)
|
|
30
|
+
return self.pytester.runpytest_subprocess(*args)
|
|
31
|
+
|
|
32
|
+
def files(self):
|
|
33
|
+
return sorted(self.dir.glob("*.jsonl")) if self.dir.exists() else []
|
|
34
|
+
|
|
35
|
+
def lines(self):
|
|
36
|
+
out = []
|
|
37
|
+
for f in self.files():
|
|
38
|
+
out.extend(f.read_bytes().decode("utf-8").splitlines(keepends=True))
|
|
39
|
+
return out
|
|
40
|
+
|
|
41
|
+
def records(self):
|
|
42
|
+
return [json.loads(line) for line in self.lines()]
|
|
43
|
+
|
|
44
|
+
def outcomes(self):
|
|
45
|
+
"""test name -> outcome, for inner files with one claim per test."""
|
|
46
|
+
return {r["test"].split("::", 1)[1]: r["outcome"] for r in self.records()}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@pytest.fixture
|
|
50
|
+
def inner(pytester, monkeypatch, tmp_path):
|
|
51
|
+
return Inner(pytester, monkeypatch, tmp_path)
|
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
"""The adapter contract (docs/adapters.md in turbolytics/temper), proven through real pytest runs."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
SUBJECT = "adapter.python"
|
|
10
|
+
CLAIM = '@pytest.mark.covers("demo.claim")\n'
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def inner_test(body, mark=CLAIM, name="test_it"):
|
|
14
|
+
return "import pytest\n\n" + mark + f"def {name}():\n" + "".join(
|
|
15
|
+
f" {line}\n" for line in body.strip().splitlines()
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@pytest.mark.covers("adapter.python")
|
|
20
|
+
@pytest.mark.covers("outcomes/pass", subject=SUBJECT)
|
|
21
|
+
def test_pass_writes_one_exact_record(inner):
|
|
22
|
+
inner.run(
|
|
23
|
+
"import pytest\n\n"
|
|
24
|
+
'@pytest.mark.covers("sink.flush.keeps_batch", subject="sink.kafka")\n'
|
|
25
|
+
"def test_flush():\n assert True\n",
|
|
26
|
+
run="ci-1",
|
|
27
|
+
)
|
|
28
|
+
(line,) = inner.lines()
|
|
29
|
+
assert line.endswith("\n") and not line.endswith("\n\n")
|
|
30
|
+
rec = json.loads(line)
|
|
31
|
+
assert list(rec) == ["covers", "subject", "test", "level", "outcome", "run", "at"]
|
|
32
|
+
assert rec["covers"] == "sink.flush.keeps_batch"
|
|
33
|
+
assert rec["subject"] == "sink.kafka"
|
|
34
|
+
assert rec["test"] == "test_inner.py::test_flush"
|
|
35
|
+
assert (rec["level"], rec["outcome"], rec["run"]) == ("unit", "pass", "ci-1")
|
|
36
|
+
assert re.fullmatch(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ", rec["at"])
|
|
37
|
+
assert line == json.dumps(rec, separators=(",", ":"), ensure_ascii=False) + "\n"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@pytest.mark.covers("adapter.python")
|
|
41
|
+
def test_empty_optional_keys_are_left_out(inner):
|
|
42
|
+
inner.run(inner_test("pass"))
|
|
43
|
+
(rec,) = inner.records()
|
|
44
|
+
assert list(rec) == ["covers", "test", "level", "outcome", "at"]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@pytest.mark.covers("adapter.python")
|
|
48
|
+
def test_unmarked_tests_record_nothing(inner):
|
|
49
|
+
inner.run("def test_plain():\n assert True\n")
|
|
50
|
+
assert inner.files() == []
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# --- adapter.failure_never_passes ------------------------------------------------
|
|
54
|
+
|
|
55
|
+
FAILURES = {
|
|
56
|
+
"assert": ("assert 1 == 2", "fail"),
|
|
57
|
+
"pytest_fail": ("pytest.fail('no')", "fail"),
|
|
58
|
+
"exception": ("raise ValueError('boom')", "error"),
|
|
59
|
+
"xfail_fails": ("pytest.xfail('expected')", "fail"),
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
|
|
64
|
+
@pytest.mark.covers("outcomes/fail", subject=SUBJECT)
|
|
65
|
+
@pytest.mark.covers("outcomes/error", subject=SUBJECT)
|
|
66
|
+
@pytest.mark.parametrize("name", sorted(FAILURES))
|
|
67
|
+
def test_a_failing_body_never_records_pass(inner, name):
|
|
68
|
+
body, want = FAILURES[name]
|
|
69
|
+
inner.run(inner_test(body))
|
|
70
|
+
(rec,) = inner.records()
|
|
71
|
+
assert rec["outcome"] == want
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
|
|
75
|
+
@pytest.mark.covers("outcomes/error", subject=SUBJECT)
|
|
76
|
+
def test_a_setup_error_records_error(inner):
|
|
77
|
+
inner.run(
|
|
78
|
+
"import pytest\n\n"
|
|
79
|
+
"@pytest.fixture\ndef broken():\n raise RuntimeError('setup')\n\n"
|
|
80
|
+
'@pytest.mark.covers("demo.claim")\n'
|
|
81
|
+
"def test_it(broken):\n assert True\n"
|
|
82
|
+
)
|
|
83
|
+
(rec,) = inner.records()
|
|
84
|
+
assert rec["outcome"] == "error"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
|
|
88
|
+
@pytest.mark.covers("outcomes/error", subject=SUBJECT)
|
|
89
|
+
def test_a_teardown_error_after_a_passing_body_records_error_not_pass(inner):
|
|
90
|
+
inner.run(
|
|
91
|
+
"import pytest\n\n"
|
|
92
|
+
"@pytest.fixture\ndef leaky():\n yield\n raise RuntimeError('teardown')\n\n"
|
|
93
|
+
'@pytest.mark.covers("demo.claim")\n'
|
|
94
|
+
"def test_it(leaky):\n assert True\n"
|
|
95
|
+
)
|
|
96
|
+
(rec,) = inner.records() # exactly one record, written after teardown
|
|
97
|
+
assert rec["outcome"] == "error"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
|
|
101
|
+
@pytest.mark.covers("outcomes/fail", subject=SUBJECT)
|
|
102
|
+
@pytest.mark.parametrize(
|
|
103
|
+
"mark",
|
|
104
|
+
["@pytest.mark.xfail", "@pytest.mark.xfail(strict=True)"],
|
|
105
|
+
ids=["xfail", "strict"],
|
|
106
|
+
)
|
|
107
|
+
def test_xfail_never_records_pass(inner, mark):
|
|
108
|
+
# fails as expected -> fail; passes unexpectedly -> fail
|
|
109
|
+
for body in ("assert False", "assert True"):
|
|
110
|
+
inner.run(inner_test(body, mark=mark + "\n" + CLAIM), "-p", "no:cacheprovider")
|
|
111
|
+
assert [r["outcome"] for r in inner.records()] == ["fail", "fail"]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
|
|
115
|
+
def test_fail_then_skip_records_fail(inner):
|
|
116
|
+
inner.run(
|
|
117
|
+
"import pytest\n\n"
|
|
118
|
+
"@pytest.fixture\ndef skipper():\n yield\n pytest.skip('late')\n\n"
|
|
119
|
+
'@pytest.mark.covers("demo.claim")\n'
|
|
120
|
+
"def test_it(skipper):\n assert False\n"
|
|
121
|
+
)
|
|
122
|
+
(rec,) = inner.records()
|
|
123
|
+
assert rec["outcome"] == "fail"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
|
|
127
|
+
def test_a_crashing_run_is_never_recorded_as_pass(inner):
|
|
128
|
+
# the process dies mid-test: nothing is written for it (no record, so no pass)
|
|
129
|
+
inner.run(
|
|
130
|
+
"import os\nimport pytest\n\n" + CLAIM + "def test_it():\n os._exit(7)\n"
|
|
131
|
+
)
|
|
132
|
+
assert inner.records() == []
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@pytest.mark.covers("adapter.failure_never_passes", subject=SUBJECT)
|
|
136
|
+
@pytest.mark.covers("outcomes/error", subject=SUBJECT)
|
|
137
|
+
@pytest.mark.parametrize("phase", ["setup", "teardown"])
|
|
138
|
+
def test_an_assertion_in_setup_or_teardown_is_an_error_not_a_test_failure(inner, phase):
|
|
139
|
+
fixture = (
|
|
140
|
+
" assert False\n yield\n" if phase == "setup" else " yield\n assert False\n"
|
|
141
|
+
)
|
|
142
|
+
inner.run(
|
|
143
|
+
"import pytest\n\n@pytest.fixture\ndef fx():\n" + fixture + "\n"
|
|
144
|
+
'@pytest.mark.covers("demo.claim")\n'
|
|
145
|
+
"def test_it(fx):\n assert True\n"
|
|
146
|
+
)
|
|
147
|
+
(rec,) = inner.records()
|
|
148
|
+
assert rec["outcome"] == "error"
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
# --- adapter.skip_records_skip ---------------------------------------------------
|
|
152
|
+
|
|
153
|
+
SKIPS = {
|
|
154
|
+
"runtime": inner_test("pytest.skip('later')"),
|
|
155
|
+
"skipif": inner_test("assert True", mark='@pytest.mark.skipif(True, reason="x")\n' + CLAIM),
|
|
156
|
+
"skip_mark": inner_test("assert True", mark='@pytest.mark.skip\n' + CLAIM),
|
|
157
|
+
"importorskip": inner_test("pytest.importorskip('no_such_module_xyz')"),
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@pytest.mark.covers("adapter.skip_records_skip", subject=SUBJECT)
|
|
162
|
+
@pytest.mark.covers("outcomes/skip", subject=SUBJECT)
|
|
163
|
+
@pytest.mark.parametrize("name", sorted(SKIPS))
|
|
164
|
+
def test_a_skipped_test_records_skip(inner, name):
|
|
165
|
+
inner.run(SKIPS[name])
|
|
166
|
+
(rec,) = inner.records()
|
|
167
|
+
assert rec["outcome"] == "skip"
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@pytest.mark.covers("adapter.skip_records_skip", subject=SUBJECT)
|
|
171
|
+
def test_a_skip_in_setup_records_skip(inner):
|
|
172
|
+
inner.run(
|
|
173
|
+
"import pytest\n\n"
|
|
174
|
+
"@pytest.fixture\ndef need():\n pytest.skip('no service')\n\n"
|
|
175
|
+
'@pytest.mark.covers("demo.claim")\n'
|
|
176
|
+
"def test_it(need):\n assert True\n"
|
|
177
|
+
)
|
|
178
|
+
(rec,) = inner.records()
|
|
179
|
+
assert rec["outcome"] == "skip"
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
# --- adapter.level_from_runner ---------------------------------------------------
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
@pytest.mark.covers("adapter.level_from_runner", subject=SUBJECT)
|
|
186
|
+
def test_the_level_is_the_runners_verbatim(inner):
|
|
187
|
+
inner.run(inner_test("pass"), level="integration")
|
|
188
|
+
(rec,) = inner.records()
|
|
189
|
+
assert rec["level"] == "integration"
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
@pytest.mark.covers("adapter.level_from_runner", subject=SUBJECT)
|
|
193
|
+
def test_a_test_that_changes_its_environment_cannot_change_level_or_place(inner, tmp_path):
|
|
194
|
+
elsewhere = tmp_path / "elsewhere"
|
|
195
|
+
inner.run(
|
|
196
|
+
inner_test(
|
|
197
|
+
"import os\n"
|
|
198
|
+
"os.environ['TEMPER_LEVEL'] = 'release'\n"
|
|
199
|
+
f"os.environ['TEMPER_EVIDENCE'] = {str(elsewhere)!r}\n"
|
|
200
|
+
"os.environ['TEMPER_RUN'] = 'forged'"
|
|
201
|
+
),
|
|
202
|
+
run="real",
|
|
203
|
+
)
|
|
204
|
+
(rec,) = inner.records()
|
|
205
|
+
assert (rec["level"], rec["run"]) == ("unit", "real")
|
|
206
|
+
assert not elsewhere.exists()
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
@pytest.mark.covers("adapter.level_from_runner", subject=SUBJECT)
|
|
210
|
+
def test_a_mark_cannot_set_a_level(inner):
|
|
211
|
+
result = inner.run(
|
|
212
|
+
"import pytest\n\n"
|
|
213
|
+
'@pytest.mark.covers("demo.claim", level="release")\n'
|
|
214
|
+
"def test_it():\n assert True\n"
|
|
215
|
+
)
|
|
216
|
+
assert result.ret != 0
|
|
217
|
+
result.stderr.fnmatch_lines(["*level*"])
|
|
218
|
+
assert inner.records() == []
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
# --- adapter.silent_without_runner -----------------------------------------------
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
@pytest.mark.covers("adapter.silent_without_runner", subject=SUBJECT)
|
|
225
|
+
@pytest.mark.parametrize("missing", ["TEMPER_LEVEL", "TEMPER_EVIDENCE", "both", "empty"])
|
|
226
|
+
def test_without_the_runner_environment_nothing_is_recorded(inner, tmp_path, monkeypatch, missing):
|
|
227
|
+
src = inner_test("assert True") + "\n" + inner_test("assert False", name="test_bad")
|
|
228
|
+
inner.pytester.makepyfile(test_inner=src)
|
|
229
|
+
for name in ("TEMPER_LEVEL", "TEMPER_EVIDENCE", "TEMPER_RUN"):
|
|
230
|
+
monkeypatch.delenv(name, raising=False)
|
|
231
|
+
if missing in ("TEMPER_EVIDENCE",):
|
|
232
|
+
monkeypatch.setenv("TEMPER_LEVEL", "unit")
|
|
233
|
+
if missing == "TEMPER_LEVEL":
|
|
234
|
+
monkeypatch.setenv("TEMPER_EVIDENCE", str(inner.dir))
|
|
235
|
+
if missing == "empty":
|
|
236
|
+
monkeypatch.setenv("TEMPER_LEVEL", "")
|
|
237
|
+
monkeypatch.setenv("TEMPER_EVIDENCE", "")
|
|
238
|
+
with_plugin = inner.pytester.runpytest_subprocess()
|
|
239
|
+
without = inner.pytester.runpytest_subprocess("-p", "no:temper")
|
|
240
|
+
assert not inner.dir.exists()
|
|
241
|
+
assert with_plugin.ret == without.ret
|
|
242
|
+
# Without the plugin the marks are unknown and pytest warns; the results are the same.
|
|
243
|
+
def results(r):
|
|
244
|
+
return {k: v for k, v in r.parseoutcomes().items() if k != "warnings"}
|
|
245
|
+
|
|
246
|
+
assert results(with_plugin) == results(without)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
@pytest.mark.covers("adapter.silent_without_runner", subject=SUBJECT)
|
|
250
|
+
def test_a_malformed_mark_is_ignored_without_the_runner(inner):
|
|
251
|
+
result = inner.run(
|
|
252
|
+
"import pytest\n\n"
|
|
253
|
+
'@pytest.mark.covers(42)\n'
|
|
254
|
+
"def test_it():\n assert True\n",
|
|
255
|
+
runner=False,
|
|
256
|
+
)
|
|
257
|
+
assert result.ret == 0
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
# --- adapter.never_truncates -----------------------------------------------------
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
@pytest.mark.covers("adapter.never_truncates", subject=SUBJECT)
|
|
264
|
+
def test_existing_evidence_is_untouched_and_each_process_has_its_own_file(inner):
|
|
265
|
+
inner.dir.mkdir()
|
|
266
|
+
old = inner.dir / "unit-1-deadbeef.jsonl"
|
|
267
|
+
old.write_bytes(b'{"covers":"old.claim","test":"t","level":"unit","outcome":"pass"}\n')
|
|
268
|
+
stray = inner.dir / "notes.txt"
|
|
269
|
+
stray.write_bytes(b"keep me")
|
|
270
|
+
inner.run(inner_test("pass"))
|
|
271
|
+
inner.run(inner_test("pass"))
|
|
272
|
+
assert old.read_bytes().startswith(b'{"covers":"old.claim"')
|
|
273
|
+
assert old.read_bytes().count(b"\n") == 1
|
|
274
|
+
assert stray.read_bytes() == b"keep me"
|
|
275
|
+
new = [f for f in inner.files() if f != old]
|
|
276
|
+
assert len(new) == 2
|
|
277
|
+
for f in new:
|
|
278
|
+
assert re.fullmatch(r"unit-\d+-[0-9a-f]{8}\.jsonl", f.name)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
@pytest.mark.covers("adapter.never_truncates", subject=SUBJECT)
|
|
282
|
+
def test_the_evidence_directory_is_created_when_missing(inner, tmp_path):
|
|
283
|
+
deep = tmp_path / "a" / "b" / "evidence"
|
|
284
|
+
inner.run(inner_test("pass"), evidence=deep)
|
|
285
|
+
assert len(list(deep.glob("*.jsonl"))) == 1
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
# --- claims ----------------------------------------------------------------------
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
@pytest.mark.covers("adapter.python")
|
|
292
|
+
def test_each_claim_is_its_own_record_with_the_same_outcome(inner):
|
|
293
|
+
inner.run(
|
|
294
|
+
"import pytest\n\n"
|
|
295
|
+
'@pytest.mark.covers("a.one")\n'
|
|
296
|
+
'@pytest.mark.covers("b.two", subject="s.x")\n'
|
|
297
|
+
'@pytest.mark.exempts("c.three", subject="s.y")\n'
|
|
298
|
+
"def test_it():\n assert False\n"
|
|
299
|
+
)
|
|
300
|
+
recs = inner.records()
|
|
301
|
+
assert len(recs) == 3
|
|
302
|
+
assert {r["outcome"] for r in recs} == {"fail"}
|
|
303
|
+
by = {r.get("covers") or r["exempts"]: r for r in recs}
|
|
304
|
+
assert "exempts" in by["c.three"] and "covers" not in by["c.three"]
|
|
305
|
+
assert by["c.three"]["subject"] == "s.y"
|
|
306
|
+
assert "subject" not in by["a.one"]
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
@pytest.mark.covers("adapter.python")
|
|
310
|
+
def test_class_and_module_marks_apply_to_each_test_inside(inner):
|
|
311
|
+
inner.run(
|
|
312
|
+
"import pytest\n\n"
|
|
313
|
+
'pytestmark = pytest.mark.covers("mod.claim")\n\n'
|
|
314
|
+
'@pytest.mark.covers("cls.claim", subject="s")\n'
|
|
315
|
+
"class TestGroup:\n"
|
|
316
|
+
" def test_a(self):\n assert True\n"
|
|
317
|
+
" def test_b(self):\n assert True\n"
|
|
318
|
+
)
|
|
319
|
+
seen = sorted((r["test"].split("::")[-1], r["covers"]) for r in inner.records())
|
|
320
|
+
assert seen == [
|
|
321
|
+
("test_a", "cls.claim"),
|
|
322
|
+
("test_a", "mod.claim"),
|
|
323
|
+
("test_b", "cls.claim"),
|
|
324
|
+
("test_b", "mod.claim"),
|
|
325
|
+
]
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
@pytest.mark.covers("adapter.python")
|
|
329
|
+
def test_the_same_claim_twice_is_one_record(inner):
|
|
330
|
+
inner.run(
|
|
331
|
+
"import pytest\n\n"
|
|
332
|
+
'pytestmark = pytest.mark.covers("same.claim", subject="s")\n\n'
|
|
333
|
+
'@pytest.mark.covers("same.claim", subject="s")\n'
|
|
334
|
+
"def test_it():\n assert True\n"
|
|
335
|
+
)
|
|
336
|
+
assert len(inner.records()) == 1
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
@pytest.mark.covers("adapter.python")
|
|
340
|
+
def test_a_scenario_is_recorded_for_parametrized_scenario_runners(inner):
|
|
341
|
+
inner.run(
|
|
342
|
+
"import pytest\nimport temper\n\n"
|
|
343
|
+
'@pytest.mark.parametrize("sid", ["alerts.sold_out", "alerts.low"])\n'
|
|
344
|
+
"def test_scenario(request, sid):\n"
|
|
345
|
+
' temper.covers(request, "alerts.low_stock", scenario=sid)\n'
|
|
346
|
+
' temper.covers(request, sid, scenario=sid)\n'
|
|
347
|
+
)
|
|
348
|
+
recs = inner.records()
|
|
349
|
+
assert len(recs) == 4
|
|
350
|
+
assert {(r["covers"], r["scenario"]) for r in recs} == {
|
|
351
|
+
("alerts.low_stock", "alerts.sold_out"),
|
|
352
|
+
("alerts.sold_out", "alerts.sold_out"),
|
|
353
|
+
("alerts.low_stock", "alerts.low"),
|
|
354
|
+
("alerts.low", "alerts.low"),
|
|
355
|
+
}
|
|
356
|
+
assert list(recs[0]) == ["covers", "scenario", "test", "level", "outcome", "at"]
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
@pytest.mark.covers("adapter.python")
|
|
360
|
+
def test_a_runtime_claim_follows_the_tests_outcome(inner):
|
|
361
|
+
inner.run(
|
|
362
|
+
"import temper\n\n"
|
|
363
|
+
"def test_it(request):\n"
|
|
364
|
+
' temper.covers(request, "late.claim")\n'
|
|
365
|
+
" assert False\n"
|
|
366
|
+
)
|
|
367
|
+
(rec,) = inner.records()
|
|
368
|
+
assert (rec["covers"], rec["outcome"]) == ("late.claim", "fail")
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
@pytest.mark.covers("adapter.python")
|
|
372
|
+
def test_exempts_has_a_runtime_form_too(inner):
|
|
373
|
+
inner.run(
|
|
374
|
+
"import temper\n\n"
|
|
375
|
+
"def test_it(request):\n"
|
|
376
|
+
' temper.exempts(request, "inv.x", subject="s")\n'
|
|
377
|
+
)
|
|
378
|
+
(rec,) = inner.records()
|
|
379
|
+
assert (rec["exempts"], rec["subject"], rec["outcome"]) == ("inv.x", "s", "pass")
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
MALFORMED = {
|
|
383
|
+
"no reference": "@pytest.mark.covers()",
|
|
384
|
+
"empty reference": '@pytest.mark.covers("")',
|
|
385
|
+
"non-string reference": "@pytest.mark.covers(42)",
|
|
386
|
+
"second positional": '@pytest.mark.covers("a.b", "c")',
|
|
387
|
+
"unknown keyword": '@pytest.mark.covers("a.b", subjcet="s")',
|
|
388
|
+
"non-string subject": '@pytest.mark.covers("a.b", subject=3)',
|
|
389
|
+
"empty subject": '@pytest.mark.covers("a.b", subject="")',
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
@pytest.mark.covers("adapter.python")
|
|
394
|
+
@pytest.mark.parametrize("name", sorted(MALFORMED))
|
|
395
|
+
def test_a_malformed_mark_stops_the_run_loudly(inner, name):
|
|
396
|
+
result = inner.run(
|
|
397
|
+
"import pytest\n\n" + MALFORMED[name] + "\ndef test_it():\n assert True\n"
|
|
398
|
+
)
|
|
399
|
+
assert result.ret != 0
|
|
400
|
+
result.stderr.fnmatch_lines(["*test_inner.py::test_it*"])
|
|
401
|
+
assert inner.records() == []
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
@pytest.mark.covers("adapter.python")
|
|
405
|
+
def test_an_unwritable_evidence_directory_stops_the_run_loudly(inner, tmp_path):
|
|
406
|
+
blocker = tmp_path / "file"
|
|
407
|
+
blocker.write_text("x")
|
|
408
|
+
result = inner.run(inner_test("pass"), evidence=blocker / "evidence")
|
|
409
|
+
assert result.ret != 0
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
@pytest.mark.covers("adapter.python")
|
|
413
|
+
def test_parallel_workers_write_separate_whole_lines(inner):
|
|
414
|
+
body = "".join(
|
|
415
|
+
f'@pytest.mark.covers("demo.claim")\ndef test_{i}():\n assert True\n\n'
|
|
416
|
+
for i in range(40)
|
|
417
|
+
)
|
|
418
|
+
result = inner.run("import pytest\n\n" + body, "-n", "3")
|
|
419
|
+
assert result.ret == 0
|
|
420
|
+
assert len(inner.files()) >= 2
|
|
421
|
+
assert len(inner.records()) == 40
|
|
422
|
+
for line in inner.lines():
|
|
423
|
+
assert line.endswith("\n")
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""The evidence file writer, directly."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
from temper import _evidence
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@pytest.mark.covers("adapter.never_truncates", subject="adapter.python")
|
|
13
|
+
def test_each_record_is_one_write_in_append_mode(tmp_path, monkeypatch):
|
|
14
|
+
writes = []
|
|
15
|
+
real = os.write
|
|
16
|
+
monkeypatch.setattr(os, "write", lambda fd, data: (writes.append(data), real(fd, data))[1])
|
|
17
|
+
w = _evidence.Writer(tmp_path, "unit")
|
|
18
|
+
for i in range(5):
|
|
19
|
+
w.write({"covers": f"c.{i}", "test": "t", "level": "unit", "outcome": "pass"})
|
|
20
|
+
w.close()
|
|
21
|
+
assert len(writes) == 5
|
|
22
|
+
assert all(d.endswith(b"\n") and d.count(b"\n") == 1 for d in writes)
|
|
23
|
+
(f,) = tmp_path.glob("*.jsonl")
|
|
24
|
+
assert re.fullmatch(r"unit-%d-[0-9a-f]{8}\.jsonl" % os.getpid(), f.name)
|
|
25
|
+
assert len(f.read_bytes().splitlines()) == 5
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@pytest.mark.covers("adapter.never_truncates", subject="adapter.python")
|
|
29
|
+
def test_two_writers_never_share_a_file(tmp_path):
|
|
30
|
+
a = _evidence.Writer(tmp_path, "unit")
|
|
31
|
+
b = _evidence.Writer(tmp_path, "unit")
|
|
32
|
+
a.write({"test": "t", "level": "unit", "outcome": "pass"})
|
|
33
|
+
b.write({"test": "t", "level": "unit", "outcome": "pass"})
|
|
34
|
+
a.close()
|
|
35
|
+
b.close()
|
|
36
|
+
assert len(list(tmp_path.glob("*.jsonl"))) == 2
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@pytest.mark.covers("adapter.never_truncates", subject="adapter.python")
|
|
40
|
+
def test_nothing_is_created_until_the_first_record(tmp_path):
|
|
41
|
+
target = tmp_path / "evidence"
|
|
42
|
+
w = _evidence.Writer(target, "unit")
|
|
43
|
+
w.close()
|
|
44
|
+
assert not target.exists()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@pytest.mark.covers("adapter.python")
|
|
48
|
+
def test_a_forked_child_writes_to_its_own_file(tmp_path):
|
|
49
|
+
if not hasattr(os, "fork"):
|
|
50
|
+
pytest.skip("no fork on this platform")
|
|
51
|
+
w = _evidence.Writer(tmp_path, "unit")
|
|
52
|
+
w.write({"test": "parent", "level": "unit", "outcome": "pass"})
|
|
53
|
+
pid = os.fork()
|
|
54
|
+
if pid == 0:
|
|
55
|
+
try:
|
|
56
|
+
w.write({"test": "child", "level": "unit", "outcome": "pass"})
|
|
57
|
+
finally:
|
|
58
|
+
os._exit(0)
|
|
59
|
+
os.waitpid(pid, 0)
|
|
60
|
+
w.close()
|
|
61
|
+
files = sorted(tmp_path.glob("*.jsonl"))
|
|
62
|
+
assert len(files) == 2
|
|
63
|
+
for f in files:
|
|
64
|
+
assert len(f.read_bytes().splitlines()) == 1
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@pytest.mark.covers("adapter.python")
|
|
68
|
+
def test_a_lone_surrogate_in_a_test_id_is_escaped_not_fatal(tmp_path):
|
|
69
|
+
w = _evidence.Writer(tmp_path, "unit")
|
|
70
|
+
w.write({"test": "bad\ud800id", "level": "unit", "outcome": "pass"})
|
|
71
|
+
w.close()
|
|
72
|
+
(f,) = tmp_path.glob("*.jsonl")
|
|
73
|
+
assert json.loads(f.read_bytes())["test"] == "bad\ud800id"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@pytest.mark.covers("adapter.python")
|
|
77
|
+
def test_non_ascii_is_written_as_utf8_not_escaped(tmp_path):
|
|
78
|
+
w = _evidence.Writer(tmp_path, "unit")
|
|
79
|
+
w.write({"test": "héllo日本", "level": "unit", "outcome": "pass"})
|
|
80
|
+
w.close()
|
|
81
|
+
(f,) = tmp_path.glob("*.jsonl")
|
|
82
|
+
assert "héllo日本".encode("utf-8") in f.read_bytes()
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@pytest.mark.covers("adapter.python")
|
|
86
|
+
def test_a_level_cannot_escape_the_directory_through_the_file_name(tmp_path):
|
|
87
|
+
w = _evidence.Writer(tmp_path / "ev", "../../x/y")
|
|
88
|
+
w.write({"test": "t", "level": "../../x/y", "outcome": "pass"})
|
|
89
|
+
w.close()
|
|
90
|
+
(f,) = (tmp_path / "ev").glob("*.jsonl")
|
|
91
|
+
assert "/" not in f.name
|