pytest-txtar 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pytest_txtar-0.1.0/.github/workflows/ci.yml +50 -0
- pytest_txtar-0.1.0/.gitignore +6 -0
- pytest_txtar-0.1.0/LICENSE +55 -0
- pytest_txtar-0.1.0/PKG-INFO +198 -0
- pytest_txtar-0.1.0/README.md +188 -0
- pytest_txtar-0.1.0/pyproject.toml +37 -0
- pytest_txtar-0.1.0/src/pytest_txtar/__init__.py +0 -0
- pytest_txtar-0.1.0/src/pytest_txtar/case.py +101 -0
- pytest_txtar-0.1.0/src/pytest_txtar/compare.py +99 -0
- pytest_txtar-0.1.0/src/pytest_txtar/hooks.py +12 -0
- pytest_txtar-0.1.0/src/pytest_txtar/plugin.py +147 -0
- pytest_txtar-0.1.0/src/pytest_txtar/py.typed +0 -0
- pytest_txtar-0.1.0/src/pytest_txtar/sandbox.py +79 -0
- pytest_txtar-0.1.0/src/pytest_txtar/txtar.py +103 -0
- pytest_txtar-0.1.0/src/pytest_txtar/update.py +27 -0
- pytest_txtar-0.1.0/tests/test_core.py +413 -0
- pytest_txtar-0.1.0/tests/test_plugin.py +152 -0
- pytest_txtar-0.1.0/tests/test_txtar.py +87 -0
- pytest_txtar-0.1.0/uv.lock +140 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
tags: ["v*"]
|
|
7
|
+
pull_request:
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
test:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
|
|
16
|
+
- name: Install uv
|
|
17
|
+
uses: astral-sh/setup-uv@v9.0.0
|
|
18
|
+
with:
|
|
19
|
+
enable-cache: true
|
|
20
|
+
cache-dependency-glob: uv.lock
|
|
21
|
+
|
|
22
|
+
- name: Sync deps
|
|
23
|
+
run: uv sync
|
|
24
|
+
|
|
25
|
+
- name: Lint
|
|
26
|
+
run: uv run ruff check .
|
|
27
|
+
|
|
28
|
+
- name: Typecheck
|
|
29
|
+
run: uv run basedpyright
|
|
30
|
+
|
|
31
|
+
- name: Test
|
|
32
|
+
run: uv run pytest
|
|
33
|
+
|
|
34
|
+
publish:
|
|
35
|
+
needs: test
|
|
36
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
37
|
+
runs-on: ubuntu-latest
|
|
38
|
+
steps:
|
|
39
|
+
- uses: actions/checkout@v4
|
|
40
|
+
|
|
41
|
+
- name: Install uv
|
|
42
|
+
uses: astral-sh/setup-uv@v9.0.0
|
|
43
|
+
|
|
44
|
+
- name: Build
|
|
45
|
+
run: uv build
|
|
46
|
+
|
|
47
|
+
- name: Publish
|
|
48
|
+
run: uv publish
|
|
49
|
+
env:
|
|
50
|
+
UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Anton
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
22
|
+
|
|
23
|
+
-------------------------------------------------------------------------------
|
|
24
|
+
|
|
25
|
+
src/pytest_txtar/txtar.py is a port of the txtar archive format from
|
|
26
|
+
golang.org/x/tools/txtar, which carries the following license:
|
|
27
|
+
|
|
28
|
+
Copyright 2018 The Go Authors.
|
|
29
|
+
Copyright (c) 2009 The Go Authors. All rights reserved.
|
|
30
|
+
|
|
31
|
+
Redistribution and use in source and binary forms, with or without
|
|
32
|
+
modification, are permitted provided that the following conditions are
|
|
33
|
+
met:
|
|
34
|
+
|
|
35
|
+
* Redistributions of source code must retain the above copyright
|
|
36
|
+
notice, this list of conditions and the following disclaimer.
|
|
37
|
+
* Redistributions in binary form must reproduce the above
|
|
38
|
+
copyright notice, this list of conditions and the following disclaimer
|
|
39
|
+
in the documentation and/or other materials provided with the
|
|
40
|
+
distribution.
|
|
41
|
+
* Neither the name of Google LLC nor the names of its
|
|
42
|
+
contributors may be used to endorse or promote products derived from
|
|
43
|
+
this software without specific prior written permission.
|
|
44
|
+
|
|
45
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
46
|
+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
47
|
+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
48
|
+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
49
|
+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
50
|
+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
51
|
+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
52
|
+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
53
|
+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
54
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
55
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pytest-txtar
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Txtar-driven contract testing for command-line tools: sandboxed runs, normalized assertions, golden update.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Python: >=3.12
|
|
8
|
+
Requires-Dist: pytest>=8
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
11
|
+
# pytest-txtar
|
|
12
|
+
|
|
13
|
+
Contract testing for command-line tools, driven by [txtar](https://pkg.go.dev/golang.org/x/tools/txtar)
|
|
14
|
+
archives: one file per case carrying the fixture tree, the invocation, and everything asserted about
|
|
15
|
+
the result — stdout, stderr, exit code, and files left behind.
|
|
16
|
+
|
|
17
|
+
Each case runs in a fresh sandbox; paths that vary per run are normalized to tokens before
|
|
18
|
+
comparison, and a golden-update flow rewrites assertions from the actual run.
|
|
19
|
+
|
|
20
|
+
The package is layered: `pytest_txtar.txtar` parses the archive format, the pytest-free core
|
|
21
|
+
(`case`, `sandbox`, `compare`, `update`) executes and compares cases parameterized by a
|
|
22
|
+
`TxtarSpec`, and a pytest plugin collects `*.txtar` files as test items.
|
|
23
|
+
|
|
24
|
+
Status: pre-release, under construction.
|
|
25
|
+
|
|
26
|
+
## Configuration
|
|
27
|
+
|
|
28
|
+
A consumer states what to run and where by implementing the `pytest_txtar_spec` hook in a
|
|
29
|
+
`conftest.py` beside its cases:
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
from pathlib import Path
|
|
33
|
+
|
|
34
|
+
import pytest
|
|
35
|
+
from pytest_txtar.case import TxtarSpec
|
|
36
|
+
|
|
37
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def pytest_txtar_spec(config: pytest.Config) -> TxtarSpec:
|
|
41
|
+
return TxtarSpec(
|
|
42
|
+
commands={"mytool": ROOT / "bin" / "mytool"},
|
|
43
|
+
roots=("home", "xdg", "cwd"),
|
|
44
|
+
cwd_root="cwd",
|
|
45
|
+
env={"HOME": "home", "XDG_CONFIG_HOME": "xdg"},
|
|
46
|
+
)
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
| Field | Meaning |
|
|
50
|
+
| --- | --- |
|
|
51
|
+
| `commands` | Command word → binary it resolves to, for the first word of a `cmd` line. |
|
|
52
|
+
| `roots` | Sandbox subdirectories a case may address under `fixtures/` and `expected/`. |
|
|
53
|
+
| `cwd_root` | The root the commands run in. |
|
|
54
|
+
| `env` | Environment variable → the root it is pointed at, on top of the ambient environment. |
|
|
55
|
+
| `tokens` | Root → normalization token; defaults to the upper-cased root in braces. |
|
|
56
|
+
|
|
57
|
+
The `home`/`xdg`/`cwd` roots above are one configuration, not a fixed set: a spec may name any roots
|
|
58
|
+
and bind any variables to them.
|
|
59
|
+
|
|
60
|
+
## Case format
|
|
61
|
+
|
|
62
|
+
This section defines the behavior; the package implements it. A runner in another language is
|
|
63
|
+
conformant when it agrees with everything below.
|
|
64
|
+
|
|
65
|
+
### Container — txtar
|
|
66
|
+
|
|
67
|
+
An archive is a leading comment followed by file sections. A section starts with a marker line
|
|
68
|
+
`-- name --`; everything up to the next marker line is its content.
|
|
69
|
+
|
|
70
|
+
- The marker line is exactly `--`, a space, the name, a space, `--`. Surrounding whitespace in the
|
|
71
|
+
name is stripped; a name may contain `/`.
|
|
72
|
+
- A line whose name would be empty (`-- --`) is content, not a marker.
|
|
73
|
+
- Content that does not end in a newline is treated as if it did. A well-formed archive therefore
|
|
74
|
+
round-trips byte-exactly through parse and serialize.
|
|
75
|
+
- A section name may appear only once per case; a repeat is a malformed case.
|
|
76
|
+
- There are no other syntax errors: any byte sequence parses.
|
|
77
|
+
|
|
78
|
+
### Sections
|
|
79
|
+
|
|
80
|
+
| Section | Required | Meaning |
|
|
81
|
+
| --- | --- | --- |
|
|
82
|
+
| `cmd` | yes | One command per line, run in order in the sandbox. |
|
|
83
|
+
| `exit` | no (default `0`) | Expected exit code of the last `cmd` line, as a decimal integer. |
|
|
84
|
+
| `stdout` | no | Expected stdout, normalized. |
|
|
85
|
+
| `stderr` | no | Expected stderr, normalized. |
|
|
86
|
+
| `fixtures/<root>/<path>` | no | File written under that sandbox root before the run. |
|
|
87
|
+
| `expected/<root>/<path>` | no | Expected content of that file after the run, normalized. |
|
|
88
|
+
|
|
89
|
+
`<root>` is one of the spec's `roots`, and `<path>` must stay inside it — an absent root, a bare
|
|
90
|
+
root with no path, or a path escaping through `..` is a malformed case.
|
|
91
|
+
|
|
92
|
+
Any other section name is a hard error — the case is malformed and collection fails. Unknown
|
|
93
|
+
sections are never ignored, so a typo (`stdOut`, `expect/cwd/x`) fails loudly instead of silently
|
|
94
|
+
asserting nothing.
|
|
95
|
+
|
|
96
|
+
Parent directories of a `fixtures/` path are created; an empty section creates an empty file.
|
|
97
|
+
|
|
98
|
+
### Sandbox
|
|
99
|
+
|
|
100
|
+
Each case runs in a fresh temporary directory holding one subdirectory per root in the spec. The
|
|
101
|
+
spec's `env` mapping points the named environment variables at those directories, and the commands
|
|
102
|
+
run in `cwd_root`.
|
|
103
|
+
|
|
104
|
+
Fixtures are materialized into the roots, then the `cmd` lines run. Nothing outside the sandbox is
|
|
105
|
+
read or written, and the sandbox is discarded after the case.
|
|
106
|
+
|
|
107
|
+
### Commands
|
|
108
|
+
|
|
109
|
+
Each `cmd` line is split with shell-like quoting and executed directly — there is no shell, so no
|
|
110
|
+
pipes, redirection, globbing or variable expansion. A first word listed in the spec's `commands`
|
|
111
|
+
mapping resolves to its configured binary path; any other first word is executed as given.
|
|
112
|
+
|
|
113
|
+
With more than one line:
|
|
114
|
+
|
|
115
|
+
- every line but the last must exit `0`; if one does not, the case fails;
|
|
116
|
+
- the last line's exit code is compared against `exit`;
|
|
117
|
+
- `stdout` and `stderr` are asserted against the concatenation of all lines' output, in order.
|
|
118
|
+
|
|
119
|
+
All lines share one sandbox, so a case can chain commands that build on each other's writes.
|
|
120
|
+
|
|
121
|
+
### Normalization
|
|
122
|
+
|
|
123
|
+
Before comparison, both sides are rewritten so that volatile values do not leak into the expected
|
|
124
|
+
text. Expected sections are written in normalized form.
|
|
125
|
+
|
|
126
|
+
Absolute sandbox paths in captured output are replaced by each root's token — `{HOME}` for the
|
|
127
|
+
`home` root by default, and whatever the spec's `tokens` mapping says otherwise.
|
|
128
|
+
|
|
129
|
+
Comparison is line by line. Within a line, the literal `[..]` matches any run of characters,
|
|
130
|
+
including none, and never crosses a line boundary — it covers commit hashes, timestamps and
|
|
131
|
+
durations. A line without `[..]` must match exactly, whitespace included.
|
|
132
|
+
|
|
133
|
+
Tokens and `[..]` apply to `stdout`, `stderr` and every `expected/` section. They are not expanded
|
|
134
|
+
in `cmd` or `fixtures/`, which are taken literally.
|
|
135
|
+
|
|
136
|
+
### Assertions
|
|
137
|
+
|
|
138
|
+
- An absent section asserts nothing. A case with no `stderr` section does not require stderr to be
|
|
139
|
+
empty; a file with no `expected/` section is not checked.
|
|
140
|
+
- An absent `exit` asserts exit `0`.
|
|
141
|
+
- An `expected/` section for a file that does not exist fails the case.
|
|
142
|
+
- Files created beyond those named by `expected/` are not flagged.
|
|
143
|
+
- A failure that needs an unwritable path or another OS-level fault is not expressible — fixtures
|
|
144
|
+
carry content, not permissions.
|
|
145
|
+
|
|
146
|
+
## Running
|
|
147
|
+
|
|
148
|
+
Cases collect as ordinary pytest items, one per `*.txtar` file, whose nodeid is the case path:
|
|
149
|
+
|
|
150
|
+
```
|
|
151
|
+
pytest tests/cases
|
|
152
|
+
pytest tests/cases -k smoke
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Selection, parallelism, reporting and exit codes are pytest's. A failing case reports one labeled
|
|
156
|
+
unified diff per mismatched section; a malformed case is a collection error.
|
|
157
|
+
|
|
158
|
+
## Authoring a case
|
|
159
|
+
|
|
160
|
+
Write the comment, `cmd` and `fixtures/` sections by hand, then let the runner fill in the rest:
|
|
161
|
+
|
|
162
|
+
```
|
|
163
|
+
pytest tests/cases/my-case.txtar --txtar-update
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
`--txtar-update` rewrites each selected case whose assertions do not hold — its `stdout`, `stderr`,
|
|
167
|
+
`exit` and `expected/` sections come from the actual normalized run, `cmd` and `fixtures/` are left
|
|
168
|
+
untouched — and lists the rewritten cases in the terminal summary. A case that already passes is
|
|
169
|
+
never rewritten, so a holding `[..]` is not flattened into the run's literal text. Review the resulting diff as you would any other change —
|
|
170
|
+
a baseline is only as good as the reading it got. Re-running `--txtar-update` on an unchanged
|
|
171
|
+
corpus must produce no diff.
|
|
172
|
+
|
|
173
|
+
## Worked example
|
|
174
|
+
|
|
175
|
+
```txtar
|
|
176
|
+
Initializes a project from the template named in the user's config.
|
|
177
|
+
|
|
178
|
+
-- fixtures/xdg/mytool/config.yaml --
|
|
179
|
+
template: minimal
|
|
180
|
+
-- cmd --
|
|
181
|
+
mytool init demo
|
|
182
|
+
-- exit --
|
|
183
|
+
0
|
|
184
|
+
-- stdout --
|
|
185
|
+
created demo/ from template minimal (config: {XDG}/mytool/config.yaml)
|
|
186
|
+
-- expected/cwd/demo/mytool.yaml --
|
|
187
|
+
template: minimal
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
The fixture lands at `<sandbox>/xdg/mytool/config.yaml`, which the spec's `XDG_CONFIG_HOME` binding
|
|
191
|
+
makes the tool read as the user's config. The absolute path it echoes back is normalized to
|
|
192
|
+
`{XDG}`, while `demo` is relative to the working directory and needs no token. The trailing
|
|
193
|
+
`expected/cwd/` section asserts the file the run left behind.
|
|
194
|
+
|
|
195
|
+
## License
|
|
196
|
+
|
|
197
|
+
MIT. `src/pytest_txtar/txtar.py` is a port of `golang.org/x/tools/txtar` and carries the upstream
|
|
198
|
+
BSD-3-Clause notice — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# pytest-txtar
|
|
2
|
+
|
|
3
|
+
Contract testing for command-line tools, driven by [txtar](https://pkg.go.dev/golang.org/x/tools/txtar)
|
|
4
|
+
archives: one file per case carrying the fixture tree, the invocation, and everything asserted about
|
|
5
|
+
the result — stdout, stderr, exit code, and files left behind.
|
|
6
|
+
|
|
7
|
+
Each case runs in a fresh sandbox; paths that vary per run are normalized to tokens before
|
|
8
|
+
comparison, and a golden-update flow rewrites assertions from the actual run.
|
|
9
|
+
|
|
10
|
+
The package is layered: `pytest_txtar.txtar` parses the archive format, the pytest-free core
|
|
11
|
+
(`case`, `sandbox`, `compare`, `update`) executes and compares cases parameterized by a
|
|
12
|
+
`TxtarSpec`, and a pytest plugin collects `*.txtar` files as test items.
|
|
13
|
+
|
|
14
|
+
Status: pre-release, under construction.
|
|
15
|
+
|
|
16
|
+
## Configuration
|
|
17
|
+
|
|
18
|
+
A consumer states what to run and where by implementing the `pytest_txtar_spec` hook in a
|
|
19
|
+
`conftest.py` beside its cases:
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
import pytest
|
|
25
|
+
from pytest_txtar.case import TxtarSpec
|
|
26
|
+
|
|
27
|
+
ROOT = Path(__file__).resolve().parents[1]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def pytest_txtar_spec(config: pytest.Config) -> TxtarSpec:
|
|
31
|
+
return TxtarSpec(
|
|
32
|
+
commands={"mytool": ROOT / "bin" / "mytool"},
|
|
33
|
+
roots=("home", "xdg", "cwd"),
|
|
34
|
+
cwd_root="cwd",
|
|
35
|
+
env={"HOME": "home", "XDG_CONFIG_HOME": "xdg"},
|
|
36
|
+
)
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
| Field | Meaning |
|
|
40
|
+
| --- | --- |
|
|
41
|
+
| `commands` | Command word → binary it resolves to, for the first word of a `cmd` line. |
|
|
42
|
+
| `roots` | Sandbox subdirectories a case may address under `fixtures/` and `expected/`. |
|
|
43
|
+
| `cwd_root` | The root the commands run in. |
|
|
44
|
+
| `env` | Environment variable → the root it is pointed at, on top of the ambient environment. |
|
|
45
|
+
| `tokens` | Root → normalization token; defaults to the upper-cased root in braces. |
|
|
46
|
+
|
|
47
|
+
The `home`/`xdg`/`cwd` roots above are one configuration, not a fixed set: a spec may name any roots
|
|
48
|
+
and bind any variables to them.
|
|
49
|
+
|
|
50
|
+
## Case format
|
|
51
|
+
|
|
52
|
+
This section defines the behavior; the package implements it. A runner in another language is
|
|
53
|
+
conformant when it agrees with everything below.
|
|
54
|
+
|
|
55
|
+
### Container — txtar
|
|
56
|
+
|
|
57
|
+
An archive is a leading comment followed by file sections. A section starts with a marker line
|
|
58
|
+
`-- name --`; everything up to the next marker line is its content.
|
|
59
|
+
|
|
60
|
+
- The marker line is exactly `--`, a space, the name, a space, `--`. Surrounding whitespace in the
|
|
61
|
+
name is stripped; a name may contain `/`.
|
|
62
|
+
- A line whose name would be empty (`-- --`) is content, not a marker.
|
|
63
|
+
- Content that does not end in a newline is treated as if it did. A well-formed archive therefore
|
|
64
|
+
round-trips byte-exactly through parse and serialize.
|
|
65
|
+
- A section name may appear only once per case; a repeat is a malformed case.
|
|
66
|
+
- There are no other syntax errors: any byte sequence parses.
|
|
67
|
+
|
|
68
|
+
### Sections
|
|
69
|
+
|
|
70
|
+
| Section | Required | Meaning |
|
|
71
|
+
| --- | --- | --- |
|
|
72
|
+
| `cmd` | yes | One command per line, run in order in the sandbox. |
|
|
73
|
+
| `exit` | no (default `0`) | Expected exit code of the last `cmd` line, as a decimal integer. |
|
|
74
|
+
| `stdout` | no | Expected stdout, normalized. |
|
|
75
|
+
| `stderr` | no | Expected stderr, normalized. |
|
|
76
|
+
| `fixtures/<root>/<path>` | no | File written under that sandbox root before the run. |
|
|
77
|
+
| `expected/<root>/<path>` | no | Expected content of that file after the run, normalized. |
|
|
78
|
+
|
|
79
|
+
`<root>` is one of the spec's `roots`, and `<path>` must stay inside it — an absent root, a bare
|
|
80
|
+
root with no path, or a path escaping through `..` is a malformed case.
|
|
81
|
+
|
|
82
|
+
Any other section name is a hard error — the case is malformed and collection fails. Unknown
|
|
83
|
+
sections are never ignored, so a typo (`stdOut`, `expect/cwd/x`) fails loudly instead of silently
|
|
84
|
+
asserting nothing.
|
|
85
|
+
|
|
86
|
+
Parent directories of a `fixtures/` path are created; an empty section creates an empty file.
|
|
87
|
+
|
|
88
|
+
### Sandbox
|
|
89
|
+
|
|
90
|
+
Each case runs in a fresh temporary directory holding one subdirectory per root in the spec. The
|
|
91
|
+
spec's `env` mapping points the named environment variables at those directories, and the commands
|
|
92
|
+
run in `cwd_root`.
|
|
93
|
+
|
|
94
|
+
Fixtures are materialized into the roots, then the `cmd` lines run. Nothing outside the sandbox is
|
|
95
|
+
read or written, and the sandbox is discarded after the case.
|
|
96
|
+
|
|
97
|
+
### Commands
|
|
98
|
+
|
|
99
|
+
Each `cmd` line is split with shell-like quoting and executed directly — there is no shell, so no
|
|
100
|
+
pipes, redirection, globbing or variable expansion. A first word listed in the spec's `commands`
|
|
101
|
+
mapping resolves to its configured binary path; any other first word is executed as given.
|
|
102
|
+
|
|
103
|
+
With more than one line:
|
|
104
|
+
|
|
105
|
+
- every line but the last must exit `0`; if one does not, the case fails;
|
|
106
|
+
- the last line's exit code is compared against `exit`;
|
|
107
|
+
- `stdout` and `stderr` are asserted against the concatenation of all lines' output, in order.
|
|
108
|
+
|
|
109
|
+
All lines share one sandbox, so a case can chain commands that build on each other's writes.
|
|
110
|
+
|
|
111
|
+
### Normalization
|
|
112
|
+
|
|
113
|
+
Before comparison, both sides are rewritten so that volatile values do not leak into the expected
|
|
114
|
+
text. Expected sections are written in normalized form.
|
|
115
|
+
|
|
116
|
+
Absolute sandbox paths in captured output are replaced by each root's token — `{HOME}` for the
|
|
117
|
+
`home` root by default, and whatever the spec's `tokens` mapping says otherwise.
|
|
118
|
+
|
|
119
|
+
Comparison is line by line. Within a line, the literal `[..]` matches any run of characters,
|
|
120
|
+
including none, and never crosses a line boundary — it covers commit hashes, timestamps and
|
|
121
|
+
durations. A line without `[..]` must match exactly, whitespace included.
|
|
122
|
+
|
|
123
|
+
Tokens and `[..]` apply to `stdout`, `stderr` and every `expected/` section. They are not expanded
|
|
124
|
+
in `cmd` or `fixtures/`, which are taken literally.
|
|
125
|
+
|
|
126
|
+
### Assertions
|
|
127
|
+
|
|
128
|
+
- An absent section asserts nothing. A case with no `stderr` section does not require stderr to be
|
|
129
|
+
empty; a file with no `expected/` section is not checked.
|
|
130
|
+
- An absent `exit` asserts exit `0`.
|
|
131
|
+
- An `expected/` section for a file that does not exist fails the case.
|
|
132
|
+
- Files created beyond those named by `expected/` are not flagged.
|
|
133
|
+
- A failure that needs an unwritable path or another OS-level fault is not expressible — fixtures
|
|
134
|
+
carry content, not permissions.
|
|
135
|
+
|
|
136
|
+
## Running
|
|
137
|
+
|
|
138
|
+
Cases collect as ordinary pytest items, one per `*.txtar` file, whose nodeid is the case path:
|
|
139
|
+
|
|
140
|
+
```
|
|
141
|
+
pytest tests/cases
|
|
142
|
+
pytest tests/cases -k smoke
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Selection, parallelism, reporting and exit codes are pytest's. A failing case reports one labeled
|
|
146
|
+
unified diff per mismatched section; a malformed case is a collection error.
|
|
147
|
+
|
|
148
|
+
## Authoring a case
|
|
149
|
+
|
|
150
|
+
Write the comment, `cmd` and `fixtures/` sections by hand, then let the runner fill in the rest:
|
|
151
|
+
|
|
152
|
+
```
|
|
153
|
+
pytest tests/cases/my-case.txtar --txtar-update
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
`--txtar-update` rewrites each selected case whose assertions do not hold — its `stdout`, `stderr`,
|
|
157
|
+
`exit` and `expected/` sections come from the actual normalized run, `cmd` and `fixtures/` are left
|
|
158
|
+
untouched — and lists the rewritten cases in the terminal summary. A case that already passes is
|
|
159
|
+
never rewritten, so a holding `[..]` is not flattened into the run's literal text. Review the resulting diff as you would any other change —
|
|
160
|
+
a baseline is only as good as the reading it got. Re-running `--txtar-update` on an unchanged
|
|
161
|
+
corpus must produce no diff.
|
|
162
|
+
|
|
163
|
+
## Worked example
|
|
164
|
+
|
|
165
|
+
```txtar
|
|
166
|
+
Initializes a project from the template named in the user's config.
|
|
167
|
+
|
|
168
|
+
-- fixtures/xdg/mytool/config.yaml --
|
|
169
|
+
template: minimal
|
|
170
|
+
-- cmd --
|
|
171
|
+
mytool init demo
|
|
172
|
+
-- exit --
|
|
173
|
+
0
|
|
174
|
+
-- stdout --
|
|
175
|
+
created demo/ from template minimal (config: {XDG}/mytool/config.yaml)
|
|
176
|
+
-- expected/cwd/demo/mytool.yaml --
|
|
177
|
+
template: minimal
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
The fixture lands at `<sandbox>/xdg/mytool/config.yaml`, which the spec's `XDG_CONFIG_HOME` binding
|
|
181
|
+
makes the tool read as the user's config. The absolute path it echoes back is normalized to
|
|
182
|
+
`{XDG}`, while `demo` is relative to the working directory and needs no token. The trailing
|
|
183
|
+
`expected/cwd/` section asserts the file the run left behind.
|
|
184
|
+
|
|
185
|
+
## License
|
|
186
|
+
|
|
187
|
+
MIT. `src/pytest_txtar/txtar.py` is a port of `golang.org/x/tools/txtar` and carries the upstream
|
|
188
|
+
BSD-3-Clause notice — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "pytest-txtar"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Txtar-driven contract testing for command-line tools: sandboxed runs, normalized assertions, golden update."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.12"
|
|
9
|
+
dependencies = ["pytest>=8"]
|
|
10
|
+
|
|
11
|
+
[project.entry-points.pytest11]
|
|
12
|
+
pytest_txtar = "pytest_txtar.plugin"
|
|
13
|
+
|
|
14
|
+
[build-system]
|
|
15
|
+
requires = ["hatchling"]
|
|
16
|
+
build-backend = "hatchling.build"
|
|
17
|
+
|
|
18
|
+
[tool.hatch.build.targets.wheel]
|
|
19
|
+
packages = ["src/pytest_txtar"]
|
|
20
|
+
|
|
21
|
+
[dependency-groups]
|
|
22
|
+
dev = [
|
|
23
|
+
"ruff",
|
|
24
|
+
"basedpyright",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[tool.ruff]
|
|
28
|
+
line-length = 100
|
|
29
|
+
|
|
30
|
+
[tool.ruff.lint]
|
|
31
|
+
select = ["E", "F", "I", "UP"]
|
|
32
|
+
|
|
33
|
+
[tool.basedpyright]
|
|
34
|
+
include = ["src", "tests"]
|
|
35
|
+
extraPaths = ["src"]
|
|
36
|
+
pythonVersion = "3.12"
|
|
37
|
+
typeCheckingMode = "strict"
|
|
File without changes
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Case model: the spec a consumer supplies, and loading a case file into it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from pytest_txtar import txtar
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class CaseError(Exception):
|
|
13
|
+
"""A malformed case file."""
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class TxtarSpec:
|
|
18
|
+
"""What a consumer must state to run cases: the tool, the sandbox, the tokens.
|
|
19
|
+
|
|
20
|
+
``commands`` maps a command word appearing first on a ``cmd`` line to the
|
|
21
|
+
binary it stands for. ``roots`` names the sandbox subdirectories a case may
|
|
22
|
+
address under ``fixtures/`` and ``expected/``; the process runs in
|
|
23
|
+
``cwd_root``, and ``env`` maps an environment variable to the root it points
|
|
24
|
+
at. ``tokens`` maps a root to the token its path is normalized to, defaulting
|
|
25
|
+
to the upper-cased root in braces.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
commands: Mapping[str, Path]
|
|
29
|
+
roots: tuple[str, ...]
|
|
30
|
+
cwd_root: str
|
|
31
|
+
env: Mapping[str, str] = field(default_factory=dict[str, str])
|
|
32
|
+
tokens: Mapping[str, str] | None = None
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def token_map(self) -> Mapping[str, str]:
|
|
36
|
+
if self.tokens is not None:
|
|
37
|
+
return self.tokens
|
|
38
|
+
return {root: "{" + root.upper() + "}" for root in self.roots}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass
|
|
42
|
+
class Case:
|
|
43
|
+
path: Path
|
|
44
|
+
archive: txtar.Archive
|
|
45
|
+
cmd: list[str]
|
|
46
|
+
exit_code: int = 0
|
|
47
|
+
stdout: str | None = None
|
|
48
|
+
stderr: str | None = None
|
|
49
|
+
fixtures: list[tuple[str, str]] = field(default_factory=list)
|
|
50
|
+
expected: list[tuple[str, str]] = field(default_factory=list)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def load_case(path: Path, spec: TxtarSpec) -> Case:
|
|
54
|
+
try:
|
|
55
|
+
archive = txtar.parse(path.read_text(encoding="utf-8"))
|
|
56
|
+
except ValueError as exc:
|
|
57
|
+
raise CaseError(str(exc)) from exc
|
|
58
|
+
|
|
59
|
+
case = Case(path=path, archive=archive, cmd=[])
|
|
60
|
+
seen_cmd = False
|
|
61
|
+
for name, data in archive.files:
|
|
62
|
+
if name == "cmd":
|
|
63
|
+
case.cmd = [line for line in data.splitlines() if line.strip()]
|
|
64
|
+
seen_cmd = True
|
|
65
|
+
elif name == "exit":
|
|
66
|
+
case.exit_code = parse_exit(data)
|
|
67
|
+
elif name == "stdout":
|
|
68
|
+
case.stdout = data
|
|
69
|
+
elif name == "stderr":
|
|
70
|
+
case.stderr = data
|
|
71
|
+
elif name.startswith("fixtures/"):
|
|
72
|
+
case.fixtures.append((check_tree_path(name, spec.roots), data))
|
|
73
|
+
elif name.startswith("expected/"):
|
|
74
|
+
case.expected.append((check_tree_path(name, spec.roots), data))
|
|
75
|
+
else:
|
|
76
|
+
raise CaseError(f"unknown section: {name}")
|
|
77
|
+
|
|
78
|
+
if not seen_cmd:
|
|
79
|
+
raise CaseError("missing section: cmd")
|
|
80
|
+
if not case.cmd:
|
|
81
|
+
raise CaseError("empty section: cmd")
|
|
82
|
+
return case
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def parse_exit(data: str) -> int:
|
|
86
|
+
text = data.strip()
|
|
87
|
+
try:
|
|
88
|
+
return int(text)
|
|
89
|
+
except ValueError as exc:
|
|
90
|
+
raise CaseError(f"exit is not an integer: {text!r}") from exc
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def check_tree_path(name: str, roots: tuple[str, ...]) -> str:
|
|
94
|
+
prefix, _, rest = name.partition("/")
|
|
95
|
+
root, _, relative = rest.partition("/")
|
|
96
|
+
if root not in roots or not relative:
|
|
97
|
+
listed = ", ".join(f"{root}/" for root in roots)
|
|
98
|
+
raise CaseError(f"{prefix}/ path must start with one of {listed}: {name}")
|
|
99
|
+
if ".." in Path(relative).parts or Path(relative).is_absolute():
|
|
100
|
+
raise CaseError(f"{prefix}/ path escapes its root: {name}")
|
|
101
|
+
return name
|