benchtare 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- benchtare-0.0.1/.gitignore +30 -0
- benchtare-0.0.1/LICENSE +21 -0
- benchtare-0.0.1/PKG-INFO +172 -0
- benchtare-0.0.1/README.md +121 -0
- benchtare-0.0.1/docs/design-brief.md +277 -0
- benchtare-0.0.1/docs/first-slice-plan.md +528 -0
- benchtare-0.0.1/docs/model.md +860 -0
- benchtare-0.0.1/docs/results-schema.md +445 -0
- benchtare-0.0.1/pyproject.toml +62 -0
- benchtare-0.0.1/src/benchtare/__init__.py +31 -0
- benchtare-0.0.1/src/benchtare/__main__.py +5 -0
- benchtare-0.0.1/src/benchtare/_yaml.py +189 -0
- benchtare-0.0.1/src/benchtare/backends/__init__.py +32 -0
- benchtare-0.0.1/src/benchtare/backends/_can_codec.py +264 -0
- benchtare-0.0.1/src/benchtare/backends/base.py +283 -0
- benchtare-0.0.1/src/benchtare/backends/sim.py +244 -0
- benchtare-0.0.1/src/benchtare/backends/sim_can.py +256 -0
- benchtare-0.0.1/src/benchtare/bench/__init__.py +5 -0
- benchtare-0.0.1/src/benchtare/bench/config.py +168 -0
- benchtare-0.0.1/src/benchtare/bench/runtime.py +850 -0
- benchtare-0.0.1/src/benchtare/cli.py +152 -0
- benchtare-0.0.1/src/benchtare/dut/__init__.py +9 -0
- benchtare-0.0.1/src/benchtare/dut/description.py +1059 -0
- benchtare-0.0.1/src/benchtare/dut/identity.py +57 -0
- benchtare-0.0.1/src/benchtare/errors.py +80 -0
- benchtare-0.0.1/src/benchtare/harness/__init__.py +11 -0
- benchtare-0.0.1/src/benchtare/harness/config.py +647 -0
- benchtare-0.0.1/src/benchtare/hookspecs.py +27 -0
- benchtare-0.0.1/src/benchtare/hygiene.py +247 -0
- benchtare-0.0.1/src/benchtare/plugin.py +1683 -0
- benchtare-0.0.1/src/benchtare/record.py +104 -0
- benchtare-0.0.1/src/benchtare/reference/__init__.py +20 -0
- benchtare-0.0.1/src/benchtare/reference/bench-sim.yaml +20 -0
- benchtare-0.0.1/src/benchtare/reference/device.py +106 -0
- benchtare-0.0.1/src/benchtare/reference/harness-can.yaml +21 -0
- benchtare-0.0.1/src/benchtare/reference/harness-io.yaml +21 -0
- benchtare-0.0.1/src/benchtare/reference/reference.dbc +27 -0
- benchtare-0.0.1/src/benchtare/reference/reference.dut.yaml +19 -0
- benchtare-0.0.1/src/benchtare/results/__init__.py +6 -0
- benchtare-0.0.1/src/benchtare/results/manifest.py +172 -0
- benchtare-0.0.1/src/benchtare/results/schema.py +488 -0
- benchtare-0.0.1/src/benchtare/results/store.py +549 -0
- benchtare-0.0.1/src/benchtare/scaffold/__init__.py +73 -0
- benchtare-0.0.1/src/benchtare/scaffold/quickstart/README.md +34 -0
- benchtare-0.0.1/src/benchtare/scaffold/quickstart/bench-sim.yaml +20 -0
- benchtare-0.0.1/src/benchtare/scaffold/quickstart/conftest.py +27 -0
- benchtare-0.0.1/src/benchtare/scaffold/quickstart/harness-can.yaml +21 -0
- benchtare-0.0.1/src/benchtare/scaffold/quickstart/pytest.ini +5 -0
- benchtare-0.0.1/src/benchtare/scaffold/quickstart/reference.dbc +27 -0
- benchtare-0.0.1/src/benchtare/scaffold/quickstart/reference.dut.yaml +26 -0
- benchtare-0.0.1/src/benchtare/scaffold/quickstart/test_reference.py +8 -0
- benchtare-0.0.1/src/benchtare/selftest/__init__.py +10 -0
- benchtare-0.0.1/src/benchtare/selftest/checks.py +264 -0
- benchtare-0.0.1/src/benchtare/signals/__init__.py +5 -0
- benchtare-0.0.1/src/benchtare/signals/model.py +182 -0
- benchtare-0.0.1/src/benchtare/sim/__init__.py +1 -0
- benchtare-0.0.1/src/benchtare/sim/model.py +158 -0
- benchtare-0.0.1/src/benchtare/sim/registry.py +9 -0
- benchtare-0.0.1/src/benchtare/sim/simulation.py +224 -0
- benchtare-0.0.1/src/benchtare/testcheck.py +142 -0
- benchtare-0.0.1/tests/conftest.py +109 -0
- benchtare-0.0.1/tests/test_bench_file.py +106 -0
- benchtare-0.0.1/tests/test_bench_runtime.py +306 -0
- benchtare-0.0.1/tests/test_check_hygiene.py +663 -0
- benchtare-0.0.1/tests/test_check_tests.py +590 -0
- benchtare-0.0.1/tests/test_clean_state.py +165 -0
- benchtare-0.0.1/tests/test_cli.py +210 -0
- benchtare-0.0.1/tests/test_dut_models.py +456 -0
- benchtare-0.0.1/tests/test_harness_file.py +178 -0
- benchtare-0.0.1/tests/test_hygiene.py +307 -0
- benchtare-0.0.1/tests/test_identity.py +243 -0
- benchtare-0.0.1/tests/test_init.py +245 -0
- benchtare-0.0.1/tests/test_input_sanity.py +397 -0
- benchtare-0.0.1/tests/test_interface_description.py +522 -0
- benchtare-0.0.1/tests/test_logging.py +313 -0
- benchtare-0.0.1/tests/test_malformed_files.py +311 -0
- benchtare-0.0.1/tests/test_manifest.py +135 -0
- benchtare-0.0.1/tests/test_package.py +9 -0
- benchtare-0.0.1/tests/test_plugin.py +750 -0
- benchtare-0.0.1/tests/test_receive_buffer.py +112 -0
- benchtare-0.0.1/tests/test_record.py +256 -0
- benchtare-0.0.1/tests/test_results_plugin.py +642 -0
- benchtare-0.0.1/tests/test_results_store.py +834 -0
- benchtare-0.0.1/tests/test_selftest.py +231 -0
- benchtare-0.0.1/tests/test_signals.py +96 -0
- benchtare-0.0.1/tests/test_testcheck.py +269 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.eggs/
|
|
6
|
+
build/
|
|
7
|
+
dist/
|
|
8
|
+
|
|
9
|
+
# Virtual environments (uv creates .venv)
|
|
10
|
+
.venv/
|
|
11
|
+
venv/
|
|
12
|
+
|
|
13
|
+
# Test and tooling caches
|
|
14
|
+
.pytest_cache/
|
|
15
|
+
.mypy_cache/
|
|
16
|
+
.ruff_cache/
|
|
17
|
+
.coverage
|
|
18
|
+
htmlcov/
|
|
19
|
+
|
|
20
|
+
# Secrets and local config
|
|
21
|
+
.env
|
|
22
|
+
.env.*
|
|
23
|
+
|
|
24
|
+
# Editors and OS
|
|
25
|
+
.vscode/
|
|
26
|
+
.idea/
|
|
27
|
+
.DS_Store
|
|
28
|
+
|
|
29
|
+
# BenchTare results databases
|
|
30
|
+
.benchtare/
|
benchtare-0.0.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Drew Scott
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
benchtare-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: benchtare
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: A self-verifying HIL/SIL test framework for embedded systems, built on pytest.
|
|
5
|
+
Project-URL: Homepage, https://github.com/yetanotherscott/BenchTare
|
|
6
|
+
Project-URL: Source, https://github.com/yetanotherscott/BenchTare
|
|
7
|
+
Project-URL: Issues, https://github.com/yetanotherscott/BenchTare/issues
|
|
8
|
+
Author: Drew Scott
|
|
9
|
+
License: MIT License
|
|
10
|
+
|
|
11
|
+
Copyright (c) 2026 Drew Scott
|
|
12
|
+
|
|
13
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
14
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
15
|
+
in the Software without restriction, including without limitation the rights
|
|
16
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
17
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
18
|
+
furnished to do so, subject to the following conditions:
|
|
19
|
+
|
|
20
|
+
The above copyright notice and this permission notice shall be included in all
|
|
21
|
+
copies or substantial portions of the Software.
|
|
22
|
+
|
|
23
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
24
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
25
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
26
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
27
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
28
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
29
|
+
SOFTWARE.
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Keywords: can,embedded,hardware-in-the-loop,hil,pytest,sil,testing
|
|
32
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
33
|
+
Classifier: Framework :: Pytest
|
|
34
|
+
Classifier: Intended Audience :: Developers
|
|
35
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
36
|
+
Classifier: Operating System :: MacOS
|
|
37
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
38
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
39
|
+
Classifier: Programming Language :: Python :: 3
|
|
40
|
+
Classifier: Topic :: Software Development :: Testing
|
|
41
|
+
Classifier: Topic :: System :: Hardware
|
|
42
|
+
Requires-Python: >=3.10
|
|
43
|
+
Requires-Dist: cantools>=45.0
|
|
44
|
+
Requires-Dist: pytest>=7.4
|
|
45
|
+
Requires-Dist: python-can>=4.6
|
|
46
|
+
Requires-Dist: pyyaml>=6.0
|
|
47
|
+
Provides-Extra: dev
|
|
48
|
+
Requires-Dist: pytest>=7.4; extra == 'dev'
|
|
49
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
50
|
+
Description-Content-Type: text/markdown
|
|
51
|
+
|
|
52
|
+
# BenchTare
|
|
53
|
+
|
|
54
|
+
A self-verifying HIL/SIL test framework for embedded systems, built on pytest.
|
|
55
|
+
|
|
56
|
+
**Status: pre-alpha.** The first slice is built:
|
|
57
|
+
everything below runs on a laptop, against a simulated reference DUT or a
|
|
58
|
+
team's own DUT model. Hardware backends come after it. It runs on Linux,
|
|
59
|
+
Windows and macOS, with Python 3.10 or later and pytest 7.4 or later.
|
|
60
|
+
|
|
61
|
+
## What it does
|
|
62
|
+
|
|
63
|
+
BenchTare gives an embedded team a complete test bench, for hardware or
|
|
64
|
+
simulation, and proves that its own results can be trusted.
|
|
65
|
+
|
|
66
|
+
- **You describe the setup, and write plain pytest.** An interface description
|
|
67
|
+
lists your device's signals (inline, or imported from DBC files); a bench
|
|
68
|
+
file lists your equipment; a harness file says which signal is on which
|
|
69
|
+
channel. Tests use named signals through a `bench` fixture, and need no
|
|
70
|
+
BenchTare imports.
|
|
71
|
+
- **It verifies everything under the device, each area on its own,** so a pass
|
|
72
|
+
or a fail reflects the device and not the setup around it:
|
|
73
|
+
|
|
74
|
+
| Area | Verified by | When |
|
|
75
|
+
| --- | --- | --- |
|
|
76
|
+
| The bench | The self-test's bench check: every equipment item is healthy | Before every run |
|
|
77
|
+
| The harness | The self-test's harness check: loopbacks read back, the DUT is present | Before every run |
|
|
78
|
+
| Test hygiene | `benchtare check-hygiene`: each test repeats, leaves the DUT clean, and starts from a meaningful state | On demand |
|
|
79
|
+
| Test logic | `benchtare check-tests`: each test can fail at every check, proven by corrupting what it reads, one reading at a time | On demand |
|
|
80
|
+
| The DUT | Your tests | Every run |
|
|
81
|
+
|
|
82
|
+
A failure in a lower area makes the areas above it "not checked", never a
|
|
83
|
+
verdict of their own.
|
|
84
|
+
- **It runs without hardware.** A simulated reference DUT, bench and harness
|
|
85
|
+
ship with the package, and you can describe your own DUT as a model in
|
|
86
|
+
Python, so tests and checks run on a laptop or in CI.
|
|
87
|
+
- **Every run is recorded** in a local results database: each test's outcome
|
|
88
|
+
and every reading, comparison and stimulus, with a manifest of what was
|
|
89
|
+
tested (file hashes, git commit), the machine it ran on, the bench equipment,
|
|
90
|
+
and the DUT's hardware and software versions.
|
|
91
|
+
- **Requirements are optional,** and planned: tests will trace to them when you
|
|
92
|
+
have them.
|
|
93
|
+
|
|
94
|
+
## Quickstart
|
|
95
|
+
|
|
96
|
+
Install BenchTare from PyPI, preferably into a virtual environment. Then write
|
|
97
|
+
the quickstart project and run it:
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
pip install benchtare
|
|
101
|
+
benchtare init demo
|
|
102
|
+
cd demo
|
|
103
|
+
pytest
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
`benchtare init` writes a complete project on the simulated reference DUT, so
|
|
107
|
+
no hardware is needed: an interface description, a bench file, a harness file,
|
|
108
|
+
staging fixtures in `conftest.py`, and one test. `pytest` runs the self-test,
|
|
109
|
+
then the test. Two more commands check the test itself:
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
benchtare check-hygiene
|
|
113
|
+
benchtare check-tests
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The first proves the test stands on its own, and the second proves it can
|
|
117
|
+
fail. The project's own README says what each file is, and suggests three ways
|
|
118
|
+
to break the project, each caught by a different check.
|
|
119
|
+
|
|
120
|
+
## Commands
|
|
121
|
+
|
|
122
|
+
BenchTare is a pytest plugin, so tests run with `pytest`. Give it the three
|
|
123
|
+
files on the command line, or set them in your pytest configuration:
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
pytest --dut my.dut.yaml --bench bench.yaml --harness harness.yaml
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Three more commands use the same files and settings:
|
|
130
|
+
|
|
131
|
+
| Command | What it does |
|
|
132
|
+
| --- | --- |
|
|
133
|
+
| `benchtare selftest` | The self-test alone: the bench check and the harness check |
|
|
134
|
+
| `benchtare check-tests` | The test check: proves each test can fail at every check |
|
|
135
|
+
| `benchtare check-hygiene` | The hygiene check: proves each test stands on its own |
|
|
136
|
+
|
|
137
|
+
## Documents
|
|
138
|
+
|
|
139
|
+
- [Model](https://github.com/yetanotherscott/BenchTare/blob/main/docs/model.md): what
|
|
140
|
+
BenchTare models, and what each file you write is for.
|
|
141
|
+
- [Design brief](https://github.com/yetanotherscott/BenchTare/blob/main/docs/design-brief.md):
|
|
142
|
+
goals, non-goals, architecture, prior art.
|
|
143
|
+
- [First-slice plan](https://github.com/yetanotherscott/BenchTare/blob/main/docs/first-slice-plan.md):
|
|
144
|
+
what is being built first, and in what order.
|
|
145
|
+
- [Results schema](https://github.com/yetanotherscott/BenchTare/blob/main/docs/results-schema.md):
|
|
146
|
+
what each run stores in the results database.
|
|
147
|
+
|
|
148
|
+
## License
|
|
149
|
+
|
|
150
|
+
MIT. Improvements are welcome back as pull requests, and are not required.
|
|
151
|
+
|
|
152
|
+
BenchTare depends on [python-can](https://github.com/hardbyte/python-can), which
|
|
153
|
+
is licensed LGPL-3.0. It is installed as a separate, unmodified package and is
|
|
154
|
+
not bundled with BenchTare, so BenchTare's own code remains MIT.
|
|
155
|
+
|
|
156
|
+
## How this project uses AI
|
|
157
|
+
|
|
158
|
+
BenchTare is not handwritten. An AI coding assistant writes most of the code,
|
|
159
|
+
tests and documentation. A person sets the goals, makes the design decisions,
|
|
160
|
+
and decides what is merged.
|
|
161
|
+
|
|
162
|
+
It is partly an experiment, to find out what it takes to get AI-generated code
|
|
163
|
+
you could trust. The same rules apply to all of it:
|
|
164
|
+
|
|
165
|
+
- Every behaviour has tests, and the whole suite runs in CI on Linux, Windows
|
|
166
|
+
and macOS.
|
|
167
|
+
- Each check BenchTare makes is itself tested by breaking it on purpose, to
|
|
168
|
+
prove that the failure it exists for would be caught.
|
|
169
|
+
- Clean-room only: public standards, public documentation and open-source
|
|
170
|
+
libraries, with each dependency's licence checked against MIT.
|
|
171
|
+
|
|
172
|
+
This covers the whole project; individual commits are not labelled.
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# BenchTare
|
|
2
|
+
|
|
3
|
+
A self-verifying HIL/SIL test framework for embedded systems, built on pytest.
|
|
4
|
+
|
|
5
|
+
**Status: pre-alpha.** The first slice is built:
|
|
6
|
+
everything below runs on a laptop, against a simulated reference DUT or a
|
|
7
|
+
team's own DUT model. Hardware backends come after it. It runs on Linux,
|
|
8
|
+
Windows and macOS, with Python 3.10 or later and pytest 7.4 or later.
|
|
9
|
+
|
|
10
|
+
## What it does
|
|
11
|
+
|
|
12
|
+
BenchTare gives an embedded team a complete test bench, for hardware or
|
|
13
|
+
simulation, and proves that its own results can be trusted.
|
|
14
|
+
|
|
15
|
+
- **You describe the setup, and write plain pytest.** An interface description
|
|
16
|
+
lists your device's signals (inline, or imported from DBC files); a bench
|
|
17
|
+
file lists your equipment; a harness file says which signal is on which
|
|
18
|
+
channel. Tests use named signals through a `bench` fixture, and need no
|
|
19
|
+
BenchTare imports.
|
|
20
|
+
- **It verifies everything under the device, each area on its own,** so a pass
|
|
21
|
+
or a fail reflects the device and not the setup around it:
|
|
22
|
+
|
|
23
|
+
| Area | Verified by | When |
|
|
24
|
+
| --- | --- | --- |
|
|
25
|
+
| The bench | The self-test's bench check: every equipment item is healthy | Before every run |
|
|
26
|
+
| The harness | The self-test's harness check: loopbacks read back, the DUT is present | Before every run |
|
|
27
|
+
| Test hygiene | `benchtare check-hygiene`: each test repeats, leaves the DUT clean, and starts from a meaningful state | On demand |
|
|
28
|
+
| Test logic | `benchtare check-tests`: each test can fail at every check, proven by corrupting what it reads, one reading at a time | On demand |
|
|
29
|
+
| The DUT | Your tests | Every run |
|
|
30
|
+
|
|
31
|
+
A failure in a lower area makes the areas above it "not checked", never a
|
|
32
|
+
verdict of their own.
|
|
33
|
+
- **It runs without hardware.** A simulated reference DUT, bench and harness
|
|
34
|
+
ship with the package, and you can describe your own DUT as a model in
|
|
35
|
+
Python, so tests and checks run on a laptop or in CI.
|
|
36
|
+
- **Every run is recorded** in a local results database: each test's outcome
|
|
37
|
+
and every reading, comparison and stimulus, with a manifest of what was
|
|
38
|
+
tested (file hashes, git commit), the machine it ran on, the bench equipment,
|
|
39
|
+
and the DUT's hardware and software versions.
|
|
40
|
+
- **Requirements are optional,** and planned: tests will trace to them when you
|
|
41
|
+
have them.
|
|
42
|
+
|
|
43
|
+
## Quickstart
|
|
44
|
+
|
|
45
|
+
Install BenchTare from PyPI, preferably into a virtual environment. Then write
|
|
46
|
+
the quickstart project and run it:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install benchtare
|
|
50
|
+
benchtare init demo
|
|
51
|
+
cd demo
|
|
52
|
+
pytest
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
`benchtare init` writes a complete project on the simulated reference DUT, so
|
|
56
|
+
no hardware is needed: an interface description, a bench file, a harness file,
|
|
57
|
+
staging fixtures in `conftest.py`, and one test. `pytest` runs the self-test,
|
|
58
|
+
then the test. Two more commands check the test itself:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
benchtare check-hygiene
|
|
62
|
+
benchtare check-tests
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The first proves the test stands on its own, and the second proves it can
|
|
66
|
+
fail. The project's own README says what each file is, and suggests three ways
|
|
67
|
+
to break the project, each caught by a different check.
|
|
68
|
+
|
|
69
|
+
## Commands
|
|
70
|
+
|
|
71
|
+
BenchTare is a pytest plugin, so tests run with `pytest`. Give it the three
|
|
72
|
+
files on the command line, or set them in your pytest configuration:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
pytest --dut my.dut.yaml --bench bench.yaml --harness harness.yaml
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Three more commands use the same files and settings:
|
|
79
|
+
|
|
80
|
+
| Command | What it does |
|
|
81
|
+
| --- | --- |
|
|
82
|
+
| `benchtare selftest` | The self-test alone: the bench check and the harness check |
|
|
83
|
+
| `benchtare check-tests` | The test check: proves each test can fail at every check |
|
|
84
|
+
| `benchtare check-hygiene` | The hygiene check: proves each test stands on its own |
|
|
85
|
+
|
|
86
|
+
## Documents
|
|
87
|
+
|
|
88
|
+
- [Model](https://github.com/yetanotherscott/BenchTare/blob/main/docs/model.md): what
|
|
89
|
+
BenchTare models, and what each file you write is for.
|
|
90
|
+
- [Design brief](https://github.com/yetanotherscott/BenchTare/blob/main/docs/design-brief.md):
|
|
91
|
+
goals, non-goals, architecture, prior art.
|
|
92
|
+
- [First-slice plan](https://github.com/yetanotherscott/BenchTare/blob/main/docs/first-slice-plan.md):
|
|
93
|
+
what is being built first, and in what order.
|
|
94
|
+
- [Results schema](https://github.com/yetanotherscott/BenchTare/blob/main/docs/results-schema.md):
|
|
95
|
+
what each run stores in the results database.
|
|
96
|
+
|
|
97
|
+
## License
|
|
98
|
+
|
|
99
|
+
MIT. Improvements are welcome back as pull requests, and are not required.
|
|
100
|
+
|
|
101
|
+
BenchTare depends on [python-can](https://github.com/hardbyte/python-can), which
|
|
102
|
+
is licensed LGPL-3.0. It is installed as a separate, unmodified package and is
|
|
103
|
+
not bundled with BenchTare, so BenchTare's own code remains MIT.
|
|
104
|
+
|
|
105
|
+
## How this project uses AI
|
|
106
|
+
|
|
107
|
+
BenchTare is not handwritten. An AI coding assistant writes most of the code,
|
|
108
|
+
tests and documentation. A person sets the goals, makes the design decisions,
|
|
109
|
+
and decides what is merged.
|
|
110
|
+
|
|
111
|
+
It is partly an experiment, to find out what it takes to get AI-generated code
|
|
112
|
+
you could trust. The same rules apply to all of it:
|
|
113
|
+
|
|
114
|
+
- Every behaviour has tests, and the whole suite runs in CI on Linux, Windows
|
|
115
|
+
and macOS.
|
|
116
|
+
- Each check BenchTare makes is itself tested by breaking it on purpose, to
|
|
117
|
+
prove that the failure it exists for would be caught.
|
|
118
|
+
- Clean-room only: public standards, public documentation and open-source
|
|
119
|
+
libraries, with each dependency's licence checked against MIT.
|
|
120
|
+
|
|
121
|
+
This covers the whole project; individual commits are not labelled.
|
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
# BenchTare Design Brief: A Self-Verifying HIL/SIL Test Framework
|
|
2
|
+
|
|
3
|
+
This is a Markdown copy of the design brief, kept with the code so that the
|
|
4
|
+
goals and constraints travel with the repository. The terms it uses (DUT,
|
|
5
|
+
signal, bench, harness, staging, self-test, hygiene check, test check) are
|
|
6
|
+
defined in [model.md](model.md).
|
|
7
|
+
|
|
8
|
+
## Purpose
|
|
9
|
+
|
|
10
|
+
An open-source Python tool that gives an embedded team a complete test bench
|
|
11
|
+
out of the box, for hardware or simulation, and proves that its own results
|
|
12
|
+
are reliable.
|
|
13
|
+
|
|
14
|
+
A team supplies a description of its device's interfaces and its bench. The
|
|
15
|
+
tool supplies named signals, pytest fixtures, a bench self-test and a results
|
|
16
|
+
history.
|
|
17
|
+
|
|
18
|
+
Requirements are optional. When a team has them they are first-class: tests
|
|
19
|
+
link to them and every report traces to them.
|
|
20
|
+
|
|
21
|
+
It is for small embedded teams that need certification-grade evidence and
|
|
22
|
+
cannot afford a commercial HIL toolchain or a dedicated test-infrastructure
|
|
23
|
+
engineer. It should also hold up for enterprise teams in large organizations.
|
|
24
|
+
|
|
25
|
+
Status: second draft.
|
|
26
|
+
|
|
27
|
+
## Goals
|
|
28
|
+
|
|
29
|
+
1. **Complete out of the box.** A team installs the tool, describes its bench,
|
|
30
|
+
and runs tests. Tests address named signals with units. Which bench channel
|
|
31
|
+
carries each signal is declared in a harness file, not coded.
|
|
32
|
+
2. **Proves its own reliability.** A test's pass or fail is worth something
|
|
33
|
+
only if everything under it works, so BenchTare verifies four areas beneath
|
|
34
|
+
the DUT, each on its own: the bench equipment and the harness (a self-test
|
|
35
|
+
before each run), test hygiene (each test repeats, leaves nothing behind and
|
|
36
|
+
starts from a meaningful state) and test logic (a test check corrupts what
|
|
37
|
+
tests read, one reading at a time, to show that each can fail at every
|
|
38
|
+
verification step). Then a pass or a fail reflects the DUT. A run manifest
|
|
39
|
+
records the framework version and the DUT, bench, harness and test files. A
|
|
40
|
+
simulated reference DUT, bench and harness ship with the project and verify
|
|
41
|
+
every framework change, like unit tests for the framework itself.
|
|
42
|
+
3. **Agnostic to target and hardware.** It works with any embedded target,
|
|
43
|
+
whatever its software architecture. Transports are plugins: CAN, serial,
|
|
44
|
+
sockets, MQTT, DAQ I/O or anything else. Backends are defined by capability,
|
|
45
|
+
and the same test runs in simulation or on hardware.
|
|
46
|
+
4. **Zero to testing with no framework knowledge.** Tests are plain pytest. One
|
|
47
|
+
command scaffolds a project, and the first test passes on a laptop with no
|
|
48
|
+
hardware attached.
|
|
49
|
+
5. **Public protocols supported out of the box.** Any publicly available
|
|
50
|
+
protocol can be added. "Supported" means tested in CI against a reference
|
|
51
|
+
implementation.
|
|
52
|
+
6. **Requirements optional and first-class.** When a team has requirements, the
|
|
53
|
+
tool imports them from open, text-based tools (Doorstop first) and from
|
|
54
|
+
commercial tools through ReqIF. Tests and results trace to them in both
|
|
55
|
+
directions.
|
|
56
|
+
7. **Results worth analysing over time.** Every run writes to a database with a
|
|
57
|
+
stable schema, local by default and remote by configuration, so pass rates,
|
|
58
|
+
durations and measured values can be tracked across runs.
|
|
59
|
+
|
|
60
|
+
## Non-goals
|
|
61
|
+
|
|
62
|
+
- **Not a requirements management tool.** It reads and links requirements.
|
|
63
|
+
Authoring and review stay in the team's existing tool.
|
|
64
|
+
- **Not a real-time simulator.** It does stimulus-and-response functional
|
|
65
|
+
testing. Closed-loop plant models at millisecond rates are out of scope.
|
|
66
|
+
- **Not a new test runner.** pytest runs the tests. The tool is a set of
|
|
67
|
+
plugins and adapters.
|
|
68
|
+
- **Not a rewrite of working protocol libraries.** It wraps python-can,
|
|
69
|
+
cantools, canopen and pymodbus first, and writes its own implementation only
|
|
70
|
+
where no adequate public library exists.
|
|
71
|
+
- **Not a board farm.** Device leasing, flashing and lab scheduling are left to
|
|
72
|
+
tools such as Jumpstarter and labgrid.
|
|
73
|
+
- **Not a write-back integrator for commercial tools.** Adapters for DOORS,
|
|
74
|
+
Jama, Polarion and similar need access the project does not have. They are
|
|
75
|
+
left to contributors who use those tools.
|
|
76
|
+
|
|
77
|
+
## How it works
|
|
78
|
+
|
|
79
|
+
The team declares its device's interfaces, its bench and the harness between
|
|
80
|
+
them, and optionally its requirements. Tests stage the device into the state
|
|
81
|
+
they need with ordinary pytest fixtures. The tool turns those into signal fixtures and requirement
|
|
82
|
+
IDs, so the only code the team writes is the tests.
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
SUPPLIED BY THE TEAM
|
|
86
|
+
+------------------------+ +------------------------+ +------------------------+
|
|
87
|
+
| Requirements (optional)| | Interface description | | Bench + harness files |
|
|
88
|
+
| Doorstop tree or ReqIF | | one file per DUT; may | | equipment, and how it |
|
|
89
|
+
| | | import DBC files | | connects to the DUT |
|
|
90
|
+
+-----------+------------+ +-----------+------------+ +-----------+------------+
|
|
91
|
+
| | |
|
|
92
|
+
v v v
|
|
93
|
+
PROVIDED BY THE TOOL
|
|
94
|
+
+------------------------+ +------------------------+ +------------------------+
|
|
95
|
+
| Requirements importer | | Signal layer |<->| Backend |
|
|
96
|
+
| loads IDs and text, | | named, typed signals | | simulated or real |
|
|
97
|
+
| validates test markers | | with units, as fixtures| | hardware, self-tested |
|
|
98
|
+
+-----------+------------+ +-----------+------------+ | before each run |
|
|
99
|
+
| | +-----------+------------+
|
|
100
|
+
| requirement IDs | signal fixtures ^
|
|
101
|
+
| v | bus and I/O
|
|
102
|
+
| +------------------------+ v
|
|
103
|
+
+--------------->| TESTS THE TEAM WRITES | +------------------------+
|
|
104
|
+
| plain pytest, tagged | | Device under test |
|
|
105
|
+
| with IDs | | real ECU or simulated |
|
|
106
|
+
+-----------+------------+ +------------------------+
|
|
107
|
+
| results
|
|
108
|
+
v
|
|
109
|
+
+------------------------+
|
|
110
|
+
| Evidence and history |
|
|
111
|
+
| results, run manifest, |
|
|
112
|
+
| trace report, trends |
|
|
113
|
+
+------------------------+
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Before each run the bench and harness check themselves: the equipment reports
|
|
117
|
+
healthy, loopbacks read back what was driven, and the DUT is present.
|
|
118
|
+
Switching the bench and harness files moves the same tests between simulation
|
|
119
|
+
and hardware, over whichever transport plugin the bench uses. Every run ends
|
|
120
|
+
in a results record with a manifest of the framework version and the DUT,
|
|
121
|
+
bench, harness and test files.
|
|
122
|
+
|
|
123
|
+
**What the interface description is.** One file per DUT lists the signals it
|
|
124
|
+
exposes. It can list them directly, import them from DBC files, or both. DBC is
|
|
125
|
+
the common format for CAN traffic: message IDs, where each signal sits in the
|
|
126
|
+
bits, and its scaling and units. Importing one gives tests named signals with
|
|
127
|
+
no hand-written wrappers. Later, the same file will import CANopen EDS files
|
|
128
|
+
and Modbus register maps, and declare the standard protocols a DUT supports,
|
|
129
|
+
such as J1939, which bring their required messages with them.
|
|
130
|
+
|
|
131
|
+
## First slice
|
|
132
|
+
|
|
133
|
+
The first release is one thin path from signals to results, small enough to
|
|
134
|
+
finish in a few weeks.
|
|
135
|
+
|
|
136
|
+
1. Describe a DUT in one interface description: signals listed directly,
|
|
137
|
+
imported from a DBC file, or both.
|
|
138
|
+
2. Run on simulated equipment (in-process I/O and a virtual CAN bus through
|
|
139
|
+
python-can) with a simulated reference DUT answering.
|
|
140
|
+
3. Run the self-test before the tests: the bench equipment is healthy, the
|
|
141
|
+
harness loopbacks read back, and the DUT is present.
|
|
142
|
+
4. Write a pytest test, with a staging fixture, that uses only signal names.
|
|
143
|
+
5. Record the run in a local database with a stable schema, with a manifest of
|
|
144
|
+
the framework version and hashes of the DUT, bench, harness and test files.
|
|
145
|
+
6. Run the test check, which corrupts what the test reads and confirms that
|
|
146
|
+
the test can fail at every check.
|
|
147
|
+
7. Run the hygiene check, which confirms that the test repeats, leaves nothing
|
|
148
|
+
behind and starts from a meaningful state.
|
|
149
|
+
8. Optional: import requirements from a Doorstop tree, tag the test with a
|
|
150
|
+
requirement ID, and get a trace report.
|
|
151
|
+
|
|
152
|
+
**Acceptance check:** a person who knows pytest and has never seen the tool
|
|
153
|
+
installs it, runs the scaffold command on the example project, and has a
|
|
154
|
+
passing test in under 15 minutes. The hygiene check then passes, and the test
|
|
155
|
+
check shows that the test can fail.
|
|
156
|
+
|
|
157
|
+
**Status:** items 1 to 7 are built, and run in CI on Linux, Windows and macOS.
|
|
158
|
+
Item 8 is optional (M10). The acceptance check by hand is still to do: it needs
|
|
159
|
+
the release on PyPI.
|
|
160
|
+
|
|
161
|
+
See [first-slice-plan.md](first-slice-plan.md) for the build order.
|
|
162
|
+
|
|
163
|
+
## Later stages
|
|
164
|
+
|
|
165
|
+
Each stage widens one area and leaves the others unchanged. The order is a
|
|
166
|
+
proposal.
|
|
167
|
+
|
|
168
|
+
| Stage | Area | Adds |
|
|
169
|
+
| --- | --- | --- |
|
|
170
|
+
| 2 | Backends | First hardware: SocketCAN plus two dissimilar low-cost DAQ devices, to prove the capability interface. The test check on real equipment needs an explicit opt-in, since a corrupted reading can lead a test to send a stimulus it normally wouldn't. The hygiene check is run on a real DUT, as acceptance |
|
|
171
|
+
| 3 | Transports | Transport plugin interface, with serial, TCP/UDP sockets, MQTT and SSH as the first plugins. SSH lets an interface description or harness reach an embedded Linux target, for signals and for test hygiene's state probes |
|
|
172
|
+
| 4 | Self-verification | Standard interface fault stimuli, such as dropped or stopped CAN messages; the shipped reference extended to cover each new backend and transport. For test hygiene: a more realistic reference (measurement noise, settling ramps, a fault that latches until a supply cycle, a free-running counter, state only a probe can read), running tests in other orders, and active disruption on DUT models. For the test check: replaying recorded readings, for signals that need it |
|
|
173
|
+
| 5 | Requirements | ReqIF, StrictDoc and Sphinx-Needs importers; trace report |
|
|
174
|
+
| 6 | Analysis | Remote database option; trends for pass rate, duration and measured values |
|
|
175
|
+
| 7 | Protocols | CANopen and Modbus, each with a CI reference; J1939 and ISOBUS considered after that. Each standard brings the valid ranges and special values (such as "error" and "not available") it defines for its signals: tests can read them instead of retyping the standard, and an interface description can override them per signal |
|
|
176
|
+
| 8 | Community | Backend and adapter guides; write-back adapters from contributors |
|
|
177
|
+
| 9 | Test development in IDEs | Setup for writing and debugging tests in common IDEs, such as VS Code and PyCharm: schemas so the editor completes and checks interface descriptions, bench files and harness files; completion of signal names in tests; run and debug configurations; `benchtare init` writing them for a new project |
|
|
178
|
+
| 10 | GUI | A graphical front end for people who don't live in a terminal, for example to run the self-test and tests, follow a run, and browse results and test records. It drives pytest and reads the results database; it doesn't replace either. Scope to be decided |
|
|
179
|
+
| 11 | Platforms | Linux, Windows and macOS supported on real hardware as well as in simulation: hardware backends that work on each (SocketCAN, from stage 2, is Linux-only, so Windows and macOS need vendor CAN adapters through python-can's interfaces, such as PCAN or Kvaser); install and setup guides for each |
|
|
180
|
+
| 12 | User guide | A comprehensive user guide with worked examples, beyond the README quickstart: recommended ways to do common things, built from the topics collected below |
|
|
181
|
+
| 13 | Test design | Helpers for boundary tests: from a signal's range, given by a standard (stage 7) or by the engineer, whose range always wins, the classic values: above and below the range, its maximum and minimum, and seeded random values within it, with the seed stored in the run so a failure can be repeated. Ranges only help write tests: BenchTare still never judges a reading by them |
|
|
182
|
+
|
|
183
|
+
Stages 9 to 13 are on the horizon rather than planned in detail. None is
|
|
184
|
+
needed for the first slice, but the first slice shouldn't make them harder: the
|
|
185
|
+
file formats, the CLI and the results schema are what they will build on. The
|
|
186
|
+
first slice already runs on all three platforms: CI tests it on Linux, Windows
|
|
187
|
+
and macOS, and the code stays portable (no OS-specific calls, paths through
|
|
188
|
+
`pathlib`).
|
|
189
|
+
|
|
190
|
+
**User guide topics so far.** Recommended practices that come up while the
|
|
191
|
+
first slice is designed, kept here until the guide is written:
|
|
192
|
+
|
|
193
|
+
- **DUT identity in CI/CD pipelines.** A pipeline knows which firmware build
|
|
194
|
+
and hardware it is testing. It can pass them on the command line
|
|
195
|
+
(`pytest --identify application=$FIRMWARE_VERSION`, with the shell filling in
|
|
196
|
+
the variable), or a session fixture in `conftest.py` can read them from
|
|
197
|
+
environment variables and call `bench.identify(...)`. BenchTare never reads
|
|
198
|
+
environment variables itself: the team's code chooses which to record.
|
|
199
|
+
- **Timing requirements are timeouts.** Set each `wait_until` and `read`
|
|
200
|
+
timeout from the requirement, not from what is comfortable: a response that
|
|
201
|
+
arrives after the timeout fails the test, so the timeout is the timing check.
|
|
202
|
+
The test record keeps how long each response took, for trends.
|
|
203
|
+
- **Boundary tests are ordinary tests.** Values above and below a range, at its
|
|
204
|
+
limits and in between come from the requirement or the standard, and go in
|
|
205
|
+
a parametrized test. The test check then proves each case can fail.
|
|
206
|
+
- **Wait for states that last, not short pulses.** A sampled channel, on a
|
|
207
|
+
DAQ or the simulated `sim` equipment, can miss a value that lasts less than
|
|
208
|
+
a sample period or two. Wait for a state the DUT holds, or for the pulse's
|
|
209
|
+
effect, rather than for the pulse itself.
|
|
210
|
+
|
|
211
|
+
## Prior art
|
|
212
|
+
|
|
213
|
+
No existing tool found combines a signal layer, a self-verifying bench and
|
|
214
|
+
optional traceability. Several cover one part well, and the plan is to
|
|
215
|
+
integrate with them, not compete.
|
|
216
|
+
|
|
217
|
+
This table comes from documentation and search results read on 14 Nov 2026,
|
|
218
|
+
not from hands-on evaluation.
|
|
219
|
+
|
|
220
|
+
| Tool | Covers | Relationship |
|
|
221
|
+
| --- | --- | --- |
|
|
222
|
+
| [jamb](https://jamb.readthedocs.io/en/latest/user-guide/pytest-integration.html) | pytest requirement markers, trace matrix and test records for IEC 62304 | Closest on traceability. Reuse its marker convention if compatible |
|
|
223
|
+
| [reqcov](https://pypi.org/project/reqcov/) | Requirement coverage from JUnit results, as a CLI and GitHub Action | Possible consumer of the results file |
|
|
224
|
+
| [Sphinx-Test-Reports](https://sphinx-test-reports.readthedocs.io/en/latest/) | Imports JUnit results into Sphinx-Needs and links them to requirements | Emit JUnit properties it can map |
|
|
225
|
+
| [StrictDoc](https://strictdoc.readthedocs.io/en/stable/) | Text-based requirements with traceability to code, tests and coverage | Import source and report target |
|
|
226
|
+
| [LOBSTER](https://github.com/bmw-software-engineering/lobster/wiki) | BMW's traceability report across requirements, code and tests | Reference for the trace model. AGPL licensed |
|
|
227
|
+
| [Jumpstarter](https://github.com/jumpstarter-dev/jumpstarter-python) | Board-farm HIL with drivers for CAN, UART, GPIO and power | Possible host for backends. No signal layer found |
|
|
228
|
+
| [OpenTAP](https://opentap.io/) | Test sequencer for bench instruments, C# with Python plugins | Different audience |
|
|
229
|
+
| [py-micro-hil](https://pypi.org/project/py-micro-hil/) | Small HIL framework for Raspberry Pi peripherals | Similar aim at smaller scope |
|
|
230
|
+
|
|
231
|
+
## Risks and open questions
|
|
232
|
+
|
|
233
|
+
**Risks**
|
|
234
|
+
|
|
235
|
+
- **Breadth.** The scope now includes self-verification, several transports and
|
|
236
|
+
run analysis, which is a lot for a small project. The first slice and
|
|
237
|
+
the plugin interfaces are the mitigation, and each later stage widens one
|
|
238
|
+
area only.
|
|
239
|
+
- **Protocol overlap.** J1939 and ISOBUS are left out of the early stages. They
|
|
240
|
+
can be added later from the public standards.
|
|
241
|
+
- **Write-back cost.** Each commercial requirements tool has its own API, and
|
|
242
|
+
the project has no way to test against them. Write-back adapters are left to
|
|
243
|
+
contributors who use those tools.
|
|
244
|
+
- **Crowded traceability space.** Several pytest traceability plugins appeared
|
|
245
|
+
in 2026. Traceability is a secondary feature here. The value is a complete
|
|
246
|
+
bench that proves its own reliability.
|
|
247
|
+
- **Self-verification depth.** Proving a test bench reliable can grow without
|
|
248
|
+
limit. The first slice limits it to the self-test, the hygiene check and the
|
|
249
|
+
test check, against simulated equipment and DUT models, plus a run manifest.
|
|
250
|
+
Deeper checks, such as active disruption and a more realistic reference, are
|
|
251
|
+
later stage 4.
|
|
252
|
+
|
|
253
|
+
**Decided**
|
|
254
|
+
|
|
255
|
+
- The license is MIT, for the widest adoption. Sharing improvements back is
|
|
256
|
+
encouraged, not required.
|
|
257
|
+
- The working name is BenchTare, installable as `benchtare`.
|
|
258
|
+
- Transports are plugins, and the framework does not care which one a bench
|
|
259
|
+
uses.
|
|
260
|
+
- The reference bench is simulated and ships with the project.
|
|
261
|
+
- Run history goes to a database, local by default and remote by
|
|
262
|
+
configuration. The local database is SQLite.
|
|
263
|
+
|
|
264
|
+
**Open questions**
|
|
265
|
+
|
|
266
|
+
- [x] Confirm the name is free on PyPI and GitHub. (Reserving it on PyPI is
|
|
267
|
+
still to do.)
|
|
268
|
+
- [ ] Standalone project, or built on Jumpstarter's driver model?
|
|
269
|
+
- [ ] Which two dissimilar low-cost DAQ devices prove the capability interface first?
|
|
270
|
+
- [ ] Which transport plugin ships second, after virtual CAN?
|
|
271
|
+
- [ ] Is a hardware reference bench needed later?
|
|
272
|
+
- [x] Which local database is the default? SQLite: the first slice is built on
|
|
273
|
+
it.
|
|
274
|
+
- [ ] Which IDEs get a ready-made setup first?
|
|
275
|
+
- [ ] What should the GUI cover first, and should it be a desktop app or a local web page?
|
|
276
|
+
- [x] When should CI start testing on Windows and macOS? From the first slice:
|
|
277
|
+
CI tests all three.
|