benchtare 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. benchtare-0.0.1/.gitignore +30 -0
  2. benchtare-0.0.1/LICENSE +21 -0
  3. benchtare-0.0.1/PKG-INFO +172 -0
  4. benchtare-0.0.1/README.md +121 -0
  5. benchtare-0.0.1/docs/design-brief.md +277 -0
  6. benchtare-0.0.1/docs/first-slice-plan.md +528 -0
  7. benchtare-0.0.1/docs/model.md +860 -0
  8. benchtare-0.0.1/docs/results-schema.md +445 -0
  9. benchtare-0.0.1/pyproject.toml +62 -0
  10. benchtare-0.0.1/src/benchtare/__init__.py +31 -0
  11. benchtare-0.0.1/src/benchtare/__main__.py +5 -0
  12. benchtare-0.0.1/src/benchtare/_yaml.py +189 -0
  13. benchtare-0.0.1/src/benchtare/backends/__init__.py +32 -0
  14. benchtare-0.0.1/src/benchtare/backends/_can_codec.py +264 -0
  15. benchtare-0.0.1/src/benchtare/backends/base.py +283 -0
  16. benchtare-0.0.1/src/benchtare/backends/sim.py +244 -0
  17. benchtare-0.0.1/src/benchtare/backends/sim_can.py +256 -0
  18. benchtare-0.0.1/src/benchtare/bench/__init__.py +5 -0
  19. benchtare-0.0.1/src/benchtare/bench/config.py +168 -0
  20. benchtare-0.0.1/src/benchtare/bench/runtime.py +850 -0
  21. benchtare-0.0.1/src/benchtare/cli.py +152 -0
  22. benchtare-0.0.1/src/benchtare/dut/__init__.py +9 -0
  23. benchtare-0.0.1/src/benchtare/dut/description.py +1059 -0
  24. benchtare-0.0.1/src/benchtare/dut/identity.py +57 -0
  25. benchtare-0.0.1/src/benchtare/errors.py +80 -0
  26. benchtare-0.0.1/src/benchtare/harness/__init__.py +11 -0
  27. benchtare-0.0.1/src/benchtare/harness/config.py +647 -0
  28. benchtare-0.0.1/src/benchtare/hookspecs.py +27 -0
  29. benchtare-0.0.1/src/benchtare/hygiene.py +247 -0
  30. benchtare-0.0.1/src/benchtare/plugin.py +1683 -0
  31. benchtare-0.0.1/src/benchtare/record.py +104 -0
  32. benchtare-0.0.1/src/benchtare/reference/__init__.py +20 -0
  33. benchtare-0.0.1/src/benchtare/reference/bench-sim.yaml +20 -0
  34. benchtare-0.0.1/src/benchtare/reference/device.py +106 -0
  35. benchtare-0.0.1/src/benchtare/reference/harness-can.yaml +21 -0
  36. benchtare-0.0.1/src/benchtare/reference/harness-io.yaml +21 -0
  37. benchtare-0.0.1/src/benchtare/reference/reference.dbc +27 -0
  38. benchtare-0.0.1/src/benchtare/reference/reference.dut.yaml +19 -0
  39. benchtare-0.0.1/src/benchtare/results/__init__.py +6 -0
  40. benchtare-0.0.1/src/benchtare/results/manifest.py +172 -0
  41. benchtare-0.0.1/src/benchtare/results/schema.py +488 -0
  42. benchtare-0.0.1/src/benchtare/results/store.py +549 -0
  43. benchtare-0.0.1/src/benchtare/scaffold/__init__.py +73 -0
  44. benchtare-0.0.1/src/benchtare/scaffold/quickstart/README.md +34 -0
  45. benchtare-0.0.1/src/benchtare/scaffold/quickstart/bench-sim.yaml +20 -0
  46. benchtare-0.0.1/src/benchtare/scaffold/quickstart/conftest.py +27 -0
  47. benchtare-0.0.1/src/benchtare/scaffold/quickstart/harness-can.yaml +21 -0
  48. benchtare-0.0.1/src/benchtare/scaffold/quickstart/pytest.ini +5 -0
  49. benchtare-0.0.1/src/benchtare/scaffold/quickstart/reference.dbc +27 -0
  50. benchtare-0.0.1/src/benchtare/scaffold/quickstart/reference.dut.yaml +26 -0
  51. benchtare-0.0.1/src/benchtare/scaffold/quickstart/test_reference.py +8 -0
  52. benchtare-0.0.1/src/benchtare/selftest/__init__.py +10 -0
  53. benchtare-0.0.1/src/benchtare/selftest/checks.py +264 -0
  54. benchtare-0.0.1/src/benchtare/signals/__init__.py +5 -0
  55. benchtare-0.0.1/src/benchtare/signals/model.py +182 -0
  56. benchtare-0.0.1/src/benchtare/sim/__init__.py +1 -0
  57. benchtare-0.0.1/src/benchtare/sim/model.py +158 -0
  58. benchtare-0.0.1/src/benchtare/sim/registry.py +9 -0
  59. benchtare-0.0.1/src/benchtare/sim/simulation.py +224 -0
  60. benchtare-0.0.1/src/benchtare/testcheck.py +142 -0
  61. benchtare-0.0.1/tests/conftest.py +109 -0
  62. benchtare-0.0.1/tests/test_bench_file.py +106 -0
  63. benchtare-0.0.1/tests/test_bench_runtime.py +306 -0
  64. benchtare-0.0.1/tests/test_check_hygiene.py +663 -0
  65. benchtare-0.0.1/tests/test_check_tests.py +590 -0
  66. benchtare-0.0.1/tests/test_clean_state.py +165 -0
  67. benchtare-0.0.1/tests/test_cli.py +210 -0
  68. benchtare-0.0.1/tests/test_dut_models.py +456 -0
  69. benchtare-0.0.1/tests/test_harness_file.py +178 -0
  70. benchtare-0.0.1/tests/test_hygiene.py +307 -0
  71. benchtare-0.0.1/tests/test_identity.py +243 -0
  72. benchtare-0.0.1/tests/test_init.py +245 -0
  73. benchtare-0.0.1/tests/test_input_sanity.py +397 -0
  74. benchtare-0.0.1/tests/test_interface_description.py +522 -0
  75. benchtare-0.0.1/tests/test_logging.py +313 -0
  76. benchtare-0.0.1/tests/test_malformed_files.py +311 -0
  77. benchtare-0.0.1/tests/test_manifest.py +135 -0
  78. benchtare-0.0.1/tests/test_package.py +9 -0
  79. benchtare-0.0.1/tests/test_plugin.py +750 -0
  80. benchtare-0.0.1/tests/test_receive_buffer.py +112 -0
  81. benchtare-0.0.1/tests/test_record.py +256 -0
  82. benchtare-0.0.1/tests/test_results_plugin.py +642 -0
  83. benchtare-0.0.1/tests/test_results_store.py +834 -0
  84. benchtare-0.0.1/tests/test_selftest.py +231 -0
  85. benchtare-0.0.1/tests/test_signals.py +96 -0
  86. benchtare-0.0.1/tests/test_testcheck.py +269 -0
@@ -0,0 +1,30 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *.egg-info/
5
+ .eggs/
6
+ build/
7
+ dist/
8
+
9
+ # Virtual environments (uv creates .venv)
10
+ .venv/
11
+ venv/
12
+
13
+ # Test and tooling caches
14
+ .pytest_cache/
15
+ .mypy_cache/
16
+ .ruff_cache/
17
+ .coverage
18
+ htmlcov/
19
+
20
+ # Secrets and local config
21
+ .env
22
+ .env.*
23
+
24
+ # Editors and OS
25
+ .vscode/
26
+ .idea/
27
+ .DS_Store
28
+
29
+ # BenchTare results databases
30
+ .benchtare/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Drew Scott
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,172 @@
1
+ Metadata-Version: 2.5
2
+ Name: benchtare
3
+ Version: 0.0.1
4
+ Summary: A self-verifying HIL/SIL test framework for embedded systems, built on pytest.
5
+ Project-URL: Homepage, https://github.com/yetanotherscott/BenchTare
6
+ Project-URL: Source, https://github.com/yetanotherscott/BenchTare
7
+ Project-URL: Issues, https://github.com/yetanotherscott/BenchTare/issues
8
+ Author: Drew Scott
9
+ License: MIT License
10
+
11
+ Copyright (c) 2026 Drew Scott
12
+
13
+ Permission is hereby granted, free of charge, to any person obtaining a copy
14
+ of this software and associated documentation files (the "Software"), to deal
15
+ in the Software without restriction, including without limitation the rights
16
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
17
+ copies of the Software, and to permit persons to whom the Software is
18
+ furnished to do so, subject to the following conditions:
19
+
20
+ The above copyright notice and this permission notice shall be included in all
21
+ copies or substantial portions of the Software.
22
+
23
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
28
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
29
+ SOFTWARE.
30
+ License-File: LICENSE
31
+ Keywords: can,embedded,hardware-in-the-loop,hil,pytest,sil,testing
32
+ Classifier: Development Status :: 2 - Pre-Alpha
33
+ Classifier: Framework :: Pytest
34
+ Classifier: Intended Audience :: Developers
35
+ Classifier: License :: OSI Approved :: MIT License
36
+ Classifier: Operating System :: MacOS
37
+ Classifier: Operating System :: Microsoft :: Windows
38
+ Classifier: Operating System :: POSIX :: Linux
39
+ Classifier: Programming Language :: Python :: 3
40
+ Classifier: Topic :: Software Development :: Testing
41
+ Classifier: Topic :: System :: Hardware
42
+ Requires-Python: >=3.10
43
+ Requires-Dist: cantools>=45.0
44
+ Requires-Dist: pytest>=7.4
45
+ Requires-Dist: python-can>=4.6
46
+ Requires-Dist: pyyaml>=6.0
47
+ Provides-Extra: dev
48
+ Requires-Dist: pytest>=7.4; extra == 'dev'
49
+ Requires-Dist: ruff; extra == 'dev'
50
+ Description-Content-Type: text/markdown
51
+
52
+ # BenchTare
53
+
54
+ A self-verifying HIL/SIL test framework for embedded systems, built on pytest.
55
+
56
+ **Status: pre-alpha.** The first slice is built:
57
+ everything below runs on a laptop, against a simulated reference DUT or a
58
+ team's own DUT model. Hardware backends come after it. It runs on Linux,
59
+ Windows and macOS, with Python 3.10 or later and pytest 7.4 or later.
60
+
61
+ ## What it does
62
+
63
+ BenchTare gives an embedded team a complete test bench, for hardware or
64
+ simulation, and proves that its own results can be trusted.
65
+
66
+ - **You describe the setup, and write plain pytest.** An interface description
67
+ lists your device's signals (inline, or imported from DBC files); a bench
68
+ file lists your equipment; a harness file says which signal is on which
69
+ channel. Tests use named signals through a `bench` fixture, and need no
70
+ BenchTare imports.
71
+ - **It verifies everything under the device, each area on its own,** so a pass
72
+ or a fail reflects the device and not the setup around it:
73
+
74
+ | Area | Verified by | When |
75
+ | --- | --- | --- |
76
+ | The bench | The self-test's bench check: every equipment item is healthy | Before every run |
77
+ | The harness | The self-test's harness check: loopbacks read back, the DUT is present | Before every run |
78
+ | Test hygiene | `benchtare check-hygiene`: each test repeats, leaves the DUT clean, and starts from a meaningful state | On demand |
79
+ | Test logic | `benchtare check-tests`: each test can fail at every check, proven by corrupting what it reads, one reading at a time | On demand |
80
+ | The DUT | Your tests | Every run |
81
+
82
+ A failure in a lower area makes the areas above it "not checked", never a
83
+ verdict of their own.
84
+ - **It runs without hardware.** A simulated reference DUT, bench and harness
85
+ ship with the package, and you can describe your own DUT as a model in
86
+ Python, so tests and checks run on a laptop or in CI.
87
+ - **Every run is recorded** in a local results database: each test's outcome
88
+ and every reading, comparison and stimulus, with a manifest of what was
89
+ tested (file hashes, git commit), the machine it ran on, the bench equipment,
90
+ and the DUT's hardware and software versions.
91
+ - **Requirements are optional,** and planned: tests will trace to them when you
92
+ have them.
93
+
94
+ ## Quickstart
95
+
96
+ Install BenchTare from PyPI, preferably into a virtual environment. Then write
97
+ the quickstart project and run it:
98
+
99
+ ```bash
100
+ pip install benchtare
101
+ benchtare init demo
102
+ cd demo
103
+ pytest
104
+ ```
105
+
106
+ `benchtare init` writes a complete project on the simulated reference DUT, so
107
+ no hardware is needed: an interface description, a bench file, a harness file,
108
+ staging fixtures in `conftest.py`, and one test. `pytest` runs the self-test,
109
+ then the test. Two more commands check the test itself:
110
+
111
+ ```bash
112
+ benchtare check-hygiene
113
+ benchtare check-tests
114
+ ```
115
+
116
+ The first proves the test stands on its own, and the second proves it can
117
+ fail. The project's own README says what each file is, and suggests three ways
118
+ to break the project, each caught by a different check.
119
+
120
+ ## Commands
121
+
122
+ BenchTare is a pytest plugin, so tests run with `pytest`. Give it the three
123
+ files on the command line, or set them in your pytest configuration:
124
+
125
+ ```bash
126
+ pytest --dut my.dut.yaml --bench bench.yaml --harness harness.yaml
127
+ ```
128
+
129
+ Three more commands use the same files and settings:
130
+
131
+ | Command | What it does |
132
+ | --- | --- |
133
+ | `benchtare selftest` | The self-test alone: the bench check and the harness check |
134
+ | `benchtare check-tests` | The test check: proves each test can fail at every check |
135
+ | `benchtare check-hygiene` | The hygiene check: proves each test stands on its own |
136
+
137
+ ## Documents
138
+
139
+ - [Model](https://github.com/yetanotherscott/BenchTare/blob/main/docs/model.md): what
140
+ BenchTare models, and what each file you write is for.
141
+ - [Design brief](https://github.com/yetanotherscott/BenchTare/blob/main/docs/design-brief.md):
142
+ goals, non-goals, architecture, prior art.
143
+ - [First-slice plan](https://github.com/yetanotherscott/BenchTare/blob/main/docs/first-slice-plan.md):
144
+ what is being built first, and in what order.
145
+ - [Results schema](https://github.com/yetanotherscott/BenchTare/blob/main/docs/results-schema.md):
146
+ what each run stores in the results database.
147
+
148
+ ## License
149
+
150
+ MIT. Improvements are welcome back as pull requests, and are not required.
151
+
152
+ BenchTare depends on [python-can](https://github.com/hardbyte/python-can), which
153
+ is licensed LGPL-3.0. It is installed as a separate, unmodified package and is
154
+ not bundled with BenchTare, so BenchTare's own code remains MIT.
155
+
156
+ ## How this project uses AI
157
+
158
+ BenchTare is not handwritten. An AI coding assistant writes most of the code,
159
+ tests and documentation. A person sets the goals, makes the design decisions,
160
+ and decides what is merged.
161
+
162
+ It is partly an experiment, to find out what it takes to get AI-generated code
163
+ you could trust. The same rules apply to all of it:
164
+
165
+ - Every behaviour has tests, and the whole suite runs in CI on Linux, Windows
166
+ and macOS.
167
+ - Each check BenchTare makes is itself tested by breaking it on purpose, to
168
+ prove that the failure it exists for would be caught.
169
+ - Clean-room only: public standards, public documentation and open-source
170
+ libraries, with each dependency's licence checked against MIT.
171
+
172
+ This covers the whole project; individual commits are not labelled.
@@ -0,0 +1,121 @@
1
+ # BenchTare
2
+
3
+ A self-verifying HIL/SIL test framework for embedded systems, built on pytest.
4
+
5
+ **Status: pre-alpha.** The first slice is built:
6
+ everything below runs on a laptop, against a simulated reference DUT or a
7
+ team's own DUT model. Hardware backends come after it. It runs on Linux,
8
+ Windows and macOS, with Python 3.10 or later and pytest 7.4 or later.
9
+
10
+ ## What it does
11
+
12
+ BenchTare gives an embedded team a complete test bench, for hardware or
13
+ simulation, and proves that its own results can be trusted.
14
+
15
+ - **You describe the setup, and write plain pytest.** An interface description
16
+ lists your device's signals (inline, or imported from DBC files); a bench
17
+ file lists your equipment; a harness file says which signal is on which
18
+ channel. Tests use named signals through a `bench` fixture, and need no
19
+ BenchTare imports.
20
+ - **It verifies everything under the device, each area on its own,** so a pass
21
+ or a fail reflects the device and not the setup around it:
22
+
23
+ | Area | Verified by | When |
24
+ | --- | --- | --- |
25
+ | The bench | The self-test's bench check: every equipment item is healthy | Before every run |
26
+ | The harness | The self-test's harness check: loopbacks read back, the DUT is present | Before every run |
27
+ | Test hygiene | `benchtare check-hygiene`: each test repeats, leaves the DUT clean, and starts from a meaningful state | On demand |
28
+ | Test logic | `benchtare check-tests`: each test can fail at every check, proven by corrupting what it reads, one reading at a time | On demand |
29
+ | The DUT | Your tests | Every run |
30
+
31
+ A failure in a lower area makes the areas above it "not checked", never a
32
+ verdict of their own.
33
+ - **It runs without hardware.** A simulated reference DUT, bench and harness
34
+ ship with the package, and you can describe your own DUT as a model in
35
+ Python, so tests and checks run on a laptop or in CI.
36
+ - **Every run is recorded** in a local results database: each test's outcome
37
+ and every reading, comparison and stimulus, with a manifest of what was
38
+ tested (file hashes, git commit), the machine it ran on, the bench equipment,
39
+ and the DUT's hardware and software versions.
40
+ - **Requirements are optional,** and planned: tests will trace to them when you
41
+ have them.
42
+
43
+ ## Quickstart
44
+
45
+ Install BenchTare from PyPI, preferably into a virtual environment. Then write
46
+ the quickstart project and run it:
47
+
48
+ ```bash
49
+ pip install benchtare
50
+ benchtare init demo
51
+ cd demo
52
+ pytest
53
+ ```
54
+
55
+ `benchtare init` writes a complete project on the simulated reference DUT, so
56
+ no hardware is needed: an interface description, a bench file, a harness file,
57
+ staging fixtures in `conftest.py`, and one test. `pytest` runs the self-test,
58
+ then the test. Two more commands check the test itself:
59
+
60
+ ```bash
61
+ benchtare check-hygiene
62
+ benchtare check-tests
63
+ ```
64
+
65
+ The first proves the test stands on its own, and the second proves it can
66
+ fail. The project's own README says what each file is, and suggests three ways
67
+ to break the project, each caught by a different check.
68
+
69
+ ## Commands
70
+
71
+ BenchTare is a pytest plugin, so tests run with `pytest`. Give it the three
72
+ files on the command line, or set them in your pytest configuration:
73
+
74
+ ```bash
75
+ pytest --dut my.dut.yaml --bench bench.yaml --harness harness.yaml
76
+ ```
77
+
78
+ Three more commands use the same files and settings:
79
+
80
+ | Command | What it does |
81
+ | --- | --- |
82
+ | `benchtare selftest` | The self-test alone: the bench check and the harness check |
83
+ | `benchtare check-tests` | The test check: proves each test can fail at every check |
84
+ | `benchtare check-hygiene` | The hygiene check: proves each test stands on its own |
85
+
86
+ ## Documents
87
+
88
+ - [Model](https://github.com/yetanotherscott/BenchTare/blob/main/docs/model.md): what
89
+ BenchTare models, and what each file you write is for.
90
+ - [Design brief](https://github.com/yetanotherscott/BenchTare/blob/main/docs/design-brief.md):
91
+ goals, non-goals, architecture, prior art.
92
+ - [First-slice plan](https://github.com/yetanotherscott/BenchTare/blob/main/docs/first-slice-plan.md):
93
+ what is being built first, and in what order.
94
+ - [Results schema](https://github.com/yetanotherscott/BenchTare/blob/main/docs/results-schema.md):
95
+ what each run stores in the results database.
96
+
97
+ ## License
98
+
99
+ MIT. Improvements are welcome back as pull requests, and are not required.
100
+
101
+ BenchTare depends on [python-can](https://github.com/hardbyte/python-can), which
102
+ is licensed LGPL-3.0. It is installed as a separate, unmodified package and is
103
+ not bundled with BenchTare, so BenchTare's own code remains MIT.
104
+
105
+ ## How this project uses AI
106
+
107
+ BenchTare is not handwritten. An AI coding assistant writes most of the code,
108
+ tests and documentation. A person sets the goals, makes the design decisions,
109
+ and decides what is merged.
110
+
111
+ It is partly an experiment, to find out what it takes to get AI-generated code
112
+ you could trust. The same rules apply to all of it:
113
+
114
+ - Every behaviour has tests, and the whole suite runs in CI on Linux, Windows
115
+ and macOS.
116
+ - Each check BenchTare makes is itself tested by breaking it on purpose, to
117
+ prove that the failure it exists for would be caught.
118
+ - Clean-room only: public standards, public documentation and open-source
119
+ libraries, with each dependency's licence checked against MIT.
120
+
121
+ This covers the whole project; individual commits are not labelled.
@@ -0,0 +1,277 @@
1
+ # BenchTare Design Brief: A Self-Verifying HIL/SIL Test Framework
2
+
3
+ This is a Markdown copy of the design brief, kept with the code so that the
4
+ goals and constraints travel with the repository. The terms it uses (DUT,
5
+ signal, bench, harness, staging, self-test, hygiene check, test check) are
6
+ defined in [model.md](model.md).
7
+
8
+ ## Purpose
9
+
10
+ An open-source Python tool that gives an embedded team a complete test bench
11
+ out of the box, for hardware or simulation, and proves that its own results
12
+ are reliable.
13
+
14
+ A team supplies a description of its device's interfaces and its bench. The
15
+ tool supplies named signals, pytest fixtures, a bench self-test and a results
16
+ history.
17
+
18
+ Requirements are optional. When a team has them they are first-class: tests
19
+ link to them and every report traces to them.
20
+
21
+ It is for small embedded teams that need certification-grade evidence and
22
+ cannot afford a commercial HIL toolchain or a dedicated test-infrastructure
23
+ engineer. It should also hold up for enterprise teams in large organizations.
24
+
25
+ Status: second draft.
26
+
27
+ ## Goals
28
+
29
+ 1. **Complete out of the box.** A team installs the tool, describes its bench,
30
+ and runs tests. Tests address named signals with units. Which bench channel
31
+ carries each signal is declared in a harness file, not coded.
32
+ 2. **Proves its own reliability.** A test's pass or fail is worth something
33
+ only if everything under it works, so BenchTare verifies four areas beneath
34
+ the DUT, each on its own: the bench equipment and the harness (a self-test
35
+ before each run), test hygiene (each test repeats, leaves nothing behind and
36
+ starts from a meaningful state) and test logic (a test check corrupts what
37
+ tests read, one reading at a time, to show that each can fail at every
38
+ verification step). Then a pass or a fail reflects the DUT. A run manifest
39
+ records the framework version and the DUT, bench, harness and test files. A
40
+ simulated reference DUT, bench and harness ship with the project and verify
41
+ every framework change, like unit tests for the framework itself.
42
+ 3. **Agnostic to target and hardware.** It works with any embedded target,
43
+ whatever its software architecture. Transports are plugins: CAN, serial,
44
+ sockets, MQTT, DAQ I/O or anything else. Backends are defined by capability,
45
+ and the same test runs in simulation or on hardware.
46
+ 4. **Zero to testing with no framework knowledge.** Tests are plain pytest. One
47
+ command scaffolds a project, and the first test passes on a laptop with no
48
+ hardware attached.
49
+ 5. **Public protocols supported out of the box.** Any publicly available
50
+ protocol can be added. "Supported" means tested in CI against a reference
51
+ implementation.
52
+ 6. **Requirements optional and first-class.** When a team has requirements, the
53
+ tool imports them from open, text-based tools (Doorstop first) and from
54
+ commercial tools through ReqIF. Tests and results trace to them in both
55
+ directions.
56
+ 7. **Results worth analysing over time.** Every run writes to a database with a
57
+ stable schema, local by default and remote by configuration, so pass rates,
58
+ durations and measured values can be tracked across runs.
59
+
60
+ ## Non-goals
61
+
62
+ - **Not a requirements management tool.** It reads and links requirements.
63
+ Authoring and review stay in the team's existing tool.
64
+ - **Not a real-time simulator.** It does stimulus-and-response functional
65
+ testing. Closed-loop plant models at millisecond rates are out of scope.
66
+ - **Not a new test runner.** pytest runs the tests. The tool is a set of
67
+ plugins and adapters.
68
+ - **Not a rewrite of working protocol libraries.** It wraps python-can,
69
+ cantools, canopen and pymodbus first, and writes its own implementation only
70
+ where no adequate public library exists.
71
+ - **Not a board farm.** Device leasing, flashing and lab scheduling are left to
72
+ tools such as Jumpstarter and labgrid.
73
+ - **Not a write-back integrator for commercial tools.** Adapters for DOORS,
74
+ Jama, Polarion and similar need access the project does not have. They are
75
+ left to contributors who use those tools.
76
+
77
+ ## How it works
78
+
79
+ The team declares its device's interfaces, its bench and the harness between
80
+ them, and optionally its requirements. Tests stage the device into the state
81
+ they need with ordinary pytest fixtures. The tool turns those into signal fixtures and requirement
82
+ IDs, so the only code the team writes is the tests.
83
+
84
+ ```
85
+ SUPPLIED BY THE TEAM
86
+ +------------------------+ +------------------------+ +------------------------+
87
+ | Requirements (optional)| | Interface description | | Bench + harness files |
88
+ | Doorstop tree or ReqIF | | one file per DUT; may | | equipment, and how it |
89
+ | | | import DBC files | | connects to the DUT |
90
+ +-----------+------------+ +-----------+------------+ +-----------+------------+
91
+ | | |
92
+ v v v
93
+ PROVIDED BY THE TOOL
94
+ +------------------------+ +------------------------+ +------------------------+
95
+ | Requirements importer | | Signal layer |<->| Backend |
96
+ | loads IDs and text, | | named, typed signals | | simulated or real |
97
+ | validates test markers | | with units, as fixtures| | hardware, self-tested |
98
+ +-----------+------------+ +-----------+------------+ | before each run |
99
+ | | +-----------+------------+
100
+ | requirement IDs | signal fixtures ^
101
+ | v | bus and I/O
102
+ | +------------------------+ v
103
+ +--------------->| TESTS THE TEAM WRITES | +------------------------+
104
+ | plain pytest, tagged | | Device under test |
105
+ | with IDs | | real ECU or simulated |
106
+ +-----------+------------+ +------------------------+
107
+ | results
108
+ v
109
+ +------------------------+
110
+ | Evidence and history |
111
+ | results, run manifest, |
112
+ | trace report, trends |
113
+ +------------------------+
114
+ ```
115
+
116
+ Before each run the bench and harness check themselves: the equipment reports
117
+ healthy, loopbacks read back what was driven, and the DUT is present.
118
+ Switching the bench and harness files moves the same tests between simulation
119
+ and hardware, over whichever transport plugin the bench uses. Every run ends
120
+ in a results record with a manifest of the framework version and the DUT,
121
+ bench, harness and test files.
122
+
123
+ **What the interface description is.** One file per DUT lists the signals it
124
+ exposes. It can list them directly, import them from DBC files, or both. DBC is
125
+ the common format for CAN traffic: message IDs, where each signal sits in the
126
+ bits, and its scaling and units. Importing one gives tests named signals with
127
+ no hand-written wrappers. Later, the same file will import CANopen EDS files
128
+ and Modbus register maps, and declare the standard protocols a DUT supports,
129
+ such as J1939, which bring their required messages with them.
130
+
131
+ ## First slice
132
+
133
+ The first release is one thin path from signals to results, small enough to
134
+ finish in a few weeks.
135
+
136
+ 1. Describe a DUT in one interface description: signals listed directly,
137
+ imported from a DBC file, or both.
138
+ 2. Run on simulated equipment (in-process I/O and a virtual CAN bus through
139
+ python-can) with a simulated reference DUT answering.
140
+ 3. Run the self-test before the tests: the bench equipment is healthy, the
141
+ harness loopbacks read back, and the DUT is present.
142
+ 4. Write a pytest test, with a staging fixture, that uses only signal names.
143
+ 5. Record the run in a local database with a stable schema, with a manifest of
144
+ the framework version and hashes of the DUT, bench, harness and test files.
145
+ 6. Run the test check, which corrupts what the test reads and confirms that
146
+ the test can fail at every check.
147
+ 7. Run the hygiene check, which confirms that the test repeats, leaves nothing
148
+ behind and starts from a meaningful state.
149
+ 8. Optional: import requirements from a Doorstop tree, tag the test with a
150
+ requirement ID, and get a trace report.
151
+
152
+ **Acceptance check:** a person who knows pytest and has never seen the tool
153
+ installs it, runs the scaffold command on the example project, and has a
154
+ passing test in under 15 minutes. The hygiene check then passes, and the test
155
+ check shows that the test can fail.
156
+
157
+ **Status:** items 1 to 7 are built, and run in CI on Linux, Windows and macOS.
158
+ Item 8 is optional (M10). The acceptance check by hand is still to do: it needs
159
+ the release on PyPI.
160
+
161
+ See [first-slice-plan.md](first-slice-plan.md) for the build order.
162
+
163
+ ## Later stages
164
+
165
+ Each stage widens one area and leaves the others unchanged. The order is a
166
+ proposal.
167
+
168
+ | Stage | Area | Adds |
169
+ | --- | --- | --- |
170
+ | 2 | Backends | First hardware: SocketCAN plus two dissimilar low-cost DAQ devices, to prove the capability interface. The test check on real equipment needs an explicit opt-in, since a corrupted reading can lead a test to send a stimulus it normally wouldn't. The hygiene check is run on a real DUT, as acceptance |
171
+ | 3 | Transports | Transport plugin interface, with serial, TCP/UDP sockets, MQTT and SSH as the first plugins. SSH lets an interface description or harness reach an embedded Linux target, for signals and for test hygiene's state probes |
172
+ | 4 | Self-verification | Standard interface fault stimuli, such as dropped or stopped CAN messages; the shipped reference extended to cover each new backend and transport. For test hygiene: a more realistic reference (measurement noise, settling ramps, a fault that latches until a supply cycle, a free-running counter, state only a probe can read), running tests in other orders, and active disruption on DUT models. For the test check: replaying recorded readings, for signals that need it |
173
+ | 5 | Requirements | ReqIF, StrictDoc and Sphinx-Needs importers; trace report |
174
+ | 6 | Analysis | Remote database option; trends for pass rate, duration and measured values |
175
+ | 7 | Protocols | CANopen and Modbus, each with a CI reference; J1939 and ISOBUS considered after that. Each standard brings the valid ranges and special values (such as "error" and "not available") it defines for its signals: tests can read them instead of retyping the standard, and an interface description can override them per signal |
176
+ | 8 | Community | Backend and adapter guides; write-back adapters from contributors |
177
+ | 9 | Test development in IDEs | Setup for writing and debugging tests in common IDEs, such as VS Code and PyCharm: schemas so the editor completes and checks interface descriptions, bench files and harness files; completion of signal names in tests; run and debug configurations; `benchtare init` writing them for a new project |
178
+ | 10 | GUI | A graphical front end for people who don't live in a terminal, for example to run the self-test and tests, follow a run, and browse results and test records. It drives pytest and reads the results database; it doesn't replace either. Scope to be decided |
179
+ | 11 | Platforms | Linux, Windows and macOS supported on real hardware as well as in simulation: hardware backends that work on each (SocketCAN, from stage 2, is Linux-only, so Windows and macOS need vendor CAN adapters through python-can's interfaces, such as PCAN or Kvaser); install and setup guides for each |
180
+ | 12 | User guide | A comprehensive user guide with worked examples, beyond the README quickstart: recommended ways to do common things, built from the topics collected below |
181
+ | 13 | Test design | Helpers for boundary tests: from a signal's range, given by a standard (stage 7) or by the engineer, whose range always wins, the classic values: above and below the range, its maximum and minimum, and seeded random values within it, with the seed stored in the run so a failure can be repeated. Ranges only help write tests: BenchTare still never judges a reading by them |
182
+
183
+ Stages 9 to 13 are on the horizon rather than planned in detail. None is
184
+ needed for the first slice, but the first slice shouldn't make them harder: the
185
+ file formats, the CLI and the results schema are what they will build on. The
186
+ first slice already runs on all three platforms: CI tests it on Linux, Windows
187
+ and macOS, and the code stays portable (no OS-specific calls, paths through
188
+ `pathlib`).
189
+
190
+ **User guide topics so far.** Recommended practices that come up while the
191
+ first slice is designed, kept here until the guide is written:
192
+
193
+ - **DUT identity in CI/CD pipelines.** A pipeline knows which firmware build
194
+ and hardware it is testing. It can pass them on the command line
195
+ (`pytest --identify application=$FIRMWARE_VERSION`, with the shell filling in
196
+ the variable), or a session fixture in `conftest.py` can read them from
197
+ environment variables and call `bench.identify(...)`. BenchTare never reads
198
+ environment variables itself: the team's code chooses which to record.
199
+ - **Timing requirements are timeouts.** Set each `wait_until` and `read`
200
+ timeout from the requirement, not from what is comfortable: a response that
201
+ arrives after the timeout fails the test, so the timeout is the timing check.
202
+ The test record keeps how long each response took, for trends.
203
+ - **Boundary tests are ordinary tests.** Values above and below a range, at its
204
+ limits and in between come from the requirement or the standard, and go in
205
+ a parametrized test. The test check then proves each case can fail.
206
+ - **Wait for states that last, not short pulses.** A sampled channel, on a
207
+ DAQ or the simulated `sim` equipment, can miss a value that lasts less than
208
+ a sample period or two. Wait for a state the DUT holds, or for the pulse's
209
+ effect, rather than for the pulse itself.
210
+
211
+ ## Prior art
212
+
213
+ No existing tool found combines a signal layer, a self-verifying bench and
214
+ optional traceability. Several cover one part well, and the plan is to
215
+ integrate with them, not compete.
216
+
217
+ This table comes from documentation and search results read on 14 Nov 2026,
218
+ not from hands-on evaluation.
219
+
220
+ | Tool | Covers | Relationship |
221
+ | --- | --- | --- |
222
+ | [jamb](https://jamb.readthedocs.io/en/latest/user-guide/pytest-integration.html) | pytest requirement markers, trace matrix and test records for IEC 62304 | Closest on traceability. Reuse its marker convention if compatible |
223
+ | [reqcov](https://pypi.org/project/reqcov/) | Requirement coverage from JUnit results, as a CLI and GitHub Action | Possible consumer of the results file |
224
+ | [Sphinx-Test-Reports](https://sphinx-test-reports.readthedocs.io/en/latest/) | Imports JUnit results into Sphinx-Needs and links them to requirements | Emit JUnit properties it can map |
225
+ | [StrictDoc](https://strictdoc.readthedocs.io/en/stable/) | Text-based requirements with traceability to code, tests and coverage | Import source and report target |
226
+ | [LOBSTER](https://github.com/bmw-software-engineering/lobster/wiki) | BMW's traceability report across requirements, code and tests | Reference for the trace model. AGPL licensed |
227
+ | [Jumpstarter](https://github.com/jumpstarter-dev/jumpstarter-python) | Board-farm HIL with drivers for CAN, UART, GPIO and power | Possible host for backends. No signal layer found |
228
+ | [OpenTAP](https://opentap.io/) | Test sequencer for bench instruments, C# with Python plugins | Different audience |
229
+ | [py-micro-hil](https://pypi.org/project/py-micro-hil/) | Small HIL framework for Raspberry Pi peripherals | Similar aim at smaller scope |
230
+
231
+ ## Risks and open questions
232
+
233
+ **Risks**
234
+
235
+ - **Breadth.** The scope now includes self-verification, several transports and
236
+ run analysis, which is a lot for a small project. The first slice and
237
+ the plugin interfaces are the mitigation, and each later stage widens one
238
+ area only.
239
+ - **Protocol overlap.** J1939 and ISOBUS are left out of the early stages. They
240
+ can be added later from the public standards.
241
+ - **Write-back cost.** Each commercial requirements tool has its own API, and
242
+ the project has no way to test against them. Write-back adapters are left to
243
+ contributors who use those tools.
244
+ - **Crowded traceability space.** Several pytest traceability plugins appeared
245
+ in 2026. Traceability is a secondary feature here. The value is a complete
246
+ bench that proves its own reliability.
247
+ - **Self-verification depth.** Proving a test bench reliable can grow without
248
+ limit. The first slice limits it to the self-test, the hygiene check and the
249
+ test check, against simulated equipment and DUT models, plus a run manifest.
250
+ Deeper checks, such as active disruption and a more realistic reference, are
251
+ later stage 4.
252
+
253
+ **Decided**
254
+
255
+ - The license is MIT, for the widest adoption. Sharing improvements back is
256
+ encouraged, not required.
257
+ - The working name is BenchTare, installable as `benchtare`.
258
+ - Transports are plugins, and the framework does not care which one a bench
259
+ uses.
260
+ - The reference bench is simulated and ships with the project.
261
+ - Run history goes to a database, local by default and remote by
262
+ configuration. The local database is SQLite.
263
+
264
+ **Open questions**
265
+
266
+ - [x] Confirm the name is free on PyPI and GitHub. (Reserving it on PyPI is
267
+ still to do.)
268
+ - [ ] Standalone project, or built on Jumpstarter's driver model?
269
+ - [ ] Which two dissimilar low-cost DAQ devices prove the capability interface first?
270
+ - [ ] Which transport plugin ships second, after virtual CAN?
271
+ - [ ] Is a hardware reference bench needed later?
272
+ - [x] Which local database is the default? SQLite: the first slice is built on
273
+ it.
274
+ - [ ] Which IDEs get a ready-made setup first?
275
+ - [ ] What should the GUI cover first, and should it be a desktop app or a local web page?
276
+ - [x] When should CI start testing on Windows and macOS? From the first slice:
277
+ CI tests all three.