testomatic-runner 0.1.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/PKG-INFO +46 -11
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/README.md +43 -9
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/pyproject.toml +7 -3
- testomatic_runner-0.3.0/testomatic/cli.py +88 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic/runner.py +28 -3
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic/steps/__init__.py +2 -0
- testomatic_runner-0.3.0/testomatic/steps/base.py +46 -0
- testomatic_runner-0.3.0/testomatic/steps/firmware.py +186 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic/steps/iomod.py +4 -2
- testomatic_runner-0.3.0/testomatic/steps/led_spectral.py +73 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic/steps/power.py +7 -2
- testomatic_runner-0.3.0/testomatic/steps/python_step.py +53 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic/suite.py +61 -20
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic_runner.egg-info/PKG-INFO +46 -11
- testomatic_runner-0.3.0/testomatic_runner.egg-info/requires.txt +7 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/tests/test_runner.py +46 -2
- testomatic_runner-0.3.0/tests/test_steps.py +503 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/tests/test_suite_parsing.py +87 -0
- testomatic_runner-0.1.0/testomatic/cli.py +0 -48
- testomatic_runner-0.1.0/testomatic/steps/base.py +0 -23
- testomatic_runner-0.1.0/testomatic/steps/firmware.py +0 -19
- testomatic_runner-0.1.0/testomatic/steps/led_spectral.py +0 -19
- testomatic_runner-0.1.0/testomatic/steps/python_step.py +0 -26
- testomatic_runner-0.1.0/testomatic_runner.egg-info/requires.txt +0 -6
- testomatic_runner-0.1.0/tests/test_steps.py +0 -150
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/LICENSE +0 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/setup.cfg +0 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic/__init__.py +0 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic/__main__.py +0 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic/steps/beep.py +0 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic/steps/delay.py +0 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic/steps/operator_intervention.py +0 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic/steps/registry.py +0 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic_runner.egg-info/SOURCES.txt +0 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic_runner.egg-info/dependency_links.txt +0 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic_runner.egg-info/entry_points.txt +0 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/testomatic_runner.egg-info/top_level.txt +0 -0
- {testomatic_runner-0.1.0 → testomatic_runner-0.3.0}/tests/test_testomatic.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: testomatic-runner
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Test runner that executes a Testomatic Test Suite Package against real hardware via testomatic-io.
|
|
5
5
|
Author: Elena Williams
|
|
6
6
|
Author-email: Jonathan Oxer <jon@oxer.com.au>
|
|
@@ -18,7 +18,8 @@ License-File: LICENSE
|
|
|
18
18
|
Provides-Extra: dev
|
|
19
19
|
Requires-Dist: pytest; extra == "dev"
|
|
20
20
|
Provides-Extra: pi
|
|
21
|
-
Requires-Dist: testomatic-io>=0.
|
|
21
|
+
Requires-Dist: testomatic-io>=0.2.0; extra == "pi"
|
|
22
|
+
Requires-Dist: esptool; extra == "pi"
|
|
22
23
|
Dynamic: license-file
|
|
23
24
|
|
|
24
25
|
# testomatic-runner
|
|
@@ -33,14 +34,14 @@ abstraction layer.
|
|
|
33
34
|
Testomatic's test pipeline has four parts, each in its own repo:
|
|
34
35
|
|
|
35
36
|
1. **Register** — a Test Suite is authored/versioned there, against a PCB `Design`.
|
|
36
|
-
2. **[`testomatic-ui`](https://github.com/SuperHouse/testomatic-ui)
|
|
37
|
+
2. **[`testomatic-ui`](https://github.com/SuperHouse/testomatic-ui)**: the on-device touchscreen
|
|
37
38
|
UI. Downloads and caches Test Suite Packages from Register so a device isn't dependent on
|
|
38
39
|
Register being reachable to run a suite it already has.
|
|
39
|
-
3. **This repo (`testomatic-runner`)
|
|
40
|
+
3. **This repo (`testomatic-runner`)**: receives a Test Suite Package (a ZIP containing
|
|
40
41
|
`test-suite-definition.json`, documented in [test-suite-package.md](test-suite-package.md)),
|
|
41
42
|
parses it with `testomatic.suite.load_suite()`, and executes each Test Step in order with
|
|
42
43
|
`testomatic.runner.TestRunner`.
|
|
43
|
-
4. **[`testomatic-io`](https://github.com/SuperHouse/testomatic-io)
|
|
44
|
+
4. **[`testomatic-io`](https://github.com/SuperHouse/testomatic-io)**: carries out each step's
|
|
44
45
|
hardware action (drive a power rail, read an IOMOD pin, beep, etc.) through its `Chassis`/
|
|
45
46
|
`TestModule` API.
|
|
46
47
|
|
|
@@ -55,24 +56,58 @@ pytest
|
|
|
55
56
|
|
|
56
57
|
# On a Raspberry Pi wired to a Testomatic chassis:
|
|
57
58
|
pip install -e ".[pi]" # pulls in testomatic-io
|
|
58
|
-
|
|
59
|
+
testomatic run path/to/suite-hw1-test-suite-v1.zip
|
|
59
60
|
```
|
|
60
61
|
|
|
61
|
-
`
|
|
62
|
-
|
|
63
|
-
`import testomatic_io` (and therefore `python -m testomatic run`) only works on real Raspberry Pi
|
|
64
|
-
hardware — `testomatic-io` is installed via the `pi` extra rather than as a core dependency, since
|
|
62
|
+
`import testomatic_io` (and therefore running a suite for real) only works on real Raspberry Pi
|
|
63
|
+
hardware. `testomatic-io` is installed via the `pi` extra rather than as a core dependency, since
|
|
65
64
|
one of its dependencies (`gpiod`) has a native extension that only builds on Linux. See
|
|
66
65
|
[TEST_RUNNER_PLAN.md](TEST_RUNNER_PLAN.md) for what's implemented, what's stubbed, and what's
|
|
67
66
|
still unverified on real hardware.
|
|
68
67
|
|
|
68
|
+
### CLI
|
|
69
|
+
|
|
70
|
+
`pip install -e ".[pi]"` registers a `testomatic` console script (`python -m testomatic` works
|
|
71
|
+
identically, per `__main__.py`). It's the same `load_suite()` → `TestRunner.run()` →
|
|
72
|
+
`format_report()` path `testomatic-ui` drives in-process. This is the path exposed standalone,
|
|
73
|
+
with no Django project or Register connection required.
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
testomatic run <suite.zip|suite.json>
|
|
77
|
+
[--avrdude-path PATH]
|
|
78
|
+
[--esptool-path PATH]
|
|
79
|
+
[--openocd-path PATH]
|
|
80
|
+
[--stm32cubeprogrammer-path PATH]
|
|
81
|
+
[--verbose]
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
- `suite_path` — a downloaded Test Suite Package `.zip`, or a bare `test-suite-definition.json`
|
|
85
|
+
(see [test-suite-package.md](test-suite-package.md)) for a suite you're editing/testing by hand.
|
|
86
|
+
- The four `--*-path` options override the executable an `UPLOAD_FIRMWARE_*` step shells out to.
|
|
87
|
+
Each defaults to the bare tool name on `$PATH` (`avrdude`, `esptool.py`, `openocd`,
|
|
88
|
+
`STM32_Programmer_CLI`) when omitted — pass a path only if that tool isn't on `$PATH` or you
|
|
89
|
+
need a specific one. These are the only device-specific settings the runner takes; everything
|
|
90
|
+
else (serial port, adapter serial, rail assignments, etc.) comes from the suite itself.
|
|
91
|
+
- `--verbose` streams an `UPLOAD_FIRMWARE_*`/`PYTHON` step's tool/console output live as it runs
|
|
92
|
+
(rather than only after the step finishes), and includes that output in the final report for
|
|
93
|
+
debugging a firmware upload that's failing or hanging. It's always captured into the step's
|
|
94
|
+
result regardless of this flag; `--verbose` only controls whether it's also echoed here.
|
|
95
|
+
- Prints the run report (and any `manual_checks`) to stdout via `format_report()`, and exits `0` if
|
|
96
|
+
the suite passed, `1` otherwise: scriptable in a shell loop or CI-style harness.
|
|
97
|
+
|
|
98
|
+
Example with an overridden esptool location:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
testomatic run suite-hw1-test-suite-v1.zip --esptool-path /opt/homebrew/bin/esptool.py
|
|
102
|
+
```
|
|
103
|
+
|
|
69
104
|
## Repo layout
|
|
70
105
|
|
|
71
106
|
- `testomatic/suite.py` — parses/validates a Test Suite Definition (JSON) into dataclasses.
|
|
72
107
|
- `testomatic/steps/` — one executor module per `step_type`, registered in `steps/registry.py`.
|
|
73
108
|
- `testomatic/runner.py` — `TestRunner.run(suite)` executes `test_steps` in order, and
|
|
74
109
|
`format_report()` renders the result plus the suite's `manual_checks`.
|
|
75
|
-
- `testomatic/cli.py` / `__main__.py` — `
|
|
110
|
+
- `testomatic/cli.py` / `__main__.py` — the `testomatic` console script (see [CLI](#cli) above).
|
|
76
111
|
- `tests/conftest.py` — fake `Chassis`/`Power`/`Beeper`/`Iomod` doubles standing in for real
|
|
77
112
|
`testomatic-io` hardware.
|
|
78
113
|
- `test-suite-package.md` — reference documentation for the Test Suite Package (ZIP) layout and
|
|
@@ -10,14 +10,14 @@ abstraction layer.
|
|
|
10
10
|
Testomatic's test pipeline has four parts, each in its own repo:
|
|
11
11
|
|
|
12
12
|
1. **Register** — a Test Suite is authored/versioned there, against a PCB `Design`.
|
|
13
|
-
2. **[`testomatic-ui`](https://github.com/SuperHouse/testomatic-ui)
|
|
13
|
+
2. **[`testomatic-ui`](https://github.com/SuperHouse/testomatic-ui)**: the on-device touchscreen
|
|
14
14
|
UI. Downloads and caches Test Suite Packages from Register so a device isn't dependent on
|
|
15
15
|
Register being reachable to run a suite it already has.
|
|
16
|
-
3. **This repo (`testomatic-runner`)
|
|
16
|
+
3. **This repo (`testomatic-runner`)**: receives a Test Suite Package (a ZIP containing
|
|
17
17
|
`test-suite-definition.json`, documented in [test-suite-package.md](test-suite-package.md)),
|
|
18
18
|
parses it with `testomatic.suite.load_suite()`, and executes each Test Step in order with
|
|
19
19
|
`testomatic.runner.TestRunner`.
|
|
20
|
-
4. **[`testomatic-io`](https://github.com/SuperHouse/testomatic-io)
|
|
20
|
+
4. **[`testomatic-io`](https://github.com/SuperHouse/testomatic-io)**: carries out each step's
|
|
21
21
|
hardware action (drive a power rail, read an IOMOD pin, beep, etc.) through its `Chassis`/
|
|
22
22
|
`TestModule` API.
|
|
23
23
|
|
|
@@ -32,24 +32,58 @@ pytest
|
|
|
32
32
|
|
|
33
33
|
# On a Raspberry Pi wired to a Testomatic chassis:
|
|
34
34
|
pip install -e ".[pi]" # pulls in testomatic-io
|
|
35
|
-
|
|
35
|
+
testomatic run path/to/suite-hw1-test-suite-v1.zip
|
|
36
36
|
```
|
|
37
37
|
|
|
38
|
-
`
|
|
39
|
-
|
|
40
|
-
`import testomatic_io` (and therefore `python -m testomatic run`) only works on real Raspberry Pi
|
|
41
|
-
hardware — `testomatic-io` is installed via the `pi` extra rather than as a core dependency, since
|
|
38
|
+
`import testomatic_io` (and therefore running a suite for real) only works on real Raspberry Pi
|
|
39
|
+
hardware. `testomatic-io` is installed via the `pi` extra rather than as a core dependency, since
|
|
42
40
|
one of its dependencies (`gpiod`) has a native extension that only builds on Linux. See
|
|
43
41
|
[TEST_RUNNER_PLAN.md](TEST_RUNNER_PLAN.md) for what's implemented, what's stubbed, and what's
|
|
44
42
|
still unverified on real hardware.
|
|
45
43
|
|
|
44
|
+
### CLI
|
|
45
|
+
|
|
46
|
+
`pip install -e ".[pi]"` registers a `testomatic` console script (`python -m testomatic` works
|
|
47
|
+
identically, per `__main__.py`). It's the same `load_suite()` → `TestRunner.run()` →
|
|
48
|
+
`format_report()` path `testomatic-ui` drives in-process. This is the path exposed standalone,
|
|
49
|
+
with no Django project or Register connection required.
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
testomatic run <suite.zip|suite.json>
|
|
53
|
+
[--avrdude-path PATH]
|
|
54
|
+
[--esptool-path PATH]
|
|
55
|
+
[--openocd-path PATH]
|
|
56
|
+
[--stm32cubeprogrammer-path PATH]
|
|
57
|
+
[--verbose]
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
- `suite_path` — a downloaded Test Suite Package `.zip`, or a bare `test-suite-definition.json`
|
|
61
|
+
(see [test-suite-package.md](test-suite-package.md)) for a suite you're editing/testing by hand.
|
|
62
|
+
- The four `--*-path` options override the executable an `UPLOAD_FIRMWARE_*` step shells out to.
|
|
63
|
+
Each defaults to the bare tool name on `$PATH` (`avrdude`, `esptool.py`, `openocd`,
|
|
64
|
+
`STM32_Programmer_CLI`) when omitted — pass a path only if that tool isn't on `$PATH` or you
|
|
65
|
+
need a specific one. These are the only device-specific settings the runner takes; everything
|
|
66
|
+
else (serial port, adapter serial, rail assignments, etc.) comes from the suite itself.
|
|
67
|
+
- `--verbose` streams an `UPLOAD_FIRMWARE_*`/`PYTHON` step's tool/console output live as it runs
|
|
68
|
+
(rather than only after the step finishes), and includes that output in the final report for
|
|
69
|
+
debugging a firmware upload that's failing or hanging. It's always captured into the step's
|
|
70
|
+
result regardless of this flag; `--verbose` only controls whether it's also echoed here.
|
|
71
|
+
- Prints the run report (and any `manual_checks`) to stdout via `format_report()`, and exits `0` if
|
|
72
|
+
the suite passed, `1` otherwise: scriptable in a shell loop or CI-style harness.
|
|
73
|
+
|
|
74
|
+
Example with an overridden esptool location:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
testomatic run suite-hw1-test-suite-v1.zip --esptool-path /opt/homebrew/bin/esptool.py
|
|
78
|
+
```
|
|
79
|
+
|
|
46
80
|
## Repo layout
|
|
47
81
|
|
|
48
82
|
- `testomatic/suite.py` — parses/validates a Test Suite Definition (JSON) into dataclasses.
|
|
49
83
|
- `testomatic/steps/` — one executor module per `step_type`, registered in `steps/registry.py`.
|
|
50
84
|
- `testomatic/runner.py` — `TestRunner.run(suite)` executes `test_steps` in order, and
|
|
51
85
|
`format_report()` renders the result plus the suite's `manual_checks`.
|
|
52
|
-
- `testomatic/cli.py` / `__main__.py` — `
|
|
86
|
+
- `testomatic/cli.py` / `__main__.py` — the `testomatic` console script (see [CLI](#cli) above).
|
|
53
87
|
- `tests/conftest.py` — fake `Chassis`/`Power`/`Beeper`/`Iomod` doubles standing in for real
|
|
54
88
|
`testomatic-io` hardware.
|
|
55
89
|
- `test-suite-package.md` — reference documentation for the Test Suite Package (ZIP) layout and
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "testomatic-runner"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.0"
|
|
8
8
|
description = "Test runner that executes a Testomatic Test Suite Package against real hardware via testomatic-io."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [
|
|
@@ -31,9 +31,13 @@ dev = ["pytest"]
|
|
|
31
31
|
# testomatic-io pulls in gpiod (libgpiod's Python bindings), which has a native extension that
|
|
32
32
|
# only builds on Linux (needs linux/const.h) -- kept out of core dependencies so `pip install
|
|
33
33
|
# -e ".[dev]"` keeps working on a Mac dev machine. Install this extra on the Raspberry Pi itself.
|
|
34
|
-
# >=0.
|
|
34
|
+
# >=0.2.0 because 0.1.0 fails a fresh
|
|
35
35
|
# install on current Raspberry Pi OS -- see testomatic-io's BLINKA_REMOVAL_PLAN.md "Motivation".
|
|
36
|
-
|
|
36
|
+
# esptool is a pure-Python PyPI package (unlike avrdude/openocd/STM32CubeProgrammer, which are
|
|
37
|
+
# native tools with no PyPI distribution) and registers an `esptool.py` console script on
|
|
38
|
+
# install, so this makes UPLOAD_FIRMWARE_ESPTOOL's default `esptool_path` (see steps/firmware.py)
|
|
39
|
+
# resolve without the caller needing to set --esptool-path/context.esptool_path themselves.
|
|
40
|
+
pi = ["testomatic-io>=0.2.0", "esptool"]
|
|
37
41
|
|
|
38
42
|
[project.urls]
|
|
39
43
|
Homepage = "https://github.com/SuperHouse/testomatic-runner"
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Command-line entry point: `python -m testomatic run <suite.zip|suite.json>`.
|
|
2
|
+
|
|
3
|
+
Verified against real Testomatic hardware for BEEP and READ_RAIL_VOLTAGE — see
|
|
4
|
+
TEST_RUNNER_PLAN.md for what's still unverified.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
|
|
11
|
+
from .runner import TestRunner, format_report
|
|
12
|
+
from .suite import load_suite
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main(argv: list[str] | None = None) -> int:
|
|
16
|
+
parser = argparse.ArgumentParser(prog="testomatic")
|
|
17
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
18
|
+
|
|
19
|
+
run_parser = subparsers.add_parser("run", help="Execute a Test Suite")
|
|
20
|
+
run_parser.add_argument(
|
|
21
|
+
"suite_path",
|
|
22
|
+
help="Path to a Test Suite Package (.zip) or a Test Suite Definition JSON file",
|
|
23
|
+
)
|
|
24
|
+
run_parser.add_argument(
|
|
25
|
+
"--avrdude-path", help="Override the avrdude executable (default: 'avrdude' on $PATH)"
|
|
26
|
+
)
|
|
27
|
+
run_parser.add_argument(
|
|
28
|
+
"--esptool-path", help="Override the esptool.py executable (default: 'esptool.py' on $PATH)"
|
|
29
|
+
)
|
|
30
|
+
run_parser.add_argument(
|
|
31
|
+
"--openocd-path", help="Override the openocd executable (default: 'openocd' on $PATH)"
|
|
32
|
+
)
|
|
33
|
+
run_parser.add_argument(
|
|
34
|
+
"--stm32cubeprogrammer-path",
|
|
35
|
+
help="Override the STM32_Programmer_CLI executable (default: 'STM32_Programmer_CLI' on $PATH)",
|
|
36
|
+
)
|
|
37
|
+
run_parser.add_argument(
|
|
38
|
+
"--verbose", action="store_true",
|
|
39
|
+
help="Stream firmware-tool/Python-step output live as steps run, and include it in the final report",
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
args = parser.parse_args(argv)
|
|
43
|
+
|
|
44
|
+
if args.command == "run":
|
|
45
|
+
return _run(
|
|
46
|
+
args.suite_path,
|
|
47
|
+
avrdude_path=args.avrdude_path,
|
|
48
|
+
esptool_path=args.esptool_path,
|
|
49
|
+
openocd_path=args.openocd_path,
|
|
50
|
+
stm32cubeprogrammer_path=args.stm32cubeprogrammer_path,
|
|
51
|
+
verbose=args.verbose,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
return 1
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _run(
|
|
58
|
+
suite_path: str,
|
|
59
|
+
*,
|
|
60
|
+
avrdude_path: str | None = None,
|
|
61
|
+
esptool_path: str | None = None,
|
|
62
|
+
openocd_path: str | None = None,
|
|
63
|
+
stm32cubeprogrammer_path: str | None = None,
|
|
64
|
+
verbose: bool = False,
|
|
65
|
+
) -> int:
|
|
66
|
+
from testomatic_io import Chassis, TestModule # imported here: only importable on real hardware
|
|
67
|
+
|
|
68
|
+
suite = load_suite(suite_path)
|
|
69
|
+
|
|
70
|
+
chassis = Chassis()
|
|
71
|
+
chassis.init()
|
|
72
|
+
test_module = TestModule()
|
|
73
|
+
test_module.init()
|
|
74
|
+
|
|
75
|
+
runner = TestRunner(
|
|
76
|
+
chassis,
|
|
77
|
+
test_module,
|
|
78
|
+
avrdude_path=avrdude_path,
|
|
79
|
+
esptool_path=esptool_path,
|
|
80
|
+
openocd_path=openocd_path,
|
|
81
|
+
stm32cubeprogrammer_path=stm32cubeprogrammer_path,
|
|
82
|
+
verbose=verbose,
|
|
83
|
+
)
|
|
84
|
+
report = runner.run(suite)
|
|
85
|
+
|
|
86
|
+
print(format_report(report, suite.manual_checks, verbose=verbose))
|
|
87
|
+
|
|
88
|
+
return 0 if report.passed else 1
|
|
@@ -25,10 +25,29 @@ class RunReport:
|
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
class TestRunner:
|
|
28
|
-
def __init__(
|
|
29
|
-
self
|
|
28
|
+
def __init__(
|
|
29
|
+
self,
|
|
30
|
+
chassis,
|
|
31
|
+
test_module=None,
|
|
32
|
+
*,
|
|
33
|
+
avrdude_path: str | None = None,
|
|
34
|
+
esptool_path: str | None = None,
|
|
35
|
+
openocd_path: str | None = None,
|
|
36
|
+
stm32cubeprogrammer_path: str | None = None,
|
|
37
|
+
verbose: bool = False,
|
|
38
|
+
):
|
|
39
|
+
self.context = ExecutionContext(
|
|
40
|
+
chassis=chassis,
|
|
41
|
+
test_module=test_module,
|
|
42
|
+
avrdude_path=avrdude_path,
|
|
43
|
+
esptool_path=esptool_path,
|
|
44
|
+
openocd_path=openocd_path,
|
|
45
|
+
stm32cubeprogrammer_path=stm32cubeprogrammer_path,
|
|
46
|
+
verbose=verbose,
|
|
47
|
+
)
|
|
30
48
|
|
|
31
49
|
def run(self, suite: TestSuiteFile) -> RunReport:
|
|
50
|
+
self.context.package_dir = suite.package_dir
|
|
32
51
|
report = RunReport()
|
|
33
52
|
|
|
34
53
|
for step in suite.test_steps:
|
|
@@ -63,13 +82,19 @@ class TestRunner:
|
|
|
63
82
|
power.rail_12v(False)
|
|
64
83
|
|
|
65
84
|
|
|
66
|
-
def format_report(report: RunReport, manual_checks: list[ManualCheck]) -> str:
|
|
85
|
+
def format_report(report: RunReport, manual_checks: list[ManualCheck], verbose: bool = False) -> str:
|
|
67
86
|
lines = []
|
|
68
87
|
|
|
69
88
|
for outcome in report.outcomes:
|
|
70
89
|
status = "PASS" if outcome.result.passed else "FAIL"
|
|
71
90
|
lines.append(f"[{status}] {outcome.step.name}: {outcome.result.message}")
|
|
72
91
|
|
|
92
|
+
output = outcome.result.measured.get("output") if verbose else None
|
|
93
|
+
if output:
|
|
94
|
+
lines.append(" --- output ---")
|
|
95
|
+
lines.extend(f" {line}" for line in output.splitlines())
|
|
96
|
+
lines.append(" --------------")
|
|
97
|
+
|
|
73
98
|
if report.aborted:
|
|
74
99
|
lines.append("ABORTED: abort-on-fail step failed, all power rails turned off")
|
|
75
100
|
|
|
@@ -13,6 +13,7 @@ from . import ( # noqa: F401 -- imported for their @register_step side effects
|
|
|
13
13
|
python_step,
|
|
14
14
|
)
|
|
15
15
|
from .base import ExecutionContext, StepResult
|
|
16
|
+
from .firmware import FIRMWARE_STEP_TYPES
|
|
16
17
|
from .registry import STEP_EXECUTORS, get_executor, register_step
|
|
17
18
|
|
|
18
19
|
__all__ = [
|
|
@@ -21,4 +22,5 @@ __all__ = [
|
|
|
21
22
|
"STEP_EXECUTORS",
|
|
22
23
|
"get_executor",
|
|
23
24
|
"register_step",
|
|
25
|
+
"FIRMWARE_STEP_TYPES",
|
|
24
26
|
]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Shared types every step executor uses."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class StepResult:
|
|
12
|
+
"""The outcome of executing one Test Step."""
|
|
13
|
+
|
|
14
|
+
passed: bool
|
|
15
|
+
message: str = ""
|
|
16
|
+
measured: dict[str, Any] = field(default_factory=dict)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class ExecutionContext:
|
|
21
|
+
"""Hardware handles, plus Test Suite Package/tool-path context, passed to every step executor.
|
|
22
|
+
|
|
23
|
+
`package_dir` is set by `TestRunner.run()` from `suite.package_dir` (see suite.py) — the
|
|
24
|
+
directory a firmware step's `firmware_file`/`images[].file` resolve relative to.
|
|
25
|
+
|
|
26
|
+
The four `*_path` fields override the executable a firmware-upload step shells out to
|
|
27
|
+
(default: the bare tool name on $PATH — see firmware.py). This is the extension point a
|
|
28
|
+
caller sets from its own per-device configuration (e.g. testomatic-ui, once it grows a
|
|
29
|
+
settings page for this); it's deliberately separate from the serial port/debug-probe fields,
|
|
30
|
+
which stay suite-side (`config["port"]` etc., as defined by Register) — see the open question
|
|
31
|
+
tracked as Register issue #122, not resolved here.
|
|
32
|
+
|
|
33
|
+
`verbose` (set via `TestRunner(...)`/`cli.py`'s `--verbose` flag) asks executors that shell
|
|
34
|
+
out or run arbitrary code (firmware.py, python_step.py) to stream their subprocess/stdout
|
|
35
|
+
output live to the console as it happens, in addition to always capturing it into
|
|
36
|
+
`StepResult.measured["output"]` regardless of this flag.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
chassis: Any
|
|
40
|
+
test_module: Any = None
|
|
41
|
+
package_dir: Path | None = None
|
|
42
|
+
avrdude_path: str | None = None
|
|
43
|
+
esptool_path: str | None = None
|
|
44
|
+
openocd_path: str | None = None
|
|
45
|
+
stm32cubeprogrammer_path: str | None = None
|
|
46
|
+
verbose: bool = False
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""UPLOAD_FIRMWARE_* steps: flash firmware onto the DUT via avrdude, esptool.py, OpenOCD, or
|
|
2
|
+
STM32CubeProgrammer — one executor per tool, since each connects to and addresses the target
|
|
3
|
+
differently. See test-suite-package.md for each step type's config fields.
|
|
4
|
+
|
|
5
|
+
Firmware bytes are resolved from the Test Suite Package: `context.package_dir` (set by
|
|
6
|
+
`TestRunner.run()` from `suite.package_dir`, itself set by `suite.load_suite()` — see suite.py)
|
|
7
|
+
is the directory `firmware_file`/`images[].file` are resolved relative to.
|
|
8
|
+
|
|
9
|
+
Tool executable paths are configurable per-`ExecutionContext` (`context.avrdude_path` etc., set
|
|
10
|
+
via `TestRunner(...)`/`cli.py`'s `--*-path` options — see base.py), defaulting to the bare tool
|
|
11
|
+
name on `$PATH` when unset. This is the extension point a caller (e.g. testomatic-ui, once it
|
|
12
|
+
grows a per-device settings page) uses to point at a tool that isn't on `$PATH`. Serial port /
|
|
13
|
+
debug-probe identification, by contrast, stays entirely suite-side (`config["port"]` etc., as
|
|
14
|
+
defined by Register) for now — see the open question tracked as Register issue #122 about
|
|
15
|
+
whether that should instead be tester-side config. Deliberately not resolved here: left until
|
|
16
|
+
the system has been exercised end-to-end using the current suite-side approach.
|
|
17
|
+
|
|
18
|
+
**Unverified against real hardware.** The exact command-line flags below are a first-pass best
|
|
19
|
+
effort from each tool's own documented CLI, not yet confirmed against a real chassis — see
|
|
20
|
+
TEST_RUNNER_PLAN.md's Phase 5 checklist.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import subprocess
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
|
|
28
|
+
from .base import ExecutionContext, StepResult
|
|
29
|
+
from .registry import register_step
|
|
30
|
+
|
|
31
|
+
# Register issue #124: a caller building a report/docket needs to pick these 4 step types out of
|
|
32
|
+
# a RunReport as a special case (collecting their names as the firmware versions on the board),
|
|
33
|
+
# separately from the pass/fail line every other step type also gets — see
|
|
34
|
+
# testomatic-ui's test_suites/docket.py, which imports this set directly rather than
|
|
35
|
+
# re-duplicating the 4 type strings, since testomatic-ui already depends on this package.
|
|
36
|
+
FIRMWARE_STEP_TYPES = frozenset({
|
|
37
|
+
"UPLOAD_FIRMWARE_AVRDUDE",
|
|
38
|
+
"UPLOAD_FIRMWARE_ESPTOOL",
|
|
39
|
+
"UPLOAD_FIRMWARE_OPENOCD",
|
|
40
|
+
"UPLOAD_FIRMWARE_STM32CUBEPROGRAMMER",
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _resolve_file(context: ExecutionContext, filename: str) -> tuple[Path | None, str | None]:
|
|
45
|
+
"""Resolves `filename` against `context.package_dir`. Returns `(path, None)` on success, or
|
|
46
|
+
`(None, error_message)` if it can't be resolved — an ordinary "file not attached/found" state
|
|
47
|
+
a caller turns straight into a failing `StepResult`, not an exceptional one."""
|
|
48
|
+
if context.package_dir is None:
|
|
49
|
+
return None, f"cannot resolve {filename!r}: no Test Suite Package directory available"
|
|
50
|
+
path = context.package_dir / filename
|
|
51
|
+
if not path.is_file():
|
|
52
|
+
return None, f"{filename!r} not found in Test Suite Package (expected at {path})"
|
|
53
|
+
return path, None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _run_tool(command: list[str], tool_label: str, context: ExecutionContext) -> StepResult:
|
|
57
|
+
"""Runs a tool's command line, turning its exit code and output into a StepResult.
|
|
58
|
+
|
|
59
|
+
Output is always captured in full into the result's `measured["output"]` (for callers such
|
|
60
|
+
as testomatic-ui to store/display later), and when `context.verbose` is set it's also
|
|
61
|
+
streamed to stdout line-by-line as the tool runs, rather than only appearing once the tool
|
|
62
|
+
exits — useful for a long-running avrdude/esptool/openocd/STM32CubeProgrammer invocation.
|
|
63
|
+
"""
|
|
64
|
+
try:
|
|
65
|
+
process = subprocess.Popen(
|
|
66
|
+
command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1,
|
|
67
|
+
)
|
|
68
|
+
except FileNotFoundError as exc:
|
|
69
|
+
return StepResult(passed=False, message=f"{tool_label} not found: {exc}")
|
|
70
|
+
|
|
71
|
+
lines = []
|
|
72
|
+
for line in process.stdout:
|
|
73
|
+
lines.append(line)
|
|
74
|
+
if context.verbose:
|
|
75
|
+
print(line, end="")
|
|
76
|
+
process.wait()
|
|
77
|
+
|
|
78
|
+
output = "".join(lines).strip()
|
|
79
|
+
last_line = output.splitlines()[-1] if output else ""
|
|
80
|
+
suffix = f": {last_line}" if last_line else ""
|
|
81
|
+
|
|
82
|
+
if process.returncode == 0:
|
|
83
|
+
return StepResult(passed=True, message=f"{tool_label} succeeded{suffix}", measured={"output": output})
|
|
84
|
+
return StepResult(
|
|
85
|
+
passed=False,
|
|
86
|
+
message=f"{tool_label} exited {process.returncode}{suffix}",
|
|
87
|
+
measured={"output": output},
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@register_step("UPLOAD_FIRMWARE_AVRDUDE")
|
|
92
|
+
def execute_avrdude(config: dict, context: ExecutionContext) -> StepResult:
|
|
93
|
+
firmware_file = config.get("firmware_file")
|
|
94
|
+
if not firmware_file:
|
|
95
|
+
return StepResult(passed=False, message="No firmware file attached to this step")
|
|
96
|
+
path, error = _resolve_file(context, firmware_file)
|
|
97
|
+
if error:
|
|
98
|
+
return StepResult(passed=False, message=error)
|
|
99
|
+
|
|
100
|
+
command = [
|
|
101
|
+
context.avrdude_path or "avrdude",
|
|
102
|
+
"-c", config["programmer_type"],
|
|
103
|
+
"-p", config["mcu"],
|
|
104
|
+
"-P", config["port"],
|
|
105
|
+
]
|
|
106
|
+
if config.get("baud_rate"):
|
|
107
|
+
command += ["-b", str(config["baud_rate"])]
|
|
108
|
+
command += ["-U", f"flash:w:{path}:i"]
|
|
109
|
+
|
|
110
|
+
return _run_tool(command, "avrdude", context)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@register_step("UPLOAD_FIRMWARE_ESPTOOL")
|
|
114
|
+
def execute_esptool(config: dict, context: ExecutionContext) -> StepResult:
|
|
115
|
+
images = config.get("images")
|
|
116
|
+
if not images:
|
|
117
|
+
return StepResult(passed=False, message="No firmware images attached to this step")
|
|
118
|
+
|
|
119
|
+
flash_args = []
|
|
120
|
+
for image in images:
|
|
121
|
+
path, error = _resolve_file(context, image["file"])
|
|
122
|
+
if error:
|
|
123
|
+
return StepResult(passed=False, message=error)
|
|
124
|
+
flash_args += [image["address"], str(path)]
|
|
125
|
+
|
|
126
|
+
command = [
|
|
127
|
+
context.esptool_path or "esptool.py",
|
|
128
|
+
"--chip", config["chip"],
|
|
129
|
+
"--port", config["port"],
|
|
130
|
+
]
|
|
131
|
+
if config.get("baud_rate"):
|
|
132
|
+
command += ["--baud", str(config["baud_rate"])]
|
|
133
|
+
command += ["write-flash", *flash_args]
|
|
134
|
+
|
|
135
|
+
return _run_tool(command, "esptool.py", context)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@register_step("UPLOAD_FIRMWARE_OPENOCD")
|
|
139
|
+
def execute_openocd(config: dict, context: ExecutionContext) -> StepResult:
|
|
140
|
+
firmware_file = config.get("firmware_file")
|
|
141
|
+
if not firmware_file:
|
|
142
|
+
return StepResult(passed=False, message="No firmware file attached to this step")
|
|
143
|
+
path, error = _resolve_file(context, firmware_file)
|
|
144
|
+
if error:
|
|
145
|
+
return StepResult(passed=False, message=error)
|
|
146
|
+
|
|
147
|
+
command = [
|
|
148
|
+
context.openocd_path or "openocd",
|
|
149
|
+
"-f", config["interface_config"],
|
|
150
|
+
"-f", config["target_config"],
|
|
151
|
+
]
|
|
152
|
+
if config.get("adapter_serial"):
|
|
153
|
+
command += ["-c", f"adapter serial {config['adapter_serial']}"]
|
|
154
|
+
|
|
155
|
+
program_command = f"program {path}"
|
|
156
|
+
if config.get("flash_address"):
|
|
157
|
+
program_command += f" {config['flash_address']}"
|
|
158
|
+
program_command += " verify reset exit"
|
|
159
|
+
command += ["-c", program_command]
|
|
160
|
+
|
|
161
|
+
return _run_tool(command, "openocd", context)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@register_step("UPLOAD_FIRMWARE_STM32CUBEPROGRAMMER")
|
|
165
|
+
def execute_stm32cubeprogrammer(config: dict, context: ExecutionContext) -> StepResult:
|
|
166
|
+
firmware_file = config.get("firmware_file")
|
|
167
|
+
if not firmware_file:
|
|
168
|
+
return StepResult(passed=False, message="No firmware file attached to this step")
|
|
169
|
+
path, error = _resolve_file(context, firmware_file)
|
|
170
|
+
if error:
|
|
171
|
+
return StepResult(passed=False, message=error)
|
|
172
|
+
|
|
173
|
+
connection = f"port={config['connection_interface']}"
|
|
174
|
+
if config.get("port"):
|
|
175
|
+
connection += f" sn={config['port']}"
|
|
176
|
+
|
|
177
|
+
command = [
|
|
178
|
+
context.stm32cubeprogrammer_path or "STM32_Programmer_CLI",
|
|
179
|
+
"-c", connection,
|
|
180
|
+
"-w", str(path),
|
|
181
|
+
]
|
|
182
|
+
if config.get("flash_address"):
|
|
183
|
+
command.append(config["flash_address"])
|
|
184
|
+
command += ["-v", "-rst"]
|
|
185
|
+
|
|
186
|
+
return _run_tool(command, "STM32CubeProgrammer", context)
|
|
@@ -19,7 +19,9 @@ def execute_digital_read(config: dict, context: ExecutionContext) -> StepResult:
|
|
|
19
19
|
return StepResult(
|
|
20
20
|
passed=passed,
|
|
21
21
|
message=f"IOMOD {iomod} pin {pin}: read {actual}, expected {expect}",
|
|
22
|
-
|
|
22
|
+
# "display" (issue testomatic-ui#13): the short value a caller like testomatic-ui's Test
|
|
23
|
+
# Docket shows next to PASS.
|
|
24
|
+
measured={"value": actual, "display": actual},
|
|
23
25
|
)
|
|
24
26
|
|
|
25
27
|
|
|
@@ -52,7 +54,7 @@ def execute_analog_read(config: dict, context: ExecutionContext) -> StepResult:
|
|
|
52
54
|
return StepResult(
|
|
53
55
|
passed=passed,
|
|
54
56
|
message=f"IOMOD {iomod} pin {pin}: read {value}",
|
|
55
|
-
measured={"value": value},
|
|
57
|
+
measured={"value": value, "display": str(value)},
|
|
56
58
|
)
|
|
57
59
|
|
|
58
60
|
|