testomatic-runner 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- testomatic/__init__.py +1 -0
- testomatic/__main__.py +6 -0
- testomatic/cli.py +48 -0
- testomatic/runner.py +85 -0
- testomatic/steps/__init__.py +24 -0
- testomatic/steps/base.py +23 -0
- testomatic/steps/beep.py +27 -0
- testomatic/steps/delay.py +15 -0
- testomatic/steps/firmware.py +19 -0
- testomatic/steps/iomod.py +67 -0
- testomatic/steps/led_spectral.py +19 -0
- testomatic/steps/operator_intervention.py +18 -0
- testomatic/steps/power.py +54 -0
- testomatic/steps/python_step.py +26 -0
- testomatic/steps/registry.py +36 -0
- testomatic/suite.py +175 -0
- testomatic_runner-0.1.0.dist-info/METADATA +81 -0
- testomatic_runner-0.1.0.dist-info/RECORD +22 -0
- testomatic_runner-0.1.0.dist-info/WHEEL +5 -0
- testomatic_runner-0.1.0.dist-info/entry_points.txt +2 -0
- testomatic_runner-0.1.0.dist-info/licenses/LICENSE +21 -0
- testomatic_runner-0.1.0.dist-info/top_level.txt +1 -0
testomatic/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Top-level package for Testomatic."""
|
testomatic/__main__.py
ADDED
testomatic/cli.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Command-line entry point: `python -m testomatic run <suite.zip|suite.json>`.
|
|
2
|
+
|
|
3
|
+
Verified against real Testomatic hardware for BEEP and READ_RAIL_VOLTAGE — see
|
|
4
|
+
TEST_RUNNER_PLAN.md for what's still unverified.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
|
|
11
|
+
from .runner import TestRunner, format_report
|
|
12
|
+
from .suite import load_suite
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main(argv: list[str] | None = None) -> int:
|
|
16
|
+
parser = argparse.ArgumentParser(prog="testomatic")
|
|
17
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
18
|
+
|
|
19
|
+
run_parser = subparsers.add_parser("run", help="Execute a Test Suite")
|
|
20
|
+
run_parser.add_argument(
|
|
21
|
+
"suite_path",
|
|
22
|
+
help="Path to a Test Suite Package (.zip) or a Test Suite Definition JSON file",
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
args = parser.parse_args(argv)
|
|
26
|
+
|
|
27
|
+
if args.command == "run":
|
|
28
|
+
return _run(args.suite_path)
|
|
29
|
+
|
|
30
|
+
return 1
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _run(suite_path: str) -> int:
|
|
34
|
+
from testomatic_io import Chassis, TestModule # imported here: only importable on real hardware
|
|
35
|
+
|
|
36
|
+
suite = load_suite(suite_path)
|
|
37
|
+
|
|
38
|
+
chassis = Chassis()
|
|
39
|
+
chassis.init()
|
|
40
|
+
test_module = TestModule()
|
|
41
|
+
test_module.init()
|
|
42
|
+
|
|
43
|
+
runner = TestRunner(chassis, test_module)
|
|
44
|
+
report = runner.run(suite)
|
|
45
|
+
|
|
46
|
+
print(format_report(report, suite.manual_checks))
|
|
47
|
+
|
|
48
|
+
return 0 if report.passed else 1
|
testomatic/runner.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Executes a parsed Test Suite's steps against real hardware."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
|
|
7
|
+
from .steps import ExecutionContext, StepResult, get_executor
|
|
8
|
+
from .suite import ManualCheck, TestStep, TestSuiteFile
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class StepOutcome:
|
|
13
|
+
step: TestStep
|
|
14
|
+
result: StepResult
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class RunReport:
|
|
19
|
+
outcomes: list[StepOutcome] = field(default_factory=list)
|
|
20
|
+
aborted: bool = False
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def passed(self) -> bool:
|
|
24
|
+
return not self.aborted and all(outcome.result.passed for outcome in self.outcomes)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class TestRunner:
|
|
28
|
+
def __init__(self, chassis, test_module=None):
|
|
29
|
+
self.context = ExecutionContext(chassis=chassis, test_module=test_module)
|
|
30
|
+
|
|
31
|
+
def run(self, suite: TestSuiteFile) -> RunReport:
|
|
32
|
+
report = RunReport()
|
|
33
|
+
|
|
34
|
+
for step in suite.test_steps:
|
|
35
|
+
result = self._run_step(step)
|
|
36
|
+
report.outcomes.append(StepOutcome(step=step, result=result))
|
|
37
|
+
|
|
38
|
+
if not result.passed and step.abort_on_fail:
|
|
39
|
+
self._all_rails_off()
|
|
40
|
+
report.aborted = True
|
|
41
|
+
break
|
|
42
|
+
|
|
43
|
+
return report
|
|
44
|
+
|
|
45
|
+
def _run_step(self, step: TestStep) -> StepResult:
|
|
46
|
+
try:
|
|
47
|
+
executor = get_executor(step.step_type)
|
|
48
|
+
except KeyError as exc:
|
|
49
|
+
return StepResult(passed=False, message=str(exc))
|
|
50
|
+
|
|
51
|
+
try:
|
|
52
|
+
return executor(step.config, self.context)
|
|
53
|
+
except Exception as exc: # a bug in one step's executor must not crash the whole run
|
|
54
|
+
return StepResult(passed=False, message=f"{step.step_type} raised: {exc}")
|
|
55
|
+
|
|
56
|
+
def _all_rails_off(self) -> None:
|
|
57
|
+
"""Safety shutdown on abort-on-fail: turn off all three power rails unconditionally,
|
|
58
|
+
regardless of what the runner believes their current state to be.
|
|
59
|
+
"""
|
|
60
|
+
power = self.context.chassis.power
|
|
61
|
+
power.rail_3v3(False)
|
|
62
|
+
power.rail_5v(False)
|
|
63
|
+
power.rail_12v(False)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def format_report(report: RunReport, manual_checks: list[ManualCheck]) -> str:
|
|
67
|
+
lines = []
|
|
68
|
+
|
|
69
|
+
for outcome in report.outcomes:
|
|
70
|
+
status = "PASS" if outcome.result.passed else "FAIL"
|
|
71
|
+
lines.append(f"[{status}] {outcome.step.name}: {outcome.result.message}")
|
|
72
|
+
|
|
73
|
+
if report.aborted:
|
|
74
|
+
lines.append("ABORTED: abort-on-fail step failed, all power rails turned off")
|
|
75
|
+
|
|
76
|
+
lines.append("")
|
|
77
|
+
lines.append(f"Result: {'PASS' if report.passed else 'FAIL'}")
|
|
78
|
+
|
|
79
|
+
if manual_checks:
|
|
80
|
+
lines.append("")
|
|
81
|
+
lines.append("Manual checks (perform by hand):")
|
|
82
|
+
for check in manual_checks:
|
|
83
|
+
lines.append(f" [ ] {check.text}")
|
|
84
|
+
|
|
85
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Step executors, registered by `step_type` in `registry.STEP_EXECUTORS`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from . import ( # noqa: F401 -- imported for their @register_step side effects
|
|
6
|
+
beep,
|
|
7
|
+
delay,
|
|
8
|
+
firmware,
|
|
9
|
+
iomod,
|
|
10
|
+
led_spectral,
|
|
11
|
+
operator_intervention,
|
|
12
|
+
power,
|
|
13
|
+
python_step,
|
|
14
|
+
)
|
|
15
|
+
from .base import ExecutionContext, StepResult
|
|
16
|
+
from .registry import STEP_EXECUTORS, get_executor, register_step
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"ExecutionContext",
|
|
20
|
+
"StepResult",
|
|
21
|
+
"STEP_EXECUTORS",
|
|
22
|
+
"get_executor",
|
|
23
|
+
"register_step",
|
|
24
|
+
]
|
testomatic/steps/base.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Shared types every step executor uses."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class StepResult:
|
|
11
|
+
"""The outcome of executing one Test Step."""
|
|
12
|
+
|
|
13
|
+
passed: bool
|
|
14
|
+
message: str = ""
|
|
15
|
+
measured: dict[str, Any] = field(default_factory=dict)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class ExecutionContext:
|
|
20
|
+
"""Hardware handles passed to every step executor."""
|
|
21
|
+
|
|
22
|
+
chassis: Any
|
|
23
|
+
test_module: Any = None
|
testomatic/steps/beep.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""BEEP step: sound the piezo beeper, optionally as a sequence of beeps.
|
|
2
|
+
|
|
3
|
+
Per TEST_RUNNER_PLAN.md's BEEP timing note: a silent gap of `duration_ms` separates each beep —
|
|
4
|
+
e.g. duration_ms=100, count=3 is beep-silence-beep-silence-beep, not three beeps back-to-back.
|
|
5
|
+
No trailing silence after the last beep.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
from .base import ExecutionContext, StepResult
|
|
13
|
+
from .registry import register_step
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@register_step("BEEP")
|
|
17
|
+
def execute(config: dict, context: ExecutionContext) -> StepResult:
|
|
18
|
+
duration_ms = config["duration_ms"]
|
|
19
|
+
count = config.get("count", 1)
|
|
20
|
+
duration_s = duration_ms / 1000
|
|
21
|
+
|
|
22
|
+
for i in range(count):
|
|
23
|
+
context.chassis.beeper.beep(duration_s)
|
|
24
|
+
if i < count - 1:
|
|
25
|
+
time.sleep(duration_s)
|
|
26
|
+
|
|
27
|
+
return StepResult(passed=True, message=f"Beeped {count}x{duration_ms}ms")
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""DELAY step: pause for a fixed number of milliseconds."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
from .base import ExecutionContext, StepResult
|
|
8
|
+
from .registry import register_step
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@register_step("DELAY")
|
|
12
|
+
def execute(config: dict, context: ExecutionContext) -> StepResult:
|
|
13
|
+
delay_ms = config["delay_ms"]
|
|
14
|
+
time.sleep(delay_ms / 1000)
|
|
15
|
+
return StepResult(passed=True, message=f"Delayed {delay_ms} ms")
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""UPLOAD_FIRMWARE step: stub.
|
|
2
|
+
|
|
3
|
+
Deferred: the design for associating firmware files with Test Suites isn't settled (the JSON
|
|
4
|
+
only carries a `firmware_file` name, not the binary), and upload-tool dispatch
|
|
5
|
+
(avrdude/esptool.py/openocd/stm32cubeprogrammer) is a moderately complex piece of work on its
|
|
6
|
+
own. See TEST_RUNNER_PLAN.md's "Deferred work" section. This stub lets suites containing an
|
|
7
|
+
UPLOAD_FIRMWARE step still run end-to-end.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from .base import ExecutionContext, StepResult
|
|
13
|
+
from .registry import register_step
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@register_step("UPLOAD_FIRMWARE")
|
|
17
|
+
def execute(config: dict, context: ExecutionContext) -> StepResult:
|
|
18
|
+
print("Firmware upload")
|
|
19
|
+
return StepResult(passed=True, message="Firmware upload (stub)")
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""IOMOD_* steps: digital/analog I/O on IOMOD expander module pins via `chassis.iomod`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .base import ExecutionContext, StepResult
|
|
6
|
+
from .registry import register_step
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@register_step("IOMOD_DIGITAL_READ")
|
|
10
|
+
def execute_digital_read(config: dict, context: ExecutionContext) -> StepResult:
|
|
11
|
+
iomod = config["iomod"]
|
|
12
|
+
pin = int(config["pin"])
|
|
13
|
+
expect = config["expect"]
|
|
14
|
+
|
|
15
|
+
value = context.chassis.iomod.digital_read(iomod, pin)
|
|
16
|
+
actual = "1" if value else "0"
|
|
17
|
+
passed = actual == expect
|
|
18
|
+
|
|
19
|
+
return StepResult(
|
|
20
|
+
passed=passed,
|
|
21
|
+
message=f"IOMOD {iomod} pin {pin}: read {actual}, expected {expect}",
|
|
22
|
+
measured={"value": actual},
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@register_step("IOMOD_DIGITAL_WRITE")
|
|
27
|
+
def execute_digital_write(config: dict, context: ExecutionContext) -> StepResult:
|
|
28
|
+
iomod = config["iomod"]
|
|
29
|
+
pin = int(config["pin"])
|
|
30
|
+
value = config["digital_write"]
|
|
31
|
+
|
|
32
|
+
context.chassis.iomod.digital_write(iomod, pin, value == "1")
|
|
33
|
+
|
|
34
|
+
return StepResult(passed=True, message=f"IOMOD {iomod} pin {pin}: wrote {value}")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@register_step("IOMOD_ANALOG_READ")
|
|
38
|
+
def execute_analog_read(config: dict, context: ExecutionContext) -> StepResult:
|
|
39
|
+
iomod = config["iomod"]
|
|
40
|
+
pin = int(config["pin"])
|
|
41
|
+
expect_min = config.get("expect_min")
|
|
42
|
+
expect_max = config.get("expect_max")
|
|
43
|
+
|
|
44
|
+
value = context.chassis.iomod.analog_read(iomod, pin)
|
|
45
|
+
|
|
46
|
+
passed = True
|
|
47
|
+
if expect_min is not None:
|
|
48
|
+
passed = passed and value >= expect_min
|
|
49
|
+
if expect_max is not None:
|
|
50
|
+
passed = passed and value <= expect_max
|
|
51
|
+
|
|
52
|
+
return StepResult(
|
|
53
|
+
passed=passed,
|
|
54
|
+
message=f"IOMOD {iomod} pin {pin}: read {value}",
|
|
55
|
+
measured={"value": value},
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@register_step("IOMOD_ANALOG_WRITE")
|
|
60
|
+
def execute_analog_write(config: dict, context: ExecutionContext) -> StepResult:
|
|
61
|
+
iomod = config["iomod"]
|
|
62
|
+
pin = int(config["pin"])
|
|
63
|
+
value = config["analog_write"]
|
|
64
|
+
|
|
65
|
+
context.chassis.iomod.analog_write(iomod, pin, value)
|
|
66
|
+
|
|
67
|
+
return StepResult(passed=True, message=f"IOMOD {iomod} pin {pin}: wrote {value}")
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""LED_SPECTRAL_READING step: stub.
|
|
2
|
+
|
|
3
|
+
Deferred: a driver for the VEML3328SL colour sensor (`ColourSensor/` in this repo) already
|
|
4
|
+
exists elsewhere and will be wired in later, either as a `chassis.colour_sensor` subsystem in
|
|
5
|
+
testomatic-io or directly in this executor — not decided yet. See TEST_RUNNER_PLAN.md's
|
|
6
|
+
"Deferred work" section. This stub lets suites containing an LED_SPECTRAL_READING step still run
|
|
7
|
+
end-to-end.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from .base import ExecutionContext, StepResult
|
|
13
|
+
from .registry import register_step
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@register_step("LED_SPECTRAL_READING")
|
|
17
|
+
def execute(config: dict, context: ExecutionContext) -> StepResult:
|
|
18
|
+
print("LED test")
|
|
19
|
+
return StepResult(passed=True, message="LED test (stub)")
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""OPERATOR_INTERVENTION step: show an instruction and wait for the operator to confirm.
|
|
2
|
+
|
|
3
|
+
CLI-only for v1 (blocks on `input()`); a future touchscreen UI can replace the confirmation
|
|
4
|
+
mechanism later without changing this executor's signature.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from .base import ExecutionContext, StepResult
|
|
10
|
+
from .registry import register_step
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@register_step("OPERATOR_INTERVENTION")
|
|
14
|
+
def execute(config: dict, context: ExecutionContext) -> StepResult:
|
|
15
|
+
message = config["message"]
|
|
16
|
+
print(message)
|
|
17
|
+
input("Press Enter once complete...")
|
|
18
|
+
return StepResult(passed=True, message=message)
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""CONTROL_POWER_RAIL / READ_RAIL_VOLTAGE / READ_RAIL_CURRENT steps: drive and read the
|
|
2
|
+
3.3V/5V/12V power rails via `chassis.power`.
|
|
3
|
+
|
|
4
|
+
The runner only ever touches a rail because a Test Suite's own CONTROL_POWER_RAIL step told it
|
|
5
|
+
to — see TEST_RUNNER_PLAN.md's "Power rail control is Test-Suite-only" note. The one exception,
|
|
6
|
+
turning every rail off on an abort-on-fail, lives in `runner.py`, not here.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from .base import ExecutionContext, StepResult
|
|
12
|
+
from .registry import register_step
|
|
13
|
+
|
|
14
|
+
_RAIL_CONTROL = {
|
|
15
|
+
"3.3V": "rail_3v3",
|
|
16
|
+
"5V": "rail_5v",
|
|
17
|
+
"12V": "rail_12v",
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
_RAIL_READ = {
|
|
21
|
+
"3.3V": "read_3v3",
|
|
22
|
+
"5V": "read_5v",
|
|
23
|
+
"12V": "read_12v",
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@register_step("CONTROL_POWER_RAIL")
|
|
28
|
+
def execute_control(config: dict, context: ExecutionContext) -> StepResult:
|
|
29
|
+
rail = config["rail"]
|
|
30
|
+
action = config["action"]
|
|
31
|
+
getattr(context.chassis.power, _RAIL_CONTROL[rail])(action == "ON")
|
|
32
|
+
return StepResult(passed=True, message=f"{rail} rail {action}")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@register_step("READ_RAIL_VOLTAGE")
|
|
36
|
+
def execute_read_voltage(config: dict, context: ExecutionContext) -> StepResult:
|
|
37
|
+
rail = config["rail"]
|
|
38
|
+
min_v = config["min_v"]
|
|
39
|
+
max_v = config["max_v"]
|
|
40
|
+
reading = getattr(context.chassis.power, _RAIL_READ[rail])()
|
|
41
|
+
passed = min_v <= reading.voltage <= max_v
|
|
42
|
+
message = f"{rail} rail: {reading.voltage:.3f}V (expected {min_v}-{max_v}V)"
|
|
43
|
+
return StepResult(passed=passed, message=message, measured={"voltage": reading.voltage})
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@register_step("READ_RAIL_CURRENT")
|
|
47
|
+
def execute_read_current(config: dict, context: ExecutionContext) -> StepResult:
|
|
48
|
+
rail = config["rail"]
|
|
49
|
+
min_ma = config["min_ma"]
|
|
50
|
+
max_ma = config["max_ma"]
|
|
51
|
+
reading = getattr(context.chassis.power, _RAIL_READ[rail])()
|
|
52
|
+
passed = min_ma <= reading.current <= max_ma
|
|
53
|
+
message = f"{rail} rail: {reading.current:.1f}mA (expected {min_ma}-{max_ma}mA)"
|
|
54
|
+
return StepResult(passed=passed, message=message, measured={"current": reading.current})
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""PYTHON step: run operator-authored Python source against the live hardware handles.
|
|
2
|
+
|
|
3
|
+
Register validates that the code parses (`ast.parse`) when the step is authored but never
|
|
4
|
+
executes it — this repo's runner is the only thing that ever runs it. The Test Suite JSON a
|
|
5
|
+
runner loads comes from a staff-only Register export, so this trust boundary is deliberate;
|
|
6
|
+
still worth knowing this executor runs arbitrary code with the same privileges as the runner
|
|
7
|
+
process. A raised exception is treated as a failed step rather than crashing the run.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from .base import ExecutionContext, StepResult
|
|
13
|
+
from .registry import register_step
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@register_step("PYTHON")
|
|
17
|
+
def execute(config: dict, context: ExecutionContext) -> StepResult:
|
|
18
|
+
python_code = config["python_code"]
|
|
19
|
+
namespace = {"chassis": context.chassis, "test_module": context.test_module}
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
exec(python_code, namespace) # noqa: S102 -- deliberate, see module docstring
|
|
23
|
+
except Exception as exc:
|
|
24
|
+
return StepResult(passed=False, message=f"Python step raised: {exc}")
|
|
25
|
+
|
|
26
|
+
return StepResult(passed=True, message="Python step executed")
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Registry mapping a Test Step's `step_type` to the callable that executes it.
|
|
2
|
+
|
|
3
|
+
Mirrors the driver-registry pattern testomatic-io uses for IOMOD expander chips (`DRIVERS` +
|
|
4
|
+
`probe()` in `testomatic_io/iomod/drivers/__init__.py`), adapted for dispatch by a known key
|
|
5
|
+
(`step_type`, already present in the parsed Test Step) rather than runtime hardware probing —
|
|
6
|
+
so this is a plain dict of functions rather than a class hierarchy with an auto-detection step.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from typing import Callable
|
|
12
|
+
|
|
13
|
+
from .base import ExecutionContext, StepResult
|
|
14
|
+
|
|
15
|
+
StepExecutor = Callable[[dict, ExecutionContext], StepResult]
|
|
16
|
+
|
|
17
|
+
STEP_EXECUTORS: dict[str, StepExecutor] = {}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def register_step(step_type: str) -> Callable[[StepExecutor], StepExecutor]:
|
|
21
|
+
"""Decorator registering the wrapped function as the executor for `step_type`."""
|
|
22
|
+
|
|
23
|
+
def decorator(executor: StepExecutor) -> StepExecutor:
|
|
24
|
+
if step_type in STEP_EXECUTORS:
|
|
25
|
+
raise ValueError(f"Step type {step_type!r} is already registered")
|
|
26
|
+
STEP_EXECUTORS[step_type] = executor
|
|
27
|
+
return executor
|
|
28
|
+
|
|
29
|
+
return decorator
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def get_executor(step_type: str) -> StepExecutor:
|
|
33
|
+
try:
|
|
34
|
+
return STEP_EXECUTORS[step_type]
|
|
35
|
+
except KeyError as exc:
|
|
36
|
+
raise KeyError(f"No executor registered for step type {step_type!r}") from exc
|
testomatic/suite.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""Parsing and validation for the Test Suite Definition JSON.
|
|
2
|
+
|
|
3
|
+
Format reference: test-suite-package.md (mirrors the format Register's
|
|
4
|
+
`testing.views._serialize_test_suite` produces).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import zipfile
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
SUPPORTED_EXPORT_SCHEMA_VERSION = 1
|
|
15
|
+
TEST_SUITE_DEFINITION_FILENAME = "test-suite-definition.json"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class SuiteFormatError(ValueError):
|
|
19
|
+
"""A Test Suite JSON file doesn't match the expected envelope shape."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class Design:
|
|
24
|
+
id: int
|
|
25
|
+
sku: str
|
|
26
|
+
name: str
|
|
27
|
+
hw_version: str
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class TestSuiteMeta:
|
|
32
|
+
version: int
|
|
33
|
+
status: str
|
|
34
|
+
notes: str | None
|
|
35
|
+
created_dt: str
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class TestStep:
|
|
40
|
+
order: int
|
|
41
|
+
step_type: str
|
|
42
|
+
name: str
|
|
43
|
+
abort_on_fail: bool
|
|
44
|
+
config_schema_version: int | None
|
|
45
|
+
config: dict
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class ManualCheck:
|
|
50
|
+
order: int
|
|
51
|
+
text: str
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True)
|
|
55
|
+
class TestSuiteFile:
|
|
56
|
+
export_schema_version: int
|
|
57
|
+
design: Design
|
|
58
|
+
test_suite: TestSuiteMeta
|
|
59
|
+
test_steps: list[TestStep]
|
|
60
|
+
manual_checks: list[ManualCheck]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def load_suite(path: str | Path) -> TestSuiteFile:
|
|
64
|
+
"""Load and validate a Test Suite Definition from `path`.
|
|
65
|
+
|
|
66
|
+
`path` may point directly at a Test Suite Definition JSON file, or at a Test Suite Package
|
|
67
|
+
ZIP archive (see test-suite-package.md) — the package's wrapped test-suite-definition.json is
|
|
68
|
+
located and parsed automatically.
|
|
69
|
+
"""
|
|
70
|
+
path = Path(path)
|
|
71
|
+
if path.suffix == ".zip":
|
|
72
|
+
text = _read_definition_from_package(path)
|
|
73
|
+
else:
|
|
74
|
+
text = path.read_text()
|
|
75
|
+
return parse_suite(json.loads(text))
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _read_definition_from_package(path: Path) -> str:
|
|
79
|
+
"""Finds and reads test-suite-definition.json from inside a Test Suite Package ZIP.
|
|
80
|
+
|
|
81
|
+
Located by filename suffix rather than a hardcoded path, since the definition sits inside a
|
|
82
|
+
top-level folder named after the package (e.g. `abc-hw1-0-test-suite-v3/test-suite-
|
|
83
|
+
definition.json`), not at the archive root.
|
|
84
|
+
"""
|
|
85
|
+
with zipfile.ZipFile(path) as archive:
|
|
86
|
+
matches = [name for name in archive.namelist() if name.endswith(TEST_SUITE_DEFINITION_FILENAME)]
|
|
87
|
+
if not matches:
|
|
88
|
+
raise SuiteFormatError(f"No {TEST_SUITE_DEFINITION_FILENAME} found in Test Suite Package {path}")
|
|
89
|
+
if len(matches) > 1:
|
|
90
|
+
raise SuiteFormatError(
|
|
91
|
+
f"Multiple {TEST_SUITE_DEFINITION_FILENAME} entries found in Test Suite Package "
|
|
92
|
+
f"{path}: {matches}"
|
|
93
|
+
)
|
|
94
|
+
return archive.read(matches[0]).decode("utf-8")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def parse_suite(data: dict) -> TestSuiteFile:
|
|
98
|
+
"""Parse and validate a Test Suite JSON export already loaded as a dict."""
|
|
99
|
+
export_schema_version = _require(data, "export_schema_version")
|
|
100
|
+
if export_schema_version != SUPPORTED_EXPORT_SCHEMA_VERSION:
|
|
101
|
+
raise SuiteFormatError(
|
|
102
|
+
f"Unsupported export_schema_version {export_schema_version!r}; "
|
|
103
|
+
f"this runner supports {SUPPORTED_EXPORT_SCHEMA_VERSION}"
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
test_steps = sorted(
|
|
107
|
+
(_parse_test_step(step) for step in _require(data, "test_steps")),
|
|
108
|
+
key=lambda step: step.order,
|
|
109
|
+
)
|
|
110
|
+
manual_checks = sorted(
|
|
111
|
+
(_parse_manual_check(check) for check in _require(data, "manual_checks")),
|
|
112
|
+
key=lambda check: check.order,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
return TestSuiteFile(
|
|
116
|
+
export_schema_version=export_schema_version,
|
|
117
|
+
design=_parse_design(_require(data, "design")),
|
|
118
|
+
test_suite=_parse_test_suite_meta(_require(data, "test_suite")),
|
|
119
|
+
test_steps=test_steps,
|
|
120
|
+
manual_checks=manual_checks,
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _require(data: dict, key: str):
|
|
125
|
+
try:
|
|
126
|
+
return data[key]
|
|
127
|
+
except KeyError as exc:
|
|
128
|
+
raise SuiteFormatError(f"Missing required key: {key}") from exc
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _parse_design(data: dict) -> Design:
|
|
132
|
+
return Design(
|
|
133
|
+
id=_require(data, "id"),
|
|
134
|
+
sku=_require(data, "sku"),
|
|
135
|
+
name=_require(data, "name"),
|
|
136
|
+
hw_version=_require(data, "hw_version"),
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _parse_test_suite_meta(data: dict) -> TestSuiteMeta:
|
|
141
|
+
return TestSuiteMeta(
|
|
142
|
+
version=_require(data, "version"),
|
|
143
|
+
status=_require(data, "status"),
|
|
144
|
+
notes=data.get("notes"),
|
|
145
|
+
created_dt=_require(data, "created_dt"),
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _parse_test_step(data: dict) -> TestStep:
|
|
150
|
+
config = _require(data, "config")
|
|
151
|
+
config_schema_version = data.get("config_schema_version")
|
|
152
|
+
inner_version = config.get("schema_version")
|
|
153
|
+
if (
|
|
154
|
+
config_schema_version is not None
|
|
155
|
+
and inner_version is not None
|
|
156
|
+
and config_schema_version != inner_version
|
|
157
|
+
):
|
|
158
|
+
raise SuiteFormatError(
|
|
159
|
+
f"Step {data.get('name')!r}: config_schema_version "
|
|
160
|
+
f"({config_schema_version}) does not match config['schema_version'] "
|
|
161
|
+
f"({inner_version})"
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
return TestStep(
|
|
165
|
+
order=_require(data, "order"),
|
|
166
|
+
step_type=_require(data, "step_type"),
|
|
167
|
+
name=_require(data, "name"),
|
|
168
|
+
abort_on_fail=data.get("abort_on_fail", False),
|
|
169
|
+
config_schema_version=config_schema_version,
|
|
170
|
+
config=config,
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _parse_manual_check(data: dict) -> ManualCheck:
|
|
175
|
+
return ManualCheck(order=_require(data, "order"), text=_require(data, "text"))
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: testomatic-runner
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Test runner that executes a Testomatic Test Suite Package against real hardware via testomatic-io.
|
|
5
|
+
Author: Elena Williams
|
|
6
|
+
Author-email: Jonathan Oxer <jon@oxer.com.au>
|
|
7
|
+
Maintainer: Elena Williams
|
|
8
|
+
Maintainer-email: Jonathan Oxer <jon@oxer.com.au>
|
|
9
|
+
License: MIT
|
|
10
|
+
Project-URL: Homepage, https://github.com/SuperHouse/testomatic-runner
|
|
11
|
+
Project-URL: bugs, https://github.com/SuperHouse/testomatic-runner/issues
|
|
12
|
+
Classifier: Topic :: Software Development :: Embedded Systems
|
|
13
|
+
Classifier: Topic :: System :: Hardware :: Hardware Drivers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Requires-Python: >=3.11
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: pytest; extra == "dev"
|
|
20
|
+
Provides-Extra: pi
|
|
21
|
+
Requires-Dist: testomatic-io>=0.1.1; extra == "pi"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# testomatic-runner
|
|
25
|
+
|
|
26
|
+
The test runner for the [Testomatic](https://testomatic.io/) PCB test jig system: it parses a
|
|
27
|
+
Test Suite Package produced by Register and executes its Test Steps in order against real
|
|
28
|
+
hardware, via the [`testomatic-io`](https://github.com/SuperHouse/testomatic-io) hardware
|
|
29
|
+
abstraction layer.
|
|
30
|
+
|
|
31
|
+
## Overview
|
|
32
|
+
|
|
33
|
+
Testomatic's test pipeline has four parts, each in its own repo:
|
|
34
|
+
|
|
35
|
+
1. **Register** — a Test Suite is authored/versioned there, against a PCB `Design`.
|
|
36
|
+
2. **[`testomatic-ui`](https://github.com/SuperHouse/testomatic-ui)** — the on-device touchscreen
|
|
37
|
+
UI. Downloads and caches Test Suite Packages from Register so a device isn't dependent on
|
|
38
|
+
Register being reachable to run a suite it already has.
|
|
39
|
+
3. **This repo (`testomatic-runner`)** — receives a Test Suite Package (a ZIP containing
|
|
40
|
+
`test-suite-definition.json`, documented in [test-suite-package.md](test-suite-package.md)),
|
|
41
|
+
parses it with `testomatic.suite.load_suite()`, and executes each Test Step in order with
|
|
42
|
+
`testomatic.runner.TestRunner`.
|
|
43
|
+
4. **[`testomatic-io`](https://github.com/SuperHouse/testomatic-io)** — carries out each step's
|
|
44
|
+
hardware action (drive a power rail, read an IOMOD pin, beep, etc.) through its `Chassis`/
|
|
45
|
+
`TestModule` API.
|
|
46
|
+
|
|
47
|
+
The physical chassis, PCB, and Test Module hardware this runner drives live in the sibling
|
|
48
|
+
[`Testomatic`](https://github.com/SuperHouse/Testomatic) repo.
|
|
49
|
+
|
|
50
|
+
## Usage
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
pip install -e ".[dev]" # unit tests, no hardware required
|
|
54
|
+
pytest
|
|
55
|
+
|
|
56
|
+
# On a Raspberry Pi wired to a Testomatic chassis:
|
|
57
|
+
pip install -e ".[pi]" # pulls in testomatic-io
|
|
58
|
+
python -m testomatic run path/to/suite-hw1-test-suite-v1.zip
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
`run` accepts either a Test Suite Package `.zip` or a bare `test-suite-definition.json` file.
|
|
62
|
+
|
|
63
|
+
`import testomatic_io` (and therefore `python -m testomatic run`) only works on real Raspberry Pi
|
|
64
|
+
hardware — `testomatic-io` is installed via the `pi` extra rather than as a core dependency, since
|
|
65
|
+
one of its dependencies (`gpiod`) has a native extension that only builds on Linux. See
|
|
66
|
+
[TEST_RUNNER_PLAN.md](TEST_RUNNER_PLAN.md) for what's implemented, what's stubbed, and what's
|
|
67
|
+
still unverified on real hardware.
|
|
68
|
+
|
|
69
|
+
## Repo layout
|
|
70
|
+
|
|
71
|
+
- `testomatic/suite.py` — parses/validates a Test Suite Definition (JSON) into dataclasses.
|
|
72
|
+
- `testomatic/steps/` — one executor module per `step_type`, registered in `steps/registry.py`.
|
|
73
|
+
- `testomatic/runner.py` — `TestRunner.run(suite)` executes `test_steps` in order, and
|
|
74
|
+
`format_report()` renders the result plus the suite's `manual_checks`.
|
|
75
|
+
- `testomatic/cli.py` / `__main__.py` — `python -m testomatic run <suite.zip|suite.json>`.
|
|
76
|
+
- `tests/conftest.py` — fake `Chassis`/`Power`/`Beeper`/`Iomod` doubles standing in for real
|
|
77
|
+
`testomatic-io` hardware.
|
|
78
|
+
- `test-suite-package.md` — reference documentation for the Test Suite Package (ZIP) layout and
|
|
79
|
+
the `test-suite-definition.json` format `suite.py` parses.
|
|
80
|
+
- `aqs-hw41-test-suite-v1.zip` — a real sample Test Suite Package, used as a test fixture.
|
|
81
|
+
- `TEST_RUNNER_PLAN.md` — the staged implementation plan and phase checklist.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
testomatic/__init__.py,sha256=hREOKAE6WarmrNKmLFRuGoB8pwiCVIrAege7oE7PioA,40
|
|
2
|
+
testomatic/__main__.py,sha256=4JMK66Wj4uLZTKbF-sT3LAxOsr6buig77PmOkJCRRxw,83
|
|
3
|
+
testomatic/cli.py,sha256=ZnCSBOvLBRUYOrtrbbzRsGRydMvEP5JvT6gGgXFmuhA,1303
|
|
4
|
+
testomatic/runner.py,sha256=603H2DvUZx1mKYdBXqy5hzEAE8UylkqxH6iwUqbE9y0,2672
|
|
5
|
+
testomatic/suite.py,sha256=dZQ6oY7_bJFxwGLlw28k5d7ybXA3rDfpUUGY55q82Gc,5332
|
|
6
|
+
testomatic/steps/__init__.py,sha256=m-iebijMXtiuirlZVXVAKIAdN9qVi_xm7JHIDg-7tGU,549
|
|
7
|
+
testomatic/steps/base.py,sha256=orw7ujGRqpOUxUp0eLhBIbwR07RhtLkGC_kel_jc9q0,467
|
|
8
|
+
testomatic/steps/beep.py,sha256=vDqKkGHFYtrpHhunUDCMOkTVncDKeouK-zfXMs6b9vU,865
|
|
9
|
+
testomatic/steps/delay.py,sha256=lI_1eTQfGy4Cj3FKY9q_fUhsXsy9wPQn80OCxA8Luc8,422
|
|
10
|
+
testomatic/steps/firmware.py,sha256=3YQwZLNmCPTCMPwewpgQNDgQWFrIkf-oDuBCgPviHfU,759
|
|
11
|
+
testomatic/steps/iomod.py,sha256=lqWNShblz0KJzghjqFqREP8B0YUaOz0YiR-fYcJOCOY,2078
|
|
12
|
+
testomatic/steps/led_spectral.py,sha256=8o4bqvsDshChYakC7qZe1fRjf5KHWn5wso1M46M44us,738
|
|
13
|
+
testomatic/steps/operator_intervention.py,sha256=6Tj4lTW7yrutwo2AqJKWSiWkSnxz0wZRHCrB9-1DYiA,621
|
|
14
|
+
testomatic/steps/power.py,sha256=FHTT0YsNEJBqFfrH6FkVpgs5xftbIr7vG3OSbUlcLD0,1997
|
|
15
|
+
testomatic/steps/python_step.py,sha256=iPIgG3y3dKQcI1LTF-4xxua5tMPsog-f1VLpfyPAHr8,1155
|
|
16
|
+
testomatic/steps/registry.py,sha256=xphrX8oJYx1oZz4ZmhQyS2NLKKYKF5wPR92QfjPlUnM,1337
|
|
17
|
+
testomatic_runner-0.1.0.dist-info/licenses/LICENSE,sha256=zHyiuFYo5fDxqAGY4e6guKUc4bwAZuucT7mZB6Dm7yk,1070
|
|
18
|
+
testomatic_runner-0.1.0.dist-info/METADATA,sha256=ZhWiqIgNl2enRwTaJHqEf12HyIZ25ZSBxkfWDECz8v0,3984
|
|
19
|
+
testomatic_runner-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
20
|
+
testomatic_runner-0.1.0.dist-info/entry_points.txt,sha256=vqVX9tGkKao6o5IRZgSYj0x0p4XSp5jt2FQD0z9KIDg,51
|
|
21
|
+
testomatic_runner-0.1.0.dist-info/top_level.txt,sha256=FdlIZiYE1JuOZVwwrO4nofFcQ91Q3bX1XKAVZZP71mA,11
|
|
22
|
+
testomatic_runner-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Jonathan Oxer
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
testomatic
|