interface-test-kit 0.1.0__py3-none-any.whl

interface_test_kit/checks/usb.py

@@ -0,0 +1,243 @@
+"""USB check with labelled test sticks: count, port, negotiated speed, write/read and throughput."""
+
+from __future__ import annotations
+
+import json
+import os
+import random
+import stat
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+from ..model import Attempt, Measurement
+from . import CheckType, fields, register
+
+BLOCK = 1 << 20
+TEST_FILE = "itk-test.bin"
+
+
+@dataclass(frozen=True)
+class UsbConfig:
+    name: str
+    count: int
+    label: str = "ITKTEST"
+    size_mb: int = 64
+    min_write_mbs: float | None = None
+    min_read_mbs: float | None = None
+    min_speed_mbps: int | None = None
+    ports: tuple[str, ...] = ()
+
+
+@dataclass
+class Stick:
+    device: str
+    label: str
+    mountpoint: str | None
+    usb_port: str | None
+    speed_mbps: int | None
+
+
+def _positive(value, name: str, whole: bool = False):
+    ok_type = int if whole else (int, float)
+    if value is None:
+        return None
+    if isinstance(value, bool) or not isinstance(value, ok_type) or value <= 0:
+        raise ValueError(f"{name}: must be a positive {'whole ' if whole else ''}number")
+    return value
+
+
+def parse(raw: dict) -> UsbConfig:
+    f = fields(raw, {"count"}, {"label": "ITKTEST", "size_mb": 64, "min_write_mbs": None,
+                                "min_read_mbs": None, "min_speed_mbps": None, "ports": []})
+    _positive(f["count"], "count", whole=True)
+    _positive(f["size_mb"], "size_mb", whole=True)
+    for key in ("min_write_mbs", "min_read_mbs"):
+        _positive(f[key], key)
+    _positive(f["min_speed_mbps"], "min_speed_mbps", whole=True)
+    if not isinstance(f["label"], str) or not f["label"]:
+        raise ValueError("label: must be a non-empty text")
+    if not isinstance(f["ports"], list):
+        raise ValueError("ports: must be a list of USB port names like 1-2")
+    return UsbConfig(str(f["name"]), f["count"], f["label"], f["size_mb"], f["min_write_mbs"],
+                     f["min_read_mbs"], f["min_speed_mbps"], tuple(str(p) for p in f["ports"]))
+
+
+def _usb_device(ctx, kname: str) -> tuple[str | None, int | None]:
+    """Walk up from the block device in sysfs to the USB device (the dir with speed and devnum)."""
+    sysfs = ctx.path("/sys").resolve()
+    try:
+        p = ctx.path(f"/sys/class/block/{kname}").resolve(strict=True)
+    except OSError:
+        return None, None
+    while p != sysfs and p != p.parent:
+        if (p / "speed").is_file() and (p / "devnum").is_file():
+            try:
+                return p.name, int(float((p / "speed").read_text().strip()))
+            except (OSError, ValueError):
+                return p.name, None
+        p = p.parent
+    return None, None
+
+
+def _walk(nodes: list[dict], tran: str | None = None):
+    for node in nodes:
+        t = node.get("tran") or tran
+        yield node, t
+        yield from _walk(node.get("children") or [], t)
+
+
+def find_sticks(cfg: UsbConfig, ctx, ignored: list[str] | None = None) -> list[Stick]:
+    result = ctx.run(["lsblk", "-J", "-o", "NAME,KNAME,PATH,LABEL,MOUNTPOINT,TRAN"])
+    if result.returncode != 0:
+        raise RuntimeError(f"lsblk failed: {result.stderr.strip()}")
+    sticks = []
+    for node, tran in _walk(json.loads(result.stdout).get("blockdevices", [])):
+        label = node.get("label")
+        if tran != "usb" or not label:
+            continue
+        if not label.startswith(cfg.label):
+            if ignored is not None:
+                ignored.append(f"ignored: {node.get('path')} label {label}")
+            continue
+        port, speed = _usb_device(ctx, node["kname"])
+        sticks.append(Stick(node["path"], label, node.get("mountpoint"), port, speed))
+    return sorted(sticks, key=lambda s: s.device)
+
+
+@lru_cache(maxsize=1)
+def _payload() -> bytes:
+    return random.Random(0).randbytes(BLOCK - 8)
+
+
+def _block(i: int) -> bytes:
+    return i.to_bytes(8, "big") + _payload()
+
+
+def _same_device(ctx, stick: Stick) -> bool:
+    """True if the mountpoint really is the stick's filesystem right now."""
+    mountpoint = ctx.path(stick.mountpoint)
+    device = ctx.path(stick.device)
+    if not device.exists():
+        # Only the simulated worlds under a fake root have no device nodes.
+        return Path(ctx.root) != Path("/")
+    try:
+        return (stat.S_ISBLK(os.stat(device).st_mode) and os.path.ismount(mountpoint)
+                and os.stat(mountpoint).st_dev == os.stat(device).st_rdev)
+    except OSError:
+        return False
+
+
+def _still_mounted(ctx, cfg: UsbConfig, stick: Stick) -> bool:
+    """Ask lsblk again right before writing: the stick may have been pulled meanwhile."""
+    try:
+        now = {s.device: s.mountpoint for s in find_sticks(cfg, ctx)}
+    except (RuntimeError, ValueError):
+        return False
+    return now.get(stick.device) == stick.mountpoint and _same_device(ctx, stick)
+
+
+def _write_file(path: Path, size_mb: int, clock) -> float:
+    if os.path.lexists(path):  # never follow a link someone left on the stick
+        os.unlink(path)
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o644)
+    try:
+        t0 = clock()
+        for i in range(size_mb):
+            os.write(fd, _block(i))
+        os.fsync(fd)
+        return clock() - t0
+    finally:
+        os.close(fd)
+
+
+def _read_back(path: Path, size_mb: int, clock) -> tuple[float, int | None]:
+    """Returns (seconds, index of the first bad block or None)."""
+    fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
+    try:
+        os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
+        bad = None
+        t0 = clock()
+        for i in range(size_mb):
+            data = os.read(fd, BLOCK)
+            if bad is None and data != _block(i):
+                bad = i
+        return clock() - t0, bad
+    finally:
+        os.close(fd)
+
+
+def _test_stick(stick: Stick, cfg: UsbConfig, ctx, details: list[str]) -> tuple[str | None, list[Measurement]]:
+    """Returns (unit-mode problem or None, measurements). Never writes outside the mountpoint."""
+    where = stick.usb_port or stick.device
+    if not _still_mounted(ctx, cfg, stick):
+        details.append(f"{stick.device}: not mounted at {stick.mountpoint} any more")
+        return f"USB: test stick in port {where} is not mounted", []
+    path = ctx.path(stick.mountpoint) / TEST_FILE
+    measurements = []
+    try:
+        try:
+            t_write = _write_file(path, cfg.size_mb, ctx.clock)
+        except OSError as e:
+            details.append(f"{stick.device}: write failed: {e}")
+            return f"USB: could not write to the test stick ({where}): full or write-protected", []
+        t_read, bad = _read_back(path, cfg.size_mb, ctx.clock)
+    finally:
+        try:
+            path.unlink()
+        except OSError:
+            pass
+    write_mbs = round(cfg.size_mb / max(t_write, 1e-6), 1)
+    read_mbs = round(cfg.size_mb / max(t_read, 1e-6), 1)
+    measurements.append(Measurement(f"{where} write", write_mbs, "MB/s", low=cfg.min_write_mbs))
+    measurements.append(Measurement(f"{where} read", read_mbs, "MB/s", low=cfg.min_read_mbs))
+    if stick.speed_mbps is not None:
+        measurements.append(Measurement(f"{where} USB speed", stick.speed_mbps, "Mbit/s",
+                                        low=cfg.min_speed_mbps))
+    details.append(f"{stick.device} ({stick.label}) on {where}: write {write_mbs} MB/s, "
+                   f"read {read_mbs} MB/s, USB {stick.speed_mbps} Mbit/s")
+    if bad is not None:
+        details.append(f"{stick.device}: block {bad} read back wrong")
+        return f"USB: stick data corrupted ({where})", measurements
+    if any(not m.passed for m in measurements[:2]):
+        return f"USB: port {where} too slow", measurements
+    if len(measurements) > 2 and not measurements[2].passed:
+        return (f"USB: port {where} runs at {stick.speed_mbps} Mbit/s, "
+                f"expected at least {cfg.min_speed_mbps}"), measurements
+    return None, measurements
+
+
+def run(cfg: UsbConfig, ctx, ui) -> Attempt:
+    details: list[str] = []
+    try:
+        sticks = find_sticks(cfg, ctx, details)
+    except (RuntimeError, ValueError) as e:
+        return Attempt.fail("USB: could not look for test sticks. Tell the test engineer.",
+                            details=[str(e)])
+    if len(sticks) != cfg.count:
+        details.append("found: " + (", ".join(f"{s.device} on {s.usb_port}" for s in sticks) or "none"))
+        return Attempt.fail(f"USB: found {len(sticks)} test stick(s), expected {cfg.count}. "
+                            f"Insert the test sticks labelled {cfg.label}.", details=details)
+    ports = {s.usb_port for s in sticks}
+    for port in cfg.ports:
+        if port not in ports:
+            return Attempt.fail(f"USB: no test stick in port {port}", details=details)
+    measurements: list[Measurement] = []
+    problems: list[tuple[str, Stick]] = []
+    for stick in sticks:
+        if not stick.mountpoint:
+            problems.append((f"USB: test stick in port {stick.usb_port or stick.device} is not mounted", stick))
+            continue
+        problem, ms = _test_stick(stick, cfg, ctx, details)
+        measurements += ms
+        if problem:
+            problems.append((problem, stick))
+    if problems:
+        unit, stick = problems[0]
+        stick_fault = "not mounted" not in unit
+        message = ctx.say(unit, f"Test stick {stick.device} faulty or too slow") if stick_fault else unit
+        return Attempt.fail(message, details=details, measurements=measurements)
+    return Attempt.ok(measurements=measurements, details=details)
+
+
+register("usb", CheckType(parse=parse, run=run))

interface_test_kit/cli.py

@@ -0,0 +1,130 @@
+"""Command line interface: ``itk run``, ``itk selftest``, ``itk demo`` and helpers."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from . import __version__
+from .profile import ProfileError, load_profile
+from .report import report_reachable, sync_spool
+from .station import load_station, station_path
+from .ui import ConsoleUI
+
+_EXIT = {"PASS": 0, "FAIL": 1, "ABORTED": 2}
+
+
+def _check_profile(files: list[str]) -> int:
+    rc = 0
+    for name in files:
+        try:
+            p = load_profile(name)
+        except (ProfileError, OSError) as e:
+            print(f"ERROR {name}: {e}")
+            rc = 2
+        else:
+            print(f"OK  {name}  {p.name} v{p.version}, {len(p.steps)} steps")
+    return rc
+
+
+def _sync(station_arg: str | None) -> int:
+    try:
+        station = load_station(station_path(station_arg))
+    except (ValueError, OSError) as e:
+        print(f"Error: {e}")
+        return 2
+    if not report_reachable(station):
+        print(f"Network drive not reachable: {station.report_dir}")
+        return 2
+    for path in sync_spool(station):
+        print(f"Uploaded {path}")
+    return 0
+
+
+def _session(args, mode: str) -> int:
+    from .session import real_context, run_session
+    try:
+        station = load_station(station_path(args.station))
+        only = [s.strip() for s in args.only.split(",") if s.strip()] if getattr(args, "only", None) else None
+        run = run_session(station, real_context(mode), ConsoleUI(),
+                          rma=getattr(args, "rma", None), only=only)
+    except KeyboardInterrupt:
+        print("\nAborted.")
+        return 2
+    except (ValueError, ProfileError, OSError) as e:
+        print(f"Error: {e}")
+        return 2
+    return _EXIT[run.status]
+
+
+def _demo(args) -> int:
+    import tempfile
+    from importlib import resources
+
+    from .runner import RETRY_PROMPT
+    from .session import run_session
+    from .sim.world import FAULTS, SERIAL_NUMBER, AutoUI, build_world
+    from .station import Station
+
+    if args.list_faults:
+        for name, text in FAULTS.items():
+            print(f"  {name:<16} {text}")
+        return 0
+    fault = None if args.fault == "none" else args.fault
+    profile = load_profile(Path(str(resources.files("interface_test_kit"))) / "profiles" / "demo.yaml")
+    with tempfile.TemporaryDirectory(prefix="itk-demo-") as tmp:
+        root = Path(tmp)
+        out = Path(args.out)
+        out.mkdir(parents=True, exist_ok=True)
+        station = Station(report_dir=out, spool_dir=root / "spool", profile_dir=root, name="demo")
+        no_to = (RETRY_PROMPT,) + (("power LED",) if fault == "led_off" else ())
+        print(f"Simulated unit, fault: {args.fault}\n")
+        run = run_session(station, build_world(fault, root), AutoUI(SERIAL_NUMBER, no_to),
+                          profiles=[profile])
+    return _EXIT[run.status]
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="itk", description=__doc__)
+    parser.add_argument("--version", action="version", version=f"itk {__version__}")
+    sub = parser.add_subparsers(dest="cmd")
+
+    p_check = sub.add_parser("check-profile", help="validate profile files")
+    p_check.add_argument("files", nargs="+", metavar="FILE")
+
+    station_help = "station file (default: $ITK_STATION or /etc/itk/station.yaml)"
+    p_run = sub.add_parser("run", help="test a unit")
+    p_run.add_argument("--station", help=station_help)
+    p_run.add_argument("--rma", help="RMA number, stored in the report")
+    p_run.add_argument("--only", help="comma-separated step ids to run, e.g. serial-1,usb")
+
+    p_self = sub.add_parser("selftest", help="check the test equipment on a reference unit")
+    p_self.add_argument("--station", help=station_help)
+
+    from .sim.world import FAULTS
+    p_demo = sub.add_parser("demo", help="run a full session against a simulated unit")
+    p_demo.add_argument("--fault", default="none", choices=["none", *FAULTS], metavar="FAULT",
+                        help="inject a fault (see --list-faults)")
+    p_demo.add_argument("--list-faults", action="store_true", help="list the faults and exit")
+    p_demo.add_argument("--out", default="reports", help="report folder (default: reports)")
+
+    p_sync = sub.add_parser("sync", help="upload reports saved locally while the network drive was down")
+    p_sync.add_argument("--station", help=station_help)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if args.cmd == "check-profile":
+        return _check_profile(args.files)
+    if args.cmd == "run":
+        return _session(args, "unit")
+    if args.cmd == "selftest":
+        return _session(args, "selftest")
+    if args.cmd == "demo":
+        return _demo(args)
+    if args.cmd == "sync":
+        return _sync(args.station)
+    parser.print_help()
+    return 2

interface_test_kit/context.py

@@ -0,0 +1,43 @@
+"""Everything a check needs from the outside world, so tests can swap it out."""
+
+from __future__ import annotations
+
+import subprocess
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable
+
+
+@dataclass
+class CommandResult:
+    returncode: int
+    stdout: str
+    stderr: str
+
+
+def run_command(args: list[str], timeout: float = 30) -> CommandResult:
+    try:
+        p = subprocess.run(args, capture_output=True, text=True, timeout=timeout)
+    except FileNotFoundError:
+        return CommandResult(127, "", f"command not found: {args[0]}")
+    except subprocess.TimeoutExpired:
+        return CommandResult(124, "", f"timed out after {timeout} s: {' '.join(args)}")
+    return CommandResult(p.returncode, p.stdout, p.stderr)
+
+
+@dataclass
+class Context:
+    root: Path = Path("/")
+    run: Callable[[list[str]], CommandResult] = run_command
+    open_serial: Callable[[str, int], Any] | None = None
+    open_can: Callable[[str], Any] | None = None
+    clock: Callable[[], float] = time.monotonic
+    sleep: Callable[[float], None] = time.sleep
+    mode: str = "unit"
+
+    def path(self, p: str) -> Path:
+        return Path(self.root) / p.lstrip("/")
+
+    def say(self, unit: str, equipment: str) -> str:
+        return equipment if self.mode == "selftest" else unit

interface_test_kit/detect.py

@@ -0,0 +1,22 @@
+"""Pick the profile that matches this device's DMI product name or device-tree model."""
+
+from __future__ import annotations
+
+from .context import Context
+from .profile import Profile
+
+_SOURCES = {"dmi_product_name": "/sys/class/dmi/id/product_name",
+            "devicetree_model": "/proc/device-tree/model"}
+
+
+def _read(ctx: Context, path: str) -> str | None:
+    try:
+        return ctx.path(path).read_bytes().decode(errors="replace").strip().rstrip("\0").strip()
+    except OSError:
+        return None
+
+
+def detect_profiles(profiles: list[Profile], ctx: Context) -> list[Profile]:
+    values = {key: _read(ctx, path) for key, path in _SOURCES.items()}
+    return [p for p in profiles
+            if p.detect and all(values.get(k) == v for k, v in p.detect.items())]

interface_test_kit/model.py

@@ -0,0 +1,108 @@
+"""Result data: measurements, attempts, step results and a whole run."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+
+
+@dataclass
+class Measurement:
+    name: str
+    value: float
+    unit: str = ""
+    low: float | None = None
+    high: float | None = None
+
+    @property
+    def passed(self) -> bool:
+        if self.low is not None and self.value < self.low:
+            return False
+        if self.high is not None and self.value > self.high:
+            return False
+        return True
+
+    def to_dict(self) -> dict:
+        return {"name": self.name, "value": self.value, "unit": self.unit,
+                "low": self.low, "high": self.high, "passed": self.passed}
+
+
+@dataclass
+class Attempt:
+    status: str
+    message: str = ""
+    details: list[str] = field(default_factory=list)
+    measurements: list[Measurement] = field(default_factory=list)
+
+    @classmethod
+    def ok(cls, message: str = "", measurements: list[Measurement] | None = None,
+           details: list[str] | None = None) -> Attempt:
+        return cls("PASS", message, list(details or []), list(measurements or []))
+
+    @classmethod
+    def fail(cls, message: str, details: list[str] | None = None,
+             measurements: list[Measurement] | None = None) -> Attempt:
+        return cls("FAIL", message, list(details or []), list(measurements or []))
+
+    def to_dict(self) -> dict:
+        return {"status": self.status, "message": self.message, "details": list(self.details),
+                "measurements": [m.to_dict() for m in self.measurements]}
+
+
+@dataclass
+class StepResult:
+    id: str
+    name: str
+    type: str
+    status: str
+    attempts: list[Attempt]
+    equipment_id: str | None = None
+
+    @property
+    def message(self) -> str:
+        if not self.attempts:
+            return ""
+        last = self.attempts[-1]
+        if self.status == "FAIL" and last.status == "PASS":
+            failed = next(a for a in reversed(self.attempts) if a.status == "FAIL")
+            return f"{failed.message} (not stable: failed again on retry)"
+        return last.message
+
+    @property
+    def passed_after_retry(self) -> bool:
+        return self.status == "PASS" and len(self.attempts) > 1
+
+    def to_dict(self) -> dict:
+        return {"id": self.id, "name": self.name, "type": self.type, "status": self.status,
+                "message": self.message, "passed_after_retry": self.passed_after_retry,
+                "equipment_id": self.equipment_id,
+                "attempts": [a.to_dict() for a in self.attempts]}
+
+
+def _iso(t: datetime | None) -> str | None:
+    return t.isoformat() if t is not None else None
+
+
+@dataclass
+class RunResult:
+    mode: str
+    serial_number: str
+    rma: str | None
+    profile_name: str
+    profile_version: int
+    station: str
+    operator: str | None
+    started: datetime
+    finished: datetime | None
+    tool_version: str
+    steps: list[StepResult]
+    status: str
+    only: list[str] | None = None
+
+    def to_dict(self) -> dict:
+        return {"mode": self.mode, "serial_number": self.serial_number, "rma": self.rma,
+                "profile": {"name": self.profile_name, "version": self.profile_version},
+                "station": self.station, "operator": self.operator,
+                "started": _iso(self.started), "finished": _iso(self.finished),
+                "tool_version": self.tool_version, "status": self.status, "only": self.only,
+                "steps": [s.to_dict() for s in self.steps]}