mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/testkit/prbuild.py

@@ -0,0 +1,148 @@
+import json
+import plistlib
+import subprocess
+from pathlib import Path
+
+BUILD_DIR = "build"            # .mobster/build: the derived data folder
+LOG_NAME = "pr-build.log"
+BUILD_TIMEOUT = 45 * 60
+LIST_TIMEOUT = 120
+SDKS = {"sim": "iphonesimulator", "device": "iphoneos"}
+DESTINATIONS = {"sim": "generic/platform=iOS Simulator", "device": "generic/platform=iOS"}
+
+
+class BuildError(Exception):
+    """The build couldn't be made: one plain sentence, and the fix."""
+
+    def __init__(self, message, fix=""):
+        super().__init__(message)
+        self.message, self.fix = message, fix
+
+
+def find_project(root):
+    """The workspace or project to build: one at the project root, else one a folder below it."""
+    root = Path(root)
+    folders = [root]
+    if root.is_dir():
+        folders += sorted(p for p in root.iterdir() if p.is_dir() and not p.name.startswith(".")
+                          and p.suffix not in (".xcodeproj", ".xcworkspace"))
+    for folder in folders:
+        workspaces = [p for p in folder.glob("*.xcworkspace") if p.is_dir()]
+        projects = [p for p in folder.glob("*.xcodeproj") if p.is_dir()]
+        if len(workspaces) > 1 or (not workspaces and len(projects) > 1):
+            names = ", ".join(p.name for p in (workspaces or projects))
+            raise BuildError(f"{folder} has more than one Xcode project ({names}).",
+                             "Pass the one to build with --project.")
+        if workspaces:
+            return "workspace", workspaces[0]
+        if projects:
+            return "project", projects[0]
+    return None
+
+
+def project_arg(path):
+    """("workspace" | "project", path) for a path given with --project."""
+    path = Path(path)
+    if path.suffix == ".xcworkspace":
+        return "workspace", path
+    if path.suffix == ".xcodeproj":
+        return "project", path
+    raise BuildError(f"{path} isn't an .xcodeproj or an .xcworkspace.", "Pass --project YourApp.xcodeproj.")
+
+
+def run_command(argv, cwd, log=None, timeout=BUILD_TIMEOUT):
+    """(exit code, output); output goes to ``log`` when given."""
+    try:
+        if log is not None:
+            Path(log).parent.mkdir(parents=True, exist_ok=True)
+            with open(log, "wb") as handle:
+                result = subprocess.run(argv, cwd=str(cwd), stdout=handle, stderr=subprocess.STDOUT,
+                                        stdin=subprocess.DEVNULL, timeout=timeout)
+            return result.returncode, Path(log).read_text(encoding="utf-8", errors="replace")[-200_000:]
+        result = subprocess.run(argv, cwd=str(cwd), capture_output=True, stdin=subprocess.DEVNULL, timeout=timeout)
+        return result.returncode, (result.stdout + result.stderr).decode("utf-8", errors="replace")
+    except FileNotFoundError:
+        return 127, "xcodebuild isn't installed."
+    except subprocess.TimeoutExpired:
+        return 124, f"xcodebuild didn't finish in {timeout // 60} minutes."
+
+
+def schemes(found, *, runner=None, cwd=None):
+    kind, path = found
+    code, output = (runner or run_command)(
+        ["xcodebuild", "-list", "-json", f"-{kind}", str(path)], cwd or path.parent)
+    # stderr follows the JSON (xcodebuild's warnings, CocoaPods' notes), so read one object.
+    start = output.find("{")
+    try:
+        data = json.JSONDecoder().raw_decode(output[start:])[0] if start >= 0 else {}
+    except ValueError:
+        data = {}
+    block = data.get(kind) or data.get("project") or data.get("workspace") or {}
+    return [name for name in block.get("schemes") or () if isinstance(name, str)]
+
+
+def pick_scheme(found, *, runner=None, cwd=None):
+    """The only scheme, or the one named like the project; else BuildError (pass --scheme)."""
+    names = schemes(found, runner=runner, cwd=cwd)
+    stem = Path(found[1]).stem
+    if len(names) == 1:
+        return names[0]
+    if stem in names:
+        return stem
+    listed = ", ".join(names[:8]) or "none"
+    raise BuildError(f"Mobster can't tell which scheme to build (schemes: {listed}).", "Pass --scheme NAME.")
+
+
+def build(root, *, scheme, project, platform="sim", configuration="Debug", runner=None, progress=None,
+          bundle=None, for_testing=False, test_plan=None):
+    """Build ``scheme`` for ``platform`` ("sim" or "device") into .mobster/build. With ``for_testing`` the same build
+    also makes the test bundles and the .xctestrun file that ``xctest.run`` runs (build-for-testing)."""
+    root = Path(root)
+    kind, path = project
+    derived = root / ".mobster" / BUILD_DIR
+    log = derived / LOG_NAME
+    argv = ["xcodebuild", f"-{kind}", str(path), "-scheme", scheme, "-configuration", configuration,
+            "-destination", DESTINATIONS[platform], "-derivedDataPath", str(derived)]
+    if platform == "sim":
+        argv.append("CODE_SIGNING_ALLOWED=NO")
+    else:
+        argv.append("-allowProvisioningUpdates")
+    if for_testing and test_plan:
+        argv += ["-testPlan", test_plan]
+    argv.append("build-for-testing" if for_testing else "build")
+    (progress or (lambda line: None))(f"Building {scheme} for the {'simulator' if platform == 'sim' else 'iPhone'}")
+    code, output = (runner or run_command)(argv, root, log)
+    if code != 0:
+        errors = [line.strip() for line in output.splitlines() if "error:" in line][:3]
+        detail = ("; ".join(errors))[:600] if errors else f"xcodebuild exited {code}"
+        fix = "Fix the build, or pass --no-build to test the builds the checks already name."
+        if platform == "device" and "signing" in output.lower():
+            fix = "Open the project in Xcode and pick a team under Signing & Capabilities, then run it again."
+        raise BuildError(f"The build failed: {detail}.", fix)
+    built = {**product(derived, configuration, platform, scheme=scheme, bundle=bundle), "log": str(log)}
+    from ..memory import writer
+    writer.from_build(root, scheme, built.get("bundle"))  # the project's scheme and bundle become memory
+    return built
+
+
+def product(derived, configuration, platform, *, scheme=None, bundle=None):
+    """The .app the build made: the one with the checks' bundle ID, else the scheme's name, else the only one."""
+    folder = Path(derived) / "Build" / "Products" / f"{configuration}-{SDKS[platform]}"
+    apps = sorted(p for p in folder.glob("*.app") if p.is_dir())
+    found = [(app, _bundle(app)) for app in apps]
+    choice = next((app for app, bid in found if bundle and bid == bundle), None) or \
+        next((app for app, _ in found if scheme and app.stem == scheme), None) or \
+        (apps[0] if len(apps) == 1 else None)
+    if choice is None:
+        raise BuildError(f"The build finished, but Mobster couldn't tell which app in {folder} to test.",
+                         "Pass --scheme with the app's scheme.")
+    return {"bundle": _bundle(choice), "path": str(choice), "platform": platform}
+
+
+def _bundle(app):
+    try:
+        with open(Path(app) / "Info.plist", "rb") as handle:
+            value = plistlib.load(handle).get("CFBundleIdentifier")
+        return value if isinstance(value, str) else None
+    except (OSError, plistlib.InvalidFileException, ValueError, AttributeError):
+        return None

mobile_agent/testkit/proof_report.py

@@ -0,0 +1,55 @@
+"""The step proof in `mobster test`'s reports: the cause line and the failing step's requests and log for JUnit,
+and each step's clip, log lines and requests for the HTML report (verify/proof.py builds the proof)."""
+
+
+def cause_html(proof):
+    """The cause line for the HTML report (verify/report.py draws it)."""
+    from ..verify.report import cause_line
+    return cause_line(proof)
+
+
+def cause_lines(proof):
+    cause = (proof or {}).get("cause")
+    return [f"Cause ({cause.get('side')}): {cause.get('line')}"] if isinstance(cause, dict) else []
+
+
+def step_lines(proof, evidence):
+    """The failing step's span for <system-out>: its text, the requests the log shows and the lines the app logged."""
+    from ..verify.evidence import log_text
+    from ..verify.proof import failing_entry
+    entry = failing_entry(proof, (evidence or {}).get("step")) if proof else None
+    if entry is None:
+        return []
+    out = [f"Step {entry.get('index')}: {entry.get('text')} ({entry.get('fromMs', 0) / 1000:.3f} to "
+           f"{entry.get('toMs', 0) / 1000:.3f} s)"]
+    rows = (entry.get("network") or {}).get("rows") or []
+    for row in rows:
+        out.append("Request: " + " ".join(str(row[key]) for key in ("method", "host", "status", "durationMs")
+                                          if row.get(key) is not None))
+    lines = (entry.get("log") or {}).get("lines") or []
+    if lines:
+        out += [f"The app's log in this step ({len(lines)} of {(entry.get('log') or {}).get('count')} lines):",
+                log_text(lines)]
+    return out
+
+
+def proof_steps(result):
+    """Each step's clip, log lines and requests, the failing step open."""
+    from ..verify.proof import clip, failing_entry
+    from ..verify.report import proof_pane
+    proof = result.get("stepProof") or {}
+    entries = proof.get("steps") or []
+    if not entries:
+        return ""
+    failing = failing_entry(proof, (result.get("evidence") or {}).get("step"))
+    video, offset = result.get("video"), result.get("videoOffsetMs")
+    panes = [proof_pane(entry, clip=clip(entry, offset), video=video,
+                        title=f"Step {entry.get('index')}: {_short(entry.get('text'))}",
+                        opened=entry is failing) for entry in entries]
+    panes = [pane for pane in panes if pane]
+    return '<div class="proof-steps">' + "".join(panes) + "</div>" if panes else ""
+
+
+def _short(text, limit=80):
+    text = " ".join(str(text or "").split())
+    return text if len(text) <= limit else text[:limit - 1] + "…"

mobile_agent/testkit/provenance.py

@@ -0,0 +1,119 @@
+"""Frozen local check inputs for comparing completed suite evidence."""
+
+import hashlib
+import json
+from dataclasses import replace
+from pathlib import Path
+
+from ..devkit import links
+from ..replay_core.heal import HealError, InputChanged, proof_assets
+from ..verify.checks import FILE_LIMIT, check_to_dict
+from ..verify.fixtures import FixtureError, with_fixtures
+
+FIELDS = ("checkSpecDigest", "baselineDigest", "fixtureDigest", "visualBaselineSha256",
+          "businessBindingSha256", "appDigest")
+
+
+def _hash(value):
+    return hashlib.sha256(json.dumps(value, sort_keys=True, ensure_ascii=False, allow_nan=False,
+                                     separators=(",", ":")).encode("utf-8")).hexdigest()
+
+
+def _source_digest(path):
+    from .input_files import digest
+    return digest(Path(path), FILE_LIMIT)
+
+
+def _app_digest(path, cache):
+    if path is None:
+        return None
+    path = Path(path)
+    if cache is not None and path in cache:
+        return cache[path]
+    # Project revisions use the same bounded, streaming file/tree digest for immutable prebuilt apps.
+    from .input_files import MAX_APP_BYTES, digest
+    try:
+        value = digest(path, MAX_APP_BYTES)
+    except (OSError, ValueError):
+        value = None
+    if cache is not None:
+        cache[path] = value
+    return value
+
+
+def app_digest(path):
+    """Bounded fingerprint of a locally preserved app artifact, or unknown."""
+    return _app_digest(path, None)
+
+
+def case_provenance(entry, root, *, asset_cache=None):
+    """Capture the effective parsed check and external proof at this admission boundary.
+
+    `app.path` bytes are an intentional comparison variable; launch and assertion semantics remain in the spec.
+    Each input freezes on its own: one that can't be read is null rather than an invented comparable identity, and the
+    others stay pinned. `inputChanged` says an input differs from what the check was admitted with, which refuses it.
+    """
+    unknown = {**dict.fromkeys(FIELDS), "sourceDigest": None, "inputChanged": False}
+    check = entry.check
+    if check is None:
+        return unknown
+    try:
+        source_digest = _source_digest(entry.path)
+    except (OSError, ValueError):
+        source_digest = None
+    frozen = {**unknown, "sourceDigest": source_digest, "appDigest": _app_digest(check.app_path, asset_cache)}
+    try:
+        effective = with_fixtures(check, root)
+        if check.fixture_origin is not None:
+            # A prepared Check retains its source. Re-resolve only fixture-owned fields here: PR builds may
+            # intentionally replace app.path, while fixture YAML and link-registry edits must still refuse.
+            try:
+                current = with_fixtures(check.fixture_origin, root)
+                if any(links.OPEN_STEP.match(step) for step in current.steps):
+                    current = replace(current,
+                                      steps=links.expand_steps(current.steps, links.discover(root, deep=False)))
+            except (FixtureError, links.LinksError) as error:
+                raise InputChanged(str(error)) from None  # it resolved when the check was prepared
+            if (current.steps, current.reset, current.launch_args, current.launch_env, current.seed) != (
+                    check.steps, check.reset, check.launch_args, check.launch_env, check.seed):
+                raise InputChanged("A fixture or linked step changed during this suite")
+        spec = check_to_dict(replace(effective, fixture_origin=None), project_root=root)
+        spec["app"].pop("path", None)
+        seed = effective.seed
+        spec["effectiveSeed"] = {"files": [(str(Path(source).relative_to(root)) if Path(source).is_relative_to(root)
+                                            else str(source), destination)
+                                           for source, destination in seed.files],
+                                 "keychain": seed.keychain, "pasteboard": seed.pasteboard,
+                                 "risky": list(seed.risky)}
+        spec["l10n"] = effective.l10n
+        spec["a11yScope"] = effective.a11y_scope
+        proof = proof_assets(effective)
+        baseline = {key: proof[key] for key in ("visualSha256", "mediaSha256", "pushSha256", "a11ySha256")}
+        fixture = {key: proof[key] for key in ("businessSha256", "seedSha256")}
+        baseline_digest = _hash(baseline) if any(value is not None for value in baseline.values()) else "none"
+        fixture_digest = _hash(fixture) if effective.business is not None or effective.seed else "none"
+        return {**frozen, "checkSpecDigest": _hash(spec), "baselineDigest": baseline_digest,
+                "visualBaselineSha256": proof["visualSha256"],
+                "fixtureDigest": fixture_digest, "businessBindingSha256": proof["businessSha256"]}
+    except InputChanged:
+        return {**frozen, "inputChanged": True}
+    except (OSError, ValueError, TypeError, FixtureError, HealError, links.LinksError):
+        return frozen
+
+
+def unpinned(frozen, check):
+    """What of a check was never frozen, as a note names it, so its report can't claim that identity."""
+    return [name for name, missing in (("check file", frozen["sourceDigest"] is None),
+                                       ("fixtures or assets", frozen["checkSpecDigest"] is None),
+                                       ("app", bool(check.app_path) and frozen["appDigest"] is None)) if missing]
+
+
+def require_case_provenance(entry, root, frozen, *, asset_cache=None):
+    """Refuse an input known to have changed, before another physical effect. A field that could not be read pins
+    nothing: the check runs, and its null keeps the report and `mobster test compare` from claiming an identity."""
+    from .targets import Refused
+    current = case_provenance(entry, root, asset_cache=asset_cache)
+    if frozen["inputChanged"] or any(pinned is not None and current[field] != pinned
+                                     for field, pinned in frozen.items()):
+        raise Refused(f"{entry.file}, its app, or a fixture or assertion asset it uses changed during this suite. "
+                      "Start a new suite.", "refused")

mobile_agent/testkit/prrun.py

@@ -0,0 +1,244 @@
+from dataclasses import dataclass, field, replace
+from pathlib import Path
+from typing import Optional
+
+from . import pr as prmod
+from . import pr_report
+from .prbuild import BuildError
+
+SMOKE = "smoke"
+DRAFT_MODES = ("run", "write", "off")
+
+
+@dataclass
+class PrOptions:
+    base: Optional[str] = None       # default: the branch the PR merges into (pr.default_base)
+    build: bool = True
+    scheme: Optional[str] = None
+    project: Optional[str] = None    # an .xcodeproj or .xcworkspace; default the one in the project
+    configuration: str = "Debug"
+    drafts: str = "run"              # run | write | off
+    gifs: bool = True
+    tested_with: bool = False        # end the PR comment with "Tested with Mobster" (opt-in)
+    github: str = "check"            # check | comment | off: Mobster Checks on the PR (github_checks.py)
+
+
+@dataclass
+class PrResult:
+    pr: dict
+    doc: dict
+    paths: dict
+    exit_code: int
+    notes: list = field(default_factory=list)
+
+
+def with_build(entries, built, testflight):
+    """Point each check of the built app at the new build; a TestFlight app's checks use the build on the phone."""
+    out = []
+    for entry in entries:
+        check = entry.check
+        if check is None:
+            out.append(entry)
+            continue
+        if check.bundle_id and check.bundle_id in testflight:
+            entry = replace(entry, check=replace(check, app_path=None))
+        elif built and built.get("path") and (
+                (check.bundle_id and check.bundle_id == built.get("bundle")) or
+                (not check.bundle_id and check.app_path and Path(check.app_path).name == Path(built["path"]).name)):
+            entry = replace(entry, check=replace(check, app_path=built["path"]))
+        out.append(entry)
+    return out
+
+
+def _all_checks(root):
+    from .discover import DiscoveryError, discover
+    if not (Path(root) / ".mobster" / "checks").is_dir():
+        return []
+    try:
+        return discover((), root)
+    except DiscoveryError:
+        return []
+
+
+def plan_pr(root, pr_options, *, git=prmod.run_git, entries=None, env=None):
+    """(PrPlan without drafts, every saved check). Raises pr.PrError."""
+    changes = prmod.read_changes(root, pr_options.base, git=git, env=env)
+    screens = prmod.screens_in(changes)
+    entries = _all_checks(root) if entries is None else entries
+    picked = prmod.pick(entries, screens, changes)
+    fallback = None
+    if not picked and not screens:
+        smoke = [entry for entry in entries if SMOKE in {tag.casefold() for tag in entry.tags}]
+        picked = [prmod.Pick(entry, ["tagged smoke"], []) for entry in smoke]
+        fallback = SMOKE if smoke else None
+    return prmod.PrPlan(changes=changes, screens=screens, picked=picked, fallback=fallback), entries
+
+
+def build_app(root, pr_options, options, *, runner=None, progress=None, bundle=None):
+    from . import prbuild
+    if not pr_options.build:
+        return None
+    platform = "device" if any(True for _ in options.devices) else "sim"
+    if not pr_options.project and not options.xctest:
+        built = _build_react_native(root, pr_options, platform, runner=runner, progress=progress, bundle=bundle)
+        if built is not None:
+            return built
+    found = prbuild.project_arg(Path(root) / pr_options.project if not Path(pr_options.project).is_absolute()
+                                else pr_options.project) if pr_options.project else prbuild.find_project(root)
+    if found is None:
+        return None
+    scheme = pr_options.scheme or prbuild.pick_scheme(found, runner=runner, cwd=root)
+    if options.xctest:   # one build-for-testing makes the app the checks drive and the tests
+        from . import xctest
+        return xctest.build(root, scheme=scheme, project=found, test_plan=options.xctest_plan,
+                            configuration=pr_options.configuration, runner=runner, progress=progress, bundle=bundle)
+    return prbuild.build(root, scheme=scheme, project=found, platform=platform,
+                         configuration=pr_options.configuration, runner=runner, progress=progress, bundle=bundle)
+
+
+def _build_react_native(root, pr_options, platform, *, runner=None, progress=None, bundle=None):
+    """An Expo or React Native app's build, or None for any other project."""
+    # A Debug React Native build loads its JavaScript from Metro, which a test run doesn't start: Debug becomes Release.
+    from ..devkit import DevError, expo
+    app = expo.detect(root)
+    if app is None or not (app.expo or app.native):
+        return None
+    configuration = "Release" if pr_options.configuration == "Debug" else pr_options.configuration
+    (progress or (lambda line: None))(f"{app.name} is a{'n Expo' if app.expo else ' React Native'} app: building "
+                                      f"{configuration}, with the JavaScript inside")
+    try:
+        return expo.build_sim(app, configuration=configuration, platform=platform, runner=runner, progress=progress,
+                              bundle=bundle)
+    except DevError as error:
+        raise BuildError(error.message, error.fix) from None
+
+
+def test_pr(pr_options, options, *, root, progress=None, git=prmod.run_git, env=None, github=None, **kwargs):
+    """Run the PR's checks."""
+    from .github_checks import GITHUB_MODES_ERROR, MODES, Poster
+    if pr_options.github not in MODES:
+        from .api import UsageError
+        raise UsageError(GITHUB_MODES_ERROR)
+    poster = github or Poster(Path(root), pr_options.github, progress, environ=env, git=git)
+    try:
+        result = _test_pr(pr_options, options, root=root, progress=progress, git=git, env=env, on_plan=poster.start,
+                          **kwargs)
+    except BaseException:
+        poster.abandon()
+        raise
+    poster.finish(result)
+    return result
+
+
+def _test_pr(pr_options, options, *, root, progress=None, stop=None, key=None, suite_id=None, on_suite=None,
+             git=prmod.run_git, build_runner=None, execute=None, gif=None, env=None, named_device=None,
+             attempts=None, gate=None, on_plan=None):
+    from .api import Planned, UsageError, check_options, resolve_targets
+    from .discover import DiscoveryError, apply_quarantine, discover, load_quarantine
+    from .reports import make_suite_dir
+    progress = progress or (lambda line: None)
+    if pr_options.drafts not in DRAFT_MODES:
+        raise UsageError(f"--drafts is one of {', '.join(DRAFT_MODES)}")
+    if options.testflight and not options.devices:
+        raise UsageError("--testflight goes with --device: a TestFlight build is on an iPhone, not a simulator.")
+    check_options(options)
+    root = Path(root)
+    # before on_plan starts the PR's check run, so a usage error leaves none behind marked cancelled
+    targets = resolve_targets(options, root=root, named_device=named_device)
+    plan, entries = plan_pr(root, pr_options, git=git, env=env)
+    if on_plan is not None:
+        on_plan(plan.changes)
+    notes = []
+    if plan.changes.truncated:
+        notes.append(f"This PR changes more than {prmod.MAX_FILES} files; Mobster read the first {prmod.MAX_FILES}.")
+    progress(_plan_line(plan))
+    preferred = next((p.entry.check.bundle_id for p in plan.picked if p.entry.check is not None
+                      and p.entry.check.bundle_id), None)
+    tested_on_phone = set(options.testflight or ())
+    every_testflight = bool(options.devices) and plan.picked and all(
+        p.entry.check is not None and p.entry.check.bundle_id in tested_on_phone for p in plan.picked)
+    built = None
+    if not every_testflight and (plan.picked or plan.screens or options.xctest):
+        try:
+            built = build_app(root, pr_options, options, runner=build_runner, progress=progress, bundle=preferred)
+        except BuildError as error:
+            return _failed_build(plan, error, root, options, suite_id, pr_options, make_suite_dir)
+    picked_entries = with_build([p.entry for p in plan.picked], built, tested_on_phone)
+    listed = load_quarantine(root / ".mobster" / "quarantine.yaml")
+    apply_quarantine(picked_entries, listed)
+    if pr_options.drafts != "off":
+        app = prmod.app_of(entries, built)
+        plan.drafts = prmod.draft_checks(plan.screens, plan.picked, app=app, root=root)
+    draft_entries = []
+    if plan.drafts and pr_options.drafts == "run":
+        try:
+            draft_entries = discover([str(path) for _, path in plan.drafts], root)
+        except DiscoveryError:
+            draft_entries = []
+        for entry in draft_entries:
+            entry.quarantined = pr_report.DRAFT_REASON
+        draft_entries = with_build(draft_entries, built, tested_on_phone)
+    run_entries = picked_entries + draft_entries
+    plan_public = plan.public(root)
+    head = plan.changes.head
+    if not run_entries and not options.xctest:
+        suite_id, folder = make_suite_dir(root / ".mobster" / "test-results", suite_id)
+        doc = {"suite": suite_id, "checks": [], "exitCode": 0, "summary": {}}
+        result = pr_report.document(plan_public, doc, folder=folder, frames={}, gifs={}, built=_public_build(built),
+                                    head=head, tested_with=pr_options.tested_with, notes=notes)
+        paths = {"folder": folder, **pr_report.write(result, folder)}
+        return PrResult(result, doc, paths, 0, notes)
+    planned = Planned(root=root, entries=run_entries, targets=targets, options=options, notes=list(notes),
+                      xctest_build=built if options.xctest else None,
+                      changed=tuple(change.path for change in plan.changes.files) if options.xctest else None)
+    if execute is None:
+        from .api import execute
+    kwargs = {"progress": progress, "stop": stop, "key": key, "suite_id": suite_id, "on_suite": on_suite}
+    if named_device is not None:
+        kwargs["named_device"] = named_device
+    if attempts is not None:
+        kwargs["attempts"] = attempts
+    if gate is not None:
+        kwargs["gate"] = gate
+    doc, paths = execute(planned, **kwargs)
+    folder = Path(paths["folder"])
+    frames = pr_report.copy_frames(doc, folder)
+    gifs = pr_report.make_gifs(doc, folder, **({"build": gif} if gif else {})) if pr_options.gifs else {}
+    result = pr_report.document(plan_public, doc, folder=folder, frames=frames, gifs=gifs,
+                                built=_public_build(built), head=head, tested_with=pr_options.tested_with,
+                                notes=list(dict.fromkeys(notes + list(doc.get("notes") or ()))))
+    paths.update(pr_report.write(result, folder))
+    return PrResult(result, doc, paths, int(doc.get("exitCode", 3)), notes)
+
+
+def _plan_line(plan):
+    checks = len(plan.picked)
+    noun = "check" if checks == 1 else "checks"
+    if plan.screens:
+        screens = len(plan.screens)
+        return (f"mobster test pr: {screens} changed screen{'' if screens == 1 else 's'}, {checks} saved {noun} "
+                "cover them")
+    if plan.fallback == SMOKE:
+        return f"mobster test pr: no screen changed, so the {checks} {noun} tagged smoke run"
+    return "mobster test pr: no screen changed and no check is tagged smoke"
+
+
+def _public_build(built):
+    if not built:
+        return None
+    return {"bundle": built.get("bundle"), "app": Path(built["path"]).name if built.get("path") else None,
+            "platform": built.get("platform")}
+
+
+def _failed_build(plan, error, root, options, suite_id, pr_options, make_suite_dir):
+    suite_id, folder = make_suite_dir(Path(root) / ".mobster" / "test-results", suite_id)
+    doc = {"suite": suite_id, "checks": [], "exitCode": 3, "summary": {}}
+    note = f"{error.message} {error.fix}".strip()
+    result = pr_report.document(plan.public(root), doc, folder=folder, frames={}, gifs={}, built=None,
+                                head=plan.changes.head, tested_with=pr_options.tested_with, notes=[note])
+    result["headline"] = "Mobster checks: the build failed"
+    paths = {"folder": folder, **pr_report.write(result, folder)}
+    return PrResult(result, doc, paths, 3, [note])
+
+
+test_pr.__test__ = False  # a function named test_* that unittest must not collect