mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/testkit/cli_arguments.py

@@ -0,0 +1,193 @@
+"""Argument definitions and usage errors for ``mobster test``."""
+
+import sys
+
+USAGE_EXIT = 3
+
+
+def add_arguments(parser, helpers):
+    """`mobster test`'s flags. ``helpers``: env_option, path_type and bounded from __main__."""
+    from . import repro, snapshots_cli
+    bounded = helpers["bounded"]
+    parser.add_argument("paths", nargs="*", metavar="PATH",
+                        help="check files or folders (default: .mobster/checks); `record` saves a task; "
+                             "`capsule` inspects evidence; `compare` reads suites; `scenario` runs ordered checks; "
+                             "`minimize` tries a paired local reproducer")
+    where = parser.add_argument_group("where checks run")
+    where.add_argument("--sim", action="append", metavar="TYPE[@RUNTIME]",
+                       help="a simulator device type, with an optional runtime, such as \"iPhone 17 Pro@iOS 26.4\"; "
+                            "repeatable (default: each check's own, or .mobster/matrix.yaml)")
+    where.add_argument("--device", action="append", metavar="NAME",
+                       help="a device from `mobster devices`, or all: every iPhone ready on this Mac, side by side "
+                            "in one report; repeatable. On a real iPhone, checks drive only builds you installed, "
+                            "never App Store apps")
+    from .. import android
+    android.add_test_flag(where)
+    where.add_argument("--testflight", action="append", metavar="BUNDLE",
+                       help="with --device: your app's bundle ID, installed from TestFlight, which checks may "
+                            "drive as your own build; repeatable")
+    where.add_argument("--matrix", metavar="FILE",
+                       help="a YAML file of simulators to run every check on (simulators only)")
+    where.add_argument("--locale", action="append", metavar="LANG[,LANG]",
+                       help="run every check once in each language, such as de,ja,ar: the app launches in that "
+                            "language and region, and the screens are read for truncated text, untranslated "
+                            "strings and, in a right-to-left language, a layout that didn't mirror")
+    where.add_argument("--spread", action="store_true",
+                       help="with two or more devices: run each check once, on whichever device is free, instead of "
+                            "on every one (the report groups checks by device)")
+    where.add_argument("--parallel", type=bounded(int, 1, 8), default=1, metavar="N",
+                       help="simulators running checks at once (default: 1); each iPhone runs one check at a time")
+    how = parser.add_argument_group("retries and flakes")
+    how.add_argument("--retries", type=bounded(int, 0, 5), metavar="N",
+                     help="attempts after a failure, from a fresh app (default: 1, or 0 with --repeat)")
+    how.add_argument("--repeat", type=bounded(int, 1, 20), default=1, metavar="K",
+                     help="run each check K times and report its pass rate (flake hunting)")
+    how.add_argument("--strict", action="store_true", help="a flaky check fails the run")
+    how.add_argument("--quarantine", metavar="FILE",
+                     help="checks that run and are reported but never fail the run (default: "
+                          ".mobster/quarantine.yaml)")
+    how.add_argument("--auto-quarantine", action="store_true",
+                     help="quarantine for this run the checks whose last 20 results say flaky; without it Mobster "
+                          "only suggests them")
+    which = parser.add_argument_group("which checks")
+    which.add_argument("--tag", action="append", metavar="TAG", help="only checks with this tag; repeatable")
+    which.add_argument("--audit", action="append", choices=("a11y",), metavar="NAME",
+                       help="also audit what each check reads: a11y finds controls with no label, tap targets "
+                            "under 44 pt, images with no label and text clipped at a large text size; no model "
+                            "is called")
+    which.add_argument("--a11y", nargs="?", const="warn", choices=("warn", "strict"), metavar="LEVEL",
+                       help="audit the screens each check reads for accessibility: warn (the default) reports "
+                            "findings and passes, strict fails on a new one; findings in "
+                            ".mobster/a11y-baseline.json never fail")
+    which.add_argument("--update-a11y-baseline", action="store_true",
+                       help="write this run's accessibility findings to .mobster/a11y-baseline.json, so later runs "
+                            "fail only on new ones")
+    snapshots_cli.add_arguments(which)
+    which.add_argument("--shard", metavar="I/N", help="run the I-th of N even shards, such as 2/4")
+    which.add_argument("--keyless", action="store_true",
+                       help="never call a model: checks with steps couldn't run")
+    replay = parser.add_argument_group("replay")
+    replay.add_argument("--replay", action="store_true",
+                        help="replay each check's recorded path with no model, and heal a step that changed (the "
+                             "default)")
+    replay.add_argument("--fresh", action="store_true",
+                        help="run every step with Smart and record the path again")
+    replay.add_argument("--no-heal", action="store_true",
+                        help="replay, and fail a check whose screen changed instead of healing it (strict CI)")
+    unit = parser.add_argument_group("your XCTest targets")
+    unit.add_argument("--xctest", action="store_true",
+                      help="also build your Xcode project once and run its XCTest and Swift Testing targets on a "
+                           "simulator beside the checks; failures join the report, the JUnit file, the history and "
+                           "the PR comment. Pass --scheme and --project to name them")
+    unit.add_argument("--test-plan", metavar="NAME", help="with --xctest: the test plan to run (default: the "
+                                                           "scheme's)")
+    unit.add_argument("--only-testing", action="append", metavar="ID",
+                      help="with --xctest: run only this Target, Target/Class or Target/Class/test; repeatable")
+    out = parser.add_argument_group("reports")
+    out.add_argument("--junit", metavar="FILE", help="also write the JUnit XML here")
+    out.add_argument("--html", metavar="DIR", help="also write the HTML report (with each run's report) here")
+    out.add_argument("--video", action="store_true",
+                     help="record each run's screen, on a simulator or an iPhone, into the report")
+    out.add_argument("--history", action="store_true",
+                     help="print each check's last 20 results (pass rate, duration trend, flake score) and run "
+                          "nothing")
+    out.add_argument("--json", action="store_true", help="print JSON on stdout instead of the summary")
+    out.add_argument("--include", action="append", dest="capsule_include", choices=("junit", "report", "visual"),
+                     metavar="KIND", help="with capsule export: copy selected raw junit, report or visual evidence "
+                                          "locally; repeatable")
+    out.add_argument("--share", action="store_true",
+                     help="also upload the report, sealed, to your team's hosted reports and print its link (Team "
+                          "plan: one of the team's Macs, or MOBSTER_TEAM_KEY in CI)")
+    out.add_argument("--share-days", type=int, choices=(7, 30, 90), default=30, metavar="DAYS",
+                     help="how long a shared report is kept: 7, 30 or 90 (default: 30)")
+    out.add_argument("--assert-timeout", type=bounded(float, 0, 60), default=5, metavar="S",
+                     help="how long expectations may wait for the screen to settle (default: 5)")
+    record = parser.add_argument_group("mobster test record")
+    record.add_argument("--run", dest="record_run", metavar="RUN_ID", help="the finished task to make a check from")
+    record.add_argument("--name", dest="record_name", metavar="TEXT", help="the check's name (default: the task)")
+    record.add_argument("-o", "--output", dest="record_output", metavar="PATH",
+                        help="with record: write the check here; with import: checks folder; with capsule export: "
+                             "new output directory")
+    imp = parser.add_argument_group("mobster test import")
+    imp.add_argument("--bundle", metavar="ID",
+                     help="the app's bundle ID for flows with no appId, and over the appId of the others")
+    imp.add_argument("--app", metavar="PATH", help="write this .app (built for the iOS Simulator) into each check")
+    imp.add_argument("--skip-unsupported", action="store_true",
+                     help="drop commands Mobster can't run instead of turning each into a step marked TODO, and list "
+                          "each one; an assertion is never dropped")
+    imp.add_argument("--dry-run", action="store_true", help="say what would be written, and write nothing")
+    imp.add_argument("--force", action="store_true", help="replace checks that already exist")
+    repro.add_arguments(parser)
+    add = parser.add_argument_group("mobster test add")
+    add.add_argument("--list", dest="add_list", action="store_true", help="list the recipes, one line each")
+    add.add_argument("--set", dest="add_set", action="append", metavar="KEY=VALUE",
+                     help="a recipe param; repeatable (`mobster test add --list` shows each recipe's)")
+    pr = parser.add_argument_group("mobster test pr")
+    pr.add_argument("--base", metavar="REF",
+                    help="the branch the PR merges into (default: $GITHUB_BASE_REF, else origin/main)")
+    pr.add_argument("--no-build", action="store_true",
+                    help="don't build the app: test the builds the checks name")
+    pr.add_argument("--scheme", metavar="NAME", help="the Xcode scheme to build (default: the only one); also the "
+                                                      "scheme --xctest builds")
+    pr.add_argument("--project", metavar="PATH",
+                    help="the .xcodeproj or .xcworkspace to build (default: the one in the project); also the one "
+                         "--xctest builds")
+    pr.add_argument("--drafts", choices=("run", "write", "off"), default="run",
+                    help="checks for changed screens no check covers: write and run them (default), only write "
+                         "them, or neither. Drafts never fail the PR")
+    pr.add_argument("--no-gifs", action="store_true", help="don't make a GIF of each failed check")
+    pr.add_argument("--tested-with", action="store_true",
+                    help="end the PR comment with a \"Tested with Mobster\" line that links to mobster.dev")
+    pr.add_argument("--comment", metavar="FILE", help="also write the PR comment's Markdown here")
+    pr.add_argument("--github", choices=("check", "comment", "off"), default="check",
+                    help="put the verdict on the pull request with Mobster Checks, the GitHub App: a check run "
+                         "(check, the default), the check run and one comment kept up to date (comment), or nothing "
+                         "(off)")
+    nightly = parser.add_argument_group("mobster test nightly")
+    nightly.add_argument("--at", metavar="HH:MM",
+                         help="with nightly on: the time to run every night, on a 24-hour clock, such as 02:00")
+    heal = parser.add_argument_group("mobster test heal")
+    heal.add_argument("--accept", action="store_true", help="keep the healed steps of these checks (default: all)")
+    heal.add_argument("--reject", action="store_true",
+                      help="keep failing: put the recorded steps back and fail when they change again")
+    reduce = parser.add_argument_group("mobster test minimize")
+    reduce.add_argument("--bad-app", dest="minimize_bad_app", metavar="PATH",
+                        help="with minimize: the exact saved bad simulator .app build")
+    reduce.add_argument("--control-app", dest="minimize_control_app", metavar="PATH",
+                        help="with minimize: the exact saved clean-control simulator .app build")
+    reduce.add_argument("--isolated", dest="minimize_isolated", action="store_true",
+                        help="with minimize: confirm the external fixture is isolated for every trial")
+    reduce.add_argument("--max-attempts", dest="minimize_max_attempts", type=bounded(int, 5, 8), metavar="N",
+                        help="with minimize: total bad and control suite dispatches, 5–8 (default: 8)")
+    helpers["env_option"](parser)
+    parser._option_string_actions["--env-file"].help = ("load KEY=VALUE lines, such as OPENAI_API_KEY or "
+                                                        "ANTHROPIC_API_KEY for Smart steps; variables already set "
+                                                        "win; default: $MOBSTER_ENV_FILE")
+    _usage_exits_3(parser)
+
+
+def _usage_exits_3(parser):
+    """argparse's usage errors exit 2, which `test` uses for needs review: they exit 3 here."""
+    parse_known = parser.parse_known_args
+
+    def parse_known_args(args=None, namespace=None):
+        namespace, extras = parse_known(args, namespace)
+        stray = [item for item in extras if item.startswith("-")]
+        if stray:
+            parser.error(f"unrecognized arguments: {' '.join(stray)}")
+        if extras:
+            namespace.paths = list(getattr(namespace, "paths", None) or []) + extras
+        return namespace, []
+
+    def error(message):
+        parser.print_usage(sys.stderr)
+        hint = ""
+        try:
+            from ..devtools import option_hint
+            hint = option_hint(parser, message)
+        except ImportError:
+            pass
+        print(f"{parser.prog}: error: {message}{hint}", file=sys.stderr, flush=True)
+        raise SystemExit(USAGE_EXIT)
+    parser.parse_known_args = parse_known_args
+    parser.error = error

mobile_agent/testkit/comparison.py

@@ -0,0 +1,288 @@
+"""Compare exact saved suites, distinguishing transitions from controlled app-state proof."""
+
+import hashlib
+import math
+import re
+from pathlib import Path
+
+from . import capsule, reports
+
+SCHEMA = "mobster.test-compare/1"
+DIGEST = re.compile(r"[a-f0-9]{64}\Z")
+REASON_CLASS = re.compile(r"[a-z][a-z0-9_]{0,39}\Z")
+FIELD = re.compile(r"[a-z][a-z0-9_.-]{0,79}\Z")
+STATUSES = frozenset({"passed", "failed", "needs_review", "couldnt_run"})
+
+
+class ComparisonError(ValueError):
+    """A named suite has no intact completed local report."""
+
+
+def _read(results_dir, suite_id):
+    if not isinstance(suite_id, str) or not capsule.SUITE_ID.fullmatch(suite_id):
+        raise ComparisonError("A suite ID must name a saved Mobster test result.")
+    try:
+        return capsule.read_completed_suite(Path(results_dir) / suite_id)
+    except (capsule.CapsuleError, OSError) as error:
+        raise ComparisonError(f"Suite {suite_id} has no intact completed result.") from error
+
+
+def compare_suites(results_dir, base_suite_id, candidate_suite_id):
+    """Read the two exact IDs under one local results directory."""
+    return compare(_read(results_dir, base_suite_id), _read(results_dir, candidate_suite_id))
+
+
+def _ref(text):
+    return hashlib.sha256(text.encode("utf-8", errors="surrogatepass")).hexdigest()[:12]
+
+
+def _digest(value):
+    return isinstance(value, str) and bool(DIGEST.fullmatch(value))
+
+
+def _frozen(value):
+    return value == "none" or _digest(value)
+
+
+def _status(cell):
+    value = cell.get("status")
+    return value if isinstance(value, str) and value in STATUSES else None
+
+
+def _reason(cell):
+    source = cell.get("reason")
+    value = source.get("class") if isinstance(source, dict) else None
+    return value if isinstance(value, str) and REASON_CLASS.fullmatch(value) else None
+
+
+def _environment(cell):
+    source = cell.get("environment")
+    if not isinstance(source, dict) or source.get("kind") not in ("simulator", "iphone") or any(
+            not isinstance(source.get(key), str) or not 1 <= len(source[key]) <= 80
+            for key in ("model", "runtime")):
+        return None
+    return {key: source[key] for key in ("kind", "model", "runtime")}
+
+
+def _policy(doc):
+    value = doc.get("executionPolicy")
+    if not isinstance(value, dict) or set(value) != {"retries", "repeat", "replay", "keyless",
+                                                     "assertTimeout", "mobster"}:
+        return None
+    if (type(value["retries"]) is not int or not 0 <= value["retries"] <= 5 or
+            type(value["repeat"]) is not int or not 1 <= value["repeat"] <= 20 or
+            value["replay"] not in ("off", "fresh", "strict", "auto") or
+            type(value["keyless"]) is not bool or
+            type(value["assertTimeout"]) not in (int, float) or
+            not math.isfinite(value["assertTimeout"]) or not 0 < value["assertTimeout"] <= 600 or
+            not isinstance(value["mobster"], str) or not 1 <= len(value["mobster"]) <= 40):
+        return None
+    return value
+
+
+def _verdict(doc):
+    value = doc.get("exitCode")
+    return value if type(value) is int and 0 <= value <= 255 else None
+
+
+def _cells(doc):
+    found, errors = {}, []
+    checks = doc.get("checks")
+    if not isinstance(checks, list):
+        return found, ["checks_missing"]
+    for check in checks:
+        if not isinstance(check, dict) or not isinstance(check.get("results"), list):
+            errors.append("check_malformed")
+            continue
+        identity = check.get("caseId") or check.get("file")
+        if not isinstance(identity, str) or not 1 <= len(identity) <= 4096:
+            errors.append("check_identity_missing")
+            continue
+        for cell in check["results"]:
+            if not isinstance(cell, dict) or not isinstance(cell.get("deviceKey"), str) or \
+                    not 1 <= len(cell["deviceKey"]) <= 256:
+                errors.append("cell_malformed")
+                continue
+            key = identity, cell["deviceKey"]
+            if key in found:
+                errors.append("duplicate_case_target")
+                found[key] = None
+            else:
+                found[key] = (check, cell)
+    return found, sorted(set(errors))
+
+
+def _clean(cell):
+    reason = cell.get("reason")
+    return (type(cell.get("quarantined")) is bool and cell["quarantined"] is False and
+            type(cell.get("flaky")) is bool and cell["flaky"] is False and
+            type(cell.get("replayed")) is bool and cell["replayed"] is False and
+            cell.get("healed") == [] and isinstance(cell.get("assertions"), list) and
+            (reason is None or isinstance(reason, dict)) and not cell.get("safetyStop") and
+            _status(cell) in ("passed", "failed"))
+
+
+def _business_fields(check, cell):
+    """Validate one selected run against its admitted binding and complete field inventory."""
+    binding = check.get("businessBindingSha256")
+    proof = cell.get("business")
+    runs = cell.get("runs")
+    if (not _digest(binding) or not isinstance(proof, dict) or proof.get("bindingSha256") != binding or
+            proof.get("setup") != "seeded" or proof.get("status") != "observed" or
+            proof.get("cleanup") != "removed" or type(proof.get("generation")) is not int or
+            proof["generation"] < 1 or not _digest(proof.get("sourceSha256")) or
+            type(cell.get("attempts")) is not int or cell["attempts"] != 1 or
+            not isinstance(runs, list) or len(runs) != 1 or not isinstance(runs[0], dict) or
+            runs[0].get("business") != proof or runs[0].get("verdict") != cell.get("status")):
+        return None
+    fields, passed, failed = (proof.get(key) for key in ("assertionFields", "passedFields", "failedFields"))
+    if (not isinstance(fields, list) or not 1 <= len(fields) <= 20 or
+            not all(isinstance(item, str) and FIELD.fullmatch(item) for item in fields) or
+            len(set(fields)) != len(fields) or not isinstance(passed, list) or not isinstance(failed, list) or
+            any(not isinstance(item, str) for item in passed + failed) or
+            len(passed) + len(failed) != len(fields) or set(passed + failed) != set(fields)):
+        return None
+    return fields, failed, passed
+
+
+def _supporting_evidence(cell, visual_required):
+    if cell.get("findings", []) != []:
+        return False
+    conditions = cell.get("conditions")
+    if conditions is not None and (not isinstance(conditions, dict) or
+                                   conditions.get("cleanup") not in ("restored", "not_needed") or
+                                   not isinstance(conditions.get("applied"), list) or
+                                   conditions.get("skipped") != []):
+        return False
+    assertions = cell.get("assertions")
+    if not isinstance(assertions, list) or any(
+            not isinstance(item, dict) or type(item.get("index")) is not int or item["index"] < 0 or
+            not isinstance(item.get("text"), str) or item.get("ok") is not True for item in assertions):
+        return False
+    if len({item["index"] for item in assertions}) != len(assertions):
+        return False
+    visual = cell.get("visual")
+    return (visual is None and not visual_required or
+            isinstance(visual, dict) and visual.get("ok") is True)
+
+
+def _case(base, candidate, key, single_repeat):
+    before_check, before = base
+    after_check, after = candidate
+    reasons = []
+    if before_check.get("provenanceDrift") or after_check.get("provenanceDrift"):
+        reasons.append("provenance_drift")
+    for field, missing, changed in (("checkSpecDigest", "check_unknown", "check_changed"),
+                                    ("baselineDigest", "baseline_unknown", "baseline_changed"),
+                                    ("fixtureDigest", "fixture_unknown", "fixture_changed")):
+        valid = _digest if field == "checkSpecDigest" else _frozen
+        if not valid(before_check.get(field)) or not valid(after_check.get(field)):
+            reasons.append(missing)
+        elif before_check[field] != after_check[field]:
+            reasons.append(changed)
+    old_env, new_env = _environment(before), _environment(after)
+    if old_env is None or new_env is None:
+        reasons.append("environment_unknown")
+    elif old_env != new_env:
+        reasons.append("environment_changed")
+    if not _clean(before) or not _clean(after):
+        reasons.append("verdict_not_stable")
+    comparable = not reasons
+    before_status, after_status = _status(before), _status(after)
+    transition = f"{before_status}_to_{after_status}" if comparable else None
+    proof_reasons = []
+    verified = False
+    if transition == "failed_to_passed":
+        binding = before_check.get("businessBindingSha256")
+        prior, later = _business_fields(before_check, before), _business_fields(after_check, after)
+        if (not single_repeat or _reason(before) != "business" or after.get("reason") is not None or
+                before.get("business") is None or after.get("business") is None or
+                before_check.get("fixtureDigest") == "none" or
+                before_check.get("businessBindingSha256") != after_check.get("businessBindingSha256") or
+                not _digest(binding) or not prior or not later or prior[0] != later[0] or
+                not prior[1] or later[1] or not set(prior[1]).issubset(later[2]) or
+                before["runs"][0].get("class") != "business" or after["runs"][0].get("class") is not None or
+                old_env["kind"] != "simulator" or
+                before["business"]["sourceSha256"] == after["business"]["sourceSha256"] or
+                "visualBaselineSha256" not in before_check or "visualBaselineSha256" not in after_check or
+                before_check["visualBaselineSha256"] != after_check["visualBaselineSha256"] or
+                (before_check["visualBaselineSha256"] is not None and
+                 not _digest(before_check["visualBaselineSha256"])) or
+                not _supporting_evidence(before, before_check.get("visualBaselineSha256") is not None) or
+                not _supporting_evidence(after, after_check.get("visualBaselineSha256") is not None)):
+            proof_reasons.append("controlled_fixture_unproven")
+        if not _digest(before.get("appDigest")) or not _digest(after.get("appDigest")):
+            proof_reasons.append("app_artifact_unknown")
+        elif before["appDigest"] == after["appDigest"]:
+            proof_reasons.append("app_artifact_unchanged")
+        verified = not proof_reasons
+    elif transition is not None:
+        proof_reasons.append("no_failed_to_passed_transition")
+    return {"caseRef": _ref(key[0]), "targetRef": _ref(key[1]),
+            "base": {"status": before_status, "reasonClass": _reason(before)},
+            "candidate": {"status": after_status, "reasonClass": _reason(after)},
+            "comparable": comparable, "observedTransition": transition,
+            "verifiedFix": verified, "proof": {"scope": "controlled_app_state" if verified else None},
+            "reasons": sorted(set(reasons)), "proofReasons": proof_reasons}
+
+
+def compare(base, candidate):
+    """Keep both actual verdicts and every incompatibility; no green-baseline search."""
+    if not isinstance(base, dict) or not isinstance(candidate, dict):
+        raise ComparisonError("Both saved suite reports must be objects.")
+    before, before_errors = _cells(base)
+    after, after_errors = _cells(candidate)
+    old_policy, new_policy = _policy(base), _policy(candidate)
+    rows = []
+    for key in sorted(before.keys() | after.keys()):
+        if key in before and key in after and before[key] is not None and after[key] is not None:
+            rows.append(_case(before[key], after[key], key, old_policy is not None and
+                              new_policy is not None and old_policy["repeat"] == new_policy["repeat"] == 1))
+            continue
+        prior = before.get(key)
+        later = after.get(key)
+        ambiguous = key in before and prior is None or key in after and later is None
+        rows.append({"caseRef": _ref(key[0]), "targetRef": _ref(key[1]),
+                     "base": {"status": _status(prior[1]) if prior else None, "reasonClass": None},
+                     "candidate": {"status": _status(later[1]) if later else None, "reasonClass": None},
+                     "comparable": False, "observedTransition": None, "verifiedFix": False,
+                     "proof": {"scope": None}, "reasons": ["duplicate_case_target" if ambiguous else
+                         "case_missing_from_candidate" if key in before else "case_missing_from_base"],
+                     "proofReasons": []})
+    reasons = before_errors + after_errors
+    if base.get("scenario") is not None or candidate.get("scenario") is not None:
+        reasons.append("scenario_not_comparable")
+    if base.get("schema") != reports.SCHEMA or candidate.get("schema") != reports.SCHEMA:
+        reasons.append("report_schema_unknown")
+    if type(base.get("finishedAt")) is not int or type(candidate.get("finishedAt")) is not int:
+        reasons.append("suite_not_terminal")
+    if base.get("stopReason") or candidate.get("stopReason") or base.get("stopped") or candidate.get("stopped"):
+        reasons.append("suite_stopped")
+    if _verdict(base) is None or _verdict(candidate) is None:
+        reasons.append("suite_verdict_missing")
+    if old_policy is None or new_policy is None:
+        reasons.append("execution_policy_unknown")
+    elif old_policy != new_policy:
+        reasons.append("execution_policy_changed")
+    if not rows:
+        reasons.append("no_cases")
+    if any(not row["comparable"] for row in rows):
+        reasons.append("cases_incomparable")
+    invalid = {"scenario_not_comparable", "suite_not_terminal", "suite_stopped", "report_schema_unknown",
+               "suite_verdict_missing", "execution_policy_unknown", "execution_policy_changed"}
+    if before_errors or after_errors or invalid.intersection(reasons):
+        for row in rows:
+            row.update(comparable=False, observedTransition=None, verifiedFix=False, proof={"scope": None})
+            row["reasons"] = sorted(set(row["reasons"] + ["suite_not_comparable"]))
+    verified = (not reasons and _verdict(base) == 1 and _verdict(candidate) == 0 and
+                any(row["verifiedFix"] for row in rows) and all(
+                    row["candidate"]["status"] == "passed" and
+                    (row["base"]["status"] != "failed" or row["verifiedFix"]) for row in rows))
+    return {"schema": SCHEMA, "baseSuite": base.get("suite") if isinstance(base.get("suite"), str) and
+            capsule.SUITE_ID.fullmatch(base["suite"]) else None,
+            "candidateSuite": candidate.get("suite") if isinstance(candidate.get("suite"), str) and
+            capsule.SUITE_ID.fullmatch(candidate["suite"]) else None,
+            "baseVerdict": _verdict(base), "candidateVerdict": _verdict(candidate),
+            "comparable": not reasons, "verifiedFix": verified,
+            "proof": {"scope": "controlled_app_state" if verified else None},
+            "reasons": sorted(set(reasons)), "cases": rows}

mobile_agent/testkit/comparison_cli.py

@@ -0,0 +1,41 @@
+"""Compare two explicitly named local terminal suite reports."""
+
+import json
+import sys
+from pathlib import Path
+
+from ..verify.checks import project_root
+from .comparison import ComparisonError, compare_suites
+
+
+def run(args, operands, *, out=None, err=None, compare=compare_suites):
+    out, err = out or sys.stdout, err or sys.stderr
+
+    def refuse(message):
+        if getattr(args, "json", False):
+            print(json.dumps({"ok": False, "error": message}), file=out)
+        else:
+            print(f"mobster test compare: {message}", file=err)
+        return 3
+
+    if len(operands) != 2:
+        return refuse("name a base and candidate: mobster test compare BASE_SUITE_ID CANDIDATE_SUITE_ID")
+    private = project_root(Path.cwd()) / ".mobster"
+    results = private / "test-results"
+    if private.is_symlink() or results.is_symlink():
+        return refuse("the project's test-results folder must not be a symlink")
+    try:
+        result = compare(results, *operands)
+    except (ComparisonError, OSError) as error:
+        return refuse(str(error))
+    if getattr(args, "json", False):
+        print(json.dumps(result, ensure_ascii=False), file=out)
+    else:
+        print(f"Base {result['baseSuite']}: exit {result['baseVerdict']}; "
+              f"candidate {result['candidateSuite']}: exit {result['candidateVerdict']}.", file=out)
+        proof = "yes, for controlled app state" if result["verifiedFix"] else "no"
+        print(f"Comparable: {'yes' if result['comparable'] else 'no'}; verified fix: {proof}.", file=out)
+        if result["reasons"]:
+            print("Reasons: " + ", ".join(result["reasons"]), file=out)
+        print(f"Compared {len(result['cases'])} case-target pairs. Use --json for each transition.", file=out)
+    return 0