mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/testkit/discover.py

@@ -0,0 +1,257 @@
+"""Which checks a `mobster test` run covers: discovery, the quarantine list, tags and shards."""
+
+import re
+from dataclasses import dataclass, field, replace
+from pathlib import Path
+from typing import Optional
+
+CHECK_SUFFIXES = (".yaml", ".yml", ".json")
+NOT_CHECKS = frozenset({"matrix.yaml", "matrix.yml", "quarantine.yaml", "quarantine.yml"})
+# What Mobster writes in .mobster/ (run results, reports, builds, the cache): never checks, though it holds .json.
+OUTPUTS = frozenset({"runs", "test-results", "build", "cache"})
+MAX_CHECKS = 500
+SHARD = re.compile(r"(\d{1,3})/(\d{1,3})")
+QUARANTINE_LIMIT = 200
+
+
+class DiscoveryError(Exception):
+    """A usage problem with the paths, the quarantine file, tags or the shard: one plain sentence."""
+
+
+@dataclass
+class Entry:
+    """One check file in the suite."""
+    path: Path                    # absolute
+    file: str                     # as reports show it: relative to the project root when inside it
+    check: object = None          # verify.checks.Check, or None when the file can't be read
+    error: Optional[str] = None   # why it can't be read
+    quarantined: Optional[str] = None  # None, or the reason it is quarantined ("" when none was given)
+    classname: str = ""
+    keys: tuple = field(default=())  # every name the quarantine file may use for it
+    locale: Optional[str] = None  # `--locale`: the language this copy of the check runs in
+
+    @property
+    def name(self):
+        return self.check.name if self.check is not None else Path(self.file).stem
+
+    @property
+    def tags(self):
+        return tuple(getattr(self.check, "tags", ()) or ())
+
+
+def in_locales(entries, locales):
+    out = []
+    for entry in entries:
+        if entry.check is None:
+            out.append(entry)
+            continue
+        for locale in locales:
+            check = replace(entry.check, name=f"{entry.check.name} [{locale}]", l10n=True,
+                            conditions={**(entry.check.conditions or {}), "locale": locale})
+            out.append(replace(entry, check=check, locale=locale, classname=f"{entry.classname}.{locale}"))
+    return out
+
+
+def in_matrix(entries):
+    """A check whose conditions list appearances, text sizes or locales becomes one entry per combination."""
+    from ..verify import conditions
+    out = []
+    for entry in entries:
+        if entry.check is None or not conditions.is_matrix(entry.check.conditions):
+            out.append(entry)
+            continue
+        for combo in conditions.combinations(entry.check.conditions):
+            tag = ", ".join(str(combo[key]) for key in conditions.MATRIX_KEYS
+                            if key in combo and isinstance(entry.check.conditions[key], list))
+            slug = re.sub(r"[^A-Za-z0-9_-]+", "_", tag.replace(", ", "."))
+            out.append(replace(entry, check=replace(entry.check, name=f"{entry.check.name} [{tag}]",
+                                                    conditions=combo),
+                               classname=f"{entry.classname}.{slug}"))
+    return out
+
+
+def with_names(entries, names):
+    """The entries whose check name or file name (without its suffix) is one of ``names``; a check that can't be read
+    is kept, so its error still shows."""
+    if not names:
+        return list(entries)
+    wanted = {name.strip().casefold() for name in names}
+    return [entry for entry in entries if entry.check is None
+            or wanted & {entry.check.name.casefold(), Path(entry.file).stem.casefold(),
+                         Path(entry.file).with_suffix("").as_posix().casefold()}]
+
+
+def _relative(path, root):
+    try:
+        return path.relative_to(root).as_posix()
+    except ValueError:
+        return path.as_posix()
+
+
+def _classname(path, root):
+    """``checks.<file>``: the file's path without its suffix, from the checks folder when it is in one."""
+    checks = root / ".mobster" / "checks"
+    try:
+        stem = path.relative_to(checks)
+    except ValueError:
+        try:
+            stem = path.relative_to(root)
+        except ValueError:
+            stem = Path(path.name)
+    parts = [re.sub(r"[^A-Za-z0-9_-]+", "_", part) for part in stem.with_suffix("").parts]
+    return "checks." + ".".join(part for part in parts if part)
+
+
+def check_files(folder):
+    out = []
+    for path in sorted(folder.rglob("*")):
+        relative = path.relative_to(folder)
+        if any(part.startswith(".") for part in relative.parts):
+            continue
+        if not path.is_file() or path.suffix.lower() not in CHECK_SUFFIXES:
+            continue
+        if path.name in NOT_CHECKS or path.name.endswith((".heal.yaml", ".heal.yml")):
+            continue
+        if any(parent == ".mobster" and child in OUTPUTS for parent, child in zip(path.parts, path.parts[1:])):
+            continue
+        out.append(path)
+    return out
+
+
+def discover(paths, root, *, cwd=None, loader=None):
+    """Every check under ``paths`` (files or folders), else ``root``/.mobster/checks, as Entries sorted by file."""
+    if loader is None:
+        from ..verify.checks import load_check as loader
+    from ..verify.checks import CheckError
+    root = Path(root).resolve()
+    cwd = Path(cwd or Path.cwd())
+    files = []
+    if not paths:
+        folder = root / ".mobster" / "checks"
+        if not folder.is_dir():
+            raise DiscoveryError(f"There's no .mobster/checks folder in {root}. Save a check there with "
+                                 "`mobster verify … --save NAME`, or pass a check file or folder.")
+        files = check_files(folder)
+        if not files:
+            raise DiscoveryError(f"{_relative(folder, cwd.resolve())} has no checks (.yaml, .yml or .json files).")
+    for given in paths or ():
+        path = Path(given).expanduser()
+        path = (path if path.is_absolute() else cwd / path).resolve()
+        if not path.exists():
+            raise DiscoveryError(f"{given} doesn't exist.")
+        if path.is_dir():
+            found = check_files(path)
+            if not found:
+                raise DiscoveryError(f"{given} has no checks (.yaml, .yml or .json files).")
+            files.extend(found)
+        elif path.suffix.lower() not in CHECK_SUFFIXES:
+            raise DiscoveryError(f"{given} isn't a check file: checks are .yaml, .yml or .json.")
+        else:
+            files.append(path)
+    unique = list(dict.fromkeys(path.resolve() for path in files))
+    if len(unique) > MAX_CHECKS:
+        raise DiscoveryError(f"That's {len(unique)} checks; a run takes at most {MAX_CHECKS}. Pass fewer paths, "
+                             "or split them with --shard.")
+    entries = []
+    for path in unique:
+        file = _relative(path, root)
+        try:
+            # A file that can't be read is kept with its error, so the suite reports it as couldn't run, not skipped.
+            check, error = loader(path), None
+        except CheckError as problem:
+            check, error = None, str(problem)
+        except (OSError, ValueError, RecursionError) as problem:
+            check, error = None, f"{file} can't be read ({type(problem).__name__})"
+        entry = Entry(path=path, file=file, check=check, error=error, classname=_classname(path, root))
+        checks = root / ".mobster" / "checks"
+        keys = {file, path.stem, Path(file).with_suffix("").as_posix()}
+        try:
+            keys.add(path.relative_to(checks).with_suffix("").as_posix())
+        except ValueError:
+            pass
+        if check is not None:
+            keys.add(check.name)
+        entry.keys = tuple(sorted(key.casefold() for key in keys))
+        entries.append(entry)
+    entries.sort(key=lambda entry: entry.file)
+    return entries
+
+
+def load_quarantine(path, *, required=False):
+    path = Path(path)
+    if not path.exists():
+        if required:
+            raise DiscoveryError(f"{path} doesn't exist.")
+        return {}
+    import yaml
+    try:
+        data = yaml.safe_load(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeDecodeError) as error:
+        raise DiscoveryError(f"{path} can't be read ({type(error).__name__}).") from None
+    except yaml.YAMLError as error:
+        mark = getattr(error, "problem_mark", None)
+        line = f" line {mark.line + 1}" if mark is not None else ""
+        raise DiscoveryError(f"{path}{line}: {getattr(error, 'problem', None) or 'not valid YAML'}") from None
+    if data is None:
+        return {}
+    if isinstance(data, dict):
+        unknown = [key for key in data if key != "quarantine"]
+        if unknown:
+            raise DiscoveryError(f"{path} has an unknown key {unknown[0]!r}; list the checks under quarantine:")
+        data = data.get("quarantine") or []
+    if not isinstance(data, list):
+        raise DiscoveryError(f"{path} must be a list of checks, such as - paywall")
+    if len(data) > QUARANTINE_LIMIT:
+        raise DiscoveryError(f"{path} lists more than {QUARANTINE_LIMIT} checks")
+    out = {}
+    for index, item in enumerate(data):
+        reason = ""
+        if isinstance(item, dict):
+            unknown = [key for key in item if key not in ("check", "reason")]
+            if unknown or not isinstance(item.get("check"), str):
+                raise DiscoveryError(f"{path} item {index + 1}: write {{check: NAME, reason: WHY}}")
+            reason = item.get("reason") or ""
+            if not isinstance(reason, str):
+                raise DiscoveryError(f"{path} item {index + 1}: reason must be text")
+            item = item["check"]
+        if not isinstance(item, str) or not item.strip():
+            raise DiscoveryError(f"{path} item {index + 1} must name a check: its file name, path or name")
+        out[item.strip().casefold()] = " ".join(reason.split())[:300]
+    return out
+
+
+def apply_quarantine(entries, listed):
+    """Mark the entries ``listed`` names; returns the names that matched no check."""
+    used = set()
+    for entry in entries:
+        for key in entry.keys:
+            if key in listed:
+                entry.quarantined = listed[key]
+                used.add(key)
+                break
+    return [key for key in listed if key not in used]
+
+
+def with_tags(entries, tags):
+    """The entries carrying any of ``tags``; a check that can't be read is kept, so its error still shows."""
+    if not tags:
+        return list(entries)
+    wanted = {tag.strip().casefold() for tag in tags}
+    return [entry for entry in entries if entry.check is None or wanted & {tag.casefold() for tag in entry.tags}]
+
+
+def parse_shard(text):
+    """``i/n`` as (i, n), 1 <= i <= n <= 100. Raises DiscoveryError."""
+    match = SHARD.fullmatch(str(text or "").strip())
+    if not match:
+        raise DiscoveryError(f"--shard {text!r}: write I/N, such as 2/4 for the second of four shards")
+    index, count = int(match[1]), int(match[2])
+    if not 1 <= index <= count <= 100:
+        raise DiscoveryError(f"--shard {text!r}: I is from 1 to N, and N at most 100")
+    return index, count
+
+
+def shard(entries, index, count):
+    # Every count-th check in file order: the shards split evenly and every machine running the same files agrees.
+    return [entry for position, entry in enumerate(sorted(entries, key=lambda e: e.file))
+            if position % count == index - 1]

mobile_agent/testkit/editor.py

@@ -0,0 +1,243 @@
+"""The editor extension's routes on `mobster serve`: a project's checks, a suite started from the editor, and heals.
+
+A suite started here runs on simulators only, never on a real iPhone: that stays `mobster test --device` and
+run_tests with a device the user allowed. One suite runs at a time per service.
+"""
+
+from pathlib import Path
+
+from .jobs import PATTERN_SUITE, Busy, PrJob, SuiteJob, Suites
+
+SUITE = r"(?P<suite>" + PATTERN_SUITE.strip("^$") + ")"
+MAX_COVERING_BYTES = 500_000
+NO_CHECKS = ("This project has no saved checks yet. Ask your agent to verify a screen with Mobster and save it "
+             "with save_check, or run `mobster verify … --save NAME`.")
+
+
+def register_routes(add_route, Route):
+    add_route(Route("testkit", "GET", "/api/tests/project", project, body="none"))
+    add_route(Route("testkit", "POST", "/api/tests/suites", start, max_bytes=16384))
+    add_route(Route("testkit", "GET", f"/api/tests/suites/{SUITE}", suite, body="none"))
+    add_route(Route("testkit", "POST", f"/api/tests/suites/{SUITE}/stop", stop, max_bytes=1024))
+    add_route(Route("testkit", "POST", "/api/tests/heals", heals, max_bytes=16384))
+
+
+def start_service(runtime):
+    runtime.editor_suites = Suites()
+
+
+def close_service(runtime):
+    found = getattr(runtime, "editor_suites", None)
+    if found is not None:
+        found.close()
+
+
+# ------------------------------------------------------------------------------------------------- the routes
+
+def project(request):
+    """The project's checks with each one's last verdict, the heals waiting, the suite going now, and with ``file``
+    (a source file) the checks that cover its screen."""
+    from ..api_routes import Response
+    from .discover import DiscoveryError, discover
+    root, bad = _root((request.query.get("root") or [None])[0])
+    if bad is not None:
+        return bad
+    out = {"root": str(root), "checks": [], "heals": [], "problem": None, "covering": None,
+           "suite": _latest(request.runtime, root)}
+    if not (root / ".mobster" / "checks").is_dir():
+        return Response.json({**out, "problem": NO_CHECKS})
+    try:
+        entries = discover([], root)
+    except DiscoveryError as error:
+        return Response.json({**out, "problem": str(error)})
+    last = _last_verdicts(root)
+    out["checks"] = [_entry_row(entry, last.get(entry.file)) for entry in entries]
+    out["heals"], out["problem"] = _waiting(root)
+    wanted = (request.query.get("file") or [None])[0]
+    if wanted is not None:
+        source, bad = _inside(root, wanted, "file")
+        if bad is not None:
+            return bad
+        out["covering"] = covering(root, source, entries)
+    return Response.json(out)
+
+
+def start(request):
+    """Start a suite: the checks named (absolute paths in the project), every check, or with ``changed`` the checks
+    that cover what changed since the last commit, after building the app."""
+    from ..api_routes import Response
+    from .api import Options
+    from .prrun import PrOptions
+    body = request.body if isinstance(request.body, dict) else {}
+    if set(body) - {"root", "checks", "changed", "sim", "heal"}:
+        return _bad("Send root and, if you like, checks, changed, sim and heal.")
+    root, bad = _root(body.get("root"))
+    if bad is not None:
+        return bad
+    checks, changed, sim, heal = body.get("checks"), body.get("changed", False), body.get("sim"), body.get("heal", True)
+    if type(changed) is not bool or type(heal) is not bool:
+        return _bad("changed and heal are true or false.")
+    if sim is not None and not (isinstance(sim, str) and 0 < len(sim) <= 128):
+        return _bad("sim is a kind of simulator, such as iPhone 17 Pro.")
+    if checks is not None:
+        if changed:
+            return _bad("Send checks or changed, not both: changed picks the checks itself.")
+        if not isinstance(checks, list) or not 0 < len(checks) <= 50:
+            return _bad("checks is a list of 1 to 50 check files.")
+        paths = []
+        for item in checks:
+            path, bad = _inside(root, item, "checks")
+            if bad is not None:
+                return bad
+            paths.append(str(path))
+    gate = getattr(request.runtime, "plan_gate", None)
+    refusal = gate.refusal() if gate is not None else None
+    if refusal is not None:
+        return Response.error(refusal, 403, "plan_required")
+    sims = (sim,) if sim else ()
+    if changed:
+        def make():
+            return PrJob(PrOptions(base="HEAD", drafts="off", github="off", gifs=False),
+                         Options(sims=sims, no_heal=not heal, command="editor: test on save"), root)
+    else:
+        def make():
+            return SuiteJob(Options(paths=tuple(paths) if checks else (str(root / ".mobster" / "checks"),),
+                                    sims=sims, no_heal=not heal, command="editor"), root=root)
+    try:
+        job = request.runtime.editor_suites.start(make)
+    except Busy as busy:
+        return Response.error(f"A suite is still running ({busy.job.public()['message']}). Wait for it, or stop it.",
+                              409, "suite_running", suite=busy.job.public())
+    return Response.json({"suite": job.public()}, 202)
+
+
+def suite(request):
+    from ..api_routes import Response
+    job = request.runtime.editor_suites.get(request.params["suite"])
+    if job is None:
+        return Response.error("Mobster has no suite with that id. It keeps the last ten.", 404, "suite_not_found")
+    return Response.json({"suite": job.public()})
+
+
+def stop(request):
+    """Stop the suite: the check going now ends, and the rest don't start."""
+    from ..api_routes import Response
+    job = request.runtime.editor_suites.get(request.params["suite"])
+    if job is None:
+        return Response.error("Mobster has no suite with that id. It keeps the last ten.", 404, "suite_not_found")
+    job.stop.set()
+    return Response.json({"suite": job.public()})
+
+
+def heals(request):
+    """Accept or reject the heals of the checks named, or of every check with heals waiting."""
+    from ..api_routes import Response
+    from ..replay_core.heal import HealError
+    from . import heal_review
+    from .discover import DiscoveryError
+    body = request.body if isinstance(request.body, dict) else {}
+    if "project" in body:  # the Mac app's project page
+        return heal_review.settled(request)
+    if set(body) - {"root", "action", "checks"} or body.get("action") not in ("accept", "reject"):
+        return _bad("Send root, action (accept or reject) and, if you like, checks.")
+    root, bad = _root(body.get("root"))
+    if bad is not None:
+        return bad
+    paths = []
+    for item in body.get("checks") or ():
+        path, bad = _inside(root, item, "checks")
+        if bad is not None:
+            return bad
+        paths.append(str(path))
+    if not (root / ".mobster" / "checks").is_dir():
+        return Response.error(NO_CHECKS, 404, "no_checks")
+    try:
+        settled = [{"check": str(entry.path), "steps": count}
+                   for entry, count in heal_review.settle(root, heal_review.pending(root, paths), body["action"])]
+    except (DiscoveryError, HealError) as error:
+        return Response.error(str(error), 409, "heals_unreadable")
+    return Response.json({"action": body["action"], "settled": settled, "heals": _waiting(root)[0]})
+
+
+# ------------------------------------------------------------------------------------------------ the pieces
+
+def covering(root, source, entries):
+    """The checks that cover the screen in ``source``, as `mobster test pr` would pick them if the whole file had
+    changed: {screen, checks: [{path, reasons}]}; None for a file that shows no screen."""
+    from . import pr
+    if source.suffix.lower() not in pr.UI_SUFFIXES + pr.STRING_SUFFIXES:
+        return None
+    try:
+        lines = source.read_text(encoding="utf-8", errors="replace")[:MAX_COVERING_BYTES].splitlines()
+    except OSError:
+        return None
+    relative = source.relative_to(root).as_posix()
+    changes = pr.Changes(base="", merge_base="", head="", top=root,
+                         files=[pr.Change(path=relative, status="M", added=lines)])
+    screens = pr.screens_in(changes, root)
+    if not screens:
+        return None
+    picked = pr.pick(entries, screens)
+    return {"screen": screens[0].name,
+            "checks": [{"path": str(item.entry.path), "name": item.entry.name, "reasons": item.reasons[:4]}
+                       for item in picked]}
+
+
+def _root(value):
+    """The project folder ``value`` names, or (None, Response 400)."""
+    if not isinstance(value, str) or not value or len(value) > 4096 or not Path(value).is_absolute():
+        return None, _bad("root is the project's absolute path.")
+    root = Path(value).resolve()
+    if not root.is_dir():
+        return None, _bad(f"{root} isn't a folder.")
+    return root, None
+
+
+def _inside(root, value, field):
+    """``value`` as an absolute path inside ``root``, or (None, Response 400)."""
+    if not isinstance(value, str) or not value or len(value) > 4096 or not Path(value).is_absolute():
+        return None, _bad(f"{field} takes absolute paths.")
+    path = Path(value).resolve()
+    if not path.is_relative_to(root) or not path.is_file():
+        return None, _bad(f"{value} isn't a file in {root}.")
+    return path, None
+
+
+def _bad(message):
+    from ..api_routes import Response
+    return Response.error(message, 400, "bad_request")
+
+
+def _last_verdicts(root):
+    """{check file: its last verdict}, the device that didn't pass first when it ran on several."""
+    from .history import report
+    last = {}
+    for row in report(root)["checks"]:
+        known = last.get(row["file"])
+        if known is None or (known["status"] == "passed" and row["lastStatus"] != "passed"):
+            last[row["file"]] = {"status": row["lastStatus"], "state": row["state"], "device": row["device"],
+                                 "runs": row["runs"]}
+    return last
+
+
+def _entry_row(entry, last):
+    check = entry.check
+    return {"path": str(entry.path), "file": entry.file, "name": entry.name, "error": entry.error,
+            "app": getattr(check, "bundle_id", None), "tags": list(entry.tags),
+            "steps": len(check.steps) if check is not None else 0, "last": last}
+
+
+def _waiting(root):
+    """(the heals waiting, None) or ([], why they can't be read)."""
+    from ..replay_core.heal import HealError
+    from . import heal_review
+    from .discover import DiscoveryError
+    try:
+        return heal_review.rows(heal_review.pending(root)), None
+    except (DiscoveryError, HealError) as error:
+        return [], str(error)
+
+
+def _latest(runtime, root):
+    job = runtime.editor_suites.latest()
+    return job.public() if job is not None and job.root is not None and Path(job.root) == root else None