mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/devices.py

@@ -0,0 +1,676 @@
+"""Every phone Mobster can drive, their ports, and naming one (SPEC §3, docs/devices.md)."""
+
+import contextlib
+import copy
+import errno
+import fcntl
+import json
+import os
+import re
+import socket
+import stat
+import tempfile
+import threading
+import time
+from pathlib import Path
+from urllib.parse import urlsplit
+
+STATES = ("ready", "busy", "connected", "needs_setup", "disconnected")
+STORE = "devices.json"
+STORE_LOCK = "devices.lock"
+LOCK_TIMEOUT = 10.0
+WDA_BASE, MJPEG_BASE = 8100, 9100
+FIRST_SLOT, LAST_SLOT = 1, 99
+# The relay and WDA listen here only. Without it iproxy listened on every interface
+# of the Mac (lsof: *:8100) and WDA on the phone's Wi-Fi address; neither has auth.
+LOOPBACK = "127.0.0.1"
+ID_PATTERN = re.compile(r"[A-Za-z0-9._-]{1,64}")
+UDID_PATTERN = re.compile(r"[0-9A-Fa-f-]{24,40}")
+
+# The one sentence each state says, when nothing more precise is known.
+REASONS = {
+    # Plain words: the app shows these under a phone's name (dashboard/src/lib/devices.ts), and the CLI's
+    # `mobster devices` prints them. "Mobster's helper" is the on-phone runner (WebDriverAgent).
+    "busy": "A task is running on it.",
+    "busy_elsewhere": "Another Mobster app or command is using it.",
+    "untrusted": "Unlock it and tap Trust on it.",
+    "not_built": "Set it up once. Mobster walks you through it.",
+    "built_for_other": "Set it up again: Mobster's helper on it was made for another iPhone.",
+    # Both are refreshed by themselves while the phone is plugged in and unlocked (runner_renewal.py).
+    "out_of_date": "It needs a quick refresh after Mobster's update. Mobster does it by itself while it's plugged in "
+                   "and unlocked.",
+    "expired": "It needs a quick refresh. Free Apple accounts ask for this every 7 days. Mobster does it by itself "
+               "while it's plugged in and unlocked.",
+    "starting": "Mobster's helper on it isn't answering yet. Keep it unlocked and connected.",
+    # iOS waits for automation mode (runner_hints.AUTOMATION_WAIT_S): the switch to turn on, said while it waits.
+    "ui_automation": "Turn on Settings › Developer › Enable UI Automation on it. If it asks for its passcode, enter "
+                     "it. Mobster's helper starts by itself after that.",
+    "stopped": "Mobster's helper isn't running on it. Start it again to see the screen.",
+    # Its helper's first start (device_manager.prearm): the switch iOS asks about, said before the first launch.
+    "prearm": "Its helper is starting for the first time. Open Settings › Developer on it and turn on Enable UI "
+              "Automation, so it starts on the first try.",
+    "unplugged": "It's unplugged. Plug it into this Mac with a cable.",
+    "sim_off": "The simulator is shut down. Start it to use it.",
+    "sim_no_wda": "The simulator is on, but Mobster's helper isn't running on it. Start it to use it.",
+    "wda_down": "Nothing answers at its address.",
+    "locked": "Unlock it to run tasks and see its screen.",
+    # Only with Wi-Fi for iPhones on (MOBSTER_WIFI_TRANSPORT) and Wi-Fi turned on for the phone (wireless/).
+    "wifi_unreachable": "It isn't reachable over Wi-Fi. Keep it unlocked and on the same Wi-Fi as this Mac, or plug "
+                        "it in.",
+    # Disconnected in Settings › Phones: still known, not used until Connect.
+    "paused": "Disconnected in Mobster. Connect it to use it again.",
+}
+
+# A phone's own settings in Mobster (devices.json "prefs", by device id): a nickname shown instead of its name, whether
+# a task on it asks before acting (None: the Mac-wide Safety setting), and the apps a task on it may use (None: all).
+NICKNAME_MAX = 64
+PREF_APPS_MAX = 64
+
+
+class DeviceNotFound(LookupError):
+    """No device has that id, UDID or name."""
+
+
+class AmbiguousDevice(LookupError):
+    """More than one device has that name."""
+
+
+def wda_url(port):
+    return f"http://{LOOPBACK}:{int(port)}"
+
+
+def port_of(url):
+    try:
+        return urlsplit(url or "").port
+    except ValueError:
+        return None
+
+
+def slot_ports(slot):
+    """(WDA port, MJPEG port) of a USB slot: 8100+N and 9100+N."""
+    return WDA_BASE + slot, MJPEG_BASE + slot
+
+
+def listening(port, host=LOOPBACK, timeout=.2):
+    """Whether something accepts connections on this Mac's loopback at ``port``."""
+    try:
+        with socket.create_connection((host, int(port)), timeout=timeout):
+            return True
+    except OSError:
+        return False
+
+
+def allocate_slot(entries, busy=None):
+    busy = busy or listening
+    taken = {int(entry["slot"]) for entry in entries if isinstance(entry.get("slot"), int)}
+    for slot in range(FIRST_SLOT, LAST_SLOT + 1):
+        if slot in taken:
+            continue
+        wda, mjpeg = slot_ports(slot)
+        if busy(wda) or busy(mjpeg):
+            continue  # another program's port: skipped, never taken over
+        return slot
+    raise LookupError(f"Every port from {WDA_BASE + FIRST_SLOT} to {WDA_BASE + LAST_SLOT} is taken, so Mobster "
+                      "has no room for another iPhone. Forget a phone you no longer use.")
+
+
+def device_id_for_url(url):
+    """The id of a device known only by its WDA address: wda-<port>."""
+    port = port_of(url)
+    return f"wda-{port}" if port else "wda"
+
+
+# -- devices.json --------------------------------------------------------------------------------------------
+
+MAX_CHOSEN = 100  # the composer's phones for one task on each (fanout's own limits apply when it runs)
+
+
+def clean_chosen(items):
+    """Device ids in order, each once, at most MAX_CHOSEN; anything else is dropped."""
+    out = []
+    for item in items:
+        if isinstance(item, str) and ID_PATTERN.fullmatch(item) and item not in out:
+            out.append(item)
+    return out[:MAX_CHOSEN]
+
+
+class DeviceStore:
+    """devices.json in the data folder: the phones set up besides the primary, their slots, and the device used last."""
+
+    def __init__(self, data_dir):
+        self.data_dir = Path(data_dir) if data_dir is not None else None
+        self.path = self.data_dir / STORE if self.data_dir is not None else None
+        self.lock_path = self.data_dir / STORE_LOCK if self.data_dir is not None else None
+        self.memory = self._empty() if self.data_dir is None else None
+        self.memory_lock = threading.Lock()
+
+    @staticmethod
+    def _empty():
+        return {"version": 1, "phones": [], "lastUsed": None, "defaultDevice": None, "disconnected": [],
+                "prefs": {}, "chosen": []}
+
+    def read(self):
+        if self.data_dir is None:
+            with self.memory_lock:
+                return copy.deepcopy(self.memory)
+        try:
+            data = json.loads(self.path.read_text())
+        except (OSError, ValueError):
+            data = None
+        if not isinstance(data, dict):  # damaged: it holds no secret and nothing that can't be set up again
+            return self._empty()
+        entries = data.get("phones")
+        if not isinstance(entries, list):
+            entries = []
+        phones = [entry for entry in entries if isinstance(entry, dict)
+                  and isinstance(entry.get("udid"), str) and UDID_PATTERN.fullmatch(entry["udid"])
+                  and type(entry.get("slot")) is int and FIRST_SLOT <= entry["slot"] <= LAST_SLOT
+                  and type(entry.get("wdaPort")) is int and type(entry.get("mjpegPort")) is int
+                  and (entry["wdaPort"], entry["mjpegPort"]) == slot_ports(entry["slot"])]
+        last, default = data.get("lastUsed"), data.get("defaultDevice")
+        paused = data.get("disconnected")
+        prefs = data.get("prefs") if isinstance(data.get("prefs"), dict) else {}
+        chosen = data.get("chosen")
+        return {"version": 1, "phones": phones,
+                "lastUsed": last if isinstance(last, str) and ID_PATTERN.fullmatch(last) else None,
+                "defaultDevice": default if isinstance(default, str) and ID_PATTERN.fullmatch(default) else None,
+                "disconnected": sorted({item for item in paused if isinstance(item, str)
+                                        and ID_PATTERN.fullmatch(item)})
+                if isinstance(paused, list) else [],
+                "prefs": {key: clean for key, value in prefs.items() if isinstance(key, str)
+                          and ID_PATTERN.fullmatch(key)
+                          for clean in [clean_prefs(value)] if clean},
+                "chosen": clean_chosen(chosen) if isinstance(chosen, list) else []}
+
+    def read_strict(self):
+        """Fresh policy for trusted suite admission; a damaged saved rule never becomes allow-all."""
+        if self.data_dir is None:
+            with self.memory_lock:
+                return copy.deepcopy(self.memory)
+        from .transport import decode_json
+        try:
+            fd = os.open(self.path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
+        except FileNotFoundError:
+            return self._empty()  # A new Mac has no device preferences yet.
+        except OSError:
+            raise ValueError("Saved device rules cannot be read") from None
+        try:
+            with os.fdopen(fd, "rb") as stream:
+                info = os.fstat(stream.fileno())
+                if not stat.S_ISREG(info.st_mode) or info.st_size > 1_000_000:
+                    raise ValueError("Saved device rules are not a bounded regular file")
+                data = decode_json(stream.read(1_000_001))
+        except (OSError, UnicodeError, ValueError, RecursionError):
+            raise ValueError("Saved device rules are damaged or unreadable") from None
+        if (not isinstance(data, dict) or type(data.get("version")) is not int or data["version"] != 1 or
+                set(data) != {"version", "phones", "lastUsed", "defaultDevice", "disconnected", "prefs", "chosen"} or
+                not isinstance(data["phones"], list) or not isinstance(data["disconnected"], list) or
+                not isinstance(data["prefs"], dict) or not isinstance(data["chosen"], list)):
+            raise ValueError("Saved device rules are damaged")
+        if any(not isinstance(entry, dict) or not isinstance(entry.get("udid"), str) or
+               not UDID_PATTERN.fullmatch(entry["udid"]) or type(entry.get("slot")) is not int or
+               not FIRST_SLOT <= entry["slot"] <= LAST_SLOT or
+               (entry.get("wdaPort"), entry.get("mjpegPort")) != slot_ports(entry["slot"])
+               for entry in data["phones"]):
+            raise ValueError("Saved device rules are damaged")
+        ids = data["disconnected"] + data["chosen"]
+        if (any(not isinstance(item, str) or not ID_PATTERN.fullmatch(item) for item in ids) or
+                any(value is not None and (not isinstance(value, str) or not ID_PATTERN.fullmatch(value))
+                    for value in (data["lastUsed"], data["defaultDevice"])) or
+                len(set(data["chosen"])) != len(data["chosen"]) or
+                len(clean_chosen(data["chosen"])) != len(data["chosen"])):
+            raise ValueError("Saved device rules are damaged")
+        for key, value in data["prefs"].items():
+            if not ID_PATTERN.fullmatch(key) or not isinstance(value, dict) or set(value) - {
+                    "nickname", "askBeforeActing", "apps"}:
+                raise ValueError("Saved device rules are damaged")
+            clean_prefs(value, strict=True)
+        return data
+
+    def phones(self):
+        return self.read()["phones"]
+
+    @contextlib.contextmanager
+    def edit(self):
+        if self.data_dir is None:
+            with self.memory_lock:
+                yield self.memory
+            return
+        self.data_dir.mkdir(parents=True, exist_ok=True)
+        fd = os.open(self.lock_path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+        try:
+            end = time.monotonic() + LOCK_TIMEOUT
+            while True:
+                try:
+                    fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                    break
+                except OSError as exc:
+                    if exc.errno not in (errno.EWOULDBLOCK, errno.EAGAIN) or time.monotonic() >= end:
+                        raise OSError("Another Mobster process is holding the device list") from None
+                    time.sleep(.05)
+            data = self.read()
+            before = json.dumps(data, sort_keys=True)
+            yield data
+            if json.dumps(data, sort_keys=True) != before:
+                self._write(data)
+        finally:
+            os.close(fd)
+
+    def _write(self, data):
+        fd, temporary = tempfile.mkstemp(prefix=".devices.", suffix=".json", dir=self.data_dir)
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as stream:
+                os.fchmod(stream.fileno(), 0o600)
+                stream.write(json.dumps(data, indent=1) + "\n")
+            os.replace(temporary, self.path)
+        except BaseException:
+            with contextlib.suppress(OSError):
+                os.unlink(temporary)
+            raise
+
+    def add(self, udid, name=None, busy=None):
+        """The phone's entry, allocating its slot the first time."""
+        with self.edit() as data:
+            entry = next((item for item in data["phones"] if item["udid"] == udid), None)
+            if entry is None:
+                slot = allocate_slot(data["phones"], busy)
+                wda, mjpeg = slot_ports(slot)
+                entry = {"udid": udid, "name": name, "slot": slot, "wdaPort": wda, "mjpegPort": mjpeg,
+                         "addedAt": round(time.time() * 1000)}
+                data["phones"].append(entry)
+            elif name and entry.get("name") != name:
+                entry["name"] = name
+            return dict(entry)
+
+    def remove(self, udid):
+        with self.edit() as data:
+            data["phones"] = [item for item in data["phones"] if item["udid"] != udid]
+            for key in ("lastUsed", "defaultDevice"):
+                if data.get(key) == udid:
+                    data[key] = None
+            data["disconnected"] = [item for item in data["disconnected"] if item != udid]
+            data["prefs"].pop(udid, None)
+            data["chosen"] = [item for item in data["chosen"] if item != udid]
+
+    def set_default(self, device_id):
+        """The phone a new task runs on unless another is picked (None: no choice, Fleet.default decides)."""
+        if device_id is not None and not ID_PATTERN.fullmatch(str(device_id)):
+            raise ValueError("A device id is 1 to 64 letters, digits, dots, dashes or underscores")
+        with self.edit() as data:
+            data["defaultDevice"] = device_id
+
+    def set_chosen(self, device_ids):
+        if not isinstance(device_ids, (list, tuple)) or len(device_ids) > MAX_CHOSEN or not all(
+                isinstance(item, str) and ID_PATTERN.fullmatch(item) for item in device_ids) or len(
+                set(device_ids)) != len(device_ids):
+            raise ValueError(f"Choose at most {MAX_CHOSEN} different phones by their ids")
+        with self.edit() as data:
+            data["chosen"] = clean_chosen(device_ids)
+
+    def set_disconnected(self, device_id, value):
+        if not ID_PATTERN.fullmatch(str(device_id)):
+            raise ValueError("A device id is 1 to 64 letters, digits, dots, dashes or underscores")
+        with self.edit() as data:
+            others = [item for item in data["disconnected"] if item != device_id]
+            data["disconnected"] = sorted(others + [device_id]) if value else others
+
+    def prefs(self, device_id):
+        return dict(self.read()["prefs"].get(device_id) or {})
+
+    def set_prefs(self, device_id, **changes):
+        """Change a phone's own settings (``clean_prefs``); a value of None removes it. The phone's settings after."""
+        if not ID_PATTERN.fullmatch(str(device_id)):
+            raise ValueError("A device id is 1 to 64 letters, digits, dots, dashes or underscores")
+        unknown = set(changes) - {"nickname", "askBeforeActing", "apps"}
+        if unknown:
+            raise ValueError("A phone's settings are nickname, askBeforeActing and apps")
+        with self.edit() as data:
+            current = dict(data["prefs"].get(device_id) or {})
+            for key, value in changes.items():
+                if value is None:
+                    current.pop(key, None)
+                else:
+                    current[key] = value
+            clean = clean_prefs(current, strict=True)
+            if clean:
+                data["prefs"][device_id] = clean
+            else:
+                data["prefs"].pop(device_id, None)
+            return dict(clean)
+
+    def set_last_used(self, device_id):
+        with self.edit() as data:
+            data["lastUsed"] = device_id
+
+    def set_wifi(self, udid, value):
+        """Set (a dict) or clear (None) the phone's ``wifi`` (wireless/store.py). LookupError when it has no entry."""
+        with self.edit() as data:
+            entry = next((item for item in data["phones"] if item["udid"].upper() == str(udid).upper()), None)
+            if entry is None:
+                raise LookupError("not_set_up")
+            if value is None:
+                entry.pop("wifi", None)
+            else:
+                entry["wifi"] = dict(value)
+
+
+def clean_nickname(value):
+    """A nickname as kept: its words joined by single spaces, 1 to 64 characters, no control characters."""
+    if not isinstance(value, str):
+        raise ValueError("A name is text")
+    text = " ".join(value.split())
+    if not 1 <= len(text) <= NICKNAME_MAX or any(ord(char) < 32 or 0x7f <= ord(char) < 0xa0 for char in text):
+        raise ValueError(f"A name is 1 to {NICKNAME_MAX} characters")
+    return text
+
+
+def clean_prefs(value, strict=False):
+    from .state import validate_bundle_id
+    if not isinstance(value, dict):
+        if strict:
+            raise ValueError("A phone's settings are an object")
+        return {}
+    out = {}
+    if value.get("nickname") is not None:
+        try:
+            out["nickname"] = clean_nickname(value["nickname"])
+        except ValueError:
+            if strict:
+                raise
+    if value.get("askBeforeActing") is not None:
+        if type(value["askBeforeActing"]) is bool:
+            out["askBeforeActing"] = value["askBeforeActing"]
+        elif strict:
+            raise ValueError("askBeforeActing is true, false or null (the Mac-wide setting)")
+    if value.get("apps") is not None:
+        apps = value["apps"]
+        try:
+            if not isinstance(apps, list) or not 1 <= len(apps) <= PREF_APPS_MAX:
+                raise ValueError
+            out["apps"] = sorted({validate_bundle_id(bundle) for bundle in apps})
+        except (ValueError, TypeError):
+            if strict:
+                raise ValueError(f"apps is null (every app) or 1 to {PREF_APPS_MAX} bundle identifiers") from None
+    return out
+
+
+def phone_folder(data_dir, udid):
+    """Where a phone other than the primary keeps its runner build, settings and logs."""
+    return Path(data_dir) / "devices" / udid
+
+
+# -- reading the machine ---------------------------------------------------------------------------------------
+
+def record(*, id, kind, name, wda_url_value, udid=None, model=None, model_name=None, ios=None, state="disconnected",
+           reason=None, mjpeg_url=None, primary=False, set_up=True, enclosure=None):
+    if state not in STATES:
+        raise ValueError(f"unknown device state {state}")
+    return {"id": id, "kind": kind, "name": name or ("iPhone" if kind == "usb" else id), "udid": udid,
+            "model": model, "modelName": model_name, "ios": ios, "enclosure": enclosure, "state": state,
+            "reason": None if state == "ready" else (reason or None), "primary": bool(primary),
+            "setUp": bool(set_up), "wdaUrl": wda_url_value, "mjpegUrl": mjpeg_url}
+
+
+def usb_state(device, manager, wda_ready, lease_free, locked=None):
+    if device is None:
+        return "disconnected", REASONS["unplugged"]
+    if not device.get("trusted"):
+        return "needs_setup", REASONS["untrusted"]
+    if manager is None:
+        return "needs_setup", REASONS["not_built"]
+    settings = manager.settings()
+    built_for = settings.get("built_for", settings.get("udid"))
+    if manager.expired():
+        return "needs_setup", REASONS["expired"]
+    if not manager.built():
+        if built_for and built_for != device.get("udid"):
+            return "needs_setup", REASONS["built_for_other"]
+        if manager.out_of_date():
+            return "needs_setup", REASONS["out_of_date"]
+        return "needs_setup", REASONS["not_built"]
+    if not wda_ready():
+        waiting = getattr(manager, "runner_waiting", None)
+        if callable(waiting) and waiting() == "unlock":
+            return "connected", REASONS["locked"]   # its helper starts when the phone is unlocked
+        running = manager.runner is not None and manager.runner.running
+        if running and getattr(manager, "runner_cause", None) == "ui_automation_off":
+            return "connected", REASONS["ui_automation"]
+        prearm = getattr(manager, "prearm", None)
+        if running and callable(prearm) and prearm():
+            return "connected", REASONS["prearm"]
+        return "connected", REASONS["starting"] if running else REASONS["stopped"]
+    if not lease_free():
+        return "busy", REASONS["busy_elsewhere"]
+    if locked:
+        return "connected", REASONS["locked"]
+    return "ready", None
+
+
+def wda_answers(url, timeout=1.0):
+    """WDA's /status says ready, asked under an absolute deadline (a phone that drips bytes can't extend it)."""
+    from .transport import read_kept
+    try:
+        return read_kept(url, "/status", timeout=timeout).get("value", {}).get("ready") is True
+    except Exception:
+        return False
+
+
+def lease_free(url, udid=None):
+    from .journal import JournalError, Lease, LeaseHeld
+    try:
+        lease = Lease.device(url, wait=0, udid=udid)
+    except LeaseHeld:
+        return False
+    except JournalError:
+        return True  # an unusable lease folder guards nothing; the run says so when it starts
+    lease.close()
+    return True
+
+
+def simulator_folder():
+    from .paths import dev_data_dir
+    return dev_data_dir()
+
+
+def simulator_rows(data_dir=None):
+    try:
+        from .sim.registry import Registry
+    except ImportError:
+        return []
+    folder = Path(data_dir) if data_dir else simulator_folder()
+    if not (folder / "simulators.json").exists():
+        return []  # never runs simctl on a Mac with no Mobster simulator
+    try:
+        entries = Registry(folder).read()
+    except Exception:
+        return []
+    states = {}
+    try:
+        from .sim import SimulatorManager
+        states = {item.udid: item.state for item in SimulatorManager(data_dir=folder)._devices()}
+    except Exception:
+        pass
+    return [{"udid": entry["udid"], "name": entry.get("name"), "device_type": entry.get("device_type"),
+             "runtime": entry.get("runtime"), "state": states.get(entry["udid"], "Missing" if states else "Unknown"),
+             "wda_url": wda_url(entry["wda_port"]), "mjpeg_url": wda_url(entry["mjpeg_port"])}
+            for entry in entries if entry.get("wda_port") and entry.get("mjpeg_port")]
+
+
+def simulator_record(row, *, ready=None, free=None, busy_here=False):
+    """One simulator row (simulator_rows) as a device. ``ready``/``free`` default to probing its WDA and lease."""
+    booted = row.get("state") == "Booted"
+    if busy_here:
+        state, reason = "busy", REASONS["busy"]
+    elif not booted:
+        state, reason = "disconnected", REASONS["sim_off"]
+    elif not (wda_answers(row["wda_url"]) if ready is None else ready):
+        state, reason = "connected", REASONS["sim_no_wda"]
+    elif not (lease_free(row["wda_url"]) if free is None else free):
+        state, reason = "busy", REASONS["busy_elsewhere"]
+    else:
+        state, reason = "ready", None
+    runtime = str(row.get("runtime") or "")
+    return record(id=row["udid"], kind="simulator", name=row.get("name"), udid=row["udid"],
+                  model=row.get("device_type"), model_name=row.get("device_type"),
+                  ios=runtime.removeprefix("iOS ").strip() or None, state=state, reason=reason,
+                  wda_url_value=row["wda_url"], mjpeg_url=row.get("mjpeg_url"))
+
+
+def simulator_records(probe=True):
+    return [simulator_record(row, **({} if probe else {"ready": True, "free": True})) for row in simulator_rows()]
+
+
+def static_specs(config=None):
+    """WDA devices given by address: ``MOBSTER_WDA_DEVICES`` (pool.wda_specs_from_env) and ``config.devices``."""
+    from .pool import wda_specs_from_config
+    try:
+        return wda_specs_from_config(config)
+    except ValueError:
+        return []
+
+
+def static_record(url, *, id=None, name=None, mjpeg_url=None, ready=None, free=None):
+    ready = wda_answers(url) if ready is None else ready
+    if not ready:
+        state, reason = "disconnected", REASONS["wda_down"]
+    elif not (lease_free(url) if free is None else free):
+        state, reason = "busy", REASONS["busy_elsewhere"]
+    else:
+        state, reason = "ready", None
+    return record(id=id or device_id_for_url(url), kind="wda", name=name or f"WebDriverAgent on port {port_of(url)}",
+                  state=state, reason=reason, wda_url_value=url.rstrip("/"), mjpeg_url=mjpeg_url)
+
+
+def discover(data_dir=None, *, sims=True, extra_urls=(), probe=True):
+    from .device_info import MODEL_NAMES
+    from .device_manager import DeviceManager
+    from .mac_tools import WDA_PORT
+    from .paths import user_data_dir
+    data_dir = Path(data_dir) if data_dir else user_data_dir()
+    records, seen_urls = [], set()
+
+    def add(item):
+        identity = (item.get("wdaUrl") or "").rstrip("/")
+        if identity and identity in seen_urls:
+            return
+        if identity:
+            seen_urls.add(identity)
+        records.append(item)
+
+    primary = DeviceManager(data_dir, wda_url(WDA_PORT))
+    store = DeviceStore(data_dir)
+    extras = store.phones()
+    claimed = {entry["udid"] for entry in extras}
+    primary.claimed = lambda: claimed
+    try:
+        connected = primary.devices()
+    except Exception:
+        connected = []
+    by_udid = {item["udid"]: item for item in connected}
+    wifi = wifi_phones(data_dir)
+
+    def usb(udid, manager, url, mjpeg, is_primary, saved_name):
+        device = by_udid.get(udid)
+        over_wifi = False
+        if device is None and wifi is not None and udid.upper() in wifi and probe and wda_answers(url):
+            # Unplugged, but the Mac app's Wi-Fi relay answers on its port: the phone is there over Wi-Fi.
+            device, over_wifi = {"udid": udid, "trusted": True}, True
+        state, reason = usb_state(device, manager, (lambda: wda_answers(url)) if probe else (lambda: True),
+                                  (lambda: lease_free(url)) if probe else (lambda: True))
+        if wifi is not None and udid.upper() in wifi and state == "disconnected":
+            reason = REASONS["wifi_unreachable"]
+        model = (device or {}).get("model")
+        item = record(id=udid, kind="usb", name=(device or {}).get("name") or saved_name, udid=udid, model=model,
+                      model_name=MODEL_NAMES.get(model), ios=(device or {}).get("ios"),
+                      enclosure=(device or {}).get("enclosure"), state=state, reason=reason,
+                      wda_url_value=url, mjpeg_url=mjpeg, primary=is_primary,
+                      set_up=manager is not None and manager.built())
+        if wifi is not None:
+            item["transport"] = "wifi" if over_wifi else "usb" if udid in by_udid else None
+        return item
+
+    chosen = primary.device(connected)
+    primary_udid = primary.settings().get("udid") or (chosen or {}).get("udid")
+    if primary_udid:
+        add(usb(primary_udid, primary, wda_url(WDA_PORT), wda_url(MJPEG_BASE), True,
+                primary.settings().get("device_name")))
+    for entry in extras:
+        if entry["udid"] == primary_udid:
+            continue
+        add(usb(entry["udid"], manager_for(entry["udid"], data_dir), wda_url(entry["wdaPort"]),
+                wda_url(entry["mjpegPort"]), False, entry.get("name")))
+    for device in connected:
+        if device["udid"] in claimed or device["udid"] == primary_udid:
+            continue
+        add(record(id=device["udid"], kind="usb", name=device.get("name"), udid=device["udid"],
+                   model=device.get("model"), model_name=MODEL_NAMES.get(device.get("model")), ios=device.get("ios"),
+                   enclosure=device.get("enclosure"), state="needs_setup", reason=REASONS["untrusted"]
+                   if not device.get("trusted") else REASONS["not_built"],
+                   wda_url_value=None, set_up=False))
+    if sims:
+        for item in simulator_records(probe=probe):
+            add(item)
+    for spec in static_specs():
+        add(static_record(spec.wda_url, id=spec.id, name=spec.label or None, mjpeg_url=spec.mjpeg_url or None,
+                          **({} if probe else {"ready": True, "free": True})))
+    for url in extra_urls:
+        if url and url.rstrip("/") not in seen_urls:
+            add(static_record(url, **({} if probe else {"ready": True, "free": True})))
+    return records
+
+
+def wifi_phones(data_dir):
+    from .wireless import store, transport_on
+    if not transport_on():
+        return None
+    try:
+        return store.enabled_udids(data_dir)
+    except Exception:
+        return set()
+
+
+def manager_for(udid, data_dir=None):
+    """The DeviceManager of a USB phone set up besides the primary one (devices.json), or None."""
+    from .device_manager import DeviceManager
+    from .paths import user_data_dir
+    data_dir = Path(data_dir) if data_dir else user_data_dir()
+    entry = next((item for item in DeviceStore(data_dir).phones() if item["udid"] == udid), None)
+    if entry is None:
+        return None
+    return DeviceManager(data_dir, wda_url(entry["wdaPort"]), udid=udid, root=phone_folder(data_dir, udid),
+                         video_port=entry["mjpegPort"])
+
+
+def resolve(records, query):
+    text = str(query or "").strip()
+    if not text:
+        raise DeviceNotFound("Name a device: its id, UDID or name.")
+    folded = text.casefold()
+    for item in records:
+        if folded in {str(item.get("id") or "").casefold(), str(item.get("udid") or "").casefold()}:
+            return item
+    named = [item for item in records if str(item.get("name") or "").casefold() == folded]
+    if len(named) == 1:
+        return named[0]
+    if named:
+        raise AmbiguousDevice(f"{len(named)} devices are named “{text}”. Name one by its id: "
+                              + ", ".join(item["id"] for item in named) + ".")
+    known = ", ".join(f"{item['name']} ({item['id']})" for item in records[:8])
+    raise DeviceNotFound(f"No device is named or has the id “{text}”." +
+                         (f" Devices: {known}." if known else " Mobster sees no device: plug in an iPhone or "
+                                                              "prepare a simulator (`mobster sim prepare`)."))
+
+
+def describe(item):
+    """One line for a terminal: name, kind, state and reason."""
+    kind = {"usb": "iPhone (USB)", "simulator": "Simulator", "wda": "WebDriverAgent",
+            "android": "Android emulator" if item.get("emulator") else "Android phone"}.get(item["kind"], item["kind"])
+    if item["kind"] == "usb" and item.get("transport") == "wifi":
+        kind = "iPhone (Wi-Fi)"
+    details = ", ".join(part for part in (item.get("modelName") or item.get("model"),
+                                          f"iOS {item['ios']}" if item.get("ios") else None) if part)
+    state = item["state"].replace("_", " ")
+    line = f"{item['name']}  [{kind}{', ' + details if details else ''}]  {state}"
+    if item.get("reason"):
+        line += f": {item['reason']}"
+    return line