mobster-cli 0.4.3__py3-none-any.whl

mobile_agent/pool.py

@@ -0,0 +1,285 @@
+"""Many phones behind one runtime: one task per phone, first come first served, unhealthy phones withdrawn."""
+
+import itertools
+import os
+import threading
+import time
+from dataclasses import dataclass
+from urllib.parse import urlsplit
+
+from .devices import UDID_PATTERN
+from .errors import MobsterError
+from .journal import JournalError, Lease
+from .wda.video import default_mjpeg_url
+
+
+class NoDeviceAvailable(MobsterError):
+    pass
+
+
+def _local_http_url(value, what):
+    parts = urlsplit(value or "")
+    if (parts.scheme not in {"http", "https"} or not parts.hostname or parts.username or parts.password
+            or parts.query or parts.fragment):
+        raise ValueError(f"{what} must be an http(s) URL without credentials")
+    return value.rstrip("/")
+
+
+@dataclass(frozen=True)
+class WdaDeviceSpec:
+    id: str
+    wda_url: str
+    mjpeg_url: str = ""
+    udid: str = ""
+    label: str = ""
+    truth: str = ""
+
+    kind = "wda"
+
+    def __post_init__(self):
+        if not isinstance(self.id, str) or not self.id or len(self.id) > 64:
+            raise ValueError("A device needs a short id")
+        object.__setattr__(self, "wda_url", _local_http_url(self.wda_url, "wda_url"))
+        mjpeg = (_local_http_url(self.mjpeg_url, "mjpeg_url") if self.mjpeg_url
+                 else default_mjpeg_url(self.wda_url) or "")
+        object.__setattr__(self, "mjpeg_url", mjpeg)
+        if self.udid and not UDID_PATTERN.fullmatch(self.udid):
+            raise ValueError("Invalid device UDID")
+
+    @property
+    def identity(self):
+        # The string direct runs lease a WDA phone by, so pooled and direct runs exclude each other.
+        return self.wda_url
+
+    def public(self):
+        # No URL or UDID: the dashboard reads this.
+        return {"id": self.id, "label": self.label or self.id, "kind": "wda", "truth": self.truth or None}
+
+
+@dataclass
+class DeviceState:
+    spec: object
+    healthy: bool = True
+    reason: str = ""
+    owner: object = None
+    lease: object = None
+    # -inf: the first allocation probes even when the host uptime is shorter than the health interval.
+    checked_at: float = float("-inf")
+    used_at: float = 0
+    failures: int = 0
+    completed: int = 0
+
+
+def probe_wda(url, timeout=3):
+    from .transport import HTTP
+    client = HTTP(url)
+    try:
+        status = client.request("GET", "/status", timeout=timeout)
+    finally:
+        client.close()
+    value = status.get("value") if isinstance(status, dict) else None
+    if not isinstance(value, dict) or value.get("ready") is False:
+        raise RuntimeError("WebDriverAgent is not ready")
+    return value
+
+
+# Probes for an extension's device kinds.
+PROBES = {}
+
+
+class DevicePool:
+    def __init__(self, specs, health_interval=20, probe=None, on_event=None):
+        specs = list(specs)
+        if not specs:
+            raise ValueError("A device pool needs at least one device")
+        if len({spec.id for spec in specs}) != len(specs):
+            raise ValueError("Device ids must be unique")
+        if len({spec.identity for spec in specs}) != len(specs):
+            raise ValueError("Two devices cannot share one route")
+        udids = [spec.udid for spec in specs if getattr(spec, "udid", "")]
+        if len(set(udids)) != len(udids):
+            raise ValueError("Two devices cannot share one UDID")
+        self.states = {spec.id: DeviceState(spec) for spec in specs}
+        self.health_interval = health_interval
+        self.probe = probe
+        self.on_event = on_event or (lambda event: None)
+        self.condition = threading.Condition(threading.RLock())
+        self.closed = False
+        self._tickets = itertools.count()
+        self._waiting = []
+
+    @property
+    def size(self):
+        return len(self.states)
+
+    def _emit(self, event, **fields):
+        try:
+            self.on_event({"event": event, **fields})
+        except Exception:
+            pass
+
+    def _refresh(self, state, force=False):
+        if state.owner is not None:
+            return state.healthy
+        if not force and time.monotonic() - state.checked_at < self.health_interval:
+            return state.healthy
+        identity = state.spec.identity
+        probe = self.probe or (probe_wda if state.spec.kind == "wda" else PROBES[state.spec.kind])
+        self.condition.release()
+        try:
+            probe(identity)
+        except Exception as error:
+            healthy, reason = False, str(error)[:120]
+        else:
+            healthy, reason = True, ""
+        finally:
+            self.condition.acquire()
+        if state.owner is not None:
+            # Taken while the probe was in flight: a stale result must not rewrite the owner's health.
+            return state.healthy
+        state.checked_at = time.monotonic()
+        if healthy != state.healthy:
+            self._emit("device_health", device=state.spec.id, healthy=healthy, reason=reason)
+        state.healthy, state.reason = healthy, reason
+        return healthy
+
+    def _take(self, owner):
+        candidates = [state for state in self.states.values()
+                      if state.owner is None and self._refresh(state)]
+        # _refresh releases the lock while probing: filter again now that it is held until ownership is assigned.
+        if self.closed:
+            raise NoDeviceAvailable("The device pool is shutting down")
+        if any(state.owner == owner for state in self.states.values()):
+            raise ValueError("This owner already holds a device")
+        candidates = [state for state in candidates if state.owner is None and state.healthy]
+        for state in sorted(candidates, key=lambda item: (item.used_at, item.spec.id)):
+            try:
+                # Another Mobster process may own this phone.
+                state.lease = Lease.device(state.spec.identity)
+            except JournalError:
+                state.healthy = False
+                state.reason = "held by another Mobster process"
+                state.checked_at = time.monotonic()
+                self._emit("device_health", device=state.spec.id, healthy=False, reason=state.reason)
+                continue
+            state.owner = owner
+            state.used_at = time.monotonic()
+            self._emit("device_acquired", device=state.spec.id, owner=str(owner))
+            return state.spec
+        return None
+
+    def acquire(self, owner, timeout=0, cancelled=None):
+        if owner is None:
+            raise ValueError("A device lease needs an owner")
+        deadline = time.monotonic() + max(0, timeout)
+        ticket = next(self._tickets)
+        with self.condition:
+            if self.closed:
+                raise NoDeviceAvailable("The device pool is shutting down")
+            if any(state.owner == owner for state in self.states.values()):
+                raise ValueError("This owner already holds a device")
+            self._waiting.append(ticket)
+            try:
+                while True:
+                    if cancelled and cancelled():
+                        raise NoDeviceAvailable("Cancelled while waiting for a device")
+                    if self.closed:
+                        raise NoDeviceAvailable("The device pool is shutting down")
+                    # Strict FIFO.
+                    if self._waiting[0] == ticket:
+                        spec = self._take(owner)
+                        if spec is not None:
+                            return spec
+                    remaining = deadline - time.monotonic()
+                    if remaining <= 0:
+                        raise NoDeviceAvailable(
+                            "Every device is busy" if any(s.owner is not None for s in self.states.values())
+                            else "No device is currently reachable")
+                    self.condition.wait(min(remaining, 1.0))
+            finally:
+                self._waiting.remove(ticket)
+                self.condition.notify_all()
+
+    def release(self, owner, failed=False):
+        with self.condition:
+            for state in self.states.values():
+                if state.owner != owner:
+                    continue
+                if state.lease is not None:
+                    try:
+                        state.lease.close()
+                    except Exception:
+                        pass
+                state.lease = None
+                state.owner = None
+                state.used_at = time.monotonic()
+                if failed:
+                    state.failures += 1
+                    # Re-probe before this device is offered again.
+                    state.checked_at = float("-inf")
+                else:
+                    state.completed += 1
+                self._emit("device_released", device=state.spec.id, owner=str(owner), failed=bool(failed))
+                self.condition.notify_all()
+                return state.spec
+            return None
+
+    def position(self):
+        with self.condition:
+            return {"waiting": len(self._waiting),
+                    "busy": sum(1 for state in self.states.values() if state.owner is not None),
+                    "healthy": sum(1 for state in self.states.values() if state.healthy),
+                    "total": len(self.states)}
+
+    def public(self):
+        with self.condition:
+            return [{**state.spec.public(), "healthy": state.healthy,
+                     "status": "busy" if state.owner is not None else
+                               "ready" if state.healthy else "unavailable",
+                     "reason": state.reason, "completed": state.completed,
+                     "failures": state.failures}
+                    for state in sorted(self.states.values(), key=lambda item: item.spec.id)]
+
+    def close(self):
+        with self.condition:
+            self.closed = True
+            for state in self.states.values():
+                if state.lease is not None:
+                    try:
+                        state.lease.close()
+                    except Exception:
+                        pass
+                state.lease = None
+            self.condition.notify_all()
+
+
+def wda_specs_from_env(value=None):
+    value = os.environ.get("MOBSTER_WDA_DEVICES", "") if value is None else value
+    specs = []
+    for index, entry in enumerate(part for part in value.split(";") if part.strip()):
+        fields = {}
+        for pair in entry.split(","):
+            if not pair.strip():
+                continue
+            key, sep, item = pair.partition("=")
+            key = key.strip().lower()
+            if not sep or key not in {"id", "udid", "wda", "mjpeg", "label", "truth"} or key in fields:
+                raise ValueError(f"MOBSTER_WDA_DEVICES entry {index + 1}: expected unique key=value pairs "
+                                 "(id, udid, wda, mjpeg, label, truth)")
+            fields[key] = item.strip()
+        if not fields.get("wda"):
+            raise ValueError(f"MOBSTER_WDA_DEVICES entry {index + 1} needs wda=<url>")
+        specs.append(WdaDeviceSpec(id=fields.get("id") or f"phone-{index + 1}", wda_url=fields["wda"],
+                                   mjpeg_url=fields.get("mjpeg", ""), udid=fields.get("udid", ""),
+                                   label=fields.get("label", ""), truth=fields.get("truth", "")))
+    return specs
+
+
+def wda_specs_from_config(config):
+    items = [item for item in getattr(config, "devices", None) or () if isinstance(item, dict) and item.get("wdaUrl")]
+    if items:
+        return [WdaDeviceSpec(id=item.get("id") or f"phone-{index + 1}", wda_url=item["wdaUrl"],
+                              mjpeg_url=item.get("mjpegUrl", ""), udid=item.get("udid", ""),
+                              label=item.get("label", ""), truth=item.get("truth", ""))
+                for index, item in enumerate(items)]
+    return wda_specs_from_env()

mobile_agent/power_watch.py

@@ -0,0 +1,275 @@
+"""Power while tasks run (Runtime.power): the Mac kept awake, its battery watched, a phone's battery watched."""
+
+import os
+import threading
+import time
+
+from . import device_status, engines, phone_ready
+from .harness import control
+
+INTERVAL = 30.0
+MAC_WARN_DEFAULT = 20
+PHONE_PAUSE_DEFAULT = 10
+PAUSE_RANGE = (1, 50)
+# A battery pause longer than this ends the task (status "stopped"): nobody came to plug the phone in.
+BATTERY_PAUSE_LIMIT = 30 * 60
+PHONE_BATTERY = "phone_battery"
+
+
+def _percent(name, default, low=1, high=99):
+    """MOBSTER_* percent: the default when unset or unreadable, None when ``off`` (or 0)."""
+    raw = os.environ.get(name, "").strip().lower()
+    if not raw:
+        return default
+    if raw in ("off", "0", "false", "no"):
+        return None
+    try:
+        value = int(float(raw))
+    except (ValueError, OverflowError):
+        return default
+    return value if low <= value <= high else default
+
+
+def mac_warn_threshold():
+    return _percent("MOBSTER_MAC_BATTERY_WARN", MAC_WARN_DEFAULT)
+
+
+def phone_pause_threshold():
+    return _percent("MOBSTER_PHONE_BATTERY_PAUSE", PHONE_PAUSE_DEFAULT, *PAUSE_RANGE)
+
+
+class AwakeHolds:
+    """One keep-awake (workflow_scheduler.KeepAwake) shared by owners, held while any owner holds it."""
+
+    def __init__(self, keep_awake=None):
+        self.keep_awake = keep_awake
+        self.owners = set()
+        self.lock = threading.Lock()
+
+    def attach(self, keep_awake):
+        with self.lock:
+            self.keep_awake = keep_awake
+            if self.owners and keep_awake is not None:
+                keep_awake.hold()
+
+    def hold(self, owner):
+        with self.lock:
+            self.owners.add(owner)
+            if self.keep_awake is not None:
+                self.keep_awake.hold()
+
+    def release(self, owner):
+        with self.lock:
+            self.owners.discard(owner)
+            if not self.owners and self.keep_awake is not None:
+                self.keep_awake.release()
+
+    def close(self):
+        with self.lock:
+            self.owners.clear()
+            if self.keep_awake is not None:
+                self.keep_awake.release()
+
+    @property
+    def held(self):
+        return bool(self.keep_awake is not None and self.keep_awake.held)
+
+    def handle(self, owner):
+        """The scheduler's keep-awake: hold() and release() as ``owner``."""
+        return _Handle(self, owner)
+
+    def status(self):
+        with self.lock:
+            owners = sorted("schedule" if owner == "schedule" else "task" for owner in self.owners)
+        enabled = bool(self.keep_awake is not None and getattr(self.keep_awake, "enabled", False))
+        return {"keptAwake": self.held, "keepAwake": enabled, "for": sorted(set(owners))}
+
+
+class _Handle:
+    def __init__(self, holds, owner):
+        self.holds, self.owner = holds, owner
+
+    @property
+    def held(self):
+        return self.owner in self.holds.owners and self.holds.held
+
+    def hold(self):
+        self.holds.hold(self.owner)
+
+    def release(self):
+        self.holds.release(self.owner)
+
+
+class PowerWatch:
+    def __init__(self, runtime, *, read_mac=None, read_phone=None, clock=time.monotonic, interval=INTERVAL):
+        self.runtime = runtime
+        self.awake = AwakeHolds()
+        self.read_mac = read_mac or device_status.read_mac
+        self.read_phone = read_phone or phone_reader
+        self.clock, self.interval = clock, interval
+        self.lock = threading.Lock()
+        self._mac = (None, None)            # (read at, value)
+        self._phones = {}                   # phone key -> (read at, value)
+        self.mac_low = None                 # the percent while the Mac's battery warning stands, else None
+        self.warned = set()                 # runs told the Mac's battery is low
+        self.paused = {}                    # run id -> the level it paused at (a battery pause this watch made)
+        self.kept_going = set()             # runs a person continued from a battery pause
+        self.stop = threading.Event()
+        self.thread = None
+
+    # -- runs ------------------------------------------------------------------------------------------------------
+
+    def run_started(self, run):
+        self.awake.hold(run.id)
+        self._ensure_thread()
+        if getattr(run, "origin", None) == "schedule" and phone_ready.enabled():
+            threading.Thread(target=self._low_power, args=(run,), name="mobster-low-power", daemon=True).start()
+
+    def _low_power(self, run):
+        """A scheduled task's one ``phone_low_power`` warning, as it starts: Low Power Mode makes its phone lock after
+        30 seconds. A read that fails says nothing (phone_ready.py); it never touches the task."""
+        try:
+            phone = self.runtime.phone_for(run)
+            warning = phone_ready.low_power_warning(phone.wda_url, phone.name or phone.id, "the next scheduled task")
+            if warning:
+                run.emit({"event": "phone_low_power", "message": warning})
+        except Exception:  # noqa: BLE001 -- a warning never stops a task
+            pass
+
+    def run_finished(self, run):
+        self.awake.release(run.id)
+        with self.lock:
+            self.warned.discard(run.id)
+            self.paused.pop(run.id, None)
+            self.kept_going.discard(run.id)
+
+    def close(self):
+        self.stop.set()
+        self.awake.close()
+
+    def _ensure_thread(self):
+        with self.lock:
+            if self.thread is not None or self.stop.is_set():
+                return
+            thread = threading.Thread(target=self._work, name="mobster-power", daemon=True)
+            thread.start()
+            self.thread = thread
+
+    def _work(self):
+        while not self.stop.wait(self.interval):
+            try:
+                self.tick()
+            except Exception:  # noqa: BLE001 -- the watch never stops a task
+                pass
+
+    # -- readings --------------------------------------------------------------------------------------------------
+
+    def mac(self, fresh=False):
+        at, value = self._mac
+        if fresh or at is None or self.clock() - at > device_status.MAC_TTL:
+            try:
+                value = self.read_mac()
+            except Exception:  # noqa: BLE001
+                value = {"power": None}
+            self._mac = (self.clock(), value)
+        return dict(value)
+
+    def phone(self, phone, fresh=False):
+        key = getattr(phone, "key", None)
+        at, value = self._phones.get(key, (None, None))
+        if fresh or at is None or self.clock() - at > device_status.PHONE_TTL:
+            try:
+                value = self.read_phone(phone)
+            except Exception:  # noqa: BLE001
+                value = {}
+            self._phones[key] = (self.clock(), value)
+        return dict(value)
+
+    def warnings(self):
+        if self.mac_low is None or not self.runtime.active_runs:
+            return []
+        return [{"code": "mac_battery_low", "percent": self.mac_low,
+                 "message": f"This Mac is on battery at {self.mac_low}%. Plug it in: a Mac that runs out of battery "
+                            "sleeps and stops its tasks."}]
+
+    # -- one look --------------------------------------------------------------------------------------------------
+
+    def tick(self):
+        runs = self._running()
+        if not runs:
+            return
+        self._mac_battery(runs)
+        threshold = phone_pause_threshold()
+        for run in runs:
+            try:
+                self._phone_battery(run, threshold)
+            except Exception:  # noqa: BLE001 -- one phone's reading never stops the others'
+                pass
+
+    def _running(self):
+        runtime = self.runtime
+        with runtime.lock:
+            ids = list(runtime.active_runs)
+            return [run for run in (runtime.runs.get(i) for i in ids) if run is not None and run.finished_at is None]
+
+    def _mac_battery(self, runs):
+        threshold = mac_warn_threshold()
+        mac = self.mac(fresh=True)
+        level = mac.get("battery")
+        low = (threshold is not None and mac.get("power") == "battery" and type(level) is int and level <= threshold)
+        if not low:
+            self.mac_low = None
+            self.warned.clear()
+            return
+        first = self.mac_low is None
+        self.mac_low = level
+        event = {"event": "mac_battery_low", "percent": level, "threshold": threshold}
+        for run in runs:
+            if run.id not in self.warned:
+                self.warned.add(run.id)
+                run.emit(dict(event))
+        if first:
+            self.runtime.publish("power", {**event, "runs": len(runs)})
+
+    def _phone_battery(self, run, threshold):
+        queue = run.steering
+        if run.id in self.paused:
+            why = control.reason(queue)
+            if not control.paused(queue) or not why or why.get("code") != PHONE_BATTERY:
+                # A person continued it (or paused it themselves): theirs from now on, and its battery doesn't pause
+                # this task again.
+                self.paused.pop(run.id, None)
+                self.kept_going.add(run.id)
+                return
+            battery = self.phone(self.runtime.phone_for(run), fresh=True).get("battery") or {}
+            if threshold is None or battery.get("state") in device_status.CHARGING:
+                self.paused.pop(run.id, None)
+                control.set_paused(queue, False)
+            return
+        if threshold is None or run.id in self.kept_going or not eligible(run) or control.paused(queue):
+            return  # off, kept going by a person, not a long or scheduled task, or a person's own pause stands
+        battery = self.phone(self.runtime.phone_for(run), fresh=True).get("battery") or {}
+        level, state = battery.get("level"), battery.get("state")
+        if type(level) is int and level <= threshold and state in device_status.DRAINING:
+            self.paused[run.id] = level
+            control.set_paused(queue, True, reason={"code": PHONE_BATTERY, "level": level,
+                                                     "limit": BATTERY_PAUSE_LIMIT})
+
+
+def eligible(run):
+    """A long task or a scheduled task on Mobster's agent, live (the only ones the battery may pause)."""
+    return (getattr(run, "engine", None) == engines.SMART and getattr(run, "mode", None) == "live"
+            and (getattr(run, "longform", None) is not None or getattr(run, "origin", None) == "schedule"))
+
+
+def phone_reader(phone):
+    """device_status.read_phone for a fleet.Phone (the primary's UDID is its device manager's)."""
+    kind = getattr(phone, "kind", None)
+    udid = getattr(phone, "udid", None)
+    manager = getattr(phone, "manager", None)
+    if kind == "usb" and not udid and manager is not None:
+        try:
+            udid = getattr(manager, "pinned", None) or manager.settings().get("udid")
+        except Exception:  # noqa: BLE001
+            udid = None
+    return device_status.read_phone(kind, udid=udid, wda_url=getattr(phone, "wda_url", None))

mobile_agent/premium_model.py

@@ -0,0 +1,22 @@
+"""``premiumModel``: Mobster is free, and Mobster's model is what you pay for. Off until the owner flips it.
+
+Off, nothing changes. On, the plan gate (plan_gate.py) lets every task through, and approvals, the journal, the device
+lock and the secret filter go on as they do on any run: they never read this flag."""
+
+import os
+import sys
+
+# The flip, together with ON in desktop/src-tauri/src/premium_model.rs (test_premium_model pins that they agree).
+ON = False
+DEV_ENV = "MOBSTER_PREMIUM_MODEL"
+
+
+def enabled(environ=None):
+    """``ON``, or ``MOBSTER_PREMIUM_MODEL=1`` in a source checkout. The frozen agent ignores the variable, because there
+    it would open the plan gate."""
+    if ON:
+        return True
+    if getattr(sys, "frozen", False):
+        return False
+    environ = os.environ if environ is None else environ
+    return environ.get(DEV_ENV) == "1"