mctrl 0.1.0__py3-none-any.whl

mindcontrol/control/bridge.py

@@ -0,0 +1,360 @@
+"""Client for the native interaction helper.
+
+The helper (``native/``, Swift) owns the cursor. This module is the near end of the
+socket to it: it finds the binary, keeps it running, and turns each gesture intent
+into one 48-byte datagram.
+
+Why the work moved out of Python at all is a measurement, not a preference. The
+snapping this exists to serve needs to ask the system what is on screen, and
+accessibility queries are synchronous IPC into other applications:
+
+===========================================  ==========  ==========
+operation                                    pyobjc      native
+===========================================  ==========  ==========
+one attribute read                              1140 us      382 us
+whole-window tree walk (2419 nodes)             2490 ms     4251 ms
+single-point hit test                                 -     0.43 ms
+===========================================  ==========  ==========
+
+Walking a window is hopeless in either language. The hit test is affordable in
+both, but it is only 3x cheaper here -- the rest is IPC, which no language avoids.
+What actually cannot be done in this process is the other half: integrating the
+cursor at display rate on a thread that is never behind the GIL while MediaPipe is
+running inference, and holding the sole write handle on the cursor so that a warp
+and a hand delta can never be posted from two threads against the same stale
+position.
+
+Sending is fire and forget. Deltas travel rather than positions, so nothing here
+ever needs to read the cursor back, which keeps the round trip count at zero.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import socket
+import struct
+import subprocess
+import time
+from dataclasses import asdict
+from pathlib import Path
+
+from ..config import STATE_DIR, NativeConfig
+
+SOCKET_PATH = STATE_DIR / "bridge.sock"
+TUNING_PATH = STATE_DIR / "bridge-tuning.json"
+
+_MAGIC = 0x4D494E44  # "MIND", the same tag the synthetic events carry
+_VERSION = 1
+# magic, version, intent, sequence, flags, a, b, sent_at, button, pad
+_FRAME = struct.Struct("<IHHIIdddII")
+
+MOVE_BY = 1
+WARP_TO_FRACTION = 2
+CLICK = 3
+PRESS = 4
+RELEASE = 5
+SCROLL = 6
+SET_MODE = 7
+RELEASE_ALL = 8
+RELOAD_CONFIG = 9
+SHUTDOWN = 10
+
+ENGAGED = 1 << 0
+POINTING = 1 << 1
+SWEEPING = 1 << 2
+
+_BUTTONS = {"left": 0, "right": 1}
+
+
+def helper_path() -> Path | None:
+    """Locate the helper binary, preferring an explicit override.
+
+    The release build is checked before the debug one so that a stale debug
+    binary left over from development does not quietly win.
+    """
+    override = os.environ.get("MINDCONTROL_BRIDGE")
+    if override:
+        candidate = Path(override).expanduser()
+        return candidate if candidate.is_file() else None
+
+    root = Path(__file__).resolve().parents[2] / "native" / ".build"
+    for build in ("release", "debug"):
+        candidate = root / build / "mindcontrol-bridge"
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+def package_dir() -> Path:
+    """The Swift package, which lives beside the Python one."""
+    return Path(__file__).resolve().parents[2] / "native"
+
+
+def build_hint() -> str:
+    return "run `mindcontrol bridge` to build it"
+
+
+def build(*, release: bool = True) -> int:
+    """Compile the helper. Returns swift's exit status, or 127 if it is absent."""
+    package = package_dir()
+    if not (package / "Package.swift").is_file():
+        print(f"[bridge] no Swift package at {package}")
+        return 2
+    command = ["swift", "build", "--package-path", str(package)]
+    if release:
+        command += ["-c", "release"]
+    print(f"[bridge] {' '.join(command)}")
+    try:
+        return subprocess.run(command, check=False).returncode
+    except FileNotFoundError:
+        print(
+            "[bridge] swift not found. Install the Xcode command line tools with "
+            "`xcode-select --install`."
+        )
+        return 127
+
+
+def run(*, rebuild: bool = False, debug: bool = False) -> int:
+    """CLI entry: build the helper if needed, then report where it stands."""
+    if rebuild or helper_path() is None:
+        status = build(release=not debug)
+        if status != 0:
+            return status
+
+    binary = helper_path()
+    if binary is None:
+        print("[bridge] built, but no binary was produced")
+        return 1
+    print(f"[bridge] helper at {binary}")
+
+    trusted = _accessibility_granted()
+    if trusted is None:
+        print("[bridge] could not check Accessibility permission")
+    elif trusted:
+        print("[bridge] Accessibility permission granted to this process")
+    else:
+        print(
+            "[bridge] no Accessibility permission yet. Snapping and highlighting need it.\n"
+            "         The helper asks for it in its own right the first time it runs, so\n"
+            "         look for it in System Settings > Privacy & Security > Accessibility."
+        )
+    return 0
+
+
+def _accessibility_granted() -> bool | None:
+    try:
+        from ApplicationServices import AXIsProcessTrusted
+    except ImportError:
+        return None
+    return bool(AXIsProcessTrusted())
+
+
+class Bridge:
+    """Owns the helper process and the socket to it.
+
+    Every method is safe to call when the helper is absent or has died; the caller
+    checks :attr:`connected` to decide whether to fall back to posting events
+    itself. Nothing here raises on a send failure -- a dropped frame costs a
+    fraction of a millimetre of cursor travel, and tearing down the pipeline over
+    one would be a far worse outcome.
+    """
+
+    def __init__(self, cfg: NativeConfig, double_click_ms: float = 400.0, *, spawn: bool = True):
+        self._cfg = cfg
+        # Lives in [gestures] rather than [native], because it is a property of how
+        # you pinch, not of the helper. The helper needs it because it is the side
+        # that stamps the click-state field two quick pinches chain through.
+        self._double_click_ms = double_click_ms
+        self._spawn = spawn
+        self._socket: socket.socket | None = None
+        self._process: subprocess.Popen[bytes] | None = None
+        self._sequence = 0
+        self._flags = -1
+        self._next_attempt = 0.0
+        self._attempts = 0
+        self.error: str | None = None
+
+    # ------------------------------------------------------------------ lifecycle
+
+    def start(self) -> bool:
+        """Write the tuning file, start the helper, and connect. False if unavailable."""
+        if not self._cfg.enabled:
+            self.error = "native bridge disabled in config"
+            return False
+        binary = helper_path()
+        if binary is None:
+            self.error = f"native helper not built; {build_hint()}"
+            return False
+
+        self.write_tuning()
+        if self._spawn and not self._launch(binary):
+            return False
+        return self._connect()
+
+    def _launch(self, binary: Path) -> bool:
+        if self._process is not None and self._process.poll() is None:
+            return True
+        STATE_DIR.mkdir(parents=True, exist_ok=True)
+        try:
+            self._process = subprocess.Popen(
+                [
+                    str(binary),
+                    "--socket",
+                    str(SOCKET_PATH),
+                    "--tuning",
+                    str(TUNING_PATH),
+                ],
+                stdout=subprocess.DEVNULL,
+                # The helper reports permission problems and its listening address
+                # on stderr; inheriting it puts those where the user is looking.
+                stderr=None,
+            )
+        except OSError as problem:
+            self.error = f"could not start native helper: {problem}"
+            return False
+        return True
+
+    # Longer than the helper's own eviction grace, which is the slowest thing that
+    # can stand between launching it and it listening: a helper that is patiently
+    # waiting for a previous one to stand down must not be given up on as broken.
+    _CONNECT_TIMEOUT = 8.0
+
+    def _connect(self) -> bool:
+        """Connect the datagram socket, waiting for the helper to bind it."""
+        deadline = time.monotonic() + self._CONNECT_TIMEOUT
+        while time.monotonic() < deadline:
+            if self._process is not None and self._process.poll() is not None:
+                self.error = f"native helper exited with status {self._process.returncode}"
+                return False
+            try:
+                handle = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
+                handle.connect(str(SOCKET_PATH))
+            except OSError:
+                time.sleep(0.05)
+                continue
+            self._socket = handle
+            self.error = None
+            self._flags = -1
+            return True
+        self.error = f"native helper did not open {SOCKET_PATH}"
+        return False
+
+    @property
+    def connected(self) -> bool:
+        return self._socket is not None
+
+    @property
+    def alive(self) -> bool:
+        """True when the helper process is still running, if we started it."""
+        return self._process is None or self._process.poll() is None
+
+    def stop(self) -> None:
+        if self._socket is not None:
+            self._send(SHUTDOWN)
+            self._socket.close()
+            self._socket = None
+        if self._process is not None:
+            try:
+                self._process.wait(timeout=2.0)
+            except subprocess.TimeoutExpired:
+                self._process.terminate()
+                try:
+                    self._process.wait(timeout=1.0)
+                except subprocess.TimeoutExpired:
+                    self._process.kill()
+            self._process = None
+
+    def write_tuning(self) -> None:
+        """Publish the native-side knobs as JSON.
+
+        The TOML stays the single place anything is tuned; this is a projection of
+        it, so the helper does not need a TOML parser to honour an edit.
+        """
+        STATE_DIR.mkdir(parents=True, exist_ok=True)
+        payload = {key: value for key, value in asdict(self._cfg).items() if key != "enabled"}
+        payload["double_click_ms"] = self._double_click_ms
+        TUNING_PATH.write_text(json.dumps(payload, indent=2))
+
+    def apply_config(self, cfg: NativeConfig, double_click_ms: float | None = None) -> None:
+        """Adopt edited settings and tell the helper to re-read them."""
+        self._cfg = cfg
+        if double_click_ms is not None:
+            self._double_click_ms = double_click_ms
+        self.write_tuning()
+        self._send(RELOAD_CONFIG)
+
+    # -------------------------------------------------------------------- sending
+
+    def _send(self, intent: int, a: float = 0.0, b: float = 0.0, flags: int = 0, button: int = 0):
+        handle = self._socket
+        if handle is None:
+            return
+        self._sequence = (self._sequence + 1) & 0xFFFFFFFF
+        frame = _FRAME.pack(
+            _MAGIC, _VERSION, intent, self._sequence, flags, a, b, time.monotonic(), button, 0
+        )
+        try:
+            handle.send(frame)
+        except OSError as problem:
+            # The helper has gone. Drop to the fallback path rather than raising
+            # into the frame loop, and let reconnect() pick it up if it returns.
+            self.error = f"native helper unreachable: {problem}"
+            self._socket = None
+            handle.close()
+
+    def reconnect(self) -> bool:
+        """Try to re-establish a dropped helper, with backoff. False if not yet."""
+        if self.connected:
+            return True
+        now = time.monotonic()
+        if now < self._next_attempt:
+            return False
+        self._attempts += 1
+        self._next_attempt = now + min(0.5 * self._attempts, 10.0)
+        binary = helper_path()
+        if binary is None:
+            return False
+        if self._spawn and not self._launch(binary):
+            return False
+        if self._connect():
+            self._attempts = 0
+            return True
+        return False
+
+    # --------------------------------------------------------------------- intents
+
+    def move_by(self, dx: float, dy: float) -> None:
+        self._send(MOVE_BY, dx, dy)
+
+    def warp_to_fraction(self, fx: float, fy: float) -> None:
+        self._send(WARP_TO_FRACTION, fx, fy)
+
+    def click(self, button: str = "left") -> None:
+        self._send(CLICK, button=_BUTTONS.get(button, 0))
+
+    def press(self, button: str = "left") -> None:
+        self._send(PRESS, button=_BUTTONS.get(button, 0))
+
+    def release(self, button: str = "left") -> None:
+        self._send(RELEASE, button=_BUTTONS.get(button, 0))
+
+    def scroll(self, dx: float, dy: float) -> None:
+        self._send(SCROLL, dx, dy)
+
+    def release_all(self) -> None:
+        self._send(RELEASE_ALL)
+
+    def set_mode(self, *, engaged: bool, pointing: bool, sweeping: bool) -> None:
+        """Tell the helper what kind of gesture is running.
+
+        Sent only on change. The helper needs this to know when to look for targets
+        at all: a scroll or a swipe is not aiming at anything, and snapping during
+        one would fight the hand.
+        """
+        flags = (ENGAGED if engaged else 0) | (POINTING if pointing else 0)
+        flags |= SWEEPING if sweeping else 0
+        if flags == self._flags:
+            return
+        self._flags = flags
+        self._send(SET_MODE, flags=flags)

mindcontrol/control/events.py

@@ -0,0 +1,34 @@
+"""Shared plumbing for posting synthetic events.
+
+Every event this app injects is stamped with the same marker so the physical-input
+watcher in `modes.py` can recognise and ignore it. If the app could not tell its
+own output apart from the user's input, it would suspend itself the moment it
+moved the cursor.
+"""
+
+from __future__ import annotations
+
+import Quartz
+
+EVENT_MARKER = 0x4D494E44  # "MIND"
+
+
+def create_source():
+    """An event source tagged as ours, or None if the system refuses one."""
+    source = Quartz.CGEventSourceCreate(Quartz.kCGEventSourceStateHIDSystemState)
+    if source is not None:
+        Quartz.CGEventSourceSetUserData(source, EVENT_MARKER)
+    return source
+
+
+def post(event) -> None:
+    """Tag and dispatch one event to the HID tap."""
+    if event is None:
+        return
+    Quartz.CGEventSetIntegerValueField(event, Quartz.kCGEventSourceUserData, EVENT_MARKER)
+    Quartz.CGEventPost(Quartz.kCGHIDEventTap, event)
+
+
+def is_ours(event) -> bool:
+    """True when an observed event was injected by this app."""
+    return Quartz.CGEventGetIntegerValueField(event, Quartz.kCGEventSourceUserData) == EVENT_MARKER

mindcontrol/control/keyboard.py

@@ -0,0 +1,101 @@
+"""Synthetic key input via Quartz.
+
+Used for the system-level gestures -- desktop switching, Mission Control,
+dictation -- where the right thing to send is the shortcut a user would type.
+Sending real shortcuts means these gestures work with whatever the user has
+already configured, instead of this app reimplementing window management.
+"""
+
+from __future__ import annotations
+
+import Quartz
+
+from ..config import KeyBinding
+from .events import create_source, post
+
+# macOS virtual key codes. Only the keys worth binding to a gesture are listed.
+KEY_CODES: dict[str, int] = {
+    "left": 0x7B,
+    "right": 0x7C,
+    "down": 0x7D,
+    "up": 0x7E,
+    "space": 0x31,
+    "tab": 0x30,
+    "return": 0x24,
+    "escape": 0x35,
+    "delete": 0x33,
+    "f1": 0x7A,
+    "f2": 0x78,
+    "f3": 0x63,
+    "f4": 0x76,
+    "f5": 0x60,
+    "f6": 0x61,
+    "f7": 0x62,
+    "f8": 0x64,
+    "f9": 0x65,
+    "f10": 0x6D,
+    "f11": 0x67,
+    "f12": 0x6F,
+    "a": 0x00,
+    "c": 0x08,
+    "d": 0x02,
+    "h": 0x04,
+    "m": 0x2E,
+    "n": 0x2D,
+    "s": 0x01,
+    "t": 0x11,
+    "v": 0x09,
+    "w": 0x0D,
+    "z": 0x06,
+}
+
+MODIFIER_FLAGS: dict[str, int] = {
+    "cmd": Quartz.kCGEventFlagMaskCommand,
+    "command": Quartz.kCGEventFlagMaskCommand,
+    "ctrl": Quartz.kCGEventFlagMaskControl,
+    "control": Quartz.kCGEventFlagMaskControl,
+    "alt": Quartz.kCGEventFlagMaskAlternate,
+    "option": Quartz.kCGEventFlagMaskAlternate,
+    "shift": Quartz.kCGEventFlagMaskShift,
+    "fn": Quartz.kCGEventFlagMaskSecondaryFn,
+}
+
+
+class Keyboard:
+    """Sends keystrokes for named actions defined in config."""
+
+    def __init__(self, keys: dict[str, KeyBinding]) -> None:
+        self._source = create_source()
+        self._keys = keys
+
+    def update_bindings(self, keys: dict[str, KeyBinding]) -> None:
+        self._keys = keys
+
+    def tap(self, binding: KeyBinding) -> bool:
+        """Press and release one key with modifiers. False if the key is unknown."""
+        code = KEY_CODES.get(binding.key.lower())
+        if code is None:
+            print(f"[keyboard] no key code for {binding.key!r}; add it to KEY_CODES")
+            return False
+        flags = 0
+        for name in binding.mods:
+            flag = MODIFIER_FLAGS.get(name.lower())
+            if flag is None:
+                print(f"[keyboard] unknown modifier {name!r}")
+                continue
+            flags |= flag
+
+        for pressed in (True, False):
+            event = Quartz.CGEventCreateKeyboardEvent(self._source, code, pressed)
+            if event is not None and flags:
+                Quartz.CGEventSetFlags(event, flags)
+            post(event)
+        return True
+
+    def run_action(self, action: str) -> bool:
+        """Fire a named action such as ``desktop_left``, if it is bound."""
+        binding = self._keys.get(action)
+        if binding is None:
+            print(f"[keyboard] action {action!r} has no key binding in [keys]")
+            return False
+        return self.tap(binding)

mindcontrol/control/modes.py

@@ -0,0 +1,194 @@
+"""Mode arbitration between your hands and your hardware.
+
+The rule that makes gesture control livable: the physical mouse and keyboard
+always win, immediately, without asking. Touch either one and gesture output
+stops mid-motion; stop touching them and it comes back on its own. You never
+"exit" hand mode, you just reach for the trackpad.
+
+Three modes:
+
+``OFF``       nothing is driven; only a held open palm is watched for.
+``ACTIVE``    hands are driving the cursor.
+``SUSPENDED`` hands are tracked but muted, because hardware was just used.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Callable
+from enum import Enum
+
+import Quartz
+
+from ..config import ModesConfig
+from .events import is_ours
+
+
+class Mode(Enum):
+    OFF = "off"
+    ACTIVE = "active"
+    SUSPENDED = "suspended"
+
+
+class PhysicalInputWatcher:
+    """Watches for real mouse and keyboard use on a private run loop.
+
+    A listen-only event tap needs a CFRunLoop to deliver callbacks, and the main
+    thread already belongs to the menu bar, so the tap gets its own thread.
+    Our own synthetic events arrive here too and are filtered out by their tag.
+    """
+
+    EVENTS = (
+        Quartz.kCGEventMouseMoved,
+        Quartz.kCGEventLeftMouseDown,
+        Quartz.kCGEventRightMouseDown,
+        Quartz.kCGEventScrollWheel,
+        Quartz.kCGEventKeyDown,
+        Quartz.kCGEventFlagsChanged,
+    )
+
+    def __init__(self, on_input: Callable[[], None]) -> None:
+        self._on_input = on_input
+        self._thread: threading.Thread | None = None
+        self._runloop = None
+        self._tap = None
+        self.error: str | None = None
+
+    def start(self) -> bool:
+        started = threading.Event()
+        self._thread = threading.Thread(target=self._run, args=(started,), name="input-watch")
+        self._thread.daemon = True
+        self._thread.start()
+        started.wait(timeout=3.0)
+        return self._tap is not None
+
+    def _run(self, started: threading.Event) -> None:
+        mask = 0
+        for event_type in self.EVENTS:
+            mask |= Quartz.CGEventMaskBit(event_type)
+        self._tap = Quartz.CGEventTapCreate(
+            Quartz.kCGSessionEventTap,
+            Quartz.kCGHeadInsertEventTap,
+            Quartz.kCGEventTapOptionListenOnly,
+            mask,
+            self._callback,
+            None,
+        )
+        if self._tap is None:
+            self.error = (
+                "could not observe input; grant Accessibility permission to enable "
+                "automatic hand-off to the mouse and keyboard"
+            )
+            started.set()
+            return
+
+        source = Quartz.CFMachPortCreateRunLoopSource(None, self._tap, 0)
+        self._runloop = Quartz.CFRunLoopGetCurrent()
+        Quartz.CFRunLoopAddSource(self._runloop, source, Quartz.kCFRunLoopCommonModes)
+        Quartz.CGEventTapEnable(self._tap, True)
+        started.set()
+        Quartz.CFRunLoopRun()
+
+    def _callback(self, proxy, event_type, event, refcon):
+        # A tap can be disabled by the system if it ever runs too slowly; the
+        # documented recovery is simply to switch it back on.
+        if event_type in (
+            Quartz.kCGEventTapDisabledByTimeout,
+            Quartz.kCGEventTapDisabledByUserInput,
+        ):
+            Quartz.CGEventTapEnable(self._tap, True)
+            return event
+        if not is_ours(event):
+            self._on_input()
+        return event
+
+    def stop(self) -> None:
+        if self._tap is not None:
+            Quartz.CGEventTapEnable(self._tap, False)
+        if self._runloop is not None:
+            Quartz.CFRunLoopStop(self._runloop)
+        if self._thread is not None:
+            self._thread.join(timeout=1.0)
+            self._thread = None
+
+
+class ModeManager:
+    """Owns the current mode and the rules for changing it."""
+
+    def __init__(self, cfg: ModesConfig, on_change: Callable[[Mode], None] | None = None) -> None:
+        self._cfg = cfg
+        self._on_change = on_change
+        self._lock = threading.Lock()
+        self._mode = Mode.ACTIVE if cfg.start_engaged else Mode.OFF
+        self._last_physical_input = 0.0
+        self._watcher: PhysicalInputWatcher | None = None
+        self.watcher_error: str | None = None
+
+    def start(self) -> None:
+        if not self._cfg.suspend_on_physical_input:
+            return
+        self._watcher = PhysicalInputWatcher(self._note_physical_input)
+        if not self._watcher.start():
+            self.watcher_error = self._watcher.error
+            print(f"[modes] {self.watcher_error}")
+            self._watcher = None
+
+    def stop(self) -> None:
+        if self._watcher is not None:
+            self._watcher.stop()
+            self._watcher = None
+
+    @property
+    def mode(self) -> Mode:
+        """Current mode, resolving an expired suspension on read."""
+        with self._lock:
+            if self._mode is Mode.SUSPENDED:
+                idle = time.monotonic() - self._last_physical_input
+                if idle >= self._cfg.resume_after_s:
+                    self._set(Mode.ACTIVE)
+            return self._mode
+
+    @property
+    def engaged(self) -> bool:
+        """True when gesture output should actually reach the system."""
+        return self.mode is Mode.ACTIVE
+
+    @property
+    def watching_for_engage(self) -> bool:
+        """True when the engage gesture is the only thing being listened for."""
+        return self.mode is Mode.OFF
+
+    def toggle(self) -> Mode:
+        """Flip between off and active, clearing any suspension."""
+        with self._lock:
+            self._set(Mode.OFF if self._mode in (Mode.ACTIVE, Mode.SUSPENDED) else Mode.ACTIVE)
+            return self._mode
+
+    def set_mode(self, mode: Mode) -> None:
+        with self._lock:
+            self._set(mode)
+
+    def _note_physical_input(self) -> None:
+        """Called from the tap thread on any real input."""
+        with self._lock:
+            self._last_physical_input = time.monotonic()
+            # Only an actively driving session gets suspended. If control is off,
+            # using the mouse should not silently arm it.
+            if self._mode is Mode.ACTIVE:
+                self._set(Mode.SUSPENDED)
+
+    def _set(self, mode: Mode) -> None:
+        """Change mode. Caller holds the lock; the callback runs outside it."""
+        if mode is self._mode:
+            return
+        self._mode = mode
+        if self._on_change is not None:
+            self._on_change(mode)
+
+    def describe(self) -> str:
+        mode = self.mode
+        if mode is Mode.SUSPENDED:
+            remaining = self._cfg.resume_after_s - (time.monotonic() - self._last_physical_input)
+            return f"suspended ({max(remaining, 0.0):.1f}s)"
+        return mode.value