mctrl 0.1.0__py3-none-any.whl

mindcontrol/control/mouse.py

@@ -0,0 +1,256 @@
+"""Where an intent leaves Python.
+
+Two ways out, same interface.
+
+*Through the native helper*, when it is running. It owns the cursor, integrates
+motion at display rate, snaps to whatever is nearest, and draws the highlight --
+none of which can be done from here, because it needs a thread that is never
+behind the GIL and a hundred accessibility hit tests a second. This is the path
+that feels smooth, and it is the default.
+
+*Straight to Quartz*, when the helper is not available -- not built yet, refused
+permission, or crashed. Not a legacy path: it is the reason a missing Swift
+toolchain degrades the feel rather than breaking the app. It posts one event per
+camera frame, which is exactly as stepped as it sounds, and is why the helper
+exists.
+
+Two details are load-bearing on both paths.
+
+*Tagging.* Every event carries a user-data marker so the physical-input watcher
+can tell our own synthetic moves from the user's real ones. Without it the app
+would see its own cursor motion, decide a human grabbed the mouse, and suspend
+itself the instant it started working. The helper stamps the same marker, which
+is why ``events.EVENT_MARKER`` and ``eventMarker`` in ``Cursor.swift`` have to
+agree.
+
+*Click chaining.* macOS decides what a double click is from the click-state field
+on the event, not from two clicks arriving close together. Real trackpads set it;
+so do we, which is what makes two quick pinches open a folder in Finder.
+"""
+
+from __future__ import annotations
+
+import time
+
+import Quartz
+
+from .bridge import Bridge
+from .events import create_source, post
+
+_BUTTON_EVENTS = {
+    "left": (Quartz.kCGEventLeftMouseDown, Quartz.kCGEventLeftMouseUp, Quartz.kCGMouseButtonLeft),
+    "right": (
+        Quartz.kCGEventRightMouseDown,
+        Quartz.kCGEventRightMouseUp,
+        Quartz.kCGMouseButtonRight,
+    ),
+}
+_DRAG_EVENTS = {
+    "left": Quartz.kCGEventLeftMouseDragged,
+    "right": Quartz.kCGEventRightMouseDragged,
+}
+
+
+def main_display_bounds() -> tuple[float, float, float, float]:
+    """The main display's rect, as (min_x, min_y, max_x, max_y).
+
+    Gaze is expressed as a fraction of this one screen rather than of the whole
+    desktop. Calibration can only teach where you look on the screen the camera
+    watched you look at, and a fraction of a three-monitor desktop would put the
+    cursor on a display you were never calibrated for.
+    """
+    box = Quartz.CGDisplayBounds(Quartz.CGMainDisplayID())
+    return (
+        box.origin.x,
+        box.origin.y,
+        box.origin.x + box.size.width,
+        box.origin.y + box.size.height,
+    )
+
+
+def desktop_bounds() -> tuple[float, float, float, float]:
+    """Union of every active display, as (min_x, min_y, max_x, max_y).
+
+    Cursor coordinates are global across displays, so clamping to the main
+    screen would trap the pointer on a multi-monitor desk. Hands can reach every
+    screen; only gaze is confined to the calibrated one.
+    """
+    error, display_ids, _ = Quartz.CGGetActiveDisplayList(16, None, None)
+    if error or not display_ids:
+        main = Quartz.CGDisplayBounds(Quartz.CGMainDisplayID())
+        return (
+            main.origin.x,
+            main.origin.y,
+            main.origin.x + main.size.width,
+            main.origin.y + main.size.height,
+        )
+    boxes = [Quartz.CGDisplayBounds(display_id) for display_id in display_ids]
+    return (
+        min(b.origin.x for b in boxes),
+        min(b.origin.y for b in boxes),
+        max(b.origin.x + b.size.width for b in boxes),
+        max(b.origin.y + b.size.height for b in boxes),
+    )
+
+
+class Mouse:
+    """Posts cursor, button and scroll events, through the helper when it is up.
+
+    Drag state is tracked here regardless of which path is in use, because the
+    press and release that bracket it are still decided in Python -- the gaze arm
+    reads :attr:`dragging` to know it must not warp mid-drag.
+    """
+
+    def __init__(self, double_click_ms: float = 400.0, bridge: Bridge | None = None) -> None:
+        self._source = create_source()
+        self._double_click_ms = double_click_ms
+        self._bridge = bridge
+        self._held: str | None = None
+        self._last_click_at = 0.0
+        self._last_click_point = (0.0, 0.0)
+        self._click_run = 0
+        self.refresh_bounds()
+
+    @property
+    def _native(self) -> Bridge | None:
+        """The helper, if it is there to take the intent."""
+        bridge = self._bridge
+        return bridge if bridge is not None and bridge.connected else None
+
+    def refresh_bounds(self) -> None:
+        """Re-read display geometry, for when a monitor is plugged in or unplugged."""
+        self._min_x, self._min_y, self._max_x, self._max_y = desktop_bounds()
+        self._gaze_box = main_display_bounds()
+
+    @property
+    def size(self) -> tuple[float, float]:
+        """Size of the whole desktop, which hand movement may roam across."""
+        return self._max_x - self._min_x, self._max_y - self._min_y
+
+    @property
+    def gaze_size(self) -> tuple[float, float]:
+        """Size of the display gaze is calibrated against."""
+        left, top, right, bottom = self._gaze_box
+        return right - left, bottom - top
+
+    @property
+    def dragging(self) -> bool:
+        return self._held is not None
+
+    def location(self) -> tuple[float, float]:
+        """Where the cursor actually is right now.
+
+        Read from the system rather than remembered, so that if you nudge the
+        real mouse, gesture movement carries on from where you left it.
+        """
+        point = Quartz.CGEventGetLocation(Quartz.CGEventCreate(None))
+        return point.x, point.y
+
+    def _clamp(self, x: float, y: float) -> tuple[float, float]:
+        return (
+            min(max(x, self._min_x), self._max_x - 1.0),
+            min(max(y, self._min_y), self._max_y - 1.0),
+        )
+
+    def _post(self, event) -> None:
+        post(event)
+
+    def _move_event(self, x: float, y: float) -> None:
+        # While a button is down the motion must be posted as a drag, or the app
+        # underneath sees the cursor teleport without ever being dragged.
+        if self._held is not None:
+            event_type = _DRAG_EVENTS[self._held]
+            button = _BUTTON_EVENTS[self._held][2]
+        else:
+            event_type = Quartz.kCGEventMouseMoved
+            button = Quartz.kCGMouseButtonLeft
+        self._post(Quartz.CGEventCreateMouseEvent(self._source, event_type, (x, y), button))
+
+    def move_by(self, dx: float, dy: float) -> None:
+        native = self._native
+        if native is not None:
+            # The helper accumulates this into a goal and walks the cursor there at
+            # display rate. Sending a delta rather than a destination is what lets
+            # it do that without ever reading the cursor back.
+            native.move_by(dx, dy)
+            return
+        x, y = self.location()
+        self._move_event(*self._clamp(x + dx, y + dy))
+
+    def move_to_fraction(self, fx: float, fy: float) -> None:
+        """Jump to a point given as fractions of the calibrated display, for gaze warps."""
+        native = self._native
+        if native is not None:
+            native.warp_to_fraction(fx, fy)
+            return
+        left, top, _, _ = self._gaze_box
+        width, height = self.gaze_size
+        self._move_event(*self._clamp(left + fx * width, top + fy * height))
+
+    def click(self, button: str = "left") -> None:
+        """Click, chaining into a double or triple click when repeated quickly."""
+        if button not in _BUTTON_EVENTS:
+            return
+        native = self._native
+        if native is not None:
+            # Chaining is done on the far side, where the click's actual landing
+            # point is known: it resolves to the snapped target, not to wherever
+            # the cursor had drifted to.
+            native.click(button)
+            return
+        down, up, index = _BUTTON_EVENTS[button]
+        x, y = self.location()
+        now = time.monotonic()
+        near = abs(x - self._last_click_point[0]) < 6 and abs(y - self._last_click_point[1]) < 6
+        in_time = (now - self._last_click_at) * 1000.0 < self._double_click_ms
+        self._click_run = self._click_run + 1 if (near and in_time) else 1
+        self._last_click_at = now
+        self._last_click_point = (x, y)
+
+        for event_type in (down, up):
+            event = Quartz.CGEventCreateMouseEvent(self._source, event_type, (x, y), index)
+            if event is not None:
+                Quartz.CGEventSetIntegerValueField(
+                    event, Quartz.kCGMouseEventClickState, min(self._click_run, 3)
+                )
+            self._post(event)
+
+    def press(self, button: str = "left") -> None:
+        if button not in _BUTTON_EVENTS or self._held is not None:
+            return
+        self._held = button
+        native = self._native
+        if native is not None:
+            native.press(button)
+            return
+        down, _, index = _BUTTON_EVENTS[button]
+        x, y = self.location()
+        self._post(Quartz.CGEventCreateMouseEvent(self._source, down, (x, y), index))
+
+    def release(self, button: str | None = None) -> None:
+        """Let go of a held button. Safe to call when nothing is held."""
+        held = self._held if button is None else button
+        self._held = None
+        if held is None or held not in _BUTTON_EVENTS:
+            return
+        native = self._native
+        if native is not None:
+            native.release(held)
+            return
+        _, up, index = _BUTTON_EVENTS[held]
+        x, y = self.location()
+        self._post(Quartz.CGEventCreateMouseEvent(self._source, up, (x, y), index))
+
+    def scroll(self, dx: float, dy: float) -> None:
+        """Scroll by a pixel delta, following the hand as if it held the page."""
+        native = self._native
+        if native is not None:
+            native.scroll(dx, dy)
+            return
+        # Pulling your hand down should drag the content down, which in wheel
+        # terms is a positive vertical value; the camera's y axis grows downward,
+        # hence the negation.
+        event = Quartz.CGEventCreateScrollWheelEvent2(
+            self._source, Quartz.kCGScrollEventUnitPixel, 2, int(-dy), int(dx), 0
+        )
+        self._post(event)

mindcontrol/debug_view.py

@@ -0,0 +1,188 @@
+"""Live overlay for tuning.
+
+Most of the thresholds in `config.toml` are only meaningful against real numbers
+from your own hands in your own lighting, so this window shows them: the
+skeleton, the classified pose, the live pinch distance, and where gaze thinks you
+are looking.
+
+On macOS an OpenCV window must own the main thread, which the menu bar already
+does, so the viewer runs in a separate process and is fed finished images. When
+running headless (`--debug`) the main thread is free and drawing happens inline.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import multiprocessing as mp
+import queue
+
+import cv2
+import numpy as np
+
+from .capture import Frame
+from .geometry import SKELETON, Pose
+from .pipeline import PipelineStatus
+
+VIEW_WIDTH = 720
+POSE_COLOURS = {
+    Pose.READY: (120, 220, 255),
+    Pose.FIST: (255, 170, 90),
+    Pose.OPEN_PALM: (140, 240, 140),
+    Pose.TELEPHONE: (220, 160, 255),
+}
+DEFAULT_COLOUR = (170, 170, 170)
+
+
+def render(
+    frame: Frame, hands: list, status: PipelineStatus, gaze: tuple[float, float] | None
+) -> np.ndarray:
+    """Draw one annotated frame, scaled down for cheap display."""
+    image = frame.image
+    scale = VIEW_WIDTH / max(image.shape[1], 1)
+    canvas = cv2.resize(image, (VIEW_WIDTH, int(image.shape[0] * scale)))
+    height, width = canvas.shape[:2]
+
+    for fused in hands:
+        features = fused.features
+        colour = POSE_COLOURS.get(features.pose, DEFAULT_COLOUR)
+        points = [(int(p[0] * width), int(p[1] * height)) for p in features.landmarks[:, :2]]
+        for start, end in SKELETON:
+            cv2.line(canvas, points[start], points[end], colour, 1, cv2.LINE_AA)
+        for point in points:
+            cv2.circle(canvas, point, 2, colour, -1, cv2.LINE_AA)
+
+        anchor = (int(features.anchor[0] * width), int(features.anchor[1] * height))
+        cv2.circle(canvas, anchor, 7, colour, 2, cv2.LINE_AA)
+        label = f"{features.handedness} {features.pose.value} pinch {features.pinch:.2f}"
+        if fused.merged:
+            label += f" x{len(fused.cameras)}"
+        cv2.putText(
+            canvas,
+            label,
+            (anchor[0] - 60, max(anchor[1] - 16, 14)),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.45,
+            colour,
+            1,
+            cv2.LINE_AA,
+        )
+
+    _draw_status(canvas, status)
+    if gaze is not None:
+        _draw_gaze_inset(canvas, gaze)
+    return canvas
+
+
+def _draw_status(canvas: np.ndarray, status: PipelineStatus) -> None:
+    lines = [
+        f"{status.fps:5.1f} fps   mode {status.mode}",
+        f"gesture {status.gesture}",
+        f"cameras {','.join(str(c) for c in status.cameras) or '-'}"
+        f"{'  merged' if status.merged else ''}"
+        f"   gaze {'ready' if status.gaze_ready else 'uncalibrated'}",
+    ]
+    cv2.rectangle(canvas, (0, 0), (canvas.shape[1], 14 + 20 * len(lines)), (20, 20, 20), -1)
+    for index, text in enumerate(lines):
+        cv2.putText(
+            canvas,
+            text,
+            (10, 20 + index * 20),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.48,
+            (230, 230, 230),
+            1,
+            cv2.LINE_AA,
+        )
+    for index, problem in enumerate(status.problems[:2]):
+        cv2.putText(
+            canvas,
+            problem[:70],
+            (10, canvas.shape[0] - 12 - index * 18),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.45,
+            (120, 140, 255),
+            1,
+            cv2.LINE_AA,
+        )
+
+
+def _draw_gaze_inset(canvas: np.ndarray, gaze: tuple[float, float]) -> None:
+    """Show gaze on a small proxy of the screen.
+
+    Gaze is a screen coordinate, not a camera one, so plotting it on the video
+    would put it somewhere meaningless. A miniature screen keeps it honest.
+    """
+    box_w, box_h = 150, 94
+    x0, y0 = canvas.shape[1] - box_w - 12, 12
+    cv2.rectangle(canvas, (x0, y0), (x0 + box_w, y0 + box_h), (70, 70, 70), 1)
+    cv2.circle(
+        canvas,
+        (int(x0 + gaze[0] * box_w), int(y0 + gaze[1] * box_h)),
+        5,
+        (110, 230, 255),
+        -1,
+        cv2.LINE_AA,
+    )
+    cv2.putText(
+        canvas, "gaze", (x0 + 4, y0 + box_h - 6), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (90, 90, 90), 1
+    )
+
+
+def _viewer(frames: mp.Queue, title: str) -> None:
+    """Child-process loop: show whatever arrives until told to stop."""
+    while True:
+        try:
+            image = frames.get(timeout=0.5)
+        except queue.Empty:
+            if cv2.waitKey(1) & 0xFF == 27:
+                break
+            continue
+        if image is None:
+            break
+        cv2.imshow(title, image)
+        if cv2.waitKey(1) & 0xFF == 27:
+            break
+    cv2.destroyAllWindows()
+    for _ in range(4):
+        cv2.waitKey(1)
+
+
+class DebugView:
+    """An overlay window hosted in its own process."""
+
+    def __init__(self, title: str = "mindcontrol") -> None:
+        self._title = title
+        self._context = mp.get_context("spawn")
+        self._queue: mp.Queue | None = None
+        self._process = None
+
+    @property
+    def running(self) -> bool:
+        return self._process is not None and self._process.is_alive()
+
+    def open(self) -> None:
+        if self.running:
+            return
+        self._queue = self._context.Queue(maxsize=1)
+        self._process = self._context.Process(
+            target=_viewer, args=(self._queue, self._title), daemon=True
+        )
+        self._process.start()
+
+    def push(self, image: np.ndarray) -> None:
+        """Offer a frame, dropping it if the viewer is still busy with the last one."""
+        if not self.running or self._queue is None:
+            return
+        with contextlib.suppress(queue.Full):
+            self._queue.put_nowait(image)
+
+    def close(self) -> None:
+        if self._queue is not None:
+            with contextlib.suppress(queue.Full):
+                self._queue.put_nowait(None)
+        if self._process is not None:
+            self._process.join(timeout=1.5)
+            if self._process.is_alive():
+                self._process.terminate()
+            self._process = None
+        self._queue = None

mindcontrol/devices.py

@@ -0,0 +1,222 @@
+"""Finding out which cameras exist.
+
+OpenCV offers no way to enumerate capture devices, only to try opening one by
+index, so discovery means probing indices and seeing what answers. macOS reports
+proper device names through AVFoundation, which is worth reading: index order is
+not stable across reboots or replugs, and picking a camera by number alone is how
+you end up gaze-tracking from the camera pointed at the wall.
+
+Names are a strong hint rather than proof. AVFoundation's discovery order has been
+observed to disagree with OpenCV's indices, and there is no identifier shared
+between the two APIs to reconcile them. So `--preview` exists: it shows a frame
+from each index, which is the only unambiguous way to learn which camera is which.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+
+import cv2
+
+MAX_PROBE = 8
+# A camera that has just been opened can take a moment to deliver its first
+# frame -- built-in webcams especially, since the indicator light has to come up.
+# Giving up after a single failed read reports a working camera as absent.
+READ_ATTEMPTS = 5
+READ_DELAY_S = 0.12
+# AVFoundation reports a placeholder frame rate until the stream settles, so a
+# camera that woke slowly can advertise 1fps while really delivering 30. Reporting
+# that is worse than saying nothing: it invites the user to exclude a good camera.
+SETTLE_READS = 5
+MEASURE_FRAMES = 15
+PLAUSIBLE_FPS = 5.0
+
+
+def _frame_rate(capture: cv2.VideoCapture) -> float:
+    """The camera's real frame rate, timed directly if it will not say."""
+    for _ in range(SETTLE_READS):
+        capture.read()
+    declared = float(capture.get(cv2.CAP_PROP_FPS))
+    if declared >= PLAUSIBLE_FPS:
+        return declared
+    start = time.perf_counter()
+    read = sum(bool(capture.read()[0]) for _ in range(MEASURE_FRAMES))
+    elapsed = time.perf_counter() - start
+    return read / elapsed if elapsed > 0 and read else declared
+
+
+@dataclass(frozen=True)
+class Device:
+    index: int
+    width: int
+    height: int
+    fps: float
+    name: str = ""
+    usable: bool = True
+    problem: str = ""
+
+    def describe(self) -> str:
+        label = self.name or "unnamed device"
+        if not self.usable:
+            return f"  [{self.index}] {label}  UNAVAILABLE - {self.problem}"
+        return f"  [{self.index}] {label}  {self.width}x{self.height} @ {self.fps:.0f}fps"
+
+
+def names() -> list[str]:
+    """Camera names in AVFoundation's order, which matches OpenCV's indices."""
+    try:
+        import AVFoundation
+    except ImportError:
+        return []
+    try:
+        # Modern discovery session; the older devicesWithMediaType_ is deprecated
+        # and returns nothing on recent macOS.
+        discover = (
+            AVFoundation.AVCaptureDeviceDiscoverySession
+            .discoverySessionWithDeviceTypes_mediaType_position_
+        )
+        session = discover(
+            [
+                AVFoundation.AVCaptureDeviceTypeBuiltInWideAngleCamera,
+                AVFoundation.AVCaptureDeviceTypeExternal,
+                AVFoundation.AVCaptureDeviceTypeContinuityCamera,
+            ],
+            AVFoundation.AVMediaTypeVideo,
+            0,
+        )
+        return [str(device.localizedName()) for device in session.devices()]
+    except (AttributeError, TypeError):
+        return []
+
+
+def probe(max_index: int = MAX_PROBE) -> list[Device]:
+    """Open each index in turn and report what it can actually deliver.
+
+    Bounded by the AVFoundation device count when that is available, because
+    probing past the last device makes OpenCV shout about indices being out of
+    bounds -- noise that looks like a failure but is just the end of the list.
+    """
+    labels = names()
+    limit = min(max_index, len(labels)) if labels else max_index
+    found: list[Device] = []
+
+    for index in range(limit):
+        name = labels[index] if index < len(labels) else ""
+        capture = cv2.VideoCapture(index, cv2.CAP_AVFOUNDATION)
+        if not capture.isOpened():
+            capture.release()
+            found.append(
+                Device(index, 0, 0, 0.0, name, usable=False, problem="could not be opened")
+            )
+            continue
+
+        # isOpened() alone is optimistic; a device that cannot produce an image
+        # is no use to us, so insist on a real frame before believing it.
+        frame = None
+        for _ in range(READ_ATTEMPTS):
+            ok, frame = capture.read()
+            if ok and frame is not None:
+                break
+            time.sleep(READ_DELAY_S)
+            frame = None
+
+        if frame is None:
+            found.append(
+                Device(
+                    index, 0, 0, 0.0, name,
+                    usable=False,
+                    problem="opened but sent no frames (in use by another app?)",
+                )
+            )
+        else:
+            found.append(
+                Device(
+                    index=index,
+                    width=int(capture.get(cv2.CAP_PROP_FRAME_WIDTH)),
+                    height=int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT)),
+                    fps=_frame_rate(capture),
+                    name=name,
+                )
+            )
+        capture.release()
+    return found
+
+
+def preview(devices: list[Device], height: int = 260) -> None:
+    """Show one frame from each usable camera, side by side and labelled.
+
+    Grabbed one device at a time: opening several cameras at once can exceed the
+    bus bandwidth and make an innocent device look broken.
+    """
+    import numpy as np
+
+    tiles = []
+    for device in devices:
+        if not device.usable:
+            continue
+        capture = cv2.VideoCapture(device.index, cv2.CAP_AVFOUNDATION)
+        frame = None
+        for _ in range(READ_ATTEMPTS):
+            ok, candidate = capture.read()
+            if ok and candidate is not None:
+                frame = candidate
+                break
+            time.sleep(READ_DELAY_S)
+        capture.release()
+        if frame is None:
+            continue
+
+        scaled = cv2.resize(frame, (int(frame.shape[1] * height / frame.shape[0]), height))
+        cv2.rectangle(scaled, (0, 0), (scaled.shape[1], 30), (20, 20, 20), -1)
+        cv2.putText(
+            scaled,
+            f"[{device.index}] {device.name}",
+            (8, 21),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.6,
+            (240, 240, 240),
+            1,
+            cv2.LINE_AA,
+        )
+        tiles.append(scaled)
+
+    if not tiles:
+        print("[cameras] nothing to preview")
+        return
+
+    print("[cameras] preview open; press any key to close")
+    cv2.imshow("mindcontrol cameras", np.hstack(tiles))
+    cv2.waitKey(0)
+    cv2.destroyAllWindows()
+    for _ in range(4):
+        cv2.waitKey(1)
+
+
+def run(max_index: int = MAX_PROBE, show: bool = False) -> int:
+    devices = probe(max_index)
+    usable = [device for device in devices if device.usable]
+
+    if not devices:
+        print("[cameras] none found; check System Settings > Privacy & Security > Camera")
+        return 2
+
+    print(f"[cameras] {len(usable)} of {len(devices)} device(s) usable:")
+    for device in devices:
+        print(device.describe())
+
+    if not usable:
+        return 2
+
+    indices = ", ".join(str(d.index) for d in usable)
+    print("\nto use them together, in config.toml:")
+    print(f"  [cameras]\n  devices = [{indices}]\n  primary_gaze = {usable[0].index}")
+    print("\nprimary_gaze should be whichever camera sits nearest the screen you look at;")
+    print("it is the one gaze is estimated from, so put it under your main display.")
+    print("\nNames come from AVFoundation and its order can disagree with these indices.")
+    print("Run 'mindcontrol cameras --preview' to see which index is really which.")
+
+    if show:
+        print()
+        preview(devices)
+    return 0