extra-desktop 0.1.1__py3-none-any.whl

extra/__init__.py

@@ -0,0 +1,5 @@
+"""
+Project Extra — Flashless Windows 10/11 Computer-Use Engine & MCP Server.
+"""
+
+__version__ = "0.1.1"

extra/cli.py

@@ -0,0 +1,242 @@
+"""
+Project Extra — Command-Line Interface (CLI) & System Diagnostic Runner
+Provides `extra doctor`, `extra test`, `extra run`, `extra inspect`, and `extra snap`.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import platform
+import sys
+import time
+from pathlib import Path
+
+# Add project root to sys.path
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+try:
+    from extra import __version__
+except Exception:
+    __version__ = "0.1.1"
+
+from extra.core.capture import capture_screen
+from extra.core.focus import (
+    find_window_by_title,
+    get_foreground_window,
+    list_windows,
+)
+from extra.core.geometry import (
+    attach_input_desktop,
+    ensure_dpi_aware,
+    get_cursor_position,
+    get_monitors_info,
+)
+from extra.core.uia_plane import SetOfMarkAnnotator, UIAutomationPlane
+from extra.fastpath.shell import resolve_executable
+
+
+def cmd_doctor() -> int:
+    """Runs a complete system health and capability diagnostic."""
+    ensure_dpi_aware()
+    attach_input_desktop()
+
+    print("=" * 65)
+    print(" EXTRA SYSTEM HEALTH & HARDWARE DIAGNOSTIC (DOCTOR)")
+    print("=" * 65)
+
+    # 1. OS & Architecture
+    print("\n[Operating System & Architecture]")
+    os_name = platform.system()
+    os_release = platform.release()
+    os_version = platform.version()
+    machine = platform.machine()
+    py_ver = sys.version.split()[0]
+    print(f"  Extra Version: {__version__}")
+    print(f"  OS:            {os_name} {os_release} (Build {os_version})")
+    print(f"  Architecture:  {machine}")
+    print(f"  Python:        {py_ver} ({'64-bit' if sys.maxsize > 2**32 else '32-bit'})")
+
+    if os_name != "Windows":
+        print("  [FAIL] Extra requires Windows 10 or Windows 11.")
+        return 1
+    else:
+        print("  [OK] Supported Windows host.")
+
+    # 2. DPI & Security Desktop Attachment
+    print("\n[DPI Geometry & Thread Desktop]")
+    dpi_ok = ensure_dpi_aware()
+    desk_ok = attach_input_desktop()
+    print(f"  DPI Awareness: {'PerMonitorV2 (Active)' if dpi_ok else 'Fallback'}")
+    print(f"  Input Desktop: {'Attached (Default)' if desk_ok else 'Standard'}")
+
+    cursor = get_cursor_position()
+    print(f"  Mouse Cursor:  Physical pixel {cursor}")
+
+    # 3. Displays & Hardware Metrics
+    print("\n[Displays & Multi-Monitor Metrics]")
+    monitors = get_monitors_info()
+    print(f"  Monitors:      {len(monitors)} connected display(s)")
+    for m in monitors:
+        print(f"    - Display {m.index}: {m.width}x{m.height} @ ({m.left}, {m.top}) | DPI: {m.dpi_x}x{m.dpi_y} ({m.scale_factor}x scale) | Primary={m.is_primary}")
+
+    # 4. Perception Engine Latency Benchmark
+    print("\n[Screen Capture Benchmark]")
+    latencies = []
+    for _ in range(3):
+        cap = capture_screen(0)
+        latencies.append(cap.duration_ms)
+    avg_latency = sum(latencies) / len(latencies)
+    print(f"  Capture Runs:  {latencies[0]:.1f}ms, {latencies[1]:.1f}ms, {latencies[2]:.1f}ms (Average: {avg_latency:.1f}ms)")
+    if avg_latency < 100:
+        print(f"  [OK] High-speed screen capture verified ({avg_latency:.1f}ms).")
+    else:
+        print("  [WARN] Screen capture took longer than expected.")
+
+    # 5. Semantic UI Automation v3 Plane
+    print("\n[Microsoft UI Automation Core COM Layer]")
+    try:
+        uia = UIAutomationPlane()
+        t0 = time.perf_counter()
+        elems = uia.inspect_window(interactive_only=True, max_elements=20)
+        t_uia = (time.perf_counter() - t0) * 1000.0
+        print(f"  UIA Status:    Active & Operational ({len(elems)} elements inspected in {t_uia:.1f}ms)")
+        print("  [OK] UIAutomationCore.dll COM interface functional.")
+    except Exception as e:
+        print(f"  [FAIL] UIA Plane Error: {e}")
+
+    # 6. Microsoft Edge & Playwright Fast-Path
+    print("\n[Browser Fast-Path (Microsoft Edge / Playwright)]")
+    edge_path = resolve_executable("edge")
+    if edge_path and os.path.exists(edge_path):
+        print(f"  Edge Binary:   {edge_path}")
+        print("  [OK] Microsoft Edge available for zero-download web fast-path.")
+    else:
+        print("  [WARN] Microsoft Edge not found at standard path.")
+
+    # 7. Active Windows
+    print("\n[Active Desktop Windows]")
+    fg = get_foreground_window()
+    print(f"  Foreground:    {repr(fg.title) if fg else 'None'} ({fg.process_name if fg else ''})")
+    wins = list_windows(visible_only=True)
+    print(f"  Total Windows: {len(wins)} visible top-level windows")
+
+    print("\n" + "=" * 65)
+    print(" DOCTOR DIAGNOSTIC COMPLETE: SYSTEM IS READY FOR EXTRA")
+    print("=" * 65)
+    return 0
+
+
+def cmd_test() -> int:
+    """Runs the full integration test suite."""
+    from extra.test_core_engine import run_tests
+    try:
+        run_tests()
+        return 0
+    except Exception as e:
+        print(f"\n[FAIL] Test suite failed: {e}")
+        return 1
+
+
+def cmd_run(args: argparse.Namespace) -> int:
+    """Starts the Model Context Protocol (MCP) server."""
+    from extra.mcp.server import main
+    main()
+    return 0
+
+
+def cmd_inspect(args: argparse.Namespace) -> int:
+    """Quickly inspects UI Automation controls on the desktop."""
+    ensure_dpi_aware()
+    attach_input_desktop()
+
+    uia = UIAutomationPlane()
+    print(f"Inspecting UI elements (window='{args.window}', max={args.limit})...")
+    hwnd = None
+    if args.window:
+        win = find_window_by_title(args.window)
+        if win:
+            hwnd = win.hwnd
+            print(f"Targeting window [{hwnd}] '{win.title}'")
+
+    elems = uia.inspect_window(hwnd=hwnd, interactive_only=not args.all, max_elements=args.limit)
+    print(f"\nDiscovered {len(elems)} elements:")
+    for el in elems:
+        safe_name = el.name.encode("ascii", "replace").decode("ascii")
+        print(f"  [{el.element_id:2d}] {el.control_type:<12} center={el.center} bbox={el.bounding_box} '{safe_name}'")
+    return 0
+
+
+def cmd_snap(args: argparse.Namespace) -> int:
+    """Captures a screenshot and optionally overlays Set-of-Mark badges."""
+    ensure_dpi_aware()
+    attach_input_desktop()
+
+    cap = capture_screen(monitor_index=args.monitor)
+    output_path = Path(args.output)
+
+    if args.som:
+        uia = UIAutomationPlane()
+        elems = uia.inspect_window(interactive_only=True, max_elements=50)
+        annotator = SetOfMarkAnnotator()
+        ann_img, mark_map = annotator.annotate(cap.image, elems)
+        ann_img.save(output_path)
+        print(f"Saved Set-of-Mark annotated screenshot ({len(mark_map)} badges) to {output_path.resolve()}")
+    else:
+        cap.image.save(output_path)
+        print(f"Saved screenshot ({cap.width}x{cap.height}, {cap.duration_ms}ms) to {output_path.resolve()}")
+    return 0
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="extra",
+        description="Extra — Flashless Windows 10/11 Computer-Use Engine & MCP Server",
+    )
+    parser.add_argument(
+        "--version", "-v", action="version", version=f"extra {__version__}"
+    )
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # doctor
+    subparsers.add_parser("doctor", help="Run system health and capability diagnostics")
+
+    # test
+    subparsers.add_parser("test", help="Run integration test suite")
+
+    # run
+    run_parser = subparsers.add_parser("run", help="Start the MCP Server on stdio")
+    run_parser.add_argument("--transport", default="stdio", choices=["stdio"], help="Transport mode")
+
+    # inspect
+    inspect_parser = subparsers.add_parser("inspect", help="Inspect Windows UI Automation tree")
+    inspect_parser.add_argument("--window", "-w", default=None, help="Filter by window title")
+    inspect_parser.add_argument("--limit", "-n", type=int, default=30, help="Max elements to display")
+    inspect_parser.add_argument("--all", "-a", action="store_true", help="Include non-interactive containers")
+
+    # snap
+    snap_parser = subparsers.add_parser("snap", help="Capture a desktop screenshot")
+    snap_parser.add_argument("--output", "-o", default="screenshot.png", help="Output file path")
+    snap_parser.add_argument("--monitor", "-m", type=int, default=0, help="Monitor index")
+    snap_parser.add_argument("--som", action="store_true", help="Overlay Set-of-Mark numbered badges")
+
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        sys.exit(0)
+
+    if args.command == "doctor":
+        sys.exit(cmd_doctor())
+    elif args.command == "test":
+        sys.exit(cmd_test())
+    elif args.command == "run":
+        sys.exit(cmd_run(args))
+    elif args.command == "inspect":
+        sys.exit(cmd_inspect(args))
+    elif args.command == "snap":
+        sys.exit(cmd_snap(args))
+
+
+if __name__ == "__main__":
+    main()

extra/core/__init__.py

@@ -0,0 +1,113 @@
+"""
+Project Extra — Core Engine Package
+High-performance Windows 10/11 desktop automation, dual-plane perception,
+zero-latency SendInput injection, and closed-loop stall breaker.
+"""
+
+from extra.core.capture import (
+    CaptureResult,
+    ScreenCaptureEngine,
+    capture_roi,
+    capture_screen,
+    get_capture_engine,
+    image_to_base64,
+)
+from extra.core.focus import (
+    WindowInfo,
+    find_window_by_title,
+    find_windows_by_process,
+    force_activate_window,
+    get_foreground_window,
+    get_window_info,
+    list_windows,
+)
+from extra.core.geometry import (
+    MonitorInfo,
+    attach_input_desktop,
+    clamp_coordinates,
+    denormalize_bbox,
+    denormalize_coordinates,
+    ensure_dpi_aware,
+    get_bbox_center,
+    get_cursor_position,
+    get_monitors_info,
+    get_primary_monitor,
+    get_virtual_screen_bounds,
+    normalize_bbox,
+    normalize_coordinates,
+)
+from extra.core.input_engine import (
+    atomic_clipboard_paste,
+    instant_type,
+    mouse_click,
+    mouse_double_click,
+    mouse_down,
+    mouse_drag,
+    mouse_move,
+    mouse_scroll,
+    mouse_up,
+    send_hotkey,
+)
+from extra.core.stall_breaker import (
+    ActionOutcome,
+    EmergencyAbortError,
+    StallBreaker,
+    StallStatus,
+)
+from extra.core.uia_plane import (
+    SetOfMarkAnnotator,
+    UIAutomationPlane,
+    UIElement,
+)
+
+__all__ = [
+    # Geometry
+    "ensure_dpi_aware",
+    "attach_input_desktop",
+    "get_monitors_info",
+    "get_primary_monitor",
+    "get_virtual_screen_bounds",
+    "normalize_coordinates",
+    "denormalize_coordinates",
+    "normalize_bbox",
+    "denormalize_bbox",
+    "get_bbox_center",
+    "clamp_coordinates",
+    "get_cursor_position",
+    "MonitorInfo",
+    # Capture
+    "ScreenCaptureEngine",
+    "get_capture_engine",
+    "capture_screen",
+    "capture_roi",
+    "image_to_base64",
+    "CaptureResult",
+    # Input
+    "instant_type",
+    "atomic_clipboard_paste",
+    "mouse_move",
+    "mouse_down",
+    "mouse_up",
+    "mouse_click",
+    "mouse_double_click",
+    "mouse_drag",
+    "mouse_scroll",
+    "send_hotkey",
+    # Focus
+    "WindowInfo",
+    "get_window_info",
+    "get_foreground_window",
+    "list_windows",
+    "find_window_by_title",
+    "find_windows_by_process",
+    "force_activate_window",
+    # UIA Plane
+    "UIAutomationPlane",
+    "SetOfMarkAnnotator",
+    "UIElement",
+    # Stall Breaker
+    "StallBreaker",
+    "StallStatus",
+    "ActionOutcome",
+    "EmergencyAbortError",
+]

extra/core/capture.py

@@ -0,0 +1,187 @@
+"""
+Project Extra — Screen Capture Engine
+High-speed screen buffer capture via hardware-accelerated and GDI pipelines,
+supporting sub-30ms frame grabs, multi-monitor selection, ROI cropping,
+and base64 encoding.
+"""
+
+from __future__ import annotations
+
+import base64
+import concurrent.futures
+import io
+import time
+from dataclasses import dataclass
+from typing import Optional, Tuple
+
+import mss
+from PIL import Image
+
+from extra.core.geometry import (
+    attach_input_desktop,
+    clamp_coordinates,
+    ensure_dpi_aware,
+)
+
+
+@dataclass
+class CaptureResult:
+    """Encapsulates captured screen image with performance metadata."""
+    image: Image.Image
+    duration_ms: float
+    monitor_index: int
+    width: int
+    height: int
+    crop_box: Optional[Tuple[int, int, int, int]] = None
+
+    def to_base64(self, format: str = "JPEG", quality: int = 85) -> str:
+        """Encodes captured PIL Image into base64 string."""
+        return image_to_base64(self.image, format=format, quality=quality)
+
+
+class ScreenCaptureEngine:
+    """
+    High-performance Windows screen capture engine.
+    Uses an isolated background capture worker thread to avoid Win32 desktop
+    security context contamination from COM apartments or UI message loops.
+    """
+
+    def __init__(self) -> None:
+        ensure_dpi_aware()
+        self._executor = concurrent.futures.ThreadPoolExecutor(
+            max_workers=1, thread_name_prefix="ExtraCaptureWorker"
+        )
+        self._sct: Optional[mss.MSS] = None
+
+    def _worker_grab(
+        self,
+        monitor_index: int,
+        crop_box: Optional[Tuple[int, int, int, int]],
+    ) -> Tuple[bytes, Tuple[int, int], str]:
+        # Always attach the clean worker thread to the active input desktop
+        attach_input_desktop()
+
+        if self._sct is None:
+            self._sct = mss.MSS()
+
+        # mss.monitors[0] is all monitors combined, mss.monitors[1] is monitor 0
+        mss_idx = monitor_index + 1
+        if mss_idx < 1 or mss_idx >= len(self._sct.monitors):
+            mss_idx = 1  # Fallback to primary monitor
+
+        mon_info = self._sct.monitors[mss_idx]
+
+        if crop_box:
+            left, top, right, bottom = crop_box
+            left, top = clamp_coordinates(left, top, monitor_index)
+            right, bottom = clamp_coordinates(right, bottom, monitor_index)
+
+            if right <= left or bottom <= top:
+                region = mon_info
+            else:
+                region = {
+                    "left": left,
+                    "top": top,
+                    "width": right - left,
+                    "height": bottom - top,
+                }
+        else:
+            region = mon_info
+
+        raw_frame = self._sct.grab(region)
+        return (bytes(raw_frame.bgra), raw_frame.size, "BGRX")
+
+    def capture(
+        self,
+        monitor_index: int = 0,
+        crop_box: Optional[Tuple[int, int, int, int]] = None,
+    ) -> CaptureResult:
+        """
+        Captures a screen frame from the specified monitor.
+        
+        Args:
+            monitor_index: 0-based index of monitor (0 = primary).
+            crop_box: Optional (left, top, right, bottom) physical pixel bounding box.
+            
+        Returns:
+            CaptureResult with PIL Image and latency stats.
+        """
+        ensure_dpi_aware()
+        t_start = time.perf_counter()
+
+        future = self._executor.submit(self._worker_grab, monitor_index, crop_box)
+        bgra_bytes, size, raw_mode = future.result()
+
+        img = Image.frombytes("RGB", size, bgra_bytes, "raw", raw_mode)
+        duration_ms = (time.perf_counter() - t_start) * 1000.0
+
+        return CaptureResult(
+            image=img,
+            duration_ms=round(duration_ms, 2),
+            monitor_index=monitor_index,
+            width=img.width,
+            height=img.height,
+            crop_box=crop_box,
+        )
+
+    def close(self) -> None:
+        """Releases capture worker thread and resources."""
+        def _cleanup():
+            if self._sct is not None:
+                try:
+                    self._sct.close()
+                except Exception:
+                    pass
+                self._sct = None
+
+        try:
+            self._executor.submit(_cleanup).result(timeout=1.0)
+        except Exception:
+            pass
+        self._executor.shutdown(wait=False)
+
+    def __enter__(self) -> "ScreenCaptureEngine":
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> None:
+        self.close()
+
+
+_default_engine: Optional[ScreenCaptureEngine] = None
+
+
+def get_capture_engine() -> ScreenCaptureEngine:
+    """Returns or initializes the shared ScreenCaptureEngine."""
+    global _default_engine
+    if _default_engine is None:
+        _default_engine = ScreenCaptureEngine()
+    return _default_engine
+
+
+def capture_screen(
+    monitor_index: int = 0,
+    crop_box: Optional[Tuple[int, int, int, int]] = None,
+) -> CaptureResult:
+    """Convenience function to capture screen via default engine."""
+    return get_capture_engine().capture(monitor_index=monitor_index, crop_box=crop_box)
+
+
+def capture_roi(
+    left: int, top: int, right: int, bottom: int, monitor_index: int = 0
+) -> CaptureResult:
+    """Convenience function to capture a specific region of interest."""
+    return capture_screen(monitor_index=monitor_index, crop_box=(left, top, right, bottom))
+
+
+def image_to_base64(image: Image.Image, format: str = "JPEG", quality: int = 85) -> str:
+    """Converts a PIL Image to a base64 encoded string."""
+    buffer = io.BytesIO()
+    if format.upper() == "JPEG":
+        if image.mode != "RGB":
+            image = image.convert("RGB")
+        image.save(buffer, format="JPEG", quality=quality, optimize=True)
+    else:
+        image.save(buffer, format=format)
+
+    encoded = base64.b64encode(buffer.getvalue()).decode("utf-8")
+    return encoded