extra-desktop 0.3.2__tar.gz → 0.3.3__tar.gz

{extra_desktop-0.3.2/extra_desktop.egg-info → extra_desktop-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: extra-desktop
-Version: 0.3.2
+Version: 0.3.3
 Summary: Flashless Linux, Windows & macOS Computer-Use Engine & Model Context Protocol (MCP) Server
 Author-email: AIYantra Engineering Team <engineering@yantraos.com>
 License-Expression: MIT
@@ -11,14 +11,15 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: NOTICE
-Requires-Dist: mcp>=1.0.0
+Requires-Dist: mcp<2,>=1.0.0
+Requires-Dist: websockets>=13.0
 Requires-Dist: playwright>=1.47.0
 Requires-Dist: pillow>=10.4.0
 Requires-Dist: beautifulsoup4>=4.12.3
 Requires-Dist: imagehash>=4.3.1
 Requires-Dist: numpy>=1.26.0
 Requires-Dist: psutil>=6.0.0
-Requires-Dist: kuzu>=0.11.0
+Requires-Dist: kuzu>=0.11.0; python_version < "3.14"
 Requires-Dist: fastembed>=0.8.0
 Requires-Dist: onnxruntime>=1.19.0
 Requires-Dist: pywin32>=306; sys_platform == "win32"
@@ -32,12 +33,19 @@ Requires-Dist: pyobjc-framework-ScreenCaptureKit>=10.0; sys_platform == "darwin"
 Requires-Dist: evdev>=1.7.0; sys_platform == "linux"
 Requires-Dist: mss>=9.0.1; sys_platform == "linux"
 Requires-Dist: jeepney>=0.8.0; sys_platform == "linux"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.24.0; extra == "dev"
+Requires-Dist: ruff>=0.6.0; extra == "dev"
+Requires-Dist: mypy>=1.11.0; extra == "dev"
+Provides-Extra: benchmark
+Requires-Dist: pytest-benchmark>=4.0.0; extra == "benchmark"
 Dynamic: license-file
 
 <div align="center">
 
 <a href="https://extra.yantraos.com">
-  <img src="https://raw.githubusercontent.com/AIYantra/extra/main/assets/logo.png" alt="Extra — Get extra from your AI" width="420" />
+  <img src="assets/logo.png" alt="Extra — Get extra from your AI" width="420" />
 </a>
 
 # Extra
@@ -54,7 +62,7 @@ Dynamic: license-file
 [![Ecosystem: yantraOS](https://img.shields.io/badge/Ecosystem-yantraOS-8A2BE2.svg)](https://yantraos.com)
 [![Status: Open Source](https://img.shields.io/badge/Open%20Source-%E2%99%A5-emerald.svg)](https://github.com/AIYantra/extra)
 
-[Website](https://extra.yantraos.com) • [Quickstart](#-quickstart-1-minute) • [Linux Guide](README_LINUX.md) • [macOS Guide](README_MACOS.md) • [Architecture](ARCHITECTURE.md) • [License](LICENSE)
+[Website](https://extra.yantraos.com) • [Quickstart](#-quickstart-1-minute) • [Linux Guide](README_LINUX.md) • [macOS Guide](README_MACOS.md) • [Architecture](docs/ARCHITECTURE.md) • [License](LICENSE)
 
 <br/>
 <br/>
@@ -278,14 +286,14 @@ extra/
 ├── cli.py                  # CLI runner (extra doctor, test, indicators, run, inspect, snap)
 ├── install.ps1             # 1-line PowerShell installer (Windows)
 ├── install.sh              # 1-line bash installer (macOS)
-├── pyproject.toml          # PyPI package manifest (extra-desktop v0.3.1)
-├── package.json            # npm package manifest (@yantraos/extra-desktop v0.3.1)
+├── pyproject.toml          # PyPI package manifest (extra-desktop v0.3.3)
+├── package.json            # npm package manifest (@yantraos/extra-desktop v0.3.3)
 ├── server.json             # Official MCP registry manifest
 ├── glama.json              # Glama MCP ecosystem manifest
 ├── smithery.yaml           # Smithery 1-click deployment manifest
 ├── requirements.txt        # Enterprise-audited dependency manifest
 ├── STARTER_PROMPT.md       # Master AI system prompt & setup directive
-├── ARCHITECTURE.md         # Full System Architecture Specification
+├── docs/                   # Full System Architecture & Specifications
 └── RELEASE_NOTES.md        # Comprehensive version release notes
 ```
 

{extra_desktop-0.3.2 → extra_desktop-0.3.3}/README.md

@@ -1,7 +1,7 @@
 <div align="center">
 
 <a href="https://extra.yantraos.com">
-  <img src="https://raw.githubusercontent.com/AIYantra/extra/main/assets/logo.png" alt="Extra — Get extra from your AI" width="420" />
+  <img src="assets/logo.png" alt="Extra — Get extra from your AI" width="420" />
 </a>
 
 # Extra
@@ -18,7 +18,7 @@
 [![Ecosystem: yantraOS](https://img.shields.io/badge/Ecosystem-yantraOS-8A2BE2.svg)](https://yantraos.com)
 [![Status: Open Source](https://img.shields.io/badge/Open%20Source-%E2%99%A5-emerald.svg)](https://github.com/AIYantra/extra)
 
-[Website](https://extra.yantraos.com) • [Quickstart](#-quickstart-1-minute) • [Linux Guide](README_LINUX.md) • [macOS Guide](README_MACOS.md) • [Architecture](ARCHITECTURE.md) • [License](LICENSE)
+[Website](https://extra.yantraos.com) • [Quickstart](#-quickstart-1-minute) • [Linux Guide](README_LINUX.md) • [macOS Guide](README_MACOS.md) • [Architecture](docs/ARCHITECTURE.md) • [License](LICENSE)
 
 <br/>
 <br/>
@@ -242,14 +242,14 @@ extra/
 ├── cli.py                  # CLI runner (extra doctor, test, indicators, run, inspect, snap)
 ├── install.ps1             # 1-line PowerShell installer (Windows)
 ├── install.sh              # 1-line bash installer (macOS)
-├── pyproject.toml          # PyPI package manifest (extra-desktop v0.3.1)
-├── package.json            # npm package manifest (@yantraos/extra-desktop v0.3.1)
+├── pyproject.toml          # PyPI package manifest (extra-desktop v0.3.3)
+├── package.json            # npm package manifest (@yantraos/extra-desktop v0.3.3)
 ├── server.json             # Official MCP registry manifest
 ├── glama.json              # Glama MCP ecosystem manifest
 ├── smithery.yaml           # Smithery 1-click deployment manifest
 ├── requirements.txt        # Enterprise-audited dependency manifest
 ├── STARTER_PROMPT.md       # Master AI system prompt & setup directive
-├── ARCHITECTURE.md         # Full System Architecture Specification
+├── docs/                   # Full System Architecture & Specifications
 └── RELEASE_NOTES.md        # Comprehensive version release notes
 ```
 

{extra_desktop-0.3.2 → extra_desktop-0.3.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "extra-desktop"
-version = "0.3.2"
+version = "0.3.3"
 description = "Flashless Linux, Windows & macOS Computer-Use Engine & Model Context Protocol (MCP) Server"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -13,14 +13,15 @@ authors = [
     { name = "AIYantra Engineering Team", email = "engineering@yantraos.com" }
 ]
 dependencies = [
-    "mcp>=1.0.0",
+    "mcp>=1.0.0,<2",
+    "websockets>=13.0",
     "playwright>=1.47.0",
     "pillow>=10.4.0",
     "beautifulsoup4>=4.12.3",
     "imagehash>=4.3.1",
     "numpy>=1.26.0",
     "psutil>=6.0.0",
-    "kuzu>=0.11.0",
+    "kuzu>=0.11.0; python_version < '3.14'",
     "fastembed>=0.8.0",
     "onnxruntime>=1.19.0",
     "pywin32>=306; sys_platform == 'win32'",
@@ -36,6 +37,17 @@ dependencies = [
     "jeepney>=0.8.0; sys_platform == 'linux'",
 ]
 
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-asyncio>=0.24.0",
+    "ruff>=0.6.0",
+    "mypy>=1.11.0",
+]
+benchmark = [
+    "pytest-benchmark>=4.0.0",
+]
+
 [project.scripts]
 extra = "extra.cli:main"
 extra-desktop = "extra.cli:main"
@@ -45,17 +57,20 @@ Homepage = "https://extra.yantraos.com"
 Repository = "https://github.com/AIYantra/extra"
 Documentation = "https://extra.yantraos.com/docs"
 
-[tool.setuptools.package-dir]
-"extra" = "."
-"extra.core" = "core"
-"extra.core.platform" = "core/platform"
-"extra.core.platform.windows" = "core/platform/windows"
-"extra.core.platform.macos" = "core/platform/macos"
-"extra.core.platform.linux" = "core/platform/linux"
-"extra.fastpath" = "fastpath"
-"extra.mcp" = "mcp"
+[tool.setuptools.packages.find]
+where = ["src"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+pythonpath = ["src"]
 addopts = "-p no:faulthandler"
+markers = [
+    "unit: Fast isolated unit tests with zero system dependencies",
+    "integration: Subsystem integration tests",
+    "windows: Tests requiring Windows Win32 / UIA subsystems",
+    "linux: Tests requiring Linux X11 / Wayland / AT-SPI2 subsystems",
+    "macos: Tests requiring macOS Quartz / Accessibility subsystems",
+    "benchmark: Performance profiling and stress benchmark suites",
+    "security: Security audit and vulnerability verification tests",
+]
 

{extra_desktop-0.3.2 → extra_desktop-0.3.3/src/extra}/__init__.py

@@ -2,4 +2,4 @@
 Project Extra — Flashless Linux, Windows & macOS Computer-Use Engine & MCP Server.
 """
 
-__version__ = "0.3.2"
+__version__ = "0.3.3"

{extra_desktop-0.3.2 → extra_desktop-0.3.3/src/extra}/cli.py

@@ -22,7 +22,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 try:
     from extra import __version__
 except Exception:
-    __version__ = "0.3.1"
+    __version__ = "0.3.3"
 
 from extra.core.capture import capture_screen
 from extra.core.focus import (
@@ -359,8 +359,45 @@ def cmd_run(args: argparse.Namespace) -> int:
         os.environ["EXTRA_ISOLATED_DESKTOP"] = "1"
     if getattr(args, "pip", False):
         os.environ["EXTRA_PIP_PREVIEW"] = "1"
+
+    transport = getattr(args, "transport", "stdio")
+    host = getattr(args, "host", "127.0.0.1")
+    port = getattr(args, "port", 8000)
+    auth_token = getattr(args, "auth_token", None)
+    allowed_hosts = getattr(args, "allowed_hosts", None)
+    tunnel = getattr(args, "tunnel", False)
+    if allowed_hosts and isinstance(allowed_hosts, str):
+        allowed_hosts = [h.strip() for h in allowed_hosts.split(",") if h.strip()]
+
     from extra.mcp.server import main
-    main()
+    main(transport=transport, host=host, port=port, auth_token=auth_token, allowed_hosts=allowed_hosts, tunnel=tunnel)
+    return 0
+
+
+def cmd_serve(args: argparse.Namespace) -> int:
+    """Starts the Extra MCP server over SSE HTTP transport (for ChatGPT plugins & web agents)."""
+    setattr(args, "transport", "sse")
+    return cmd_run(args)
+
+
+def cmd_connect(args: argparse.Namespace) -> int:
+    """Connects this desktop to the Extra Cloud Hub for ChatGPT automation."""
+    from extra.relay import start_relay, DEFAULT_RELAY_URL
+    hub_url = getattr(args, "hub", None) or os.environ.get("EXTRA_HUB_URL") or DEFAULT_RELAY_URL
+    token = getattr(args, "token", None)
+    if not token or token == "anonymous":
+        token = os.environ.get("EXTRA_KEY") or os.environ.get("EXTRA_RELAY_KEY")
+    if not token or token == "anonymous":
+        print("[!] Error: Workstation link key is required to connect to the Extra Cloud Hub.")
+        print("    Get your key from your dashboard at: https://extra.yantraos.com/dashboard")
+        print("    Then run:")
+        print("      extra connect --key <your_key>")
+        return 1
+    start_relay(
+        hub_url=hub_url,
+        token=token,
+        device_name=getattr(args, "name", None),
+    )
     return 0
 
 
@@ -836,7 +873,7 @@ def cmd_update(args: argparse.Namespace) -> int:
         if pyproject_file.exists():
             try:
                 subprocess.run(
-                    [sys.executable, "-m", "pip", "install", "--quiet", "-e", str(repo_dir)],
+                    [sys.executable, "-m", "pip", "install", "--quiet", "--no-deps", "-e", str(repo_dir)],
                     check=True,
                 )
                 print("  [OK] Package editable link verified.")
@@ -845,6 +882,32 @@ def cmd_update(args: argparse.Namespace) -> int:
     else:
         print("\n[Dependency Refresh] Skipped (--skip-deps specified).")
 
+    # Phase 2b: Refresh CLI executable wrappers with updated PYTHONPATH
+    try:
+        bin_dir = Path.home() / ".extra" / "bin"
+        if bin_dir.exists():
+            if platform.system() == "Windows":
+                cmd_wrapper = bin_dir / "extra.cmd"
+                cmd_content = (
+                    f"@echo off\n"
+                    f"set PYTHONPATH={repo_dir}\\src;{repo_dir};%PYTHONPATH%\n"
+                    f'"{sys.executable}" -m extra.cli %*\n'
+                )
+                cmd_wrapper.write_text(cmd_content, encoding="ascii")
+                print("  [OK] Updated Windows CLI wrapper in ~/.extra/bin/extra.cmd.")
+            else:
+                sh_wrapper = bin_dir / "extra"
+                sh_content = (
+                    f"#!/usr/bin/env bash\n"
+                    f'export PYTHONPATH="{repo_dir}/src:{repo_dir}:$PYTHONPATH"\n'
+                    f'exec "{sys.executable}" -m extra.cli "$@"\n'
+                )
+                sh_wrapper.write_text(sh_content, encoding="utf-8")
+                sh_wrapper.chmod(0o755)
+                print("  [OK] Updated Unix CLI wrapper in ~/.extra/bin/extra.")
+    except Exception as ex:
+        print(f"  [WARN] Could not refresh CLI wrapper: {ex}")
+
     # Phase 3: AI Rules & Skills Synchronization
     sync_ai_rules_and_skills(repo_dir)
 
@@ -1035,7 +1098,8 @@ def check_first_run_consent() -> None:
         save_config(cfg)
 
 
-def main() -> None:
+def build_parser() -> argparse.ArgumentParser:
+    """Builds and returns the top-level ArgumentParser for Extra CLI."""
     parser = argparse.ArgumentParser(
         prog="extra",
         description="Extra — Flashless macOS, Windows & Linux Computer-Use Engine & MCP Server",
@@ -1069,11 +1133,32 @@ def main() -> None:
     update_parser.add_argument("--force", action="store_true", help="Force update and reset local changes to upstream")
 
     # run
-    run_parser = subparsers.add_parser("run", help="Start the MCP Server on stdio")
-    run_parser.add_argument("--transport", default="stdio", choices=["stdio"], help="Transport mode")
+    run_parser = subparsers.add_parser("run", help="Start the MCP Server on stdio or SSE")
+    run_parser.add_argument("--transport", default="stdio", choices=["stdio", "sse", "streamable-http"], help="Transport mode (default: stdio)")
+    run_parser.add_argument("--host", default="127.0.0.1", help="Host IP to bind to when running SSE or HTTP transport (default: 127.0.0.1)")
+    run_parser.add_argument("--port", "-p", type=int, default=8000, help="Port to bind to when running SSE or HTTP transport (default: 8000)")
+    run_parser.add_argument("--auth-token", "--auth", default=None, help="Bearer authorization token to secure SSE transport")
+    run_parser.add_argument("--allowed-hosts", default=None, help="Comma-separated allowed host header values for DNS rebinding protection")
+    run_parser.add_argument("--tunnel", "-t", action="store_true", help="Launch secure Cloudflare Quick Tunnel (public HTTPS) for pairing with ChatGPT or remote agents")
     run_parser.add_argument("--isolate", action="store_true", help="Confine automation to isolated child session / virtual desktop")
     run_parser.add_argument("--pip", action="store_true", help="Launch live Picture-in-Picture floating preview window")
 
+    # serve (ChatGPT Plugin & Remote Web Agent bridge)
+    serve_parser = subparsers.add_parser("serve", help="Start the Extra MCP Server over SSE HTTP transport (for ChatGPT plugins & web agents)")
+    serve_parser.add_argument("--host", default="127.0.0.1", help="Host IP to bind to (default: 127.0.0.1; use 0.0.0.0 for public/tunnel)")
+    serve_parser.add_argument("--port", "-p", type=int, default=8000, help="Port to bind to (default: 8000)")
+    serve_parser.add_argument("--auth-token", "--auth", default=None, help="Bearer authorization token to secure SSE transport")
+    serve_parser.add_argument("--allowed-hosts", default=None, help="Comma-separated allowed host headers (e.g. *.trycloudflare.com)")
+    serve_parser.add_argument("--tunnel", "-t", action="store_true", help="Launch secure Cloudflare Quick Tunnel (public HTTPS) for pairing with ChatGPT or remote agents")
+    serve_parser.add_argument("--isolate", action="store_true", help="Confine automation to isolated child session / virtual desktop")
+    serve_parser.add_argument("--pip", action="store_true", help="Launch live Picture-in-Picture floating preview window")
+
+    # connect (Desktop Relay Client for ChatGPT & Cloud Hub)
+    connect_parser = subparsers.add_parser("connect", help="Connect this desktop to the Extra Cloud Hub for ChatGPT automation")
+    connect_parser.add_argument("--hub", default=None, help="WebSocket Relay Hub URL (default: official extra.yantraos.com relay)")
+    connect_parser.add_argument("--key", "--token", dest="token", default="anonymous", help="Setup key or authentication token from extra.yantraos.com account")
+    connect_parser.add_argument("--name", default=None, help="Custom friendly device name for this workstation")
+
     # pip (Picture-in-Picture floating preview window)
     pip_parser = subparsers.add_parser("pip", help="Control hardware-accelerated Picture-in-Picture floating preview window")
     pip_parser.add_argument("action", nargs="?", default="demo", choices=["demo", "start", "stop"], help="Action: demo (live 3s preview), start, stop")
@@ -1154,6 +1239,11 @@ def main() -> None:
     audit_parser.add_argument("--record", "-r", action="store_true", help="Record audited mission metrics into local impact_ledger.json")
     audit_parser.add_argument("--json", action="store_true", help="Output audit results in raw JSON")
 
+    return parser
+
+
+def main() -> None:
+    parser = build_parser()
     args = parser.parse_args()
 
     if not args.command:
@@ -1195,6 +1285,10 @@ def main() -> None:
         sys.exit(cmd_uninstall(args))
     elif args.command == "run":
         sys.exit(cmd_run(args))
+    elif args.command == "serve":
+        sys.exit(cmd_serve(args))
+    elif args.command == "connect":
+        sys.exit(cmd_connect(args))
     elif args.command == "pip":
         sys.exit(cmd_pip(args))
     elif args.command == "inspect":

{extra_desktop-0.3.2 → extra_desktop-0.3.3/src/extra}/core/distiller.py

@@ -242,19 +242,7 @@ class UIDistiller:
             except Exception as ex:
                 logger.debug("Edge Vision fallback skipped: %s", ex)
 
-        # Truncate to max_elements limit
-        final_elements = filtered_elements[:max_elements]
-
-        # 4. Spatially Sort (Top-to-Bottom, Left-to-Right by rows)
-        sorted_elements = self._spatial_sort(final_elements)
-
-        # Count interactives
-        interactive_count = sum(
-            1 for el in sorted_elements
-            if el.control_type in ACTION_CONTROL_TYPES or el.control_type in INPUT_CONTROL_TYPES
-        )
-
-        # 4.5 Discover High-Contrast Work Surface / Canvas Viewport (< 5ms)
+        # 4. Discover High-Contrast Work Surface / Custom Canvas Viewport (< 5ms)
         work_surface_dict = None
         if target_hwnd:
             try:
@@ -264,9 +252,51 @@ class UIDistiller:
                 ws = self._vision_parser.detect_work_surface_in_window(target_hwnd)
                 if ws:
                     work_surface_dict = ws.to_dict()
+
+                    # Universal Hybrid Canvas Vision:
+                    # Detect opaque, custom-rendered surfaces (e.g. Qt/WebGL/DirectX canvas viewports,
+                    # chat lists, artboards) where accessibility provides zero/minimal child elements.
+                    ws_left, ws_top, ws_right, ws_bottom = ws.left, ws.top, ws.right, ws.bottom
+                    elements_inside_ws = [
+                        el for el in filtered_elements
+                        if ws_left <= el.center[0] <= ws_right and ws_top <= el.center[1] <= ws_bottom
+                    ]
+
+                    # If the work surface spans a major area (>= 160x160 px) but has <= 2 accessibility controls inside it:
+                    if ws.width >= 160 and ws.height >= 160 and len(elements_inside_ws) <= 2:
+                        try:
+                            canvas_crop = (ws_left, ws_top, ws_right, ws_bottom)
+                            vision_canvas_elements = self._vision_parser.parse_window(
+                                target_hwnd, max_elements=max_elements, crop_box=canvas_crop
+                            )
+                            if vision_canvas_elements:
+                                existing_names = {
+                                    (el.name or "").lower().strip()
+                                    for el in filtered_elements if el.name
+                                }
+                                for vel in vision_canvas_elements:
+                                    vel_name = (vel.name or "").strip().lower()
+                                    if not vel_name or vel_name not in existing_names:
+                                        filtered_elements.append(vel)
+                                        if vel_name:
+                                            existing_names.add(vel_name)
+                        except Exception as c_ex:
+                            logger.debug("Canvas vision parsing skipped: %s", c_ex)
             except Exception as ex:
                 logger.debug("Work surface viewport detection skipped: %s", ex)
 
+        # Truncate to max_elements limit
+        final_elements = filtered_elements[:max_elements]
+
+        # 4.5 Spatially Sort (Top-to-Bottom, Left-to-Right by rows)
+        sorted_elements = self._spatial_sort(final_elements)
+
+        # Count interactives
+        interactive_count = sum(
+            1 for el in sorted_elements
+            if el.control_type in ACTION_CONTROL_TYPES or el.control_type in INPUT_CONTROL_TYPES
+        )
+
         # 5. Format Content
         fmt_clean = format.lower().strip()
         if fmt_clean == "compact":

{extra_desktop-0.3.2 → extra_desktop-0.3.3/src/extra}/core/platform/linux/shell.py

@@ -400,6 +400,7 @@ BUILTIN_APP_FALLBACKS: Dict[str, Dict[str, Any]] = {
     },
     "blender": {"candidates": ["blender"], "type": "bin", "proc": "blender"},
     "vlc": {"candidates": ["vlc"], "type": "bin", "proc": "vlc"},
+    "canva": {"candidates": ["canva"], "type": "bin", "proc": "canva"},
 }
 
 # Compatibility dictionary mapping directly to target names

{extra_desktop-0.3.2 → extra_desktop-0.3.3/src/extra}/core/platform/macos/shell.py

@@ -41,6 +41,9 @@ BUILTIN_MAC_APP_REGISTRY: Dict[str, Dict[str, str]] = {
     "photos": {"target": "Photos", "bundle": "/System/Applications/Photos.app", "proc": "Photos", "type": "app"},
     "code": {"target": "Visual Studio Code", "bundle": "/Applications/Visual Studio Code.app", "proc": "Code", "type": "app"},
     "vscode": {"target": "Visual Studio Code", "bundle": "/Applications/Visual Studio Code.app", "proc": "Code", "type": "app"},
+    "vlc": {"target": "VLC", "bundle": "/Applications/VLC.app", "proc": "VLC", "type": "app"},
+    "blender": {"target": "Blender", "bundle": "/Applications/Blender.app", "proc": "Blender", "type": "app"},
+    "canva": {"target": "Canva", "bundle": "/Applications/Canva.app", "proc": "Canva", "type": "app"},
 }
 
 MAC_APP_REGISTRY: Dict[str, Dict[str, str]] = dict(BUILTIN_MAC_APP_REGISTRY)

{extra_desktop-0.3.2 → extra_desktop-0.3.3/src/extra}/core/platform/windows/input_engine.py

@@ -211,52 +211,71 @@ def _send_inputs(inputs: List[INPUT]) -> int:
     return int(user32.SendInput(n, arr, ctypes.sizeof(INPUT)))
 
 
+def _release_stuck_modifiers() -> None:
+    """Releases any lingering modifier keys (Ctrl, Alt, Shift, Win) to avoid keystroke corruption."""
+    mods = (
+        win32con.VK_CONTROL, win32con.VK_LCONTROL, win32con.VK_RCONTROL,
+        win32con.VK_MENU, win32con.VK_LMENU, win32con.VK_RMENU,
+        win32con.VK_SHIFT, win32con.VK_LSHIFT, win32con.VK_RSHIFT,
+        win32con.VK_LWIN, win32con.VK_RWIN,
+    )
+    for vk in mods:
+        if user32.GetAsyncKeyState(vk) & 0x8000:
+            ku = INPUT(type=INPUT_KEYBOARD)
+            ku.ki = KEYBDINPUT(wVk=vk, wScan=0, dwFlags=KEYEVENTF_KEYUP, time=0, dwExtraInfo=0)
+            _send_inputs([ku])
+
+
 def instant_type(text: str, press_enter: bool = False) -> None:
     """
-    Injects Unicode strings directly into the Windows input queue in sub-5ms.
+    Injects Unicode strings directly into the Windows input queue with atomic hardware delivery.
     Uses Win32 KEYEVENTF_UNICODE with UTF-16 code units (supporting surrogate pairs for emojis).
+    Batches KeyDown and KeyUp pairs atomically to prevent WinUI 3 input buffer desynchronization.
+    Translates newlines to physical Enter keystrokes to ensure 100% fidelity across native editors.
     """
     if not text and not press_enter:
         return
 
-    inputs: List[INPUT] = []
+    _release_stuck_modifiers()
 
-    # Encode to UTF-16-LE words to cleanly capture surrogate pairs (e.g., emojis)
-    utf16_bytes = text.encode("utf-16-le")
-    code_units = [
-        int.from_bytes(utf16_bytes[i : i + 2], "little")
-        for i in range(0, len(utf16_bytes), 2)
-    ]
+    # Normalize line endings to avoid duplicate carriage returns
+    clean_text = text.replace("\r\n", "\n").replace("\r", "\n")
 
-    for unit in code_units:
-        # Key down
-        kd = INPUT(type=INPUT_KEYBOARD)
-        kd.ki = KEYBDINPUT(
-            wVk=0,
-            wScan=unit,
-            dwFlags=KEYEVENTF_UNICODE,
-            time=0,
-            dwExtraInfo=0,
-        )
-        # Key up
-        ku = INPUT(type=INPUT_KEYBOARD)
-        ku.ki = KEYBDINPUT(
-            wVk=0,
-            wScan=unit,
-            dwFlags=KEYEVENTF_UNICODE | KEYEVENTF_KEYUP,
-            time=0,
-            dwExtraInfo=0,
-        )
-        inputs.extend([kd, ku])
+    for ch in clean_text:
+        if ch == "\n":
+            # Translate newline to physical Enter keystroke for native Win32 & WinUI edit controls
+            kd_enter = INPUT(type=INPUT_KEYBOARD)
+            kd_enter.ki = KEYBDINPUT(wVk=win32con.VK_RETURN, wScan=0, dwFlags=0, time=0, dwExtraInfo=0)
+            ku_enter = INPUT(type=INPUT_KEYBOARD)
+            ku_enter.ki = KEYBDINPUT(wVk=win32con.VK_RETURN, wScan=0, dwFlags=KEYEVENTF_KEYUP, time=0, dwExtraInfo=0)
+            _send_inputs([kd_enter, ku_enter])
+            time.sleep(0.003)
+            continue
+
+        # Encode character to UTF-16-LE words to capture surrogate pairs
+        utf16_bytes = ch.encode("utf-16-le")
+        code_units = [
+            int.from_bytes(utf16_bytes[i : i + 2], "little")
+            for i in range(0, len(utf16_bytes), 2)
+        ]
+
+        for unit in code_units:
+            kd = INPUT(type=INPUT_KEYBOARD)
+            kd.ki = KEYBDINPUT(wVk=0, wScan=unit, dwFlags=KEYEVENTF_UNICODE, time=0, dwExtraInfo=0)
+            ku = INPUT(type=INPUT_KEYBOARD)
+            ku.ki = KEYBDINPUT(wVk=0, wScan=unit, dwFlags=KEYEVENTF_UNICODE | KEYEVENTF_KEYUP, time=0, dwExtraInfo=0)
+            # Dispatch KeyDown and KeyUp atomically together in a single OS SendInput batch
+            _send_inputs([kd, ku])
+            time.sleep(0.001)
 
     if press_enter:
+        time.sleep(0.005)
         kd_enter = INPUT(type=INPUT_KEYBOARD)
         kd_enter.ki = KEYBDINPUT(wVk=win32con.VK_RETURN, wScan=0, dwFlags=0, time=0, dwExtraInfo=0)
         ku_enter = INPUT(type=INPUT_KEYBOARD)
         ku_enter.ki = KEYBDINPUT(wVk=win32con.VK_RETURN, wScan=0, dwFlags=KEYEVENTF_KEYUP, time=0, dwExtraInfo=0)
-        inputs.extend([kd_enter, ku_enter])
+        _send_inputs([kd_enter, ku_enter])
 
-    _send_inputs(inputs)
     _notify_indicator("type")
 
 

{extra_desktop-0.3.2 → extra_desktop-0.3.3/src/extra}/core/platform/windows/shell.py

@@ -63,6 +63,9 @@ BUILTIN_APP_REGISTRY: Dict[str, Dict[str, str]] = {
     "mspaint": {"target": "mspaint.exe", "type": "exe", "proc": "mspaint.exe"},
     "photos": {"target": "ms-photos:", "type": "uri", "proc": "Photos.exe"},
     "store": {"target": "ms-windows-store:", "type": "uri", "proc": "WinStore.App.exe"},
+    "vlc": {"target": "vlc.exe", "type": "exe", "proc": "vlc.exe"},
+    "blender": {"target": "blender.exe", "type": "exe", "proc": "blender.exe"},
+    "canva": {"target": "canva.exe", "type": "exe", "proc": "canva.exe"},
 }
 
 APP_REGISTRY: Dict[str, Dict[str, str]] = dict(BUILTIN_APP_REGISTRY)
@@ -511,12 +514,23 @@ def launch_app(
     if wait_for_window:
         deadline = time.perf_counter() + timeout
         expected_proc = APP_REGISTRY.get(clean_app, {}).get("proc", "")
+        EXCLUDED_SHELL_CLASSES = {
+            "progman", "workerw", "shell_traywnd", "shell_secondarytraywnd",
+            "windows.ui.core.corewindow", "dv2controlhost",
+        }
+        EXCLUDED_SHELL_TITLES = {"program manager", "", "taskbar", "start"}
 
         while time.perf_counter() < deadline:
             time.sleep(0.1)
             current_windows = list_windows(visible_only=True)
 
             for win in current_windows:
+                # Universal System Shell Filter: ignore desktop backdrops and taskbars
+                win_class = (getattr(win, "class_name", "") or "").lower()
+                win_title = (getattr(win, "title", "") or "").strip().lower()
+                if win_class in EXCLUDED_SHELL_CLASSES or win_title in EXCLUDED_SHELL_TITLES:
+                    continue
+
                 # Prioritize newly created window handles
                 if win.hwnd not in existing_hwnds:
                     matched_hwnd = win.hwnd

{extra_desktop-0.3.2 → extra_desktop-0.3.3/src/extra}/core/platform/windows/uia_plane.py

@@ -114,6 +114,7 @@ CONTROL_TYPE_NAMES: Dict[int, str] = {
 INTERACTIVE_TYPES: Set[str] = {
     "Button",
     "Edit",
+    "Document",
     "Hyperlink",
     "CheckBox",
     "RadioButton",
@@ -129,6 +130,7 @@ INTERACTIVE_TYPES: Set[str] = {
 UIA_INVOKE_PATTERN_ID = 10000
 UIA_TOGGLE_PATTERN_ID = 10015
 UIA_VALUE_PATTERN_ID = 10002
+UIA_TEXT_PATTERN_ID = 10014
 
 
 class UIAutomationPlane(AbstractAccessibilityPlane):
@@ -240,6 +242,31 @@ class UIAutomationPlane(AbstractAccessibilityPlane):
                         continue
 
                     name = str(elem.CurrentName or "").strip()
+
+                    # Fallback A: ValuePattern (for Edit / ComboBox controls with empty Name)
+                    if not name:
+                        try:
+                            val_pat = elem.GetCurrentPattern(UIA_VALUE_PATTERN_ID)
+                            if val_pat:
+                                val_iface = val_pat.QueryInterface(self._mod.IUIAutomationValuePattern)
+                                name = str(val_iface.CurrentValue or "").strip()
+                        except Exception:
+                            pass
+
+                    # Fallback B: TextPattern (for Terminal buffers, PowerShell console, and rich text documents)
+                    if not name and c_type_name in ("Document", "Text", "Edit"):
+                        try:
+                            txt_pat = elem.GetCurrentPattern(UIA_TEXT_PATTERN_ID)
+                            if txt_pat:
+                                txt_iface = txt_pat.QueryInterface(self._mod.IUIAutomationTextPattern)
+                                doc_range = txt_iface.DocumentRange
+                                full_txt = str(doc_range.GetText(300) or "").strip()
+                                if full_txt:
+                                    lines = [ln.strip() for ln in full_txt.splitlines() if ln.strip()]
+                                    name = " \n ".join(lines[-3:]) if len(lines) > 3 else full_txt
+                        except Exception:
+                            pass
+
                     auto_id = str(elem.CurrentAutomationId or "").strip()
                     class_name = str(elem.CurrentClassName or "").strip()
                     is_enabled = bool(elem.CurrentIsEnabled)