gillm 0.1.3__py3-none-any.whl

gillm/__init__.py

@@ -0,0 +1,20 @@
+"""gillm: GUI Control Plugin with NLP & Intent Contracts.
+
+This package consolidates GUI control logic across the Koru ecosystem,
+integrates NLP2DSL client SDK for parsing intents, and supports
+contract validation via Intract.
+"""
+
+from gillm import capture, focus, injection, intents, nlp_bridge, orchestrator
+from gillm.orchestrator import DriveOrchestrator
+
+__version__ = "0.1.3"
+__all__ = [
+    "DriveOrchestrator",
+    "capture",
+    "focus",
+    "injection",
+    "intents",
+    "nlp_bridge",
+    "orchestrator",
+]

gillm/capture/__init__.py

@@ -0,0 +1,16 @@
+"""Screen capture and vision logic."""
+
+from gillm.capture.mss_backend import (
+    CapturedImage,
+    capture_primary_rgb,
+    capture_primary_rgb_wayland_fallback,
+)
+from gillm.capture.portal_backend import PortalCaptureError, capture_portal_png
+
+__all__ = [
+    "CapturedImage",
+    "PortalCaptureError",
+    "capture_portal_png",
+    "capture_primary_rgb",
+    "capture_primary_rgb_wayland_fallback",
+]

gillm/capture/mss_backend.py

@@ -0,0 +1,130 @@
+"""Cross-platform screenshot capture with mss backend."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+from gillm.capture.portal_backend import PortalCaptureError, capture_portal_png
+
+
+def resolve_scale(override: float | None) -> float:
+    if override is not None:
+        value = override
+    else:
+        raw = os.environ.get("KORU_VISION_SCALE", "0.2").strip() or "0.2"
+        try:
+            value = float(raw)
+        except ValueError:
+            value = 0.2
+    return max(0.05, min(1.0, value))
+
+
+def downscale_rgb_nearest(
+    rgb: bytes, src_w: int, src_h: int, dst_w: int, dst_h: int
+) -> bytes:
+    if dst_w >= src_w and dst_h >= src_h:
+        return rgb
+    src_stride = src_w * 3
+    cols = [(x * src_w // dst_w) * 3 for x in range(dst_w)]
+    out = bytearray(dst_w * dst_h * 3)
+    out_off = 0
+    src_view = memoryview(rgb)
+    for y in range(dst_h):
+        row_base = (y * src_h // dst_h) * src_stride
+        for col_off in cols:
+            src_off = row_base + col_off
+            out[out_off : out_off + 3] = src_view[src_off : src_off + 3]
+            out_off += 3
+    return bytes(out)
+
+
+def rgb_mostly_black(rgb: bytes, *, threshold: float = 0.98) -> bool:
+    if not rgb:
+        return True
+    step = max(1, (len(rgb) // 3) // 8000)
+    black = 0
+    total = 0
+    for offset in range(0, len(rgb) - 2, 3 * step):
+        total += 1
+        if rgb[offset : offset + 3] == b"\x00\x00\x00":
+            black += 1
+    return total > 0 and (black / total) >= threshold
+
+
+@dataclass(frozen=True)
+class CapturedImage:
+    """Portable raw image container."""
+
+    width: int
+    height: int
+    rgb: bytes
+    scale: float
+
+
+def capture_primary_rgb(*, scale: float | None = None) -> CapturedImage:
+    """Capture the primary monitor as RGB pixels, scaled by ``scale``."""
+    import mss
+
+    scale_val = resolve_scale(scale)
+    with mss.mss() as sct:
+        monitor = sct.monitors[1]  # primary monitor
+        img = sct.grab(monitor)
+        src_w = img.width
+        src_h = img.height
+        dst_w = int(src_w * scale_val)
+        dst_h = int(src_h * scale_val)
+        rgb_raw = img.rgb
+        if scale_val < 1.0:
+            rgb_scaled = downscale_rgb_nearest(rgb_raw, src_w, src_h, dst_w, dst_h)
+            return CapturedImage(
+                width=dst_w,
+                height=dst_h,
+                rgb=rgb_scaled,
+                scale=scale_val,
+            )
+        return CapturedImage(
+            width=src_w,
+            height=src_h,
+            rgb=rgb_raw,
+            scale=1.0,
+        )
+
+
+def capture_primary_rgb_wayland_fallback(*, scale: float | None = None) -> CapturedImage:
+    """Try mss, fallback to XDG Desktop Portal if mostly black (typical on Wayland)."""
+    try:
+        img = capture_primary_rgb(scale=scale)
+        if not rgb_mostly_black(img.rgb):
+            return img
+    except Exception:
+        pass
+
+    # Portal path
+    png_bytes = capture_portal_png()
+    return _parse_png_to_rgb(png_bytes, scale=scale)
+
+
+def _parse_png_to_rgb(png_bytes: bytes, *, scale: float | None = None) -> CapturedImage:
+    # Use pillow dynamically if available
+    try:
+        from io import BytesIO
+        from PIL import Image
+
+        img = Image.open(BytesIO(png_bytes))
+        if img.mode != "RGB":
+            img = img.convert("RGB")
+        src_w, src_h = img.size
+        scale_val = resolve_scale(scale)
+        dst_w = int(src_w * scale_val)
+        dst_h = int(src_h * scale_val)
+        if scale_val < 1.0:
+            img = img.resize((dst_w, dst_h), Image.Resampling.NEAREST)
+        return CapturedImage(
+            width=img.size[0],
+            height=img.size[1],
+            rgb=img.tobytes(),
+            scale=scale_val,
+        )
+    except Exception as exc:
+        raise PortalCaptureError(f"cannot parse PNG bytes to RGB (PIL required for portal parsing): {exc}") from exc

gillm/capture/portal_backend.py

@@ -0,0 +1,113 @@
+"""Wayland-friendly screenshot via XDG Desktop Portal.
+
+Extracted from ``koruvision.portal_capture``.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+import urllib.parse
+from pathlib import Path
+
+
+class PortalCaptureError(RuntimeError):
+    """Portal screenshot failed."""
+
+
+_INLINE_SCRIPT = r"""
+import sys
+import dbus
+import dbus.mainloop.glib
+from gi.repository import GLib
+
+dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
+bus = dbus.SessionBus()
+token = "koruvision_portal"
+sender = bus.get_unique_name()[1:].replace(".", "_")
+request_path = f"/org/freedesktop/portal/desktop/request/{sender}/{token}"
+state = {"uri": None, "error": None}
+
+def _on_response(response, results):
+    if int(response) != 0:
+        state["error"] = f"portal response code {response}"
+    elif "uri" in results:
+        state["uri"] = str(results["uri"])
+    else:
+        state["error"] = "portal response missing uri"
+    loop.quit()
+
+bus.add_signal_receiver(
+    _on_response,
+    dbus_interface="org.freedesktop.portal.Request",
+    path=request_path,
+    signal_name="Response",
+)
+proxy = bus.get_object("org.freedesktop.portal.Desktop", "/org/freedesktop/portal/desktop")
+iface = dbus.Interface(proxy, "org.freedesktop.portal.Screenshot")
+iface.Screenshot("", {"handle_token": token, "interactive": False})
+
+loop = GLib.MainLoop()
+GLib.timeout_add(12000, lambda: (loop.quit(), False)[1])
+loop.run()
+
+if state["error"]:
+    print(state["error"], file=sys.stderr)
+    sys.exit(2)
+if not state["uri"]:
+    print("portal screenshot timed out", file=sys.stderr)
+    sys.exit(3)
+print(state["uri"])
+"""
+
+
+def _portal_python() -> str:
+    override = os.environ.get("KORU_PORTAL_PYTHON", "").strip()
+    if override:
+        return override
+    candidates = ["/usr/bin/python3", shutil.which("python3"), sys.executable]
+    for candidate in candidates:
+        if not candidate:
+            continue
+        try:
+            proc = subprocess.run(
+                [candidate, "-c", "import dbus; import gi"],
+                capture_output=True,
+                timeout=5,
+                check=False,
+            )
+        except (OSError, subprocess.TimeoutExpired):
+            continue
+        if proc.returncode == 0:
+            return candidate
+    return sys.executable
+
+
+def capture_portal_png(*, timeout_seconds: float = 15.0) -> bytes:
+    """Capture the screen through ``org.freedesktop.portal.Screenshot``."""
+    python = _portal_python()
+    try:
+        proc = subprocess.run(
+            [python, "-c", _INLINE_SCRIPT],
+            capture_output=True,
+            timeout=timeout_seconds,
+            text=True,
+            check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired) as exc:
+        raise PortalCaptureError(f"portal subprocess failed: {exc}") from exc
+    if proc.returncode != 0:
+        detail = (proc.stderr or "").strip() or f"exit code {proc.returncode}"
+        raise PortalCaptureError(
+            f"portal capture failed via {python}: {detail}."
+        )
+    uri = (proc.stdout or "").strip()
+    if not uri:
+        raise PortalCaptureError("portal returned an empty URI")
+    path = Path(urllib.parse.urlparse(uri).path)
+    try:
+        return path.read_bytes()
+    except OSError as exc:
+        raise PortalCaptureError(f"cannot read portal screenshot at {path}") from exc

gillm/cli/__init__.py

@@ -0,0 +1,5 @@
+"""CLI entry point package."""
+
+from gillm.cli.main import main
+
+__all__ = ["main"]

gillm/cli/main.py

@@ -0,0 +1,80 @@
+"""Command-line interface for the gillm GUI control engine."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from gillm.orchestrator.drive import DriveOrchestrator
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="gillm: LLM/NLP-driven GUI Control Engine"
+    )
+    subparsers = parser.add_subparsers(dest="command", help="Sub-commands")
+
+    # Command: run (execute direct actions or DSL file)
+    run_parser = subparsers.add_parser("run", help="Execute GUI actions from file")
+    run_parser.add_argument("file", help="Path to JSON file containing GUI steps")
+    run_parser.add_argument("--dry-run", action="store_true", help="Log actions without executing them")
+
+    # Command: nlp (execute natural language command)
+    nlp_parser = subparsers.add_parser("nlp", help="Translate and execute natural language GUI commands")
+    nlp_parser.add_argument("instruction", help="Natural language instruction (e.g. 'focus vscode and type hello')")
+    nlp_parser.add_argument("--dry-run", action="store_true", help="Log actions without executing them")
+
+    # Command: capture (take screen capture)
+    capture_parser = subparsers.add_parser("capture", help="Take a primary display screen capture")
+    capture_parser.add_argument("--scale", type=float, default=0.2, help="Image scale factor (default: 0.2)")
+
+    args = parser.parse_args()
+
+    if not args.command:
+        parser.print_help()
+        return 0
+
+    orchestrator = DriveOrchestrator(log_fn=lambda msg: print(f"[*] {msg}"))
+
+    if args.command == "run":
+        try:
+            with open(args.file, encoding="utf-8") as f:
+                steps = json.load(f)
+            if not isinstance(steps, list):
+                print("[-] Error: Workflow file must contain a JSON list of steps", file=sys.stderr)
+                return 1
+            results = orchestrator.execute_workflow(steps, dry_run=args.dry_run)
+            print("[+] Execution finished:")
+            print(json.dumps(results, indent=2))
+            return 0
+        except Exception as exc:
+            print(f"[-] Error: {exc}", file=sys.stderr)
+            return 1
+
+    elif args.command == "nlp":
+        try:
+            print(f"[*] Parsing and executing instruction: {args.instruction!r}")
+            results = orchestrator.drive_natural_language(args.instruction, dry_run=args.dry_run)
+            print("[+] Execution finished:")
+            print(json.dumps(results, indent=2))
+            return 0
+        except Exception as exc:
+            print(f"[-] Error: {exc}", file=sys.stderr)
+            return 1
+
+    elif args.command == "capture":
+        try:
+            print(f"[*] Capturing screenshot (scale={args.scale})...")
+            img = orchestrator.capture_screenshot(scale=args.scale)
+            print(f"[+] Success: Captured {img.width}x{img.height} screen at scale {img.scale}")
+            return 0
+        except Exception as exc:
+            print(f"[-] Error: {exc}", file=sys.stderr)
+            return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

gillm/config.py

@@ -0,0 +1,112 @@
+"""User-tunable autopilot config.
+
+Self-contained config module for gillm.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import tomllib
+from dataclasses import dataclass, field
+from functools import lru_cache
+from pathlib import Path
+
+
+def resolve_xdg_path(relative_path: str) -> Path:
+    """Resolve an XDG-style config path."""
+    xdg = os.environ.get("XDG_CONFIG_HOME")
+    base = Path(xdg) if xdg else Path.home() / ".config"
+    return base / relative_path
+
+
+# Single source of truth for built-in submit keys.
+_BUILTIN_SUBMIT_KEYS: dict[str, str] = {
+    "default": "Return",
+    "antigravity": "Return",
+    "windsurf": "Return",
+    "vscode": "Return",
+    "vscodium": "Return",
+    "cursor": "Return",
+    "jetbrains": "ctrl+Return",
+    "zed": "Return",
+}
+
+
+@dataclass(frozen=True)
+class AutopilotConfig:
+    """In-memory view of ``autopilot.toml`` (or defaults)."""
+
+    submit_keys: dict[str, str] = field(default_factory=lambda: dict(_BUILTIN_SUBMIT_KEYS))
+    source: Path | None = None
+
+    def submit_key_for(self, ide: str) -> str:
+        """Return the configured submit key for ``ide`` (or the default)."""
+        return self.submit_keys.get(ide) or self.submit_keys.get("default", "Return")
+
+
+def default_config_path() -> Path:
+    """Resolve the XDG-style config path for autopilot."""
+    return resolve_xdg_path("koru/autopilot.toml")
+
+
+def _merge_submit_keys(raw: object) -> dict[str, str]:
+    """Validate and merge user-provided ``[submit_keys]`` over defaults."""
+    merged = dict(_BUILTIN_SUBMIT_KEYS)
+    if not isinstance(raw, dict):
+        return merged
+    for ide, key in raw.items():
+        if not isinstance(ide, str) or not ide:
+            continue
+        if not isinstance(key, str) or not key:
+            continue
+        merged[ide] = key
+    return merged
+
+
+def load_config(path: Path | None = None) -> AutopilotConfig:
+    """Read the TOML config from ``path`` (or the default location)."""
+    config_path = path or default_config_path()
+    if not config_path.is_file():
+        return AutopilotConfig()
+    try:
+        data = tomllib.loads(config_path.read_text(encoding="utf-8"))
+    except (OSError, tomllib.TOMLDecodeError, UnicodeDecodeError) as exc:
+        print(
+            f"gillm autopilot: ignoring malformed config {config_path}: {exc}",
+            file=sys.stderr,
+        )
+        return AutopilotConfig()
+    submit_keys = _merge_submit_keys(data.get("submit_keys"))
+    return AutopilotConfig(submit_keys=submit_keys, source=config_path)
+
+
+@lru_cache(maxsize=1)
+def _cached_config() -> AutopilotConfig:
+    return load_config()
+
+
+def cached_config() -> AutopilotConfig:
+    """Process-lifetime memoised :func:`load_config`."""
+    for mod_name in ("koru.autopilot.injector", "koruide.injector", "koru.autopilot.config"):
+        if mod_name in sys.modules:
+            mod = sys.modules[mod_name]
+            if hasattr(mod, "cached_config"):
+                val = getattr(mod, "cached_config")
+                if val is not cached_config and callable(val):
+                    return val()
+    return _cached_config()
+
+
+def clear_config_cache() -> None:
+    """Drop the cached config."""
+    _cached_config.cache_clear()
+
+
+__all__ = [
+    "AutopilotConfig",
+    "default_config_path",
+    "load_config",
+    "cached_config",
+    "clear_config_cache",
+]

gillm/focus/__init__.py

@@ -0,0 +1,30 @@
+"""OS strategies for window focus and key injection."""
+
+from gillm.focus.registry import (
+    get_os_strategy,
+    list_os_strategy_ids,
+    register_os_strategy,
+    resolve_active_os_strategy,
+)
+from gillm.focus.strategy import (
+    FocusOutcome,
+    KeySequence,
+    OsCapabilities,
+    OsStrategy,
+    StaticOsIdentityMixin,
+)
+
+# Auto-import strategies to register them
+from gillm.focus import darwin, wayland, windows, x11
+
+__all__ = [
+    "FocusOutcome",
+    "KeySequence",
+    "OsCapabilities",
+    "OsStrategy",
+    "StaticOsIdentityMixin",
+    "get_os_strategy",
+    "list_os_strategy_ids",
+    "register_os_strategy",
+    "resolve_active_os_strategy",
+]

gillm/focus/darwin.py

@@ -0,0 +1,72 @@
+"""macOS strategy."""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+import sys
+from dataclasses import dataclass
+
+from gillm.focus.registry import register_os_strategy
+from gillm.focus.strategy import (
+    FocusOutcome,
+    KeySequence,
+    OsCapabilities,
+    OsStrategy,
+    StaticOsIdentityMixin,
+)
+
+
+def _run(argv: list[str], *, timeout: float = 10.0) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        argv,
+        capture_output=True,
+        text=True,
+        check=False,
+        timeout=timeout,
+    )
+
+
+@dataclass(frozen=True)
+class DarwinStrategy(StaticOsIdentityMixin, OsStrategy):
+    OS_ID = "darwin"
+    OS_LABEL = "macOS"
+
+    def matches_current_environment(self) -> bool:
+        return sys.platform == "darwin"
+
+    def capabilities(self) -> OsCapabilities:
+        osascript = shutil.which("osascript")
+        focus_methods = ("osascript",) if osascript else ()
+        keyboard_tool: str | None = None
+        if shutil.which("cliclick"):
+            keyboard_tool = "cliclick"
+        elif osascript:
+            keyboard_tool = "osascript"
+        return OsCapabilities(
+            can_focus_window=bool(osascript),
+            can_inject_keys=keyboard_tool is not None,
+            can_paste_clipboard=bool(shutil.which("pbpaste")),
+            focus_methods=focus_methods,
+            keyboard_tool=keyboard_tool,
+        )
+
+    def focus_window(self, window_name_hints: tuple[str, ...]) -> FocusOutcome:
+        if not shutil.which("osascript"):
+            return FocusOutcome(ok=False, detail="darwin: osascript not on PATH")
+        for hint in window_name_hints:
+            script = f'tell application "{hint}" to activate'
+            if _run(["osascript", "-e", script]).returncode == 0:
+                return FocusOutcome(ok=True, method="osascript")
+        return FocusOutcome(
+            ok=False,
+            detail="darwin: no application activated for the given hints",
+        )
+
+    def inject_keys(self, sequence: KeySequence) -> bool:
+        return False
+
+
+register_os_strategy(DarwinStrategy())
+
+__all__ = ["DarwinStrategy"]

gillm/focus/registry.py

@@ -0,0 +1,50 @@
+"""Registry + resolver for :class:`OsStrategy` implementations."""
+
+from __future__ import annotations
+
+from gillm.focus.strategy import OsStrategy
+
+_REGISTRY: list[OsStrategy] = []
+
+
+def register_os_strategy(strategy: OsStrategy) -> None:
+    """Register ``strategy``."""
+    existing = [s for s in _REGISTRY if s.id == strategy.id]
+    if existing:
+        for old in existing:
+            _REGISTRY.remove(old)
+    _REGISTRY.append(strategy)
+
+
+def get_os_strategy(strategy_id: str) -> OsStrategy | None:
+    """Return the strategy with ``id == strategy_id`` if registered."""
+    for strategy in _REGISTRY:
+        if strategy.id == strategy_id:
+            return strategy
+    return None
+
+
+def list_os_strategy_ids() -> tuple[str, ...]:
+    """Return the ids of every registered strategy."""
+    return tuple(s.id for s in _REGISTRY)
+
+
+def resolve_active_os_strategy() -> OsStrategy:
+    """Return the strategy whose ``matches_current_environment`` is true."""
+    for strategy in _REGISTRY:
+        if strategy.matches_current_environment():
+            return strategy
+    if not _REGISTRY:
+        raise RuntimeError(
+            "gillm.focus.registry: no OsStrategy registered; "
+            "did import of gillm.focus fail?"
+        )
+    return _REGISTRY[-1]
+
+
+__all__ = [
+    "get_os_strategy",
+    "list_os_strategy_ids",
+    "register_os_strategy",
+    "resolve_active_os_strategy",
+]