oagi-core 0.14.1__py3-none-any.whl → 0.15.0__py3-none-any.whl

oagi/handler/utils.py

@@ -5,6 +5,593 @@
 #  This file is part of the official API project.
 #  Licensed under the MIT License.
 # -----------------------------------------------------------------------------
+"""Shared utilities for action handling and conversion.
+
+This module provides common functionality used by both PyautoguiActionHandler
+(for local execution) and action converters (for remote execution).
+"""
+
+import sys
+
+# =============================================================================
+# Key Normalization Mapping
+# =============================================================================
+
+# Minimal key mapping - only normalizes common variations to pyautogui names
+# Matches original PyautoguiActionHandler.hotkey_variations_mapping behavior exactly:
+#   "capslock": ["caps_lock", "caps", "capslock"] -> capslock
+#   "pgup": ["page_up", "pageup"] -> pgup
+#   "pgdn": ["page_down", "pagedown"] -> pgdn
+KEY_MAP: dict[str, str] = {
+    # Caps lock variations -> capslock
+    "caps_lock": "capslock",
+    "caps": "capslock",
+    # Page up variations -> pgup (short form, matching original)
+    "page_up": "pgup",
+    "pageup": "pgup",
+    # Page down variations -> pgdn (short form, matching original)
+    "page_down": "pgdn",
+    "pagedown": "pgdn",
+}
+
+# Valid pyautogui key names
+PYAUTOGUI_VALID_KEYS: frozenset[str] = frozenset(
+    {
+        # Alphabet keys
+        "a",
+        "b",
+        "c",
+        "d",
+        "e",
+        "f",
+        "g",
+        "h",
+        "i",
+        "j",
+        "k",
+        "l",
+        "m",
+        "n",
+        "o",
+        "p",
+        "q",
+        "r",
+        "s",
+        "t",
+        "u",
+        "v",
+        "w",
+        "x",
+        "y",
+        "z",
+        # Number keys
+        "0",
+        "1",
+        "2",
+        "3",
+        "4",
+        "5",
+        "6",
+        "7",
+        "8",
+        "9",
+        # Function keys
+        "f1",
+        "f2",
+        "f3",
+        "f4",
+        "f5",
+        "f6",
+        "f7",
+        "f8",
+        "f9",
+        "f10",
+        "f11",
+        "f12",
+        "f13",
+        "f14",
+        "f15",
+        "f16",
+        "f17",
+        "f18",
+        "f19",
+        "f20",
+        "f21",
+        "f22",
+        "f23",
+        "f24",
+        # Navigation keys
+        "up",
+        "down",
+        "left",
+        "right",
+        "home",
+        "end",
+        "pageup",
+        "pagedown",
+        "pgup",
+        "pgdn",
+        # Editing keys
+        "backspace",
+        "delete",
+        "del",
+        "insert",
+        "enter",
+        "return",
+        "tab",
+        "space",
+        # Modifier keys (with left/right variants)
+        "shift",
+        "shiftleft",
+        "shiftright",
+        "ctrl",
+        "ctrlleft",
+        "ctrlright",
+        "alt",
+        "altleft",
+        "altright",
+        "option",
+        "optionleft",
+        "optionright",
+        "command",
+        "win",
+        "winleft",
+        "winright",
+        "fn",
+        # Lock keys
+        "capslock",
+        "numlock",
+        "scrolllock",
+        # Special keys
+        "esc",
+        "escape",
+        "pause",
+        "printscreen",
+        "prtsc",
+        "prtscr",
+        "prntscrn",
+        "print",
+        "apps",
+        "clear",
+        "sleep",
+        # Symbols
+        "!",
+        "@",
+        "#",
+        "$",
+        "%",
+        "^",
+        "&",
+        "*",
+        "(",
+        ")",
+        "-",
+        "_",
+        "=",
+        "+",
+        "[",
+        "]",
+        "{",
+        "}",
+        "\\",
+        "|",
+        ";",
+        ":",
+        "'",
+        '"',
+        ",",
+        ".",
+        "<",
+        ">",
+        "/",
+        "?",
+        "`",
+        "~",
+        # Numpad keys
+        "num0",
+        "num1",
+        "num2",
+        "num3",
+        "num4",
+        "num5",
+        "num6",
+        "num7",
+        "num8",
+        "num9",
+        "divide",
+        "multiply",
+        "subtract",
+        "add",
+        "decimal",
+        # Media keys
+        "volumeup",
+        "volumedown",
+        "volumemute",
+        "playpause",
+        "stop",
+        "nexttrack",
+        "prevtrack",
+        # Browser keys
+        "browserback",
+        "browserforward",
+        "browserrefresh",
+        "browserstop",
+        "browsersearch",
+        "browserfavorites",
+        "browserhome",
+        # Application launch keys
+        "launchapp1",
+        "launchapp2",
+        "launchmail",
+        "launchmediaselect",
+    }
+)
+
+
+# =============================================================================
+# Coordinate Scaling
+# =============================================================================
+
+
+class CoordinateScaler:
+    """Handles coordinate scaling between different coordinate systems.
+
+    This class provides reusable coordinate transformation logic used by both
+    PyautoguiActionHandler (local execution) and action converters (remote execution).
+
+    Args:
+        source_width: Width of the source coordinate space (e.g., 1000 for OAGI)
+        source_height: Height of the source coordinate space
+        target_width: Width of the target coordinate space (e.g., screen width)
+        target_height: Height of the target coordinate space
+        origin_x: X offset of the target coordinate origin (for multi-monitor)
+        origin_y: Y offset of the target coordinate origin (for multi-monitor)
+    """
+
+    def __init__(
+        self,
+        source_width: int,
+        source_height: int,
+        target_width: int,
+        target_height: int,
+        origin_x: int = 0,
+        origin_y: int = 0,
+    ):
+        self.source_width = source_width
+        self.source_height = source_height
+        self.target_width = target_width
+        self.target_height = target_height
+        self.origin_x = origin_x
+        self.origin_y = origin_y
+        self.scale_x = target_width / source_width
+        self.scale_y = target_height / source_height
+
+    def scale(
+        self,
+        x: int | float,
+        y: int | float,
+        *,
+        clamp: bool = True,
+        prevent_failsafe: bool = False,
+        strict: bool = False,
+    ) -> tuple[int, int]:
+        """Scale coordinates from source to target space.
+
+        Args:
+            x: X coordinate in source space
+            y: Y coordinate in source space
+            clamp: If True, clamp to valid target range
+            prevent_failsafe: If True, offset corner coordinates by 1 pixel
+                             (prevents PyAutoGUI fail-safe trigger)
+            strict: If True, raise ValueError when coordinates are outside
+                   valid source range [0, source_width] x [0, source_height]
+
+        Returns:
+            Tuple of (target_x, target_y) in target coordinate space
+
+        Raises:
+            ValueError: If strict=True and coordinates are outside valid range
+        """
+        # Strict validation: check if coordinates are in valid source range
+        if strict:
+            if x < 0 or x > self.source_width:
+                raise ValueError(
+                    f"x coordinate {x} out of valid range [0, {self.source_width}]. "
+                    f"Coordinates must be normalized between 0 and {self.source_width}."
+                )
+            if y < 0 or y > self.source_height:
+                raise ValueError(
+                    f"y coordinate {y} out of valid range [0, {self.source_height}]. "
+                    f"Coordinates must be normalized between 0 and {self.source_height}."
+                )
+
+        scaled_x = round(x * self.scale_x)
+        scaled_y = round(y * self.scale_y)
+
+        if clamp:
+            # Clamp to valid range
+            scaled_x = max(0, min(scaled_x, self.target_width - 1))
+            scaled_y = max(0, min(scaled_y, self.target_height - 1))
+
+        if prevent_failsafe:
+            # Prevent PyAutoGUI fail-safe by adjusting corner coordinates
+            if scaled_x < 1:
+                scaled_x = 1
+            elif scaled_x > self.target_width - 2:
+                scaled_x = self.target_width - 2
+            if scaled_y < 1:
+                scaled_y = 1
+            elif scaled_y > self.target_height - 2:
+                scaled_y = self.target_height - 2
+
+        # Add origin offset (for multi-monitor support)
+        return scaled_x + self.origin_x, scaled_y + self.origin_y
+
+    def set_origin(self, origin_x: int, origin_y: int) -> None:
+        """Update the origin offset."""
+        self.origin_x = origin_x
+        self.origin_y = origin_y
+
+    def set_target_size(self, width: int, height: int) -> None:
+        """Update the target size and recalculate scale factors."""
+        self.target_width = width
+        self.target_height = height
+        self.scale_x = width / self.source_width
+        self.scale_y = height / self.source_height
+
+
+# =============================================================================
+# Key Normalization Functions
+# =============================================================================
+
+
+def normalize_key(key: str, *, macos_ctrl_to_cmd: bool = False) -> str:
+    """Normalize a key name to pyautogui format.
+
+    Args:
+        key: Key name to normalize (e.g., "ctrl", "Control", "page_down")
+        macos_ctrl_to_cmd: If True and on macOS, remap 'ctrl' to 'command'
+
+    Returns:
+        Normalized key name (e.g., "ctrl", "pagedown")
+    """
+    key = key.strip().lower()
+    normalized = KEY_MAP.get(key, key)
+
+    # Remap ctrl to command on macOS if enabled
+    if macos_ctrl_to_cmd and sys.platform == "darwin" and normalized == "ctrl":
+        return "command"
+
+    return normalized
+
+
+def parse_hotkey(
+    hotkey_str: str,
+    *,
+    macos_ctrl_to_cmd: bool = False,
+    validate: bool = True,
+) -> list[str]:
+    """Parse a hotkey string into a list of normalized key names.
+
+    Args:
+        hotkey_str: Hotkey string (e.g., "ctrl+c", "alt, tab", "Shift+Enter")
+        macos_ctrl_to_cmd: If True and on macOS, remap 'ctrl' to 'command'
+        validate: If True, validate keys against PYAUTOGUI_VALID_KEYS
+
+    Returns:
+        List of normalized key names (e.g., ["ctrl", "c"])
+
+    Raises:
+        ValueError: If validate=True and any key is invalid
+    """
+    # Remove parentheses if present
+    hotkey_str = hotkey_str.strip("()")
+
+    # Split by '+' or ',' to get individual keys
+    if "+" in hotkey_str:
+        keys = [
+            normalize_key(k, macos_ctrl_to_cmd=macos_ctrl_to_cmd)
+            for k in hotkey_str.split("+")
+        ]
+    else:
+        keys = [
+            normalize_key(k, macos_ctrl_to_cmd=macos_ctrl_to_cmd)
+            for k in hotkey_str.split(",")
+        ]
+
+    # Filter empty strings
+    keys = [k for k in keys if k]
+
+    if validate:
+        validate_keys(keys)
+
+    return keys
+
+
+def validate_keys(keys: list[str]) -> None:
+    """Validate that all keys are recognized by pyautogui.
+
+    Args:
+        keys: List of normalized key names
+
+    Raises:
+        ValueError: If any key is invalid, with helpful suggestions
+    """
+    invalid_keys = [k for k in keys if k and k not in PYAUTOGUI_VALID_KEYS]
+
+    if invalid_keys:
+        suggestions = []
+        for invalid_key in invalid_keys:
+            if invalid_key in ("ret",):
+                suggestions.append(f"'{invalid_key}' -> use 'enter' or 'return'")
+            elif invalid_key.startswith("num") and len(invalid_key) > 3:
+                suggestions.append(
+                    f"'{invalid_key}' -> numpad keys use format 'num0'-'num9'"
+                )
+            else:
+                suggestions.append(f"'{invalid_key}' is not a valid key name")
+
+        error_msg = "Invalid key name(s) in hotkey: " + ", ".join(suggestions)
+        valid_sample = ", ".join(sorted(list(PYAUTOGUI_VALID_KEYS)[:30]))
+        error_msg += f"\n\nValid keys include: {valid_sample}... (and more)"
+        raise ValueError(error_msg)
+
+
+# =============================================================================
+# Coordinate Parsing Functions
+# =============================================================================
+
+
+def parse_click_coords(
+    argument: str,
+    scaler: CoordinateScaler,
+    *,
+    prevent_failsafe: bool = False,
+    strict: bool = False,
+) -> tuple[int, int]:
+    """Parse click coordinates from argument string.
+
+    Args:
+        argument: Coordinate string in format "x, y"
+        scaler: CoordinateScaler instance for coordinate transformation
+        prevent_failsafe: If True, offset corner coordinates
+        strict: If True, raise ValueError for out-of-range coordinates
+
+    Returns:
+        Tuple of (x, y) in target coordinate space
+
+    Raises:
+        ValueError: If coordinate format is invalid or (strict=True) out of range
+    """
+    # Check for common format errors
+    if " and " in argument.lower() or " then " in argument.lower():
+        raise ValueError(
+            f"Invalid click format: '{argument}'. "
+            "Cannot combine multiple actions with 'and' or 'then'."
+        )
+
+    parts = argument.split(",") if argument else []
+    if len(parts) < 2:
+        raise ValueError(
+            f"Invalid click coordinate format: '{argument}'. "
+            "Expected 'x, y' (comma-separated numeric values)"
+        )
+
+    try:
+        x = float(parts[0].strip())
+        y = float(parts[1].strip())
+        return scaler.scale(x, y, prevent_failsafe=prevent_failsafe, strict=strict)
+    except (ValueError, IndexError) as e:
+        raise ValueError(
+            f"Failed to parse click coords '{argument}': {e}. "
+            "Coordinates must be comma-separated numeric values."
+        ) from e
+
+
+def parse_drag_coords(
+    argument: str,
+    scaler: CoordinateScaler,
+    *,
+    prevent_failsafe: bool = False,
+    strict: bool = False,
+) -> tuple[int, int, int, int]:
+    """Parse drag coordinates from argument string.
+
+    Args:
+        argument: Coordinate string in format "x1, y1, x2, y2"
+        scaler: CoordinateScaler instance for coordinate transformation
+        prevent_failsafe: If True, offset corner coordinates
+        strict: If True, raise ValueError for out-of-range coordinates
+
+    Returns:
+        Tuple of (x1, y1, x2, y2) in target coordinate space
+
+    Raises:
+        ValueError: If coordinate format is invalid or (strict=True) out of range
+    """
+    # Check for common format errors
+    if " and " in argument.lower() or " then " in argument.lower():
+        raise ValueError(
+            f"Invalid drag format: '{argument}'. "
+            "Cannot combine multiple actions with 'and' or 'then'."
+        )
+
+    parts = argument.split(",") if argument else []
+    if len(parts) != 4:
+        raise ValueError(
+            f"Invalid drag coordinate format: '{argument}'. "
+            "Expected 'x1, y1, x2, y2' (4 comma-separated numeric values)"
+        )
+
+    try:
+        sx = float(parts[0].strip())
+        sy = float(parts[1].strip())
+        ex = float(parts[2].strip())
+        ey = float(parts[3].strip())
+        x1, y1 = scaler.scale(sx, sy, prevent_failsafe=prevent_failsafe, strict=strict)
+        x2, y2 = scaler.scale(ex, ey, prevent_failsafe=prevent_failsafe, strict=strict)
+        return x1, y1, x2, y2
+    except (ValueError, IndexError) as e:
+        raise ValueError(
+            f"Failed to parse drag coords '{argument}': {e}. "
+            "Coordinates must be comma-separated numeric values."
+        ) from e
+
+
+def parse_scroll_coords(
+    argument: str,
+    scaler: CoordinateScaler,
+    *,
+    prevent_failsafe: bool = False,
+    strict: bool = False,
+) -> tuple[int, int, str]:
+    """Parse scroll coordinates and direction from argument string.
+
+    Args:
+        argument: Scroll string in format "x, y, direction"
+        scaler: CoordinateScaler instance for coordinate transformation
+        prevent_failsafe: If True, offset corner coordinates
+        strict: If True, raise ValueError for out-of-range coordinates
+
+    Returns:
+        Tuple of (x, y, direction) where direction is 'up' or 'down'
+
+    Raises:
+        ValueError: If format is invalid or (strict=True) coordinates out of range
+    """
+    parts = [p.strip() for p in argument.split(",")]
+    if len(parts) != 3:
+        raise ValueError(
+            f"Invalid scroll format: '{argument}'. "
+            "Expected 'x, y, direction' (e.g., '500, 300, up')"
+        )
+
+    try:
+        x = float(parts[0])
+        y = float(parts[1])
+        direction = parts[2].lower()
+
+        if direction not in ("up", "down"):
+            raise ValueError(
+                f"Invalid scroll direction: '{direction}'. Use 'up' or 'down'."
+            )
+
+        scaled_x, scaled_y = scaler.scale(
+            x, y, prevent_failsafe=prevent_failsafe, strict=strict
+        )
+        return scaled_x, scaled_y, direction
+    except (ValueError, IndexError) as e:
+        if "scroll direction" in str(e):
+            raise
+        raise ValueError(
+            f"Failed to parse scroll coords '{argument}': {e}. "
+            "Format: 'x, y, direction'"
+        ) from e
+
+
+# =============================================================================
+# Handler Utility Functions
+# =============================================================================
 
 
 def reset_handler(handler) -> None:

oagi/handler/ydotool_action_handler.py

@@ -15,6 +15,7 @@ from oagi.handler.screen_manager import Screen
 from ..constants import DEFAULT_STEP_DELAY
 from ..types import Action, ActionType, parse_coords, parse_drag_coords, parse_scroll
 from .capslock_manager import CapsLockManager
+from .utils import CoordinateScaler, normalize_key, parse_hotkey
 from .wayland_support import Ydotool, get_screen_size
 
 
@@ -73,6 +74,13 @@ class YdotoolActionHandler(Ydotool):
         self.caps_manager = CapsLockManager(mode=self.config.capslock_mode)
         # The origin position of coordinates (the top-left corner of the screen)
         self.origin_x, self.origin_y = 0, 0
+        # Initialize coordinate scaler
+        self._coord_scaler = CoordinateScaler(
+            source_width=1000,
+            source_height=1000,
+            target_width=self.screen_width,
+            target_height=self.screen_height,
+        )
 
     def reset(self):
         """Reset handler state.
@@ -90,6 +98,12 @@ class YdotoolActionHandler(Ydotool):
         """
         self.screen_width, self.screen_height = screen.width, screen.height
         self.origin_x, self.origin_y = screen.x, screen.y
+        self._coord_scaler = CoordinateScaler(
+            source_width=1000,
+            source_height=1000,
+            target_width=self.screen_width,
+            target_height=self.screen_height,
+        )
 
     def _execute_action(self, action: Action) -> bool:
         """
@@ -150,8 +164,8 @@ class YdotoolActionHandler(Ydotool):
                 text = self.caps_manager.transform_text(text)
                 self._run_ydotool(["type", text], count=count)
 
-            case ActionType.FINISH:
-                # Task completion - reset handler state
+            case ActionType.FINISH | ActionType.FAIL:
+                # Task completion or infeasible - reset handler state
                 self.reset()
 
             case ActionType.WAIT:
@@ -168,45 +182,14 @@ class YdotoolActionHandler(Ydotool):
         return finished
 
     def _denormalize_coords(self, x: float, y: float) -> tuple[int, int]:
-        """Convert coordinates from 0-1000 range to actual screen coordinates.
-
-        Also handles corner coordinates to prevent PyAutoGUI fail-safe trigger.
-        Corner coordinates (0,0), (0,max), (max,0), (max,max) are offset by 1 pixel.
-        """
-        screen_x = int(x * self.screen_width / 1000)
-        screen_y = int(y * self.screen_height / 1000)
-
-        # Prevent fail-safe by adjusting corner coordinates
-        # Check if coordinates are at screen corners (with small tolerance)
-        if screen_x < 1:
-            screen_x = 1
-        elif screen_x > self.screen_width - 1:
-            screen_x = self.screen_width - 1
-
-        if screen_y < 1:
-            screen_y = 1
-        elif screen_y > self.screen_height - 1:
-            screen_y = self.screen_height - 1
-
-        # Add origin offset to convert relative to top-left corner
-        screen_x += self.origin_x
-        screen_y += self.origin_y
-
-        return screen_x, screen_y
+        """Convert coordinates from 0-1000 range to actual screen coordinates."""
+        screen_x, screen_y = self._coord_scaler.scale(x, y, prevent_failsafe=True)
+        # Add origin offset for multi-screen support
+        return screen_x + self.origin_x, screen_y + self.origin_y
 
     def _normalize_key(self, key: str) -> str:
         """Normalize key names for consistency."""
-        key = key.strip().lower()
-        # Normalize caps lock variations
-        hotkey_variations_mapping = {
-            "capslock": ["caps_lock", "caps", "capslock"],
-            "pgup": ["page_up", "pageup"],
-            "pgdn": ["page_down", "pagedown"],
-        }
-        for normalized, variations in hotkey_variations_mapping.items():
-            if key in variations:
-                return normalized
-        return key
+        return normalize_key(key)
 
     def _parse_coords(self, args_str: str) -> tuple[int, int]:
         """Extract x, y coordinates from argument string."""
@@ -234,11 +217,7 @@ class YdotoolActionHandler(Ydotool):
 
     def _parse_hotkey(self, args_str: str) -> list[str]:
         """Parse hotkey string into list of keys."""
-        # Remove parentheses if present
-        args_str = args_str.strip("()")
-        # Split by '+' to get individual keys
-        keys = [self._normalize_key(key) for key in args_str.split("+")]
-        return keys
+        return parse_hotkey(args_str.strip("()"), validate=False)
 
     def __call__(self, actions: list[Action]) -> None:
         """Execute the provided list of actions."""

oagi/server/socketio_server.py

@@ -364,7 +364,7 @@ class SessionNamespace(socketio.AsyncNamespace):
                     timeout=self.config.socketio_timeout,
                 )
 
-            case ActionType.FINISH:
+            case ActionType.FINISH | ActionType.FAIL:
                 return await self.call(
                     "finish",
                     FinishEventData(**common).model_dump(),