oagi-core 0.14.1__py3-none-any.whl → 0.15.0__py3-none-any.whl

oagi/__init__.py

@@ -38,6 +38,10 @@ from oagi.types.models import (
 # Format: name -> (module_path, package_to_check, extra_name)
 # package_to_check is None if no optional dependency is required
 _LAZY_IMPORTS_DATA: dict[str, tuple[str, str | None, str | None]] = {
+    # Action converters (no optional dependencies)
+    "OagiActionConverter": ("oagi.converters.oagi", None, None),
+    "ConverterConfig": ("oagi.converters.base", None, None),
+    "BaseActionConverter": ("oagi.converters.base", None, None),
     # Desktop handlers (require pyautogui/PIL)
     "AsyncPyautoguiActionHandler": (
         "oagi.handler.async_pyautogui_action_handler",
@@ -88,6 +92,8 @@ if TYPE_CHECKING:
     from oagi.agent.default import AsyncDefaultAgent
     from oagi.agent.observer.agent_observer import AsyncAgentObserver
     from oagi.agent.tasker import TaskerAgent
+    from oagi.converters.base import BaseActionConverter, ConverterConfig
+    from oagi.converters.oagi import OagiActionConverter
     from oagi.handler.async_pyautogui_action_handler import AsyncPyautoguiActionHandler
     from oagi.handler.async_screenshot_maker import AsyncScreenshotMaker
     from oagi.handler.async_ydotool_action_handler import AsyncYdotoolActionHandler
@@ -174,4 +180,8 @@ __all__ = [
     "YdotoolConfig",
     # Lazy imports - Screen manager
     "ScreenManager",
+    # Lazy imports - Action converters
+    "OagiActionConverter",
+    "ConverterConfig",
+    "BaseActionConverter",
 ]

oagi/converters/__init__.py

@@ -0,0 +1,56 @@
+# -----------------------------------------------------------------------------
+#  Copyright (c) OpenAGI Foundation
+#  All rights reserved.
+#
+#  This file is part of the official API project.
+#  Licensed under the MIT License.
+# -----------------------------------------------------------------------------
+"""Action converters for VLM support.
+
+This module provides the base class and OAGI implementation for action converters.
+Third parties can inherit from BaseActionConverter to create custom converters.
+
+Example usage:
+    from oagi.converters import OagiActionConverter, ConverterConfig
+
+    # Configure for 1920x1080 sandbox
+    config = ConverterConfig(sandbox_width=1920, sandbox_height=1080)
+    converter = OagiActionConverter(config=config)
+
+    # Convert OAGI actions to pyautogui strings
+    result = converter(actions)  # list[str]
+
+    # Convert to runtime API steps
+    for cmd in result:
+        step = converter.action_string_to_step(cmd)
+        # Execute step via runtime API...
+
+Creating custom converters:
+    from oagi.converters import BaseActionConverter, ConverterConfig
+
+    class MyActionConverter(BaseActionConverter[MyAction]):
+        @property
+        def coord_width(self) -> int:
+            return 1000  # Your model's coordinate width
+
+        @property
+        def coord_height(self) -> int:
+            return 1000  # Your model's coordinate height
+
+        def _convert_single_action(self, action: MyAction) -> list[str]:
+            # Convert action to pyautogui command strings
+            ...
+
+        def serialize_actions(self, actions: list[MyAction]) -> list[dict]:
+            # Serialize actions for trajectory logging
+            ...
+"""
+
+from .base import BaseActionConverter, ConverterConfig
+from .oagi import OagiActionConverter
+
+__all__ = [
+    "BaseActionConverter",
+    "ConverterConfig",
+    "OagiActionConverter",
+]

oagi/converters/base.py

@@ -0,0 +1,292 @@
+# -----------------------------------------------------------------------------
+#  Copyright (c) OpenAGI Foundation
+#  All rights reserved.
+#
+#  This file is part of the official API project.
+#  Licensed under the MIT License.
+# -----------------------------------------------------------------------------
+"""Base class for action converters.
+
+This module provides the abstract base class for converting model-specific
+actions to pyautogui command strings for remote execution.
+"""
+
+import re
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Generic, TypeVar
+
+from ..handler.capslock_manager import CapsLockManager
+from ..handler.utils import (
+    CoordinateScaler,
+    normalize_key,
+    parse_hotkey,
+    validate_keys,
+)
+
+T = TypeVar("T")
+
+
+@dataclass
+class ConverterConfig:
+    """Configuration for action converters.
+
+    Matches the configuration options in PyautoguiConfig for consistency.
+    """
+
+    sandbox_width: int = 1920
+    sandbox_height: int = 1080
+    drag_duration: float = 0.5
+    scroll_amount: int = 2
+    wait_duration: float = 1.0
+    hotkey_interval: float = 0.1
+    capslock_mode: str = "session"
+    strict_coordinate_validation: bool = False
+    """If True, raise ValueError when coordinates are outside valid range.
+    If False (default), clamp coordinates to valid range (original behavior)."""
+
+
+class BaseActionConverter(ABC, Generic[T]):
+    """Abstract base class for action converters.
+
+    Subclasses must implement:
+    - coord_width/coord_height properties for input coordinate space
+    - _convert_single_action() for model-specific conversion logic
+    - serialize_actions() for trajectory logging
+
+    Provides common functionality:
+    - Coordinate scaling via CoordinateScaler
+    - Key normalization via shared utils
+    - __call__ interface returning list of action strings
+    - action_string_to_step() for runtime API format
+    """
+
+    def __init__(
+        self,
+        *,
+        config: ConverterConfig | None = None,
+        logger: Any | None = None,
+    ):
+        """Initialize the converter.
+
+        Args:
+            config: Converter configuration. Uses defaults if not provided.
+            logger: Optional logger instance for debug/error logging.
+        """
+        self.config = config or ConverterConfig()
+        self.logger = logger
+
+        # Initialize coordinate scaler
+        self._coord_scaler = CoordinateScaler(
+            source_width=self.coord_width,
+            source_height=self.coord_height,
+            target_width=self.config.sandbox_width,
+            target_height=self.config.sandbox_height,
+        )
+
+        # Initialize caps lock manager
+        self.caps_manager = CapsLockManager(mode=self.config.capslock_mode)
+
+        # Track last cursor position (for actions without explicit coordinates)
+        self._last_x: int | None = None
+        self._last_y: int | None = None
+
+    @property
+    @abstractmethod
+    def coord_width(self) -> int:
+        """Input coordinate space width (e.g., 1024 for XGA, 1000 for OAGI)."""
+        ...
+
+    @property
+    @abstractmethod
+    def coord_height(self) -> int:
+        """Input coordinate space height (e.g., 768 for XGA, 1000 for OAGI)."""
+        ...
+
+    @property
+    def scale_x(self) -> float:
+        """X scaling factor from input to sandbox coordinates."""
+        return self._coord_scaler.scale_x
+
+    @property
+    def scale_y(self) -> float:
+        """Y scaling factor from input to sandbox coordinates."""
+        return self._coord_scaler.scale_y
+
+    def scale_coordinate(self, x: int | float, y: int | float) -> tuple[int, int]:
+        """Scale coordinates from model space to sandbox space.
+
+        Args:
+            x: X coordinate in model space
+            y: Y coordinate in model space
+
+        Returns:
+            Tuple of (scaled_x, scaled_y) in sandbox space
+        """
+        return self._coord_scaler.scale(x, y)
+
+    def normalize_key(self, key: str) -> str:
+        """Normalize a key name to pyautogui format.
+
+        Args:
+            key: Key name to normalize
+
+        Returns:
+            Normalized key name
+        """
+        return normalize_key(key)
+
+    def parse_hotkey(self, hotkey_str: str, *, validate: bool = True) -> list[str]:
+        """Parse a hotkey string into a list of normalized key names.
+
+        Args:
+            hotkey_str: Hotkey string (e.g., "ctrl+c")
+            validate: If True, validate keys against PYAUTOGUI_VALID_KEYS
+
+        Returns:
+            List of normalized key names
+        """
+        return parse_hotkey(hotkey_str, validate=validate)
+
+    def validate_keys(self, keys: list[str]) -> None:
+        """Validate that all keys are recognized by pyautogui.
+
+        Args:
+            keys: List of key names to validate
+
+        Raises:
+            ValueError: If any key is invalid
+        """
+        validate_keys(keys)
+
+    def _get_last_or_center(self) -> tuple[int, int]:
+        """Get last cursor position or screen center as fallback.
+
+        Returns:
+            Tuple of (x, y) coordinates
+        """
+        if self._last_x is not None and self._last_y is not None:
+            return self._last_x, self._last_y
+        return self.config.sandbox_width // 2, self.config.sandbox_height // 2
+
+    def _log_error(self, message: str) -> None:
+        """Log an error message if logger is available."""
+        if self.logger:
+            self.logger.error(message)
+
+    def _log_info(self, message: str) -> None:
+        """Log an info message if logger is available."""
+        if self.logger:
+            self.logger.info(message)
+
+    def _log_debug(self, message: str) -> None:
+        """Log a debug message if logger is available."""
+        if self.logger:
+            self.logger.debug(message)
+
+    def __call__(self, actions: list[T]) -> list[str]:
+        """Convert actions to list of pyautogui command strings.
+
+        Args:
+            actions: List of model-specific action objects
+
+        Returns:
+            List of pyautogui command strings
+
+        Raises:
+            RuntimeError: If all action conversions failed
+        """
+        converted: list[str] = []
+        failed: list[tuple[str, str]] = []
+        skipped: list[str] = []
+
+        if not actions:
+            return converted
+
+        for action in actions:
+            try:
+                action_strings = self._convert_single_action(action)
+
+                if not action_strings:
+                    # No-op action (e.g., screenshot, cursor_position)
+                    action_type = getattr(action, "action_type", repr(action))
+                    skipped.append(str(action_type))
+                    continue
+
+                converted.extend(action_strings)
+
+            except Exception as e:
+                action_repr = repr(action)
+                self._log_error(f"Failed to convert action: {action_repr}, error: {e}")
+                failed.append((action_repr, str(e)))
+
+        if skipped:
+            self._log_debug(f"Skipped no-op actions: {skipped}")
+
+        if not converted and actions and failed:
+            raise RuntimeError(
+                f"All action conversions failed ({len(failed)}/{len(actions)}): {failed}"
+            )
+
+        return converted
+
+    @abstractmethod
+    def _convert_single_action(self, action: T) -> list[str]:
+        """Convert a single action to pyautogui command string(s).
+
+        Args:
+            action: Model-specific action object
+
+        Returns:
+            List of pyautogui command strings (may be empty for no-op actions)
+
+        Raises:
+            ValueError: If action format is invalid
+        """
+        ...
+
+    @abstractmethod
+    def serialize_actions(self, actions: list[T]) -> list[dict[str, Any]]:
+        """Serialize actions for trajectory logging.
+
+        Args:
+            actions: List of model-specific action objects
+
+        Returns:
+            List of serialized action dictionaries
+        """
+        ...
+
+    def action_string_to_step(self, action: str) -> dict[str, Any]:
+        """Convert an action string into a step for runtime/do API.
+
+        Args:
+            action: Action string (e.g., "pyautogui.click(x=100, y=200)")
+
+        Returns:
+            Step dict for runtime API
+        """
+        action_str = str(action).strip()
+
+        # Special markers
+        upper = action_str.upper()
+        if upper in ["DONE", "FAIL"]:
+            return {"type": "sleep", "parameters": {"seconds": 0}}
+
+        # WAIT(seconds)
+        wait_match = re.match(
+            r"^WAIT\((?P<sec>[0-9]*\.?[0-9]+)\)$", action_str, re.IGNORECASE
+        )
+        if wait_match:
+            seconds = float(wait_match.group("sec"))
+            return {"type": "sleep", "parameters": {"seconds": seconds}}
+
+        # pyautogui code path
+        if "pyautogui" in action_str.lower():
+            return {
+                "type": "pyautogui",
+                "parameters": {"code": action_str},
+            }
+
+        # Default: shell command
+        return {"type": "execute", "parameters": {"command": action_str, "shell": True}}

oagi/converters/oagi.py

@@ -0,0 +1,198 @@
+# -----------------------------------------------------------------------------
+#  Copyright (c) OpenAGI Foundation
+#  All rights reserved.
+#
+#  This file is part of the official API project.
+#  Licensed under the MIT License.
+# -----------------------------------------------------------------------------
+"""OAGI action converter.
+
+This module provides the OagiActionConverter for converting OAGI actions
+to pyautogui command strings for remote execution.
+"""
+
+from typing import Any
+
+from ..handler.utils import (
+    parse_click_coords,
+    parse_drag_coords,
+    parse_scroll_coords,
+)
+from ..types import Action, ActionType
+from .base import BaseActionConverter
+
+# OAGI uses normalized 0-1000 coordinate space
+OAGI_COORD_SIZE = 1000
+
+
+class OagiActionConverter(BaseActionConverter[Action]):
+    """Convert OAGI actions to pyautogui command strings.
+
+    This converter handles:
+    1. Coordinate scaling from 0-1000 space to sandbox dimensions (1920x1080)
+    2. Action format conversion from OAGI Action format to pyautogui strings
+    3. Key name normalization for hotkey combinations
+
+    The output can be converted to runtime API steps via action_string_to_step().
+    """
+
+    @property
+    def coord_width(self) -> int:
+        return OAGI_COORD_SIZE
+
+    @property
+    def coord_height(self) -> int:
+        return OAGI_COORD_SIZE
+
+    def __call__(self, actions: list[Action]) -> list[str]:
+        """Convert OAGI actions to list of pyautogui command strings.
+
+        Extends base implementation to handle action count and finish detection.
+        """
+        converted: list[str] = []
+        failed: list[tuple[str, str]] = []
+        has_terminal = False
+
+        if not actions:
+            return converted
+
+        for action in actions:
+            # Check for duplicate finish()/fail() during iteration
+            is_terminal = action.type in (ActionType.FINISH, ActionType.FAIL)
+            if is_terminal:
+                if has_terminal:
+                    raise ValueError(
+                        "Duplicate finish()/fail() detected. "
+                        "Only one finish() or fail() is allowed per action sequence."
+                    )
+                has_terminal = True
+
+            try:
+                converted.extend(self._convert_action(action))
+            except Exception as e:
+                action_repr = f"{action.type.value}({action.argument})"
+                self._log_error(f"Failed to convert action: {action_repr}, error: {e}")
+                failed.append((action_repr, str(e)))
+
+        if not converted and actions and failed:
+            raise RuntimeError(
+                f"All action conversions failed ({len(failed)}/{len(actions)}): {failed}"
+            )
+        return converted
+
+    def _convert_action(self, action: Action) -> list[str]:
+        """Convert action to list of pyautogui command strings.
+
+        Handles action.count for repeat support.
+        """
+        count = action.count or 1
+        single_actions = self._convert_single_action(action)
+
+        # Repeat the actions count times
+        return single_actions * int(count)
+
+    def _convert_single_action(self, action: Action) -> list[str]:
+        """Convert a single OAGI action to pyautogui command string(s)."""
+        action_type = action.type.value
+        argument = (action.argument or "").strip("()")
+
+        drag_duration = self.config.drag_duration
+        scroll_amount = self.config.scroll_amount
+        wait_duration = self.config.wait_duration
+        hotkey_interval = self.config.hotkey_interval
+        strict = self.config.strict_coordinate_validation
+
+        if action_type == ActionType.CLICK.value:
+            x, y = parse_click_coords(argument, self._coord_scaler, strict=strict)
+            return [f"pyautogui.click(x={x}, y={y})"]
+
+        if action_type == ActionType.LEFT_DOUBLE.value:
+            x, y = parse_click_coords(argument, self._coord_scaler, strict=strict)
+            return [f"pyautogui.doubleClick(x={x}, y={y})"]
+
+        if action_type == ActionType.LEFT_TRIPLE.value:
+            x, y = parse_click_coords(argument, self._coord_scaler, strict=strict)
+            return [f"pyautogui.tripleClick(x={x}, y={y})"]
+
+        if action_type == ActionType.RIGHT_SINGLE.value:
+            x, y = parse_click_coords(argument, self._coord_scaler, strict=strict)
+            return [f"pyautogui.rightClick(x={x}, y={y})"]
+
+        if action_type == ActionType.DRAG.value:
+            sx, sy, ex, ey = parse_drag_coords(
+                argument, self._coord_scaler, strict=strict
+            )
+            return [
+                f"pyautogui.moveTo({sx}, {sy})",
+                f"pyautogui.dragTo({ex}, {ey}, duration={drag_duration})",
+            ]
+
+        if action_type == ActionType.HOTKEY.value:
+            keys = self.parse_hotkey(argument, validate=True)
+            valid_keys = [k for k in keys if k]
+            if not valid_keys:
+                raise ValueError(
+                    f"Invalid hotkey format: '{argument}'. "
+                    "Expected key names like 'ctrl+c', 'alt+tab'"
+                )
+            # Check if this is a caps lock key press
+            if len(valid_keys) == 1 and valid_keys[0] == "capslock":
+                if self.caps_manager.should_use_system_capslock():
+                    return [f"pyautogui.hotkey('capslock', interval={hotkey_interval})"]
+                else:
+                    self.caps_manager.toggle()
+                    return []  # No pyautogui command for session mode
+            else:
+                keys_str = ", ".join(repr(k) for k in valid_keys)
+                return [f"pyautogui.hotkey({keys_str}, interval={hotkey_interval})"]
+
+        if action_type == ActionType.TYPE.value:
+            text = argument.strip("\"'")
+            text = self.caps_manager.transform_text(text)
+            return [f"pyautogui.typewrite({text!r})"]
+
+        if action_type == ActionType.SCROLL.value:
+            x, y, direction = parse_scroll_coords(
+                argument, self._coord_scaler, strict=strict
+            )
+            amount = scroll_amount if direction == "up" else -scroll_amount
+            return [f"pyautogui.moveTo({x}, {y})", f"pyautogui.scroll({amount})"]
+
+        if action_type == ActionType.WAIT.value:
+            try:
+                seconds = float(argument) if argument else wait_duration
+            except ValueError:
+                raise ValueError(
+                    f"Invalid wait duration: '{argument}'. "
+                    "Expected numeric value in seconds."
+                )
+            return [f"WAIT({seconds})"]
+
+        if action_type == ActionType.FINISH.value:
+            self._log_info("Task completion action -> DONE")
+            return ["DONE"]
+
+        if action_type == ActionType.FAIL.value:
+            self._log_info("Task infeasible action -> FAIL")
+            return ["FAIL"]
+
+        if action_type == ActionType.CALL_USER.value:
+            self._log_info("User intervention requested")
+            return []
+
+        raise ValueError(
+            f"Unknown action type: '{action_type}'. "
+            "Supported: click, left_double, left_triple, right_single, drag, "
+            "hotkey, type, scroll, wait, finish, fail, call_user"
+        )
+
+    def serialize_actions(self, actions: list[Action]) -> list[dict[str, Any]]:
+        """Serialize OAGI actions for trajectory logging."""
+        return [
+            {
+                "type": action.type.value,
+                "argument": action.argument,
+                "count": action.count,
+            }
+            for action in (actions or [])
+        ]

oagi/handler/pyautogui_action_handler.py

@@ -17,6 +17,7 @@ from ..constants import DEFAULT_STEP_DELAY
 from ..exceptions import check_optional_dependency
 from ..types import Action, ActionType, parse_coords, parse_drag_coords, parse_scroll
 from .capslock_manager import CapsLockManager
+from .utils import CoordinateScaler, normalize_key, parse_hotkey
 
 check_optional_dependency("pyautogui", "PyautoguiActionHandler", "desktop")
 import pyautogui  # noqa: E402
@@ -92,6 +93,15 @@ class PyautoguiActionHandler:
         self.caps_manager = CapsLockManager(mode=self.config.capslock_mode)
         # The origin position of coordinates (the top-left corner of the target screen)
         self.origin_x, self.origin_y = 0, 0
+        # Initialize coordinate scaler (OAGI uses 0-1000 normalized coordinates)
+        self._coord_scaler = CoordinateScaler(
+            source_width=1000,
+            source_height=1000,
+            target_width=self.screen_width,
+            target_height=self.screen_height,
+            origin_x=self.origin_x,
+            origin_y=self.origin_y,
+        )
 
     def reset(self):
         """Reset handler state.
@@ -109,6 +119,9 @@ class PyautoguiActionHandler:
         """
         self.screen_width, self.screen_height = screen.width, screen.height
         self.origin_x, self.origin_y = screen.x, screen.y
+        # Update coordinate scaler
+        self._coord_scaler.set_target_size(screen.width, screen.height)
+        self._coord_scaler.set_origin(screen.x, screen.y)
 
     def _denormalize_coords(self, x: float, y: float) -> tuple[int, int]:
         """Convert coordinates from 0-1000 range to actual screen coordinates.
@@ -116,26 +129,7 @@ class PyautoguiActionHandler:
         Also handles corner coordinates to prevent PyAutoGUI fail-safe trigger.
         Corner coordinates (0,0), (0,max), (max,0), (max,max) are offset by 1 pixel.
         """
-        screen_x = int(x * self.screen_width / 1000)
-        screen_y = int(y * self.screen_height / 1000)
-
-        # Prevent fail-safe by adjusting corner coordinates
-        # Check if coordinates are at screen corners (with small tolerance)
-        if screen_x < 1:
-            screen_x = 1
-        elif screen_x > self.screen_width - 1:
-            screen_x = self.screen_width - 1
-
-        if screen_y < 1:
-            screen_y = 1
-        elif screen_y > self.screen_height - 1:
-            screen_y = self.screen_height - 1
-
-        # Add origin offset to convert relative to top-left corner
-        screen_x += self.origin_x
-        screen_y += self.origin_y
-
-        return screen_x, screen_y
+        return self._coord_scaler.scale(x, y, prevent_failsafe=True)
 
     def _parse_coords(self, args_str: str) -> tuple[int, int]:
         """Extract x, y coordinates from argument string."""
@@ -163,28 +157,15 @@ class PyautoguiActionHandler:
 
     def _normalize_key(self, key: str) -> str:
         """Normalize key names for consistency."""
-        key = key.strip().lower()
-        # Normalize caps lock variations
-        hotkey_variations_mapping = {
-            "capslock": ["caps_lock", "caps", "capslock"],
-            "pgup": ["page_up", "pageup"],
-            "pgdn": ["page_down", "pagedown"],
-        }
-        for normalized, variations in hotkey_variations_mapping.items():
-            if key in variations:
-                return normalized
-        # Remap ctrl to command on macOS if enabled
-        if self.config.macos_ctrl_to_cmd and sys.platform == "darwin" and key == "ctrl":
-            return "command"
-        return key
+        return normalize_key(key, macos_ctrl_to_cmd=self.config.macos_ctrl_to_cmd)
 
     def _parse_hotkey(self, args_str: str) -> list[str]:
         """Parse hotkey string into list of keys."""
-        # Remove parentheses if present
-        args_str = args_str.strip("()")
-        # Split by '+' to get individual keys
-        keys = [self._normalize_key(key) for key in args_str.split("+")]
-        return keys
+        return parse_hotkey(
+            args_str,
+            macos_ctrl_to_cmd=self.config.macos_ctrl_to_cmd,
+            validate=False,  # Don't validate, let pyautogui handle invalid keys
+        )
 
     def _move_and_wait(self, x: int, y: int) -> None:
         """Move cursor to position and wait before clicking."""
@@ -269,8 +250,8 @@ class PyautoguiActionHandler:
                 )
                 pyautogui.scroll(scroll_amount)
 
-            case ActionType.FINISH:
-                # Task completion - reset handler state
+            case ActionType.FINISH | ActionType.FAIL:
+                # Task completion or infeasible - reset handler state
                 self.reset()
 
             case ActionType.WAIT: