oagi-core 0.9.0__py3-none-any.whl

oagi/pyautogui_action_handler.py

@@ -0,0 +1,268 @@
+# -----------------------------------------------------------------------------
+#  Copyright (c) OpenAGI Foundation
+#  All rights reserved.
+#
+#  This file is part of the official API project.
+#  Licensed under the MIT License.
+# -----------------------------------------------------------------------------
+
+import re
+import time
+
+from pydantic import BaseModel, Field
+
+from .exceptions import check_optional_dependency
+from .types import Action, ActionType
+
+check_optional_dependency("pyautogui", "PyautoguiActionHandler", "desktop")
+import pyautogui  # noqa: E402
+
+
+class CapsLockManager:
+    """Manages caps lock state for text transformation."""
+
+    def __init__(self, mode: str = "session"):
+        """Initialize caps lock manager.
+
+        Args:
+            mode: Either "session" (internal state) or "system" (OS-level)
+        """
+        self.mode = mode
+        self.caps_enabled = False
+
+    def toggle(self):
+        """Toggle caps lock state in session mode."""
+        if self.mode == "session":
+            self.caps_enabled = not self.caps_enabled
+
+    def transform_text(self, text: str) -> str:
+        """Transform text based on caps lock state.
+
+        Args:
+            text: Input text to transform
+
+        Returns:
+            Transformed text (uppercase if caps enabled in session mode)
+        """
+        if self.mode == "session" and self.caps_enabled:
+            # Transform letters to uppercase, preserve special characters
+            return "".join(c.upper() if c.isalpha() else c for c in text)
+        return text
+
+    def should_use_system_capslock(self) -> bool:
+        """Check if system-level caps lock should be used."""
+        return self.mode == "system"
+
+
+class PyautoguiConfig(BaseModel):
+    """Configuration for PyautoguiActionHandler."""
+
+    drag_duration: float = Field(
+        default=0.5, description="Duration for drag operations in seconds"
+    )
+    scroll_amount: int = Field(
+        default=30, description="Amount to scroll (positive for up, negative for down)"
+    )
+    wait_duration: float = Field(
+        default=1.0, description="Duration for wait actions in seconds"
+    )
+    action_pause: float = Field(
+        default=0.1, description="Pause between PyAutoGUI actions in seconds"
+    )
+    hotkey_interval: float = Field(
+        default=0.1, description="Interval between key presses in hotkey combinations"
+    )
+    capslock_mode: str = Field(
+        default="session",
+        description="Caps lock handling mode: 'session' (internal state) or 'system' (OS-level)",
+    )
+
+
+class PyautoguiActionHandler:
+    """
+    Handles actions to be executed using PyAutoGUI.
+
+    This class provides functionality for handling and executing a sequence of
+    actions using the PyAutoGUI library. It processes a list of actions and executes
+    them as per the implementation.
+
+    Methods:
+        __call__: Executes the provided list of actions.
+
+    Args:
+        actions (list[Action]): List of actions to be processed and executed.
+    """
+
+    def __init__(self, config: PyautoguiConfig | None = None):
+        # Use default config if none provided
+        self.config = config or PyautoguiConfig()
+        # Get screen dimensions for coordinate denormalization
+        self.screen_width, self.screen_height = pyautogui.size()
+        # Set default delay between actions
+        pyautogui.PAUSE = self.config.action_pause
+        # Initialize caps lock manager
+        self.caps_manager = CapsLockManager(mode=self.config.capslock_mode)
+
+    def _denormalize_coords(self, x: float, y: float) -> tuple[int, int]:
+        """Convert coordinates from 0-1000 range to actual screen coordinates.
+
+        Also handles corner coordinates to prevent PyAutoGUI fail-safe trigger.
+        Corner coordinates (0,0), (0,max), (max,0), (max,max) are offset by 1 pixel.
+        """
+        screen_x = int(x * self.screen_width / 1000)
+        screen_y = int(y * self.screen_height / 1000)
+
+        # Prevent fail-safe by adjusting corner coordinates
+        # Check if coordinates are at screen corners (with small tolerance)
+        if screen_x < 1:
+            screen_x = 1
+        elif screen_x > self.screen_width - 1:
+            screen_x = self.screen_width - 1
+
+        if screen_y < 1:
+            screen_y = 1
+        elif screen_y > self.screen_height - 1:
+            screen_y = self.screen_height - 1
+
+        return screen_x, screen_y
+
+    def _parse_coords(self, args_str: str) -> tuple[int, int]:
+        """Extract x, y coordinates from argument string."""
+        match = re.match(r"(\d+),\s*(\d+)", args_str)
+        if not match:
+            raise ValueError(f"Invalid coordinates format: {args_str}")
+        x, y = int(match.group(1)), int(match.group(2))
+        return self._denormalize_coords(x, y)
+
+    def _parse_drag_coords(self, args_str: str) -> tuple[int, int, int, int]:
+        """Extract x1, y1, x2, y2 coordinates from drag argument string."""
+        match = re.match(r"(\d+),\s*(\d+),\s*(\d+),\s*(\d+)", args_str)
+        if not match:
+            raise ValueError(f"Invalid drag coordinates format: {args_str}")
+        x1, y1, x2, y2 = (
+            int(match.group(1)),
+            int(match.group(2)),
+            int(match.group(3)),
+            int(match.group(4)),
+        )
+        x1, y1 = self._denormalize_coords(x1, y1)
+        x2, y2 = self._denormalize_coords(x2, y2)
+        return x1, y1, x2, y2
+
+    def _parse_scroll(self, args_str: str) -> tuple[int, int, str]:
+        """Extract x, y, direction from scroll argument string."""
+        match = re.match(r"(\d+),\s*(\d+),\s*(\w+)", args_str)
+        if not match:
+            raise ValueError(f"Invalid scroll format: {args_str}")
+        x, y = int(match.group(1)), int(match.group(2))
+        x, y = self._denormalize_coords(x, y)
+        direction = match.group(3).lower()
+        return x, y, direction
+
+    def _normalize_key(self, key: str) -> str:
+        """Normalize key names for consistency."""
+        key = key.strip().lower()
+        # Normalize caps lock variations
+        if key in ["caps_lock", "caps", "capslock"]:
+            return "capslock"
+        return key
+
+    def _parse_hotkey(self, args_str: str) -> list[str]:
+        """Parse hotkey string into list of keys."""
+        # Remove parentheses if present
+        args_str = args_str.strip("()")
+        # Split by '+' to get individual keys
+        keys = [self._normalize_key(key) for key in args_str.split("+")]
+        return keys
+
+    def _execute_single_action(self, action: Action) -> None:
+        """Execute a single action once."""
+        arg = action.argument.strip("()")  # Remove outer parentheses if present
+
+        match action.type:
+            case ActionType.CLICK:
+                x, y = self._parse_coords(arg)
+                pyautogui.click(x, y)
+
+            case ActionType.LEFT_DOUBLE:
+                x, y = self._parse_coords(arg)
+                pyautogui.doubleClick(x, y)
+
+            case ActionType.LEFT_TRIPLE:
+                x, y = self._parse_coords(arg)
+                pyautogui.tripleClick(x, y)
+
+            case ActionType.RIGHT_SINGLE:
+                x, y = self._parse_coords(arg)
+                pyautogui.rightClick(x, y)
+
+            case ActionType.DRAG:
+                x1, y1, x2, y2 = self._parse_drag_coords(arg)
+                pyautogui.moveTo(x1, y1)
+                pyautogui.dragTo(
+                    x2, y2, duration=self.config.drag_duration, button="left"
+                )
+
+            case ActionType.HOTKEY:
+                keys = self._parse_hotkey(arg)
+                # Check if this is a caps lock key press
+                if len(keys) == 1 and keys[0] == "capslock":
+                    if self.caps_manager.should_use_system_capslock():
+                        # System mode: use OS-level caps lock
+                        pyautogui.hotkey(
+                            "capslock", interval=self.config.hotkey_interval
+                        )
+                    else:
+                        # Session mode: toggle internal state
+                        self.caps_manager.toggle()
+                else:
+                    # Regular hotkey combination
+                    pyautogui.hotkey(*keys, interval=self.config.hotkey_interval)
+
+            case ActionType.TYPE:
+                # Remove quotes if present
+                text = arg.strip("\"'")
+                # Apply caps lock transformation if needed
+                text = self.caps_manager.transform_text(text)
+                pyautogui.typewrite(text)
+
+            case ActionType.SCROLL:
+                x, y, direction = self._parse_scroll(arg)
+                pyautogui.moveTo(x, y)
+                scroll_amount = (
+                    self.config.scroll_amount
+                    if direction == "up"
+                    else -self.config.scroll_amount
+                )
+                pyautogui.scroll(scroll_amount)
+
+            case ActionType.FINISH:
+                # Task completion - no action needed
+                pass
+
+            case ActionType.WAIT:
+                # Wait for a short period
+                time.sleep(self.config.wait_duration)
+
+            case ActionType.CALL_USER:
+                # Call user - implementation depends on requirements
+                print("User intervention requested")
+
+            case _:
+                print(f"Unknown action type: {action.type}")
+
+    def _execute_action(self, action: Action) -> None:
+        """Execute an action, potentially multiple times."""
+        count = action.count or 1
+
+        for _ in range(count):
+            self._execute_single_action(action)
+
+    def __call__(self, actions: list[Action]) -> None:
+        """Execute the provided list of actions."""
+        for action in actions:
+            try:
+                self._execute_action(action)
+            except Exception as e:
+                print(f"Error executing action {action.type}: {e}")
+                raise

oagi/screenshot_maker.py

@@ -0,0 +1,41 @@
+# -----------------------------------------------------------------------------
+#  Copyright (c) OpenAGI Foundation
+#  All rights reserved.
+#
+#  This file is part of the official API project.
+#  Licensed under the MIT License.
+# -----------------------------------------------------------------------------
+
+from typing import Optional
+
+from .pil_image import PILImage
+from .types import Image
+from .types.models.image_config import ImageConfig
+
+
+class ScreenshotMaker:
+    """Takes screenshots using pyautogui."""
+
+    def __init__(self, config: ImageConfig | None = None):
+        self.config = config or ImageConfig()
+        self._last_image: Optional[PILImage] = None
+
+    def __call__(self) -> Image:
+        """Take and process a screenshot."""
+        # Create PILImage from screenshot
+        pil_image = PILImage.from_screenshot()
+
+        # Apply transformation if config is set
+        if self.config:
+            pil_image = pil_image.transform(self.config)
+
+        # Store as the last image
+        self._last_image = pil_image
+
+        return pil_image
+
+    def last_image(self) -> Image:
+        """Return the last screenshot taken, or take a new one if none exists."""
+        if self._last_image is None:
+            return self()
+        return self._last_image

oagi/server/__init__.py

@@ -0,0 +1,13 @@
+# -----------------------------------------------------------------------------
+#  Copyright (c) OpenAGI Foundation
+#  All rights reserved.
+#
+#  This file is part of the official API project.
+#  Licensed under the MIT License.
+# -----------------------------------------------------------------------------
+
+from .config import ServerConfig
+from .main import create_app
+from .socketio_server import sio
+
+__all__ = ["create_app", "sio", "ServerConfig"]

oagi/server/agent_wrappers.py

@@ -0,0 +1,98 @@
+# -----------------------------------------------------------------------------
+#  Copyright (c) OpenAGI Foundation
+#  All rights reserved.
+#
+#  This file is part of the official API project.
+#  Licensed under the MIT License.
+# -----------------------------------------------------------------------------
+
+import logging
+from typing import TYPE_CHECKING
+
+from ..types import URLImage
+from ..types.models.action import Action
+from .models import ScreenshotRequestData, ScreenshotResponseData
+
+if TYPE_CHECKING:
+    from .session_store import Session
+    from .socketio_server import SessionNamespace
+
+logger = logging.getLogger(__name__)
+
+
+class SocketIOActionHandler:
+    """Wraps Socket.IO connection as an AsyncActionHandler.
+
+    This handler emits actions through the Socket.IO connection to the client.
+    """
+
+    def __init__(self, namespace: "SessionNamespace", session: "Session"):
+        self.namespace = namespace
+        self.session = session
+
+    async def __call__(self, actions: list[Action]) -> None:
+        if not actions:
+            logger.debug("No actions to execute")
+            return
+
+        logger.debug(f"Executing {len(actions)} actions via Socket.IO")
+        await self.namespace._emit_actions(self.session, actions)
+
+
+class SocketIOImageProvider:
+    """Wraps Socket.IO connection as an AsyncImageProvider.
+
+    This provider requests screenshots from the client through Socket.IO.
+    """
+
+    def __init__(
+        self,
+        namespace: "SessionNamespace",
+        session: "Session",
+        oagi_client,
+    ):
+        self.namespace = namespace
+        self.session = session
+        self.oagi_client = oagi_client
+        self._last_url: str | None = None
+
+    async def __call__(self) -> URLImage:
+        logger.debug("Requesting screenshot via Socket.IO")
+
+        # Get S3 presigned URL from OAGI
+        upload_response = await self.oagi_client.get_s3_presigned_url()
+
+        # Request screenshot from client with the presigned URL
+        screenshot_data = await self.namespace.call(
+            "request_screenshot",
+            ScreenshotRequestData(
+                presigned_url=upload_response.url,
+                uuid=upload_response.uuid,
+                expires_at=str(upload_response.expires_at),  # Convert int to string
+            ).model_dump(),
+            to=self.session.socket_id,
+            timeout=self.namespace.config.socketio_timeout,
+        )
+
+        if not screenshot_data:
+            raise Exception("No response from screenshot request")
+
+        # Validate response
+        ack = ScreenshotResponseData(**screenshot_data)
+        if not ack.success:
+            raise Exception(f"Screenshot upload failed: {ack.error}")
+
+        # Store the URL for last_image()
+        self._last_url = upload_response.download_url
+        self.session.current_screenshot_url = upload_response.download_url
+
+        logger.debug(f"Screenshot captured successfully: {upload_response.uuid}")
+        return URLImage(upload_response.download_url)
+
+    async def last_image(self) -> URLImage:
+        if self._last_url:
+            logger.debug("Returning last captured screenshot")
+            return URLImage(self._last_url)
+
+        logger.debug("No previous screenshot, capturing new one")
+        return await self()

oagi/server/config.py

@@ -0,0 +1,46 @@
+# -----------------------------------------------------------------------------
+#  Copyright (c) OpenAGI Foundation
+#  All rights reserved.
+#
+#  This file is part of the official API project.
+#  Licensed under the MIT License.
+# -----------------------------------------------------------------------------
+
+from pydantic import Field
+
+from ..exceptions import check_optional_dependency
+
+check_optional_dependency("pydantic_settings", "Server features", "server")
+from pydantic_settings import BaseSettings  # noqa: E402
+
+
+class ServerConfig(BaseSettings):
+    # OAGI API settings
+    oagi_api_key: str = Field(..., alias="OAGI_API_KEY")
+    oagi_base_url: str = Field(default="https://api.agiopen.org", alias="OAGI_BASE_URL")
+
+    # Server settings
+    server_host: str = Field(default="0.0.0.0", alias="OAGI_SERVER_HOST")
+    server_port: int = Field(default=8000, alias="OAGI_SERVER_PORT")
+    cors_allowed_origins: str = Field(default="*", alias="OAGI_CORS_ORIGINS")
+
+    # Session settings
+    session_timeout_seconds: float = Field(default=10.0)
+
+    # Model settings
+    default_model: str = Field(default="lux-v1", alias="OAGI_DEFAULT_MODEL")
+    default_temperature: float = Field(default=0.0, ge=0.0, le=2.0)
+
+    # Agent settings
+    max_steps: int = Field(default=30, alias="OAGI_MAX_STEPS", ge=1, le=100)
+
+    # Socket.IO settings
+    socketio_path: str = Field(default="/socket.io")
+    socketio_timeout: float = Field(default=30.0)
+
+    model_config = {
+        "env_file": ".env",
+        "env_file_encoding": "utf-8",
+        "populate_by_name": True,
+        "extra": "ignore",
+    }

oagi/server/main.py

@@ -0,0 +1,157 @@
+# -----------------------------------------------------------------------------
+#  Copyright (c) OpenAGI Foundation
+#  All rights reserved.
+#
+#  This file is part of the official API project.
+#  Licensed under the MIT License.
+# -----------------------------------------------------------------------------
+
+import logging
+from datetime import datetime
+from typing import Any
+
+from ..exceptions import check_optional_dependency
+from .config import ServerConfig
+from .models import SessionStatusData
+from .session_store import session_store
+from .socketio_server import socket_app
+
+check_optional_dependency("fastapi", "Server features", "server")
+check_optional_dependency("uvicorn", "Server features", "server")
+
+import uvicorn  # noqa: E402
+from fastapi import FastAPI, HTTPException  # noqa: E402
+from fastapi.middleware.cors import CORSMiddleware  # noqa: E402
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+
+logger = logging.getLogger(__name__)
+
+
+def create_app(config: ServerConfig | None = None) -> FastAPI:
+    if config is None:
+        config = ServerConfig()
+
+    app = FastAPI(
+        title="OAGI Socket.IO Server",
+        description="Real-time task automation server for OAGI SDK",
+        version="0.1.0",
+    )
+
+    cors_origins = (
+        config.cors_allowed_origins.split(",")
+        if config.cors_allowed_origins != "*"
+        else ["*"]
+    )
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=cors_origins,
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    @app.get("/")
+    async def root() -> dict[str, str]:
+        return {
+            "name": "OAGI Socket.IO Server",
+            "version": "0.1.0",
+            "status": "running",
+        }
+
+    @app.get("/health")
+    async def health_check() -> dict[str, Any]:
+        return {
+            "status": "healthy",
+            "server": {
+                "name": "OAGI Socket.IO Server",
+                "version": "0.1.0",
+            },
+            "config": {
+                "base_url": config.oagi_base_url,
+                "default_model": config.default_model,
+            },
+            "sessions": {
+                "active": len(session_store.sessions),
+                "connected": sum(
+                    1 for s in session_store.sessions.values() if s.socket_id
+                ),
+            },
+        }
+
+    @app.get("/sessions")
+    async def list_sessions() -> dict[str, Any]:
+        return {
+            "sessions": session_store.list_sessions(),
+            "total": len(session_store.sessions),
+        }
+
+    @app.get("/sessions/{session_id}")
+    async def get_session(session_id: str) -> SessionStatusData:
+        session = session_store.get_session(session_id)
+        if not session:
+            raise HTTPException(
+                status_code=404, detail=f"Session {session_id} not found"
+            )
+
+        return SessionStatusData(
+            session_id=session.session_id,
+            status=session.status,  # type: ignore
+            instruction=session.instruction,
+            created_at=session.created_at,
+            actions_executed=session.actions_executed,
+            last_activity=datetime.fromtimestamp(session.last_activity).isoformat(),
+        )
+
+    @app.delete("/sessions/{session_id}")
+    async def delete_session(session_id: str) -> dict[str, str]:
+        session = session_store.get_session(session_id)
+        if not session:
+            raise HTTPException(
+                status_code=404, detail=f"Session {session_id} not found"
+            )
+
+        if session.oagi_client:
+            try:
+                await session.oagi_client.close()
+            except Exception as e:
+                logger.warning(f"Error closing OAGI client: {e}")
+
+        deleted = session_store.delete_session(session_id)
+        if deleted:
+            return {"message": f"Session {session_id} deleted"}
+        else:
+            raise HTTPException(status_code=500, detail="Failed to delete session")
+
+    @app.post("/sessions/cleanup")
+    async def cleanup_sessions(timeout_hours: float = 1.0) -> dict[str, Any]:
+        timeout_seconds = timeout_hours * 3600
+        cleaned = session_store.cleanup_inactive_sessions(timeout_seconds)
+        return {
+            "cleaned": cleaned,
+            "remaining": len(session_store.sessions),
+        }
+
+    # Mount Socket.IO application
+    app.mount("/", socket_app)
+
+    logger.info(
+        f"Server created - will listen on {config.server_host}:{config.server_port}"
+    )
+
+    return app
+
+
+if __name__ == "__main__":
+    config = ServerConfig()
+    app = create_app(config)
+    uvicorn.run(
+        app,
+        host=config.server_host,
+        port=config.server_port,
+        log_level="info",
+    )