hud-python 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

hud/__init__.py

@@ -5,19 +5,21 @@ HUD Gym SDK - A Python SDK for interacting with HUD environments.
 from __future__ import annotations
 
 from . import agent, env, gym, settings, task, taskset, types, utils
-from .job import create_job, job, load_job
+from .job import create_job, load_job, run_job
+from .job import job as register_job
 from .taskset import load_taskset
 
-__version__ = "0.2.1"
+__version__ = "0.2.3"
 
 __all__ = [
     "agent",
     "create_job",
     "env",
     "gym",
-    "job",
     "load_job",
     "load_taskset",
+    "register_job",
+    "run_job",
     "settings",
     "task",
     "taskset",

hud/adapters/__init__.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from .claude import ClaudeAdapter
 from .common import CLA, Adapter
+from .common.types import ResponseAction
 from .operator import OperatorAdapter
 
-__all__ = ["CLA", "Adapter", "ClaudeAdapter", "OperatorAdapter"]
+__all__ = ["CLA", "Adapter", "ClaudeAdapter", "OperatorAdapter", "ResponseAction"]

hud/adapters/claude/adapter.py

@@ -23,9 +23,13 @@ from hud.adapters.common.types import (
 
 class ClaudeAdapter(Adapter):
     KEY_MAP: ClassVar[dict[str, CLAKey]] = {
-        "Return": "enter",
-        "Super": "win",
-        }
+        "return": "enter",
+        "super": "win",
+        "super_l": "win",
+        "super_r": "win",
+        "right shift": "shift",
+        "left shift": "shift",
+    }
 
     def __init__(self) -> None:
         super().__init__()
@@ -34,7 +38,8 @@ class ClaudeAdapter(Adapter):
 
     def _map_key(self, key: str) -> CLAKey:
         """Map a key to its standardized form."""
-        return self.KEY_MAP.get(key, key.lower())  # type: ignore
+        return self.KEY_MAP.get(key.lower(), key.lower())  # type: ignore
+
     def convert(self, data: Any) -> CLA:
         try:
             action_type = data.get("action")
@@ -42,9 +47,7 @@ class ClaudeAdapter(Adapter):
             if action_type == "key":
                 assert "text" in data
                 if "+" in data["text"]:
-                    keys: list[CLAKey] = [
-                        self._map_key(k) for k in (data["text"].split("+"))
-                    ]
+                    keys: list[CLAKey] = [self._map_key(k) for k in (data["text"].split("+"))]
                     assert len(keys) > 0
                     return PressAction(keys=keys)
                 return PressAction(keys=[self._map_key(data["text"])])
@@ -78,19 +81,12 @@ class ClaudeAdapter(Adapter):
                 assert len(coord) == 2
                 if (
                     len(self.memory) == 0
-                    or (
-                        self.memory[-1] is not MoveAction
-                        and self.memory[-1] is not ClickAction
-                    )
+                    or (self.memory[-1] is not MoveAction and self.memory[-1] is not ClickAction)
                     or self.memory[-1].point is None
                 ):
-                    raise ValueError(
-                        "Left click drag must be preceded by a move or click action"
-                    )
+                    raise ValueError("Left click drag must be preceded by a move or click action")
                 else:
-                    return DragAction(
-                        path=[self.memory[-1].point, Point(x=coord[0], y=coord[1])]
-                    )
+                    return DragAction(path=[self.memory[-1].point, Point(x=coord[0], y=coord[1])])
 
             elif action_type == "right_click":
                 assert "coordinate" in data

hud/adapters/common/adapter.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, TypeAlias
 
 import numpy as np
 from PIL import Image
@@ -11,7 +11,7 @@ from .types import CLA
 if TYPE_CHECKING:
     from typing_extensions import TypeIs
 
-ImageType = np.ndarray[Any, Any] | Image.Image | str | None
+ImageType: TypeAlias = np.ndarray[Any, Any] | Image.Image | str | None
 
 
 def _is_numpy_array(observation: Any) -> TypeIs[np.ndarray]:
@@ -164,5 +164,5 @@ class Adapter:
     def adapt_list(self, actions: list[Any]) -> list[CLA]:
         if not isinstance(actions, list):
             raise ValueError("Please provide a list of actions")
-        
+
         return [self.adapt(action) for action in actions]

hud/adapters/common/tests/test_adapter.py

@@ -0,0 +1,277 @@
+from __future__ import annotations
+
+import base64
+import io
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import pytest
+from PIL import Image
+
+from hud.adapters.common import Adapter
+from hud.adapters.common.types import ClickAction, Point, TypeAction
+
+
+@pytest.fixture
+def adapter():
+    """Fixture providing a clean adapter instance."""
+    return Adapter()
+
+
+@pytest.fixture
+def test_image():
+    """Fixture providing test image in various formats."""
+    img = Image.new("RGB", (100, 80), color="red")
+    img_bytes = io.BytesIO()
+    img.save(img_bytes, format="PNG")
+    img_base64 = base64.b64encode(img_bytes.getvalue()).decode("utf-8")
+    img_array = np.array(img)
+
+    return {
+        "pil": img,
+        "bytes": img_bytes.getvalue(),
+        "base64": img_base64,
+        "array": img_array,
+    }
+
+
+def test_init(adapter):
+    """Test adapter initialization."""
+    assert adapter.agent_width == 1920
+    assert adapter.agent_height == 1080
+    assert adapter.env_width == 1920
+    assert adapter.env_height == 1080
+    assert adapter.memory == []
+
+
+def test_preprocess(adapter):
+    """Test preprocess method (default implementation)."""
+    action = {"type": "click", "point": {"x": 100, "y": 100}}
+    result = adapter.preprocess(action)
+    assert result == action  # Default implementation returns unchanged
+
+
+def test_convert_valid(adapter):
+    """Test convert method with valid action."""
+    action = ClickAction(point=Point(x=100, y=100))
+    result = adapter.convert(action)
+    # Fix: Instead of checking against CLA, check it's the same type as the input
+    assert isinstance(result, ClickAction)
+    assert result == action
+
+
+def test_convert_invalid(adapter):
+    """Test convert method with invalid action."""
+    with pytest.raises(ValueError):
+        adapter.convert(None)  # type: ignore
+
+
+def test_json_valid(adapter):
+    """Test json method with valid action."""
+    action = ClickAction(point=Point(x=100, y=100))
+    result = adapter.json(action)
+    assert isinstance(result, dict)
+    assert result["type"] == "click"
+    assert result["point"]["x"] == 100
+    assert result["point"]["y"] == 100
+
+
+def test_json_invalid(adapter):
+    """Test json method with invalid action."""
+    with pytest.raises(ValueError):
+        adapter.json(None)  # type: ignore
+
+
+def test_rescale_pil_image(adapter, test_image):
+    """Test rescaling PIL Image."""
+    result = adapter.rescale(test_image["pil"])
+
+    # Verify result is base64 string
+    assert isinstance(result, str)
+
+    # Verify environment dimensions were updated
+    assert adapter.env_width == 100
+    assert adapter.env_height == 80
+
+    # Decode and verify image dimensions
+    img_bytes = base64.b64decode(result)
+    img = Image.open(io.BytesIO(img_bytes))
+    assert img.size == (adapter.agent_width, adapter.agent_height)
+
+
+def test_rescale_numpy_array(adapter, test_image):
+    """Test rescaling numpy array."""
+    result = adapter.rescale(test_image["array"])
+
+    # Verify result is base64 string
+    assert isinstance(result, str)
+
+    # Verify environment dimensions were updated
+    assert adapter.env_width == 100
+    assert adapter.env_height == 80
+
+
+def test_rescale_base64(adapter, test_image):
+    """Test rescaling base64 string."""
+    result = adapter.rescale(test_image["base64"])
+
+    # Verify result is base64 string
+    assert isinstance(result, str)
+
+    # Verify environment dimensions were updated
+    assert adapter.env_width == 100
+    assert adapter.env_height == 80
+
+
+def test_rescale_base64_with_header(adapter, test_image):
+    """Test rescaling base64 string with header."""
+    base64_with_header = f"data:image/png;base64,{test_image['base64']}"
+    result = adapter.rescale(base64_with_header)
+
+    # Verify result is base64 string
+    assert isinstance(result, str)
+
+    # Verify environment dimensions were updated
+    assert adapter.env_width == 100
+    assert adapter.env_height == 80
+
+
+def test_rescale_invalid_type(adapter):
+    """Test rescaling with invalid type."""
+    with pytest.raises(ValueError):
+        adapter.rescale(123)  # type: ignore
+
+
+def test_rescale_none(adapter):
+    """Test rescaling with None."""
+    result = adapter.rescale(None)
+    assert result is None
+
+
+def test_postprocess_action_click(adapter):
+    """Test postprocess_action with click action."""
+    # Set different agent and env dimensions
+    adapter.agent_width = 1000
+    adapter.agent_height = 800
+    adapter.env_width = 2000
+    adapter.env_height = 1600
+
+    action = {"type": "click", "point": {"x": 500, "y": 400}}
+    result = adapter.postprocess_action(action)
+
+    # Coordinates should be doubled
+    assert result["point"]["x"] == 1000
+    assert result["point"]["y"] == 800
+
+
+def test_postprocess_action_drag(adapter):
+    """Test postprocess_action with drag action."""
+    # Set different agent and env dimensions
+    adapter.agent_width = 1000
+    adapter.agent_height = 800
+    adapter.env_width = 2000
+    adapter.env_height = 1600
+
+    action = {"type": "drag", "path": [{"x": 100, "y": 200}, {"x": 300, "y": 400}]}
+    result = adapter.postprocess_action(action)
+
+    # Coordinates should be doubled
+    assert result["path"][0]["x"] == 200
+    assert result["path"][0]["y"] == 400
+    assert result["path"][1]["x"] == 600
+    assert result["path"][1]["y"] == 800
+
+
+def test_postprocess_action_scroll(adapter):
+    """Test postprocess_action with scroll action."""
+    # Set different agent and env dimensions
+    adapter.agent_width = 1000
+    adapter.agent_height = 800
+    adapter.env_width = 2000
+    adapter.env_height = 1600
+
+    action = {"type": "scroll", "point": {"x": 500, "y": 400}, "scroll": {"x": 0, "y": 10}}
+    result = adapter.postprocess_action(action)
+
+    # Point coordinates should be doubled
+    assert result["point"]["x"] == 1000
+    assert result["point"]["y"] == 800
+    # Scroll amount should be scaled
+    assert result["scroll"]["x"] == 0
+    assert result["scroll"]["y"] == 20
+
+
+def test_postprocess_action_empty(adapter):
+    """Test postprocess_action with empty action."""
+    result = adapter.postprocess_action({})
+    assert result == {}
+
+
+def test_adapt(adapter):
+    """Test adapt method."""
+    # Mock the needed methods
+    with (
+        patch.object(adapter, "preprocess", return_value={"preprocessed": True}),
+        patch.object(adapter, "convert", return_value=TypeAction(text="test")),
+        patch.object(adapter, "json", return_value={"type": "type", "text": "test"}),
+        patch.object(adapter, "postprocess_action", return_value={"type": "type", "text": "test"}),
+        patch("hud.adapters.common.adapter.TypeAdapter") as mock_adapter,
+    ):
+        mock_validator = MagicMock()
+        mock_adapter.return_value = mock_validator
+        mock_validator.validate_python.return_value = TypeAction(text="test")
+
+        adapter.adapt({"raw": "action"})
+
+        # Verify the method chain was called correctly
+        adapter.preprocess.assert_called_once_with({"raw": "action"})
+        adapter.convert.assert_called_once_with({"preprocessed": True})
+        adapter.json.assert_called_once_with(TypeAction(text="test"))
+        adapter.postprocess_action.assert_called_once_with({"type": "type", "text": "test"})
+
+        # Verify the memory was updated
+        assert len(adapter.memory) == 1
+        assert adapter.memory[0] == TypeAction(text="test")
+
+
+def test_adapt_list(adapter):
+    """Test adapt_list method."""
+    # Fix: Use side_effect to return different values for each call to adapt
+    click_action = ClickAction(point=Point(x=100, y=100))
+    type_action = TypeAction(text="test")
+
+    mock_adapt = MagicMock(side_effect=[click_action, type_action])
+    with patch.object(adapter, "adapt", mock_adapt):
+        actions = [{"type": "click"}, {"type": "type"}]
+        result = adapter.adapt_list(actions)
+
+        assert adapter.adapt.call_count == 2
+        assert len(result) == 2
+        assert result[0] == click_action
+        assert result[1] == type_action
+
+
+def test_adapt_list_invalid(adapter):
+    """Test adapt_list with invalid input."""
+    with pytest.raises(ValueError):
+        adapter.adapt_list("not a list")  # type: ignore
+
+
+def test_integration(adapter):
+    """Integration test for the full adapter pipeline."""
+    adapter.agent_width = 1000
+    adapter.agent_height = 800
+    adapter.env_width = 2000
+    adapter.env_height = 1600
+
+    # Create a click action
+    action = ClickAction(point=Point(x=500, y=400))
+
+    result = adapter.adapt(action)
+
+    assert isinstance(result, ClickAction)
+    assert result.point is not None
+    assert result.point.x == 1000  # Scaled from 500 to 1000
+    assert result.point.y == 800  # Scaled from 400 to 800
+
+    assert len(adapter.memory) == 1

hud/adapters/common/types.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Annotated, Literal
+from typing import Annotated, Literal, TypeAlias
 
 from pydantic import BaseModel, Field
 
@@ -20,7 +20,6 @@ class Point(BaseModel):
 class ClickAction(CLAAction):
     type: Literal["click"] = "click"
     point: Point | None = None
-    selector: str | None = None
     button: Literal["left", "right", "wheel", "back", "forward"] = "left"
     pattern: list[int] | None = None  # [delay_1, delay_2, ...]
     hold_keys: list[CLAKey] | None = None
@@ -48,7 +47,6 @@ class KeyUpAction(CLAAction):
 class TypeAction(CLAAction):
     type: Literal["type"] = "type"
     text: str
-    selector: str | None = None
     enter_after: bool | None = False
 
 
@@ -64,7 +62,6 @@ class ScrollAction(CLAAction):
 class MoveAction(CLAAction):
     type: Literal["move"] = "move"
     point: Point | None = None
-    selector: str | None = None
     offset: Point | None = None
 
 
@@ -85,7 +82,7 @@ class DragAction(CLAAction):
 # RESPONSE ACTION from agent
 class ResponseAction(CLAAction):
     type: Literal["response"] = "response"
-    text: str # The final textual response from the agent
+    text: str  # The final textual response from the agent
 
 
 # SCREENSHOT ACTION
@@ -121,7 +118,7 @@ CLA = Annotated[
 ]
 
 
-CLAKey = Literal[
+CLAKey: TypeAlias = Literal[
     # Control keys
     "backspace",
     "tab",

hud/adapters/operator/adapter.py

@@ -20,78 +20,71 @@ from hud.adapters.common.types import (
 
 class OperatorAdapter(Adapter):
     KEY_MAP: ClassVar[dict[str, CLAKey]] = {
-        "Return": "enter",
-        "ArrowUp": "up",
-        "ArrowDown": "down",
-        "ArrowLeft": "left",
-        "ArrowRight": "right",
+        "return": "enter",
+        "arrowup": "up",
+        "arrowdown": "down",
+        "arrowleft": "left",
+        "arrowright": "right",
     }
-    
+
     def __init__(self) -> None:
         super().__init__()
         # OpenAI Computer Use default dimensions
         self.agent_width = 1024
         self.agent_height = 768
-        
+
     def _map_key(self, key: str) -> CLAKey:
         """Map a key to its standardized form."""
-        return self.KEY_MAP.get(key, key.lower())  # type: ignore
-        
+        return self.KEY_MAP.get(key.lower(), key.lower())  # type: ignore
+
     def convert(self, data: Any) -> CLA:
         """Convert a Computer Use action to a HUD action"""
         try:
             action_type = data.get("type")
-            
+
             if action_type == "click":
                 x, y = data.get("x", 0), data.get("y", 0)
                 button = data.get("button", "left")
                 return ClickAction(point=Point(x=x, y=y), button=button)
-                
+
             elif action_type == "double_click":
                 x, y = data.get("x", 0), data.get("y", 0)
-                return ClickAction(
-                    point=Point(x=x, y=y),
-                    button="left",
-                    pattern=[100]
-                )
-                
+                return ClickAction(point=Point(x=x, y=y), button="left", pattern=[100])
+
             elif action_type == "scroll":
                 x, y = data.get("x", 0), data.get("y", 0)
                 scroll_x = data.get("scroll_x", 0)
                 scroll_y = data.get("scroll_y", 0)
-                return ScrollAction(
-                    point=Point(x=x, y=y),
-                    scroll=Point(x=scroll_x, y=scroll_y)
-                )
-                
+                return ScrollAction(point=Point(x=x, y=y), scroll=Point(x=scroll_x, y=scroll_y))
+
             elif action_type == "type":
                 text = data.get("text", "")
                 return TypeAction(text=text, enter_after=False)
-                
+
             elif action_type == "wait":
                 ms = data.get("ms", 1000)
                 return WaitAction(time=ms)
-                
+
             elif action_type == "move":
                 x, y = data.get("x", 0), data.get("y", 0)
                 return MoveAction(point=Point(x=x, y=y))
-                
+
             elif action_type == "keypress":
                 keys = data.get("keys", [])
                 return PressAction(keys=[self._map_key(k) for k in keys])
-                
+
             elif action_type == "drag":
                 path = data.get("path", [])
                 points = [Point(x=p.get("x", 0), y=p.get("y", 0)) for p in path]
                 return DragAction(path=points)
-            
+
             elif action_type == "screenshot":
                 return ScreenshotFetch()
-            
+
             elif action_type == "response":
                 return ResponseAction(text=data.get("text", ""))
             else:
                 raise ValueError(f"Unsupported action type: {action_type}")
-                
+
         except Exception as e:
             raise ValueError(f"Invalid action: {data}. Error: {e!s}") from e

hud/agent/__init__.py

@@ -1,7 +1,15 @@
 from .base import Agent
 from .claude import ClaudeAgent
 from .operator import OperatorAgent
+from .langchain import LangchainAgent
 
 from hud.adapters import OperatorAdapter, ClaudeAdapter
 
-__all__ = ["Agent", "ClaudeAgent", "OperatorAgent", "OperatorAdapter", "ClaudeAdapter"]
+__all__ = [
+    "Agent",
+    "ClaudeAgent",
+    "OperatorAgent",
+    "OperatorAdapter",
+    "ClaudeAdapter",
+    "LangchainAgent",
+]