hud-python 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

hud/__init__.py

@@ -5,19 +5,20 @@ HUD Gym SDK - A Python SDK for interacting with HUD environments.
 from __future__ import annotations
 
 from . import agent, env, gym, settings, task, taskset, types, utils
-from .job import create_job, job, load_job, run_job
+from .job import create_job, load_job, run_job
+from .job import job as register_job
 from .taskset import load_taskset
 
-__version__ = "0.2.2"
+__version__ = "0.2.4"
 
 __all__ = [
     "agent",
     "create_job",
     "env",
     "gym",
-    "job",
     "load_job",
     "load_taskset",
+    "register_job",
     "run_job",
     "settings",
     "task",

hud/adapters/claude/adapter.py

@@ -39,7 +39,7 @@ class ClaudeAdapter(Adapter):
     def _map_key(self, key: str) -> CLAKey:
         """Map a key to its standardized form."""
         return self.KEY_MAP.get(key.lower(), key.lower())  # type: ignore
-    
+
     def convert(self, data: Any) -> CLA:
         try:
             action_type = data.get("action")
@@ -47,9 +47,7 @@ class ClaudeAdapter(Adapter):
             if action_type == "key":
                 assert "text" in data
                 if "+" in data["text"]:
-                    keys: list[CLAKey] = [
-                        self._map_key(k) for k in (data["text"].split("+"))
-                    ]
+                    keys: list[CLAKey] = [self._map_key(k) for k in (data["text"].split("+"))]
                     assert len(keys) > 0
                     return PressAction(keys=keys)
                 return PressAction(keys=[self._map_key(data["text"])])
@@ -83,19 +81,12 @@ class ClaudeAdapter(Adapter):
                 assert len(coord) == 2
                 if (
                     len(self.memory) == 0
-                    or (
-                        self.memory[-1] is not MoveAction
-                        and self.memory[-1] is not ClickAction
-                    )
+                    or (self.memory[-1] is not MoveAction and self.memory[-1] is not ClickAction)
                     or self.memory[-1].point is None
                 ):
-                    raise ValueError(
-                        "Left click drag must be preceded by a move or click action"
-                    )
+                    raise ValueError("Left click drag must be preceded by a move or click action")
                 else:
-                    return DragAction(
-                        path=[self.memory[-1].point, Point(x=coord[0], y=coord[1])]
-                    )
+                    return DragAction(path=[self.memory[-1].point, Point(x=coord[0], y=coord[1])])
 
             elif action_type == "right_click":
                 assert "coordinate" in data

hud/adapters/common/adapter.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, TypeAlias
 
 import numpy as np
 from PIL import Image
@@ -11,7 +11,7 @@ from .types import CLA
 if TYPE_CHECKING:
     from typing_extensions import TypeIs
 
-ImageType = np.ndarray[Any, Any] | Image.Image | str | None
+ImageType: TypeAlias = np.ndarray[Any, Any] | Image.Image | str | None
 
 
 def _is_numpy_array(observation: Any) -> TypeIs[np.ndarray]:
@@ -164,5 +164,5 @@ class Adapter:
     def adapt_list(self, actions: list[Any]) -> list[CLA]:
         if not isinstance(actions, list):
             raise ValueError("Please provide a list of actions")
-        
+
         return [self.adapt(action) for action in actions]

hud/adapters/common/tests/test_adapter.py

@@ -0,0 +1,277 @@
+from __future__ import annotations
+
+import base64
+import io
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import pytest
+from PIL import Image
+
+from hud.adapters.common import Adapter
+from hud.adapters.common.types import ClickAction, Point, TypeAction
+
+
+@pytest.fixture
+def adapter():
+    """Fixture providing a clean adapter instance."""
+    return Adapter()
+
+
+@pytest.fixture
+def test_image():
+    """Fixture providing test image in various formats."""
+    img = Image.new("RGB", (100, 80), color="red")
+    img_bytes = io.BytesIO()
+    img.save(img_bytes, format="PNG")
+    img_base64 = base64.b64encode(img_bytes.getvalue()).decode("utf-8")
+    img_array = np.array(img)
+
+    return {
+        "pil": img,
+        "bytes": img_bytes.getvalue(),
+        "base64": img_base64,
+        "array": img_array,
+    }
+
+
+def test_init(adapter):
+    """Test adapter initialization."""
+    assert adapter.agent_width == 1920
+    assert adapter.agent_height == 1080
+    assert adapter.env_width == 1920
+    assert adapter.env_height == 1080
+    assert adapter.memory == []
+
+
+def test_preprocess(adapter):
+    """Test preprocess method (default implementation)."""
+    action = {"type": "click", "point": {"x": 100, "y": 100}}
+    result = adapter.preprocess(action)
+    assert result == action  # Default implementation returns unchanged
+
+
+def test_convert_valid(adapter):
+    """Test convert method with valid action."""
+    action = ClickAction(point=Point(x=100, y=100))
+    result = adapter.convert(action)
+    # Fix: Instead of checking against CLA, check it's the same type as the input
+    assert isinstance(result, ClickAction)
+    assert result == action
+
+
+def test_convert_invalid(adapter):
+    """Test convert method with invalid action."""
+    with pytest.raises(ValueError):
+        adapter.convert(None)  # type: ignore
+
+
+def test_json_valid(adapter):
+    """Test json method with valid action."""
+    action = ClickAction(point=Point(x=100, y=100))
+    result = adapter.json(action)
+    assert isinstance(result, dict)
+    assert result["type"] == "click"
+    assert result["point"]["x"] == 100
+    assert result["point"]["y"] == 100
+
+
+def test_json_invalid(adapter):
+    """Test json method with invalid action."""
+    with pytest.raises(ValueError):
+        adapter.json(None)  # type: ignore
+
+
+def test_rescale_pil_image(adapter, test_image):
+    """Test rescaling PIL Image."""
+    result = adapter.rescale(test_image["pil"])
+
+    # Verify result is base64 string
+    assert isinstance(result, str)
+
+    # Verify environment dimensions were updated
+    assert adapter.env_width == 100
+    assert adapter.env_height == 80
+
+    # Decode and verify image dimensions
+    img_bytes = base64.b64decode(result)
+    img = Image.open(io.BytesIO(img_bytes))
+    assert img.size == (adapter.agent_width, adapter.agent_height)
+
+
+def test_rescale_numpy_array(adapter, test_image):
+    """Test rescaling numpy array."""
+    result = adapter.rescale(test_image["array"])
+
+    # Verify result is base64 string
+    assert isinstance(result, str)
+
+    # Verify environment dimensions were updated
+    assert adapter.env_width == 100
+    assert adapter.env_height == 80
+
+
+def test_rescale_base64(adapter, test_image):
+    """Test rescaling base64 string."""
+    result = adapter.rescale(test_image["base64"])
+
+    # Verify result is base64 string
+    assert isinstance(result, str)
+
+    # Verify environment dimensions were updated
+    assert adapter.env_width == 100
+    assert adapter.env_height == 80
+
+
+def test_rescale_base64_with_header(adapter, test_image):
+    """Test rescaling base64 string with header."""
+    base64_with_header = f"data:image/png;base64,{test_image['base64']}"
+    result = adapter.rescale(base64_with_header)
+
+    # Verify result is base64 string
+    assert isinstance(result, str)
+
+    # Verify environment dimensions were updated
+    assert adapter.env_width == 100
+    assert adapter.env_height == 80
+
+
+def test_rescale_invalid_type(adapter):
+    """Test rescaling with invalid type."""
+    with pytest.raises(ValueError):
+        adapter.rescale(123)  # type: ignore
+
+
+def test_rescale_none(adapter):
+    """Test rescaling with None."""
+    result = adapter.rescale(None)
+    assert result is None
+
+
+def test_postprocess_action_click(adapter):
+    """Test postprocess_action with click action."""
+    # Set different agent and env dimensions
+    adapter.agent_width = 1000
+    adapter.agent_height = 800
+    adapter.env_width = 2000
+    adapter.env_height = 1600
+
+    action = {"type": "click", "point": {"x": 500, "y": 400}}
+    result = adapter.postprocess_action(action)
+
+    # Coordinates should be doubled
+    assert result["point"]["x"] == 1000
+    assert result["point"]["y"] == 800
+
+
+def test_postprocess_action_drag(adapter):
+    """Test postprocess_action with drag action."""
+    # Set different agent and env dimensions
+    adapter.agent_width = 1000
+    adapter.agent_height = 800
+    adapter.env_width = 2000
+    adapter.env_height = 1600
+
+    action = {"type": "drag", "path": [{"x": 100, "y": 200}, {"x": 300, "y": 400}]}
+    result = adapter.postprocess_action(action)
+
+    # Coordinates should be doubled
+    assert result["path"][0]["x"] == 200
+    assert result["path"][0]["y"] == 400
+    assert result["path"][1]["x"] == 600
+    assert result["path"][1]["y"] == 800
+
+
+def test_postprocess_action_scroll(adapter):
+    """Test postprocess_action with scroll action."""
+    # Set different agent and env dimensions
+    adapter.agent_width = 1000
+    adapter.agent_height = 800
+    adapter.env_width = 2000
+    adapter.env_height = 1600
+
+    action = {"type": "scroll", "point": {"x": 500, "y": 400}, "scroll": {"x": 0, "y": 10}}
+    result = adapter.postprocess_action(action)
+
+    # Point coordinates should be doubled
+    assert result["point"]["x"] == 1000
+    assert result["point"]["y"] == 800
+    # Scroll amount should be scaled
+    assert result["scroll"]["x"] == 0
+    assert result["scroll"]["y"] == 20
+
+
+def test_postprocess_action_empty(adapter):
+    """Test postprocess_action with empty action."""
+    result = adapter.postprocess_action({})
+    assert result == {}
+
+
+def test_adapt(adapter):
+    """Test adapt method."""
+    # Mock the needed methods
+    with (
+        patch.object(adapter, "preprocess", return_value={"preprocessed": True}),
+        patch.object(adapter, "convert", return_value=TypeAction(text="test")),
+        patch.object(adapter, "json", return_value={"type": "type", "text": "test"}),
+        patch.object(adapter, "postprocess_action", return_value={"type": "type", "text": "test"}),
+        patch("hud.adapters.common.adapter.TypeAdapter") as mock_adapter,
+    ):
+        mock_validator = MagicMock()
+        mock_adapter.return_value = mock_validator
+        mock_validator.validate_python.return_value = TypeAction(text="test")
+
+        adapter.adapt({"raw": "action"})
+
+        # Verify the method chain was called correctly
+        adapter.preprocess.assert_called_once_with({"raw": "action"})
+        adapter.convert.assert_called_once_with({"preprocessed": True})
+        adapter.json.assert_called_once_with(TypeAction(text="test"))
+        adapter.postprocess_action.assert_called_once_with({"type": "type", "text": "test"})
+
+        # Verify the memory was updated
+        assert len(adapter.memory) == 1
+        assert adapter.memory[0] == TypeAction(text="test")
+
+
+def test_adapt_list(adapter):
+    """Test adapt_list method."""
+    # Fix: Use side_effect to return different values for each call to adapt
+    click_action = ClickAction(point=Point(x=100, y=100))
+    type_action = TypeAction(text="test")
+
+    mock_adapt = MagicMock(side_effect=[click_action, type_action])
+    with patch.object(adapter, "adapt", mock_adapt):
+        actions = [{"type": "click"}, {"type": "type"}]
+        result = adapter.adapt_list(actions)
+
+        assert adapter.adapt.call_count == 2
+        assert len(result) == 2
+        assert result[0] == click_action
+        assert result[1] == type_action
+
+
+def test_adapt_list_invalid(adapter):
+    """Test adapt_list with invalid input."""
+    with pytest.raises(ValueError):
+        adapter.adapt_list("not a list")  # type: ignore
+
+
+def test_integration(adapter):
+    """Integration test for the full adapter pipeline."""
+    adapter.agent_width = 1000
+    adapter.agent_height = 800
+    adapter.env_width = 2000
+    adapter.env_height = 1600
+
+    # Create a click action
+    action = ClickAction(point=Point(x=500, y=400))
+
+    result = adapter.adapt(action)
+
+    assert isinstance(result, ClickAction)
+    assert result.point is not None
+    assert result.point.x == 1000  # Scaled from 500 to 1000
+    assert result.point.y == 800  # Scaled from 400 to 800
+
+    assert len(adapter.memory) == 1

hud/adapters/common/types.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Annotated, Literal
+from typing import Annotated, Literal, TypeAlias
 
 from pydantic import BaseModel, Field
 
@@ -82,7 +82,7 @@ class DragAction(CLAAction):
 # RESPONSE ACTION from agent
 class ResponseAction(CLAAction):
     type: Literal["response"] = "response"
-    text: str # The final textual response from the agent
+    text: str  # The final textual response from the agent
 
 
 # SCREENSHOT ACTION
@@ -118,7 +118,7 @@ CLA = Annotated[
 ]
 
 
-CLAKey = Literal[
+CLAKey: TypeAlias = Literal[
     # Control keys
     "backspace",
     "tab",

hud/adapters/operator/adapter.py

@@ -26,72 +26,65 @@ class OperatorAdapter(Adapter):
         "arrowleft": "left",
         "arrowright": "right",
     }
-    
+
     def __init__(self) -> None:
         super().__init__()
         # OpenAI Computer Use default dimensions
         self.agent_width = 1024
         self.agent_height = 768
-        
+
     def _map_key(self, key: str) -> CLAKey:
         """Map a key to its standardized form."""
         return self.KEY_MAP.get(key.lower(), key.lower())  # type: ignore
-        
+
     def convert(self, data: Any) -> CLA:
         """Convert a Computer Use action to a HUD action"""
         try:
             action_type = data.get("type")
-            
+
             if action_type == "click":
                 x, y = data.get("x", 0), data.get("y", 0)
                 button = data.get("button", "left")
                 return ClickAction(point=Point(x=x, y=y), button=button)
-                
+
             elif action_type == "double_click":
                 x, y = data.get("x", 0), data.get("y", 0)
-                return ClickAction(
-                    point=Point(x=x, y=y),
-                    button="left",
-                    pattern=[100]
-                )
-                
+                return ClickAction(point=Point(x=x, y=y), button="left", pattern=[100])
+
             elif action_type == "scroll":
                 x, y = data.get("x", 0), data.get("y", 0)
                 scroll_x = data.get("scroll_x", 0)
                 scroll_y = data.get("scroll_y", 0)
-                return ScrollAction(
-                    point=Point(x=x, y=y),
-                    scroll=Point(x=scroll_x, y=scroll_y)
-                )
-                
+                return ScrollAction(point=Point(x=x, y=y), scroll=Point(x=scroll_x, y=scroll_y))
+
             elif action_type == "type":
                 text = data.get("text", "")
                 return TypeAction(text=text, enter_after=False)
-                
+
             elif action_type == "wait":
                 ms = data.get("ms", 1000)
                 return WaitAction(time=ms)
-                
+
             elif action_type == "move":
                 x, y = data.get("x", 0), data.get("y", 0)
                 return MoveAction(point=Point(x=x, y=y))
-                
+
             elif action_type == "keypress":
                 keys = data.get("keys", [])
                 return PressAction(keys=[self._map_key(k) for k in keys])
-                
+
             elif action_type == "drag":
                 path = data.get("path", [])
                 points = [Point(x=p.get("x", 0), y=p.get("y", 0)) for p in path]
                 return DragAction(path=points)
-            
+
             elif action_type == "screenshot":
                 return ScreenshotFetch()
-            
+
             elif action_type == "response":
                 return ResponseAction(text=data.get("text", ""))
             else:
                 raise ValueError(f"Unsupported action type: {action_type}")
-                
+
         except Exception as e:
             raise ValueError(f"Invalid action: {data}. Error: {e!s}") from e

hud/agent/__init__.py

@@ -5,4 +5,11 @@ from .langchain import LangchainAgent
 
 from hud.adapters import OperatorAdapter, ClaudeAdapter
 
-__all__ = ["Agent", "ClaudeAgent", "OperatorAgent", "OperatorAdapter", "ClaudeAdapter", "LangchainAgent"]
+__all__ = [
+    "Agent",
+    "ClaudeAgent",
+    "OperatorAgent",
+    "OperatorAdapter",
+    "ClaudeAdapter",
+    "LangchainAgent",
+]

hud/agent/base.py

@@ -2,108 +2,108 @@ from abc import ABC, abstractmethod
 from typing import Sequence, TypeVar, Generic
 
 from hud.adapters import Adapter, CLA
-from hud.env.environment import Observation
+from hud.utils.common import Observation
 
 # Generic type for different client types (Anthropic, OpenAI, etc.)
-ClientT = TypeVar('ClientT')
-ActionT = TypeVar('ActionT')
+ClientT = TypeVar("ClientT")
+ActionT = TypeVar("ActionT")
+
 
 class Agent(Generic[ClientT, ActionT], ABC):
     """
     Base class for all agents.
-    
+
     Implements a three-stage prediction process:
     1. preprocess - Prepare observation data (e.g., rescale screenshot)
     2. fetch_response - Make API calls to get model response
     3. postprocess - Convert model actions to HUD format
-    
+
     Subclasses only need to implement the fetch_response method.
     """
-    
+
     def __init__(self, client: ClientT | None = None, adapter: Adapter | None = None):
         """
         Initialize the agent.
-        
+
         Args:
             client: The client to use for API calls
             adapter: The adapter to use for preprocessing and postprocessing
         """
         self.client = client
         self.adapter = adapter
-    
+
     def preprocess(self, observation: Observation) -> Observation:
         """
         Preprocess the observation before sending to the model.
-        
+
         Args:
             observation: The raw observation from the environment
-            
+
         Returns:
             Observation: The processed observation ready for the model
         """
         if not self.adapter or not observation.screenshot:
             return observation
-            
+
         # Create a new observation with the rescaled screenshot
         processed_obs = Observation(
-            text=observation.text,
-            screenshot=self.adapter.rescale(observation.screenshot)
+            text=observation.text, screenshot=self.adapter.rescale(observation.screenshot)
         )
         return processed_obs
-    
+
     @abstractmethod
     async def fetch_response(self, observation: Observation) -> tuple[list[ActionT], bool]:
         """
         Fetch a response from the model based on the observation.
-        
+
         Args:
             observation: The preprocessed observation
-            
+
         Returns:
             tuple[list[ActionT], bool]: A tuple containing the list of raw actions and a
                                        boolean indicating if the agent believes it has
                                        completed the task
         """
         pass
-    
+
     def postprocess(self, actions: list[ActionT]) -> list[CLA]:
         """
         Convert model actions to HUD actions.
-        
+
         Args:
             actions: The raw actions from the model
-            
+
         Returns:
             Sequence[CLA]: The actions converted to HUD format
         """
         if not self.adapter:
             raise ValueError("Cannot postprocess actions without an adapter")
-        
+
         return self.adapter.adapt_list(actions)
-    
+
     async def predict(self, observation: Observation) -> tuple[list[CLA] | list[ActionT], bool]:
         """
         Predict the next action based on the observation.
-        
+
         Implements the full three-stage prediction process.
-        
+
         Args:
             observation: The observation from the environment
-            
+
         Returns:
             tuple[list[CLA] | list[ActionT], bool]: A tuple containing the list of actions and a boolean
                                                        indicating if the agent believes it has completed the task
         """
         # Stage 1: Preprocess the observation
         processed_obs = self.preprocess(observation)
-        
+
         # Stage 2: Fetch response from the model
         actions, done = await self.fetch_response(processed_obs)
-        
+
         # Stage 3: Postprocess the actions if we have an adapter
         if self.adapter and actions:
             hud_actions = self.postprocess(actions)
             return hud_actions, done
-            
+
         # If no adapter, return actions as is
-        return actions, done
+        return actions, done