hud-python 0.3.5__py3-none-any.whl → 0.4.0__py3-none-any.whl

hud/adapters/common/adapter.py

@@ -1,178 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Any, TypeAlias
-
-from pydantic import TypeAdapter, ValidationError
-
-from .types import CLA
-
-if TYPE_CHECKING:
-    import numpy as np
-    from PIL import Image
-    from typing_extensions import TypeIs
-
-    ImageType: TypeAlias = np.ndarray[Any, Any] | Image.Image | str | None
-else:
-    ImageType: TypeAlias = Any | str | None
-
-
-def _is_numpy_array(observation: Any) -> TypeIs[np.ndarray]:
-    """Check if the observation is a numpy array, without requiring numpy."""
-    try:
-        import numpy as np  # type: ignore
-
-        return isinstance(observation, np.ndarray)
-    except (ModuleNotFoundError, NameError):
-        return False
-
-
-class Adapter:
-    def __init__(self) -> None:
-        self.memory = []
-
-        self.agent_width = 1920
-        self.agent_height = 1080
-        self.env_width = 1920
-        self.env_height = 1080
-
-    def preprocess(self, action: Any) -> Any:
-        return action
-
-    def convert(self, action: Any) -> CLA:
-        if action is None:
-            raise ValueError("Please provide a valid action")
-        try:
-            return TypeAdapter(CLA).validate_python(action)
-        except ValidationError as e:
-            raise ValueError(f"Invalid action type in conversion: {action}") from e
-
-    def json(self, action: CLA) -> Any:
-        if action is None:
-            raise ValueError("Please provide a valid action")
-        try:
-            validated = TypeAdapter(CLA).validate_python(action)
-            return validated.model_dump()
-        except ValidationError as e:
-            raise ValueError(f"Invalid action type in json creation: {action}") from e
-
-    def rescale(self, observation: ImageType) -> str | None:
-        """
-        Resize the observation (image) to agent-specific dimensions.
-
-        Args:
-            observation: Image data, which can be:
-                - numpy array
-                - PIL Image
-                - base64 string (PNG) # TODO: JPG
-
-        Returns:
-            Base64-encoded string of the resized image (PNG format)
-        """
-        if observation is None:
-            return None
-
-        # Import PIL only when needed
-        try:
-            from PIL import Image
-        except ImportError:
-            raise ImportError(
-                "PIL (Pillow) is required for image processing. "
-                "Please install it with 'pip install Pillow'"
-            ) from None
-
-        # Handle different input types.
-        if _is_numpy_array(observation):
-            # Convert numpy array to PIL Image
-            img = Image.fromarray(observation)
-        elif isinstance(observation, Image.Image):
-            img = observation
-        elif isinstance(observation, str):
-            # Assume it's a base64 string
-            try:
-                import base64
-                import io
-
-                # Remove header if present (e.g., 'data:image/png;base64,')
-                if "," in observation:
-                    observation = observation.split(",")[1]
-                # Decode base64 string to bytes
-                img_bytes = base64.b64decode(observation)
-                # Convert to PIL Image
-                img = Image.open(io.BytesIO(img_bytes))
-            except Exception as e:
-                raise ValueError(f"Failed to decode base64 image: {e}") from None
-        else:
-            raise ValueError(f"Unsupported observation type: {type(observation)}")
-
-        # Update environment dimensions
-        self.env_width, self.env_height = img.size
-
-        # Resize to agent dimensions
-        resized_img = img.resize((self.agent_width, self.agent_height), Image.Resampling.LANCZOS)
-
-        # Always convert to base64 string
-        import base64
-        import io
-
-        buffered = io.BytesIO()
-        resized_img.save(buffered, format="PNG")
-        return base64.b64encode(buffered.getvalue()).decode("utf-8")
-
-    def postprocess_action(self, action: dict[str, Any]) -> dict[str, Any]:
-        """
-        Rescale action coordinates from agent dimensions to environment dimensions.
-
-        Args:
-            action: Action dictionary with coordinates
-
-        Returns:
-            Action with rescaled coordinates
-        """
-        if not action:
-            return action
-
-        # Calculate scaling factors
-        x_scale = self.env_width / self.agent_width
-        y_scale = self.env_height / self.agent_height
-
-        # Deep copy to avoid modifying the original
-        processed_action = action.copy()
-
-        # Rescale based on action type and structure
-        if "point" in processed_action and processed_action["point"] is not None:
-            # For actions with a single point (click, move)
-            processed_action["point"]["x"] = int(processed_action["point"]["x"] * x_scale)
-            processed_action["point"]["y"] = int(processed_action["point"]["y"] * y_scale)
-
-        if (path := processed_action.get("path")) is not None:
-            # For actions with a path (drag)
-            for point in path:
-                point["x"] = int(point["x"] * x_scale)
-                point["y"] = int(point["y"] * y_scale)
-
-        if "scroll" in processed_action and processed_action["scroll"] is not None:
-            # For scroll actions
-            processed_action["scroll"]["x"] = int(processed_action["scroll"]["x"] * x_scale)
-            processed_action["scroll"]["y"] = int(processed_action["scroll"]["y"] * y_scale)
-
-        return processed_action
-
-    def adapt(self, action: Any) -> CLA:
-        # any preprocessing steps
-        action = self.preprocess(action)
-
-        # convert to CLA
-        action = self.convert(action)
-        self.memory.append(action)
-
-        # convert to json and apply coordinate rescaling
-        action_dict = self.json(action)
-        rescaled_action = self.postprocess_action(action_dict)
-
-        # convert back to CLA
-        return TypeAdapter(CLA).validate_python(rescaled_action)
-
-    def adapt_list(self, actions: list[Any]) -> list[CLA]:
-        if not isinstance(actions, list):
-            raise ValueError("Please provide a list of actions")
-        return [self.adapt(action) for action in actions]

hud/adapters/common/tests/test_adapter.py

@@ -1,289 +0,0 @@
-from __future__ import annotations
-
-import base64
-import io
-from unittest.mock import MagicMock, patch
-
-import pytest
-from PIL import Image
-
-try:
-    import numpy as np
-
-    HAS_NUMPY = True
-except ImportError:
-    HAS_NUMPY = False
-    np = None
-
-from hud.adapters.common import Adapter
-from hud.adapters.common.types import ClickAction, Point, TypeAction
-
-
-@pytest.fixture
-def adapter():
-    """Fixture providing a clean adapter instance."""
-    return Adapter()
-
-
-@pytest.fixture
-def test_image():
-    """Fixture providing test image in various formats."""
-    img = Image.new("RGB", (100, 80), color="red")
-    img_bytes = io.BytesIO()
-    img.save(img_bytes, format="PNG")
-    img_base64 = base64.b64encode(img_bytes.getvalue()).decode("utf-8")
-
-    result = {
-        "pil": img,
-        "bytes": img_bytes.getvalue(),
-        "base64": img_base64,
-    }
-
-    if HAS_NUMPY:
-        img_array = np.array(img)  # type: ignore
-        result["array"] = img_array
-
-    return result
-
-
-def test_init(adapter):
-    """Test adapter initialization."""
-    assert adapter.agent_width == 1920
-    assert adapter.agent_height == 1080
-    assert adapter.env_width == 1920
-    assert adapter.env_height == 1080
-    assert adapter.memory == []
-
-
-def test_preprocess(adapter):
-    """Test preprocess method (default implementation)."""
-    action = {"type": "click", "point": {"x": 100, "y": 100}}
-    result = adapter.preprocess(action)
-    assert result == action  # Default implementation returns unchanged
-
-
-def test_convert_valid(adapter):
-    """Test convert method with valid action."""
-    action = ClickAction(point=Point(x=100, y=100))
-    result = adapter.convert(action)
-    # Fix: Instead of checking against CLA, check it's the same type as the input
-    assert isinstance(result, ClickAction)
-    assert result == action
-
-
-def test_convert_invalid(adapter):
-    """Test convert method with invalid action."""
-    with pytest.raises(ValueError):
-        adapter.convert(None)  # type: ignore
-
-
-def test_json_valid(adapter):
-    """Test json method with valid action."""
-    action = ClickAction(point=Point(x=100, y=100))
-    result = adapter.json(action)
-    assert isinstance(result, dict)
-    assert result["type"] == "click"
-    assert result["point"]["x"] == 100
-    assert result["point"]["y"] == 100
-
-
-def test_json_invalid(adapter):
-    """Test json method with invalid action."""
-    with pytest.raises(ValueError):
-        adapter.json(None)  # type: ignore
-
-
-def test_rescale_pil_image(adapter, test_image):
-    """Test rescaling PIL Image."""
-    result = adapter.rescale(test_image["pil"])
-
-    # Verify result is base64 string
-    assert isinstance(result, str)
-
-    # Verify environment dimensions were updated
-    assert adapter.env_width == 100
-    assert adapter.env_height == 80
-
-    # Decode and verify image dimensions
-    img_bytes = base64.b64decode(result)
-    img = Image.open(io.BytesIO(img_bytes))
-    assert img.size == (adapter.agent_width, adapter.agent_height)
-
-
-@pytest.mark.skipif(not HAS_NUMPY, reason="numpy not available")
-def test_rescale_numpy_array(adapter, test_image):
-    """Test rescaling numpy array."""
-    result = adapter.rescale(test_image["array"])
-
-    # Verify result is base64 string
-    assert isinstance(result, str)
-
-    # Verify environment dimensions were updated
-    assert adapter.env_width == 100
-    assert adapter.env_height == 80
-
-
-def test_rescale_base64(adapter, test_image):
-    """Test rescaling base64 string."""
-    result = adapter.rescale(test_image["base64"])
-
-    # Verify result is base64 string
-    assert isinstance(result, str)
-
-    # Verify environment dimensions were updated
-    assert adapter.env_width == 100
-    assert adapter.env_height == 80
-
-
-def test_rescale_base64_with_header(adapter, test_image):
-    """Test rescaling base64 string with header."""
-    base64_with_header = f"data:image/png;base64,{test_image['base64']}"
-    result = adapter.rescale(base64_with_header)
-
-    # Verify result is base64 string
-    assert isinstance(result, str)
-
-    # Verify environment dimensions were updated
-    assert adapter.env_width == 100
-    assert adapter.env_height == 80
-
-
-def test_rescale_invalid_type(adapter):
-    """Test rescaling with invalid type."""
-    with pytest.raises(ValueError):
-        adapter.rescale(123)  # type: ignore
-
-
-def test_rescale_none(adapter):
-    """Test rescaling with None."""
-    result = adapter.rescale(None)
-    assert result is None
-
-
-def test_postprocess_action_click(adapter):
-    """Test postprocess_action with click action."""
-    # Set different agent and env dimensions
-    adapter.agent_width = 1000
-    adapter.agent_height = 800
-    adapter.env_width = 2000
-    adapter.env_height = 1600
-
-    action = {"type": "click", "point": {"x": 500, "y": 400}}
-    result = adapter.postprocess_action(action)
-
-    # Coordinates should be doubled
-    assert result["point"]["x"] == 1000
-    assert result["point"]["y"] == 800
-
-
-def test_postprocess_action_drag(adapter):
-    """Test postprocess_action with drag action."""
-    # Set different agent and env dimensions
-    adapter.agent_width = 1000
-    adapter.agent_height = 800
-    adapter.env_width = 2000
-    adapter.env_height = 1600
-
-    action = {"type": "drag", "path": [{"x": 100, "y": 200}, {"x": 300, "y": 400}]}
-    result = adapter.postprocess_action(action)
-
-    # Coordinates should be doubled
-    assert result["path"][0]["x"] == 200
-    assert result["path"][0]["y"] == 400
-    assert result["path"][1]["x"] == 600
-    assert result["path"][1]["y"] == 800
-
-
-def test_postprocess_action_scroll(adapter):
-    """Test postprocess_action with scroll action."""
-    # Set different agent and env dimensions
-    adapter.agent_width = 1000
-    adapter.agent_height = 800
-    adapter.env_width = 2000
-    adapter.env_height = 1600
-
-    action = {"type": "scroll", "point": {"x": 500, "y": 400}, "scroll": {"x": 0, "y": 10}}
-    result = adapter.postprocess_action(action)
-
-    # Point coordinates should be doubled
-    assert result["point"]["x"] == 1000
-    assert result["point"]["y"] == 800
-    # Scroll amount should be scaled
-    assert result["scroll"]["x"] == 0
-    assert result["scroll"]["y"] == 20
-
-
-def test_postprocess_action_empty(adapter):
-    """Test postprocess_action with empty action."""
-    result = adapter.postprocess_action({})
-    assert result == {}
-
-
-def test_adapt(adapter):
-    """Test adapt method."""
-    # Mock the needed methods
-    with (
-        patch.object(adapter, "preprocess", return_value={"preprocessed": True}),
-        patch.object(adapter, "convert", return_value=TypeAction(text="test")),
-        patch.object(adapter, "json", return_value={"type": "type", "text": "test"}),
-        patch.object(adapter, "postprocess_action", return_value={"type": "type", "text": "test"}),
-        patch("hud.adapters.common.adapter.TypeAdapter") as mock_adapter,
-    ):
-        mock_validator = MagicMock()
-        mock_adapter.return_value = mock_validator
-        mock_validator.validate_python.return_value = TypeAction(text="test")
-
-        adapter.adapt({"raw": "action"})
-
-        # Verify the method chain was called correctly
-        adapter.preprocess.assert_called_once_with({"raw": "action"})
-        adapter.convert.assert_called_once_with({"preprocessed": True})
-        adapter.json.assert_called_once_with(TypeAction(text="test"))
-        adapter.postprocess_action.assert_called_once_with({"type": "type", "text": "test"})
-
-        # Verify the memory was updated
-        assert len(adapter.memory) == 1
-        assert adapter.memory[0] == TypeAction(text="test")
-
-
-def test_adapt_list(adapter):
-    """Test adapt_list method."""
-    # Fix: Use side_effect to return different values for each call to adapt
-    click_action = ClickAction(point=Point(x=100, y=100))
-    type_action = TypeAction(text="test")
-
-    mock_adapt = MagicMock(side_effect=[click_action, type_action])
-    with patch.object(adapter, "adapt", mock_adapt):
-        actions = [{"type": "click"}, {"type": "type"}]
-        result = adapter.adapt_list(actions)
-
-        assert adapter.adapt.call_count == 2
-        assert len(result) == 2
-        assert result[0] == click_action
-        assert result[1] == type_action
-
-
-def test_adapt_list_invalid(adapter):
-    """Test adapt_list with invalid input."""
-    with pytest.raises(ValueError):
-        adapter.adapt_list("not a list")  # type: ignore
-
-
-def test_integration(adapter):
-    """Integration test for the full adapter pipeline."""
-    adapter.agent_width = 1000
-    adapter.agent_height = 800
-    adapter.env_width = 2000
-    adapter.env_height = 1600
-
-    # Create a click action
-    action = ClickAction(point=Point(x=500, y=400))
-
-    result = adapter.adapt(action)
-
-    assert isinstance(result, ClickAction)
-    assert result.point is not None
-    assert result.point.x == 1000  # Scaled from 500 to 1000
-    assert result.point.y == 800  # Scaled from 400 to 800
-
-    assert len(adapter.memory) == 1