hud 0.6.19.dev1__tar.gz → 0.6.19.dev2__tar.gz

{hud-0.6.19.dev1 → hud-0.6.19.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hud
-Version: 0.6.19.dev1
+Version: 0.6.19.dev2
 Summary: SDK for the HUD platform.
 Project-URL: Homepage, https://github.com/hud-evals/hud-python
 Project-URL: Bug Tracker, https://github.com/hud-evals/hud-python/issues

{hud-0.6.19.dev1 → hud-0.6.19.dev2}/hud/agents/claude/agent.py

@@ -8,7 +8,7 @@ import json
 import logging
 import math
 from random import SystemRandom
-from typing import TYPE_CHECKING, Literal, cast
+from typing import TYPE_CHECKING, cast
 
 import httpx
 import httpx2
@@ -23,7 +23,6 @@ from anthropic import (
 )
 from anthropic.types import CacheControlEphemeralParam
 from anthropic.types.beta import (
-    BetaBase64ImageSourceParam,
     BetaBase64PDFSourceParam,
     BetaImageBlockParam,
     BetaMessage,
@@ -35,12 +34,14 @@ from anthropic.types.beta import (
     BetaToolResultBlockParam,
     BetaToolUnionParam,
 )
+from PIL import Image
 
 from hud.agents.tool_agent import RunState, ToolAgent
 from hud.agents.types import AgentStep, Citation, ClaudeConfig, Usage
 from hud.types import MCPToolCall, MCPToolResult
 from hud.utils import gateway
 
+from .images import image_source
 from .tools.coding import ClaudeBashTool, ClaudeTextEditorTool
 from .tools.computer import ClaudeComputerTool
 from .tools.mcp_proxy import ClaudeMCPProxyTool
@@ -51,7 +52,6 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
-ClaudeImageMediaType = Literal["image/jpeg", "image/png", "image/gif", "image/webp"]
 ClaudeToolResultContent = BetaTextBlockParam | BetaImageBlockParam | BetaRequestDocumentBlockParam
 
 _RETRYABLE_STREAM_ERROR_TYPES = frozenset(
@@ -140,6 +140,7 @@ class ClaudeAgent(ToolAgent[BetaMessageParam, ClaudeConfig]):
             return None
 
         result_content = result.content
+        image_error = False
         if result.isError:
             error_msg = next(
                 (c.text for c in result.content if isinstance(c, mcp_types.TextContent)),
@@ -168,14 +169,19 @@ class ClaudeAgent(ToolAgent[BetaMessageParam, ClaudeConfig]):
                             citations={"enabled": True},
                         )
                 case mcp_types.ImageContent():
-                    block = BetaImageBlockParam(
-                        type="image",
-                        source=BetaBase64ImageSourceParam(
-                            type="base64",
-                            media_type=cast("ClaudeImageMediaType", content.mimeType),
-                            data=content.data,
-                        ),
-                    )
+                    try:
+                        block = BetaImageBlockParam(
+                            type="image",
+                            source=image_source(
+                                content,
+                                preserve_dimensions=isinstance(
+                                    state.tools.get(call.name), ClaudeComputerTool
+                                ),
+                            ),
+                        )
+                    except (ValueError, OSError, Image.DecompressionBombError) as exc:
+                        image_error = True
+                        block = BetaTextBlockParam(type="text", text=f"Cannot view image: {exc}")
                 case mcp_types.EmbeddedResource(
                     resource=mcp_types.BlobResourceContents(mimeType="application/pdf") as resource,
                 ):
@@ -207,6 +213,7 @@ class ClaudeAgent(ToolAgent[BetaMessageParam, ClaudeConfig]):
                     type="tool_result",
                     tool_use_id=tool_use_id,
                     content=claude_blocks,
+                    is_error=bool(result.isError) or image_error,
                 ),
             ],
         )

hud-0.6.19.dev2/hud/agents/claude/images.py

@@ -0,0 +1,65 @@
+"""Image input preparation for Anthropic Messages."""
+
+from __future__ import annotations
+
+import base64
+import math
+from io import BytesIO
+from typing import TYPE_CHECKING, Literal, cast
+
+from anthropic.types.beta import BetaBase64ImageSourceParam
+from PIL import Image, ImageOps
+
+if TYPE_CHECKING:
+    from mcp.types import ImageContent
+
+ClaudeImageMediaType = Literal["image/jpeg", "image/png", "image/gif", "image/webp"]
+MAX_SOURCE_IMAGE_PIXELS = 16_000_000
+MAX_IMAGE_EDGE = 1568
+MAX_IMAGE_PIXELS = 1_150_000
+MAX_IMAGE_BASE64_BYTES = 5_000_000
+
+
+def image_source(
+    content: ImageContent, *, preserve_dimensions: bool = False
+) -> BetaBase64ImageSourceParam:
+    """Prepare a tool image, preserving computer-tool coordinate space when requested."""
+    if preserve_dimensions:
+        return BetaBase64ImageSourceParam(
+            type="base64",
+            media_type=cast("ClaudeImageMediaType", content.mimeType),
+            data=content.data,
+        )
+    with Image.open(BytesIO(base64.b64decode(content.data, validate=True))) as image:
+        if image.width * image.height > MAX_SOURCE_IMAGE_PIXELS:
+            raise ValueError(
+                f"Image exceeds the {MAX_SOURCE_IMAGE_PIXELS:,}-pixel source limit. "
+                "Resize or crop it before viewing."
+            )
+        return _encode_image(image)
+
+
+def _encode_image(image: Image.Image) -> BetaBase64ImageSourceParam:
+    scale = min(
+        1.0,
+        MAX_IMAGE_EDGE / max(image.size),
+        math.sqrt(MAX_IMAGE_PIXELS / (image.width * image.height)),
+    )
+    image.thumbnail(
+        (max(1, int(image.width * scale)), max(1, int(image.height * scale))),
+        Image.Resampling.LANCZOS,
+    )
+    image = ImageOps.exif_transpose(image)
+    image = image.convert("RGBA" if image.has_transparency_data else "RGB")
+    buffer = BytesIO()
+    image.save(buffer, format="PNG", optimize=True)
+    data = base64.b64encode(buffer.getvalue()).decode("ascii")
+    mime_type: ClaudeImageMediaType = "image/png"
+    if len(data) > MAX_IMAGE_BASE64_BYTES:
+        background = Image.new("RGB", image.size, "white")
+        background.paste(image, mask=image.getchannel("A") if image.mode == "RGBA" else None)
+        buffer = BytesIO()
+        background.save(buffer, format="JPEG", quality=85, optimize=True)
+        data = base64.b64encode(buffer.getvalue()).decode("ascii")
+        mime_type = "image/jpeg"
+    return BetaBase64ImageSourceParam(type="base64", media_type=mime_type, data=data)

{hud-0.6.19.dev1 → hud-0.6.19.dev2}/hud/agents/claude/tools/coding.py

@@ -171,7 +171,7 @@ class ClaudeTextEditorTool(SSHTool):
 
         match command:
             case "view":
-                return await self.file_read(path)
+                return await self.file_view(path)
             case "create":
                 content = arguments.get("file_text", "")
                 return await self.file_write(path, str(content))

{hud-0.6.19.dev1 → hud-0.6.19.dev2}/hud/agents/openai_compatible/tools/filesystem.py

@@ -39,7 +39,8 @@ class _FilesystemTool(SSHTool):
 class ReadTool(_FilesystemTool):
     name = "read"
     description = (
-        "Reads a file or directory from the workspace. Use offset and limit for pagination."
+        "Reads text, images, or directories from the workspace. "
+        "Use offset and limit for text pagination."
     )
     parameters: ClassVar[dict[str, Any]] = {
         "type": "object",
@@ -70,8 +71,8 @@ class ReadTool(_FilesystemTool):
         limit = _positive_int(arguments.get("limit"), default=DEFAULT_READ_LIMIT, name="limit")
         if not (await self.bash(f"test -d {shlex.quote(path)}")).isError:
             return await self._read_directory(path, offset=offset, limit=limit)
-        result = await self.file_read(path)
-        if result.isError:
+        result = await self.file_view(path)
+        if result.isError or any(isinstance(c, mcp_types.ImageContent) for c in result.content):
             return result
         text = result_text(result)
         lines = text.splitlines()

{hud-0.6.19.dev1 → hud-0.6.19.dev2}/hud/agents/tests/test_claude_agent.py

@@ -4,6 +4,11 @@ plus the pure ``_citation`` / ``_cache_last_user_block`` helpers.
 
 from __future__ import annotations
 
+import base64
+import random
+import struct
+import zlib
+from io import BytesIO
 from types import SimpleNamespace
 from typing import Any, cast
 from unittest.mock import AsyncMock, Mock, call
@@ -12,8 +17,14 @@ import httpx
 import httpx2
 import pytest
 from anthropic import APIStatusError
+from mcp.types import ImageContent
+from PIL import Image
 
 from hud.agents.claude.agent import ClaudeAgent
+from hud.agents.claude.tools.computer import ClaudeComputerTool
+from hud.agents.tool_agent import RunState
+from hud.capabilities import RFBClient
+from hud.types import MCPToolCall, MCPToolResult
 
 
 class FakeStream:
@@ -344,3 +355,109 @@ def test_cache_last_user_block_marks_content() -> None:
     content = cast("list[Any]", out[-1]["content"])
     block = cast("dict[str, Any]", content[0])
     assert block.get("cache_control") == {"type": "ephemeral"}
+
+
+def _image_result(data: bytes, *, computer: bool = False) -> Any:
+    agent = _agent()
+    state = RunState()
+    if computer:
+        spec = ClaudeComputerTool.default_spec("claude")
+        assert spec is not None
+        tool = ClaudeComputerTool(spec=spec, client=Mock(spec=RFBClient))
+        state.tools[tool.provider_name] = tool
+        name = tool.provider_name
+    else:
+        name = "read_file"
+    content = ImageContent(type="image", mimeType="image/png", data=base64.b64encode(data).decode())
+    result = MCPToolResult(content=[content])
+    original = result.model_dump()
+    message = agent._format_result(
+        MCPToolCall(id="image-call", name=name, arguments={}), result, state
+    )
+    assert result.model_dump() == original
+    assert isinstance(message, dict)
+    return next(iter(message["content"]))
+
+
+@pytest.mark.parametrize("size", [(4000, 2000), (2000, 4000), (2000, 2000), (32, 24)])
+def test_tool_images_fit_claude_limits_without_upscaling(size: tuple[int, int]) -> None:
+    buffer = BytesIO()
+    Image.new("RGB", size, "white").save(buffer, format="PNG")
+    result = _image_result(buffer.getvalue())
+    assert not result["is_error"]
+    source = result["content"][0]["source"]
+    assert len(source["data"]) <= 5_000_000
+    with Image.open(BytesIO(base64.b64decode(source["data"]))) as image:
+        assert max(image.size) <= 1568
+        assert image.width * image.height <= 1_150_000
+        assert image.width <= size[0] and image.height <= size[1]
+        assert image.width / image.height == pytest.approx(size[0] / size[1], rel=0.005)
+
+
+def test_claude_compresses_noisy_transparent_tool_images() -> None:
+    size = (1072, 1072)
+    image = Image.frombytes("RGBA", size, random.Random(0).randbytes(size[0] * size[1] * 4))
+    buffer = BytesIO()
+    image.save(buffer, format="PNG")
+    assert len(base64.b64encode(buffer.getvalue())) > 5_000_000
+    result = _image_result(buffer.getvalue())
+    assert not result["is_error"]
+    source = result["content"][0]["source"]
+    assert source["media_type"] == "image/jpeg"
+    assert len(source["data"]) <= 5_000_000
+    with Image.open(BytesIO(base64.b64decode(source["data"]))) as viewed:
+        assert viewed.size == size
+        assert viewed.mode == "RGB"
+
+
+def test_claude_applies_image_orientation() -> None:
+    image = Image.new("RGB", (40, 20), "red")
+    exif = Image.Exif()
+    exif[274] = 6
+    buffer = BytesIO()
+    image.save(buffer, format="JPEG", exif=exif)
+    source = _image_result(buffer.getvalue())["content"][0]["source"]
+    with Image.open(BytesIO(base64.b64decode(source["data"]))) as viewed:
+        assert viewed.size == (20, 40)
+        assert viewed.getexif().get(274, 1) == 1
+
+
+def test_claude_preserves_small_png_transparency() -> None:
+    buffer = BytesIO()
+    Image.new("RGBA", (32, 24), (255, 0, 0, 128)).save(buffer, format="PNG")
+    source = _image_result(buffer.getvalue())["content"][0]["source"]
+    assert source["media_type"] == "image/png"
+    with Image.open(BytesIO(base64.b64decode(source["data"]))) as viewed:
+        assert viewed.getpixel((0, 0)) == (255, 0, 0, 128)
+
+
+@pytest.mark.filterwarnings("ignore:Image size.*:PIL.Image.DecompressionBombWarning")
+@pytest.mark.parametrize("size", [(4001, 4000), (10000, 10000)])
+def test_claude_rejects_oversized_source_before_decoding(size: tuple[int, int]) -> None:
+    buffer = BytesIO()
+    Image.new("RGB", (1, 1)).save(buffer, format="PNG")
+    data = bytearray(buffer.getvalue())
+    data[16:24] = struct.pack(">II", *size)
+    data[29:33] = struct.pack(">I", zlib.crc32(data[12:29]))
+    result = _image_result(bytes(data))
+    assert result["is_error"]
+    assert result["content"][0]["type"] == "text"
+    assert "16,000,000-pixel source limit" in result["content"][0]["text"]
+
+
+def test_claude_returns_invalid_image_as_tool_error() -> None:
+    result = _image_result(b"invalid image")
+    assert result["is_error"]
+    assert result["content"][0]["type"] == "text"
+    assert "Cannot view image" in result["content"][0]["text"]
+
+
+def test_claude_preserves_computer_screenshot_coordinate_space() -> None:
+    buffer = BytesIO()
+    Image.new("RGB", (2000, 1000), "white").save(buffer, format="PNG")
+    data = buffer.getvalue()
+    result = _image_result(data, computer=True)
+    assert not result["is_error"]
+    source = result["content"][0]["source"]
+    assert source["media_type"] == "image/png"
+    assert base64.b64decode(source["data"]) == data

{hud-0.6.19.dev1 → hud-0.6.19.dev2}/hud/agents/tests/test_provider_native_tools.py

@@ -7,13 +7,16 @@ client and assert the command translation + result shape, fully offline.
 
 from __future__ import annotations
 
+import base64
 import shlex
+from io import BytesIO
 from types import SimpleNamespace
 from typing import Any, cast
 
 import asyncssh
 import mcp.types as mcp_types
 import pytest
+from PIL import Image
 
 from hud.agents.claude.tools.coding import ClaudeBashTool, ClaudeTextEditorTool
 from hud.agents.gemini.tools.coding import GeminiEditTool, GeminiShellTool
@@ -29,7 +32,7 @@ from hud.types import MCPToolCall
 
 
 class _Completed:
-    def __init__(self, *, stdout: str = "", stderr: str = "", exit_status: int = 0) -> None:
+    def __init__(self, *, stdout: str | bytes = "", stderr: str = "", exit_status: int = 0) -> None:
         self.stdout = stdout
         self.stderr = stderr
         self.exit_status = exit_status
@@ -71,7 +74,8 @@ class _Conn:
                 return _Completed(
                     stderr=f"cat: {parts[2]}: No such file or directory", exit_status=1
                 )
-            return _Completed(stdout=self._store[parts[2]].decode())
+            data = self._store[parts[2]]
+            return _Completed(stdout=data if encoding is None else data.decode(encoding))
         if len(parts) == 3 and parts[:2] == ["cat", ">"]:
             assert input is not None
             self._store[parts[2]] = input.encode()
@@ -645,3 +649,52 @@ async def test_reading_a_missing_file_is_a_tool_error_not_a_raised_traceback() -
 
     assert result.isError is True
     assert "No such file or directory" in result_text(result)
+
+
+@pytest.mark.parametrize("image_format", ["PNG", "JPEG", "GIF", "WEBP"])
+@pytest.mark.parametrize("tool_class", [ClaudeTextEditorTool, ReadTool])
+async def test_file_view_returns_images_as_image_content(
+    image_format: str, tool_class: type[ClaudeTextEditorTool] | type[ReadTool]
+) -> None:
+    buffer = BytesIO()
+    Image.new("RGB", (32, 24), "red").save(buffer, format=image_format)
+    data = buffer.getvalue()
+    ssh = _FakeSSH(files={"/image with no extension": data})
+    tool = tool_class(spec=tool_class.default_spec("claude"), client=ssh)
+
+    result = await tool.execute(
+        {
+            "command": "view",
+            "path": "/image with no extension",
+            "filePath": "/image with no extension",
+        }
+    )
+
+    assert not result.isError
+    assert len(result.content) == 1
+    content = result.content[0]
+    assert isinstance(content, mcp_types.ImageContent)
+    assert content.mimeType == Image.MIME[image_format]
+    assert base64.b64decode(content.data) == data
+
+
+@pytest.mark.parametrize("data", [b"\x00binary", b"\xff\xfeinvalid"])
+async def test_file_view_rejects_non_image_binary(data: bytes) -> None:
+    ssh = _FakeSSH(files={"/binary": data})
+    tool = ClaudeTextEditorTool(spec=ClaudeTextEditorTool.default_spec("claude"), client=ssh)
+
+    result = await tool.execute({"command": "view", "path": "/binary"})
+
+    assert result.isError
+    assert "expected UTF-8 text or an image" in result_text(result)
+
+
+async def test_file_view_preserves_unicode_text() -> None:
+    text = "Hello, 世界\n"
+    ssh = _FakeSSH(files={"/text.png": text.encode()})
+    tool = ClaudeTextEditorTool(spec=ClaudeTextEditorTool.default_spec("claude"), client=ssh)
+
+    result = await tool.execute({"command": "view", "path": "/text.png"})
+
+    assert not result.isError
+    assert result_text(result) == text

hud-0.6.19.dev2/hud/agents/tools/file_view.py

@@ -0,0 +1,38 @@
+"""Typed file content for agent tools."""
+
+from __future__ import annotations
+
+import base64
+
+from mcp.types import ImageContent
+
+from hud.agents.tools.base import tool_err, tool_ok
+from hud.types import MCPToolResult
+
+
+def view_file(data: bytes) -> MCPToolResult:
+    """Identify supported images by signature and preserve their encoded bytes."""
+    mime_type = None
+    if data.startswith(b"\x89PNG\r\n\x1a\n"):
+        mime_type = "image/png"
+    elif data.startswith(b"\xff\xd8\xff"):
+        mime_type = "image/jpeg"
+    elif data.startswith((b"GIF87a", b"GIF89a")):
+        mime_type = "image/gif"
+    elif data.startswith(b"RIFF") and data[8:12] == b"WEBP":
+        mime_type = "image/webp"
+    if mime_type is not None:
+        return MCPToolResult(
+            content=[
+                ImageContent(
+                    type="image", mimeType=mime_type, data=base64.b64encode(data).decode("ascii")
+                )
+            ]
+        )
+    try:
+        text = data.decode("utf-8")
+    except UnicodeDecodeError:
+        return tool_err("Cannot view this file: expected UTF-8 text or an image.")
+    if "\x00" in text:
+        return tool_err("Cannot view this file: expected UTF-8 text or an image.")
+    return tool_ok(text)

{hud-0.6.19.dev1 → hud-0.6.19.dev2}/hud/agents/tools/ssh.py

@@ -11,6 +11,7 @@ import asyncssh
 import mcp.types as mcp_types
 
 from hud.agents.tools.base import AgentTool, tool_err, tool_ok
+from hud.agents.tools.file_view import view_file
 from hud.capabilities import SSHClient
 from hud.types import MCPToolResult
 
@@ -59,6 +60,14 @@ class SSHTool(AgentTool[SSHClient]):
         except asyncssh.ProcessError as e:
             return tool_err(_remote_error(e))
 
+    async def file_view(self, path: str) -> MCPToolResult:
+        """Read a file as text or image content through SSH exec."""
+        try:
+            data = await self.client.read_bytes(path)
+        except asyncssh.ProcessError as e:
+            return tool_err(_remote_error(e))
+        return view_file(data)
+
     async def file_write(self, path: str, content: str) -> MCPToolResult:
         """Write a text file through SSH exec."""
         try:

{hud-0.6.19.dev1 → hud-0.6.19.dev2}/hud/capabilities/ssh.py

@@ -7,7 +7,7 @@ import base64
 import contextlib
 import json
 import shlex
-from typing import TYPE_CHECKING, Any, ClassVar, Self
+from typing import TYPE_CHECKING, Any, ClassVar, Self, cast
 from urllib.parse import urlsplit
 
 import asyncssh
@@ -204,20 +204,22 @@ class SSHClient(CapabilityClient):
 
     async def read_text(self, path: str, *, timeout_s: float | None = None) -> str:
         """Read a UTF-8 text file through the exec channel."""
+        return _decode(await self.read_bytes(path, timeout_s=timeout_s))
+
+    async def read_bytes(self, path: str, *, timeout_s: float | None = None) -> bytes:
+        """Read a file through the exec channel without decoding it."""
         if self._is_windows:
             quoted = _powershell_quote(path)
             script = f"[Convert]::ToBase64String([IO.File]::ReadAllBytes({quoted}))"
             result = await self.run(_powershell(script), check=True, timeout=timeout_s)
-            return base64.b64decode(_stdout(result)).decode("utf-8", errors="replace")
-        # encoding=None transports raw bytes: a strict connection-level UTF-8
-        # decode would raise on files with invalid UTF-8 instead of replacing.
+            return base64.b64decode(_stdout(result))
         result = await self.run(
             f"cat -- {shlex.quote(path)}",
             check=True,
             encoding=None,
             timeout=timeout_s,
         )
-        return _decode(result.stdout)
+        return cast("bytes", result.stdout)
 
     async def write_text(
         self,

{hud-0.6.19.dev1 → hud-0.6.19.dev2}/hud/version.py

@@ -4,4 +4,4 @@ Version information for the HUD SDK.
 
 from __future__ import annotations
 
-__version__ = "0.6.19.dev1"
+__version__ = "0.6.19.dev2"