hud-python 0.2.6__py3-none-any.whl → 0.2.8__py3-none-any.whl

hud/types.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import enum
 from pathlib import Path
-from typing import Literal, TypeAlias
+from typing import Any, Literal, TypeAlias
 
 from pydantic import BaseModel
 
@@ -28,6 +28,9 @@ class CustomGym(BaseModel):
     # B. If string, then it is the uri of the docker image to use.
     #    The controller must already be installed in the image.
     image_or_build_context: str | Path
+    # host_config will be passed to the docker client when creating the environment.
+    # refer to official docker api documentation for available configs.
+    host_config: dict[str, Any] | None = None
 
 
 class EnvironmentStatus(str, enum.Enum):
@@ -48,7 +51,30 @@ class EnvironmentStatus(str, enum.Enum):
 
 
 # Available HUD gyms
-ServerGym: TypeAlias = Literal["qa", "hud-browser", "OSWorld-Ubuntu"]
+ServerGym: TypeAlias = Literal["qa", "hud-browser", "OSWorld-Ubuntu", "docker"]
 
 # Gyms can be either custom or server-side
 Gym: TypeAlias = CustomGym | ServerGym
+
+
+# Metadata keys for the environment.
+# partial: Whether the environment evaluator should give partial grades.
+# eval_model: The model to use for evaluation when running a VLM. Wraps langchain.
+# agent_name: The name of the agent that was used for running this task.
+ServerMetadataKeys: TypeAlias = Literal["partial", "eval_model", "agent_name"]
+MetadataKeys: TypeAlias = str | ServerMetadataKeys
+
+
+# Dictionary of sensitive data (only supported for hud-browser environments)
+# key: website name or page identifier
+# value: Dictionary of credentials for the sensitive data
+# Example:
+# {
+#     "google.com": {
+#         "google_username": "my_username",
+#         "google_password": "my_password"
+#     }
+# }
+# The agent only has access to the key of the credential, not the value. (i.e. google_username)
+# The value is only available to the environment. (i.e. my_username)
+SensitiveData: TypeAlias = dict[str, dict[str, str]]

hud/utils/agent.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from hud.task import Task
+
+AGENT_PROMPT = (
+    "You are an AI agent whose goal is to accomplish the ultimate task following the instructions."
+)
+
+
+def format_agent_prompt(environment_prompt: str | None, task: Task | None) -> str:
+    """
+    Format the agent prompt with the environment prompt and the task prompt.
+    """
+    prompt = AGENT_PROMPT
+
+    # User-provided system prompt takes precedence over environment prompt
+    if task and task.system_prompt:
+        prompt += f"\n\n{task.system_prompt}"
+    elif environment_prompt:
+        prompt += f"\n\n{environment_prompt}"
+
+    if task:
+        if task.sensitive_data:
+            prompt += "\n\nHere are placeholders for sensitive data for each domain:"
+            for domain, credentials in task.sensitive_data.items():
+                prompt += f"\n{domain}: "
+                placeholders = [f"{key}" for key in credentials]
+                prompt += f"{', '.join(placeholders)}"
+            prompt += "\n\nYou can type these placeholders to enter the sensitive data when needed."
+
+        if task.prompt:
+            prompt += f"\n\n{task.prompt}"
+
+    return prompt

hud/utils/common.py

@@ -6,6 +6,7 @@ import tarfile
 import zipfile
 from typing import TYPE_CHECKING, Any, TypedDict
 
+from pathspec import PathSpec
 from pydantic import BaseModel
 
 from hud.server.requests import make_request
@@ -67,8 +68,8 @@ class Observation(BaseModel):
 
     def __str__(self) -> str:
         return f"""Observation(screenshot={
-            self.screenshot[:100] if self.screenshot else "None"
-        }..., text={self.text}...)"""
+            f"{self.screenshot[:100]}..." if self.screenshot else "None"
+        }, text={f"{self.text[:100]}..." if self.text else "None"})"""
 
 
 class ExecuteResult(TypedDict):
@@ -86,44 +87,159 @@ class ExecuteResult(TypedDict):
     exit_code: int
 
 
-def directory_to_tar_bytes(directory_path: Path) -> bytes:
+# ---------------------------------------------------------------------------
+# Helper functions for handling ignore patterns
+# ---------------------------------------------------------------------------
+
+
+def _read_ignore_file(file_path: Path) -> list[str]:
+    """Return patterns from *file_path* (ignoring blanks / comments)."""
+    if not file_path.exists():
+        return []
+
+    patterns: list[str] = []
+    for line in file_path.read_text().splitlines():
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        patterns.append(stripped)
+    return patterns
+
+
+def _gather_ignore_patterns(root_dir: Path, filename: str) -> list[str]:
+    """Collect *filename* patterns throughout *root_dir* respecting hierarchy.
+
+    For a nested ignore file located at ``sub/dir/.gitignore`` containing the
+    pattern ``foo/``, the returned pattern will be ``sub/dir/foo/`` so that it
+    is evaluated relative to *root_dir* when passed to ``PathSpec``.
     """
-    Converts a directory to a tar archive and returns it as bytes.
+    gathered: list[str] = []
+
+    root_dir = root_dir.resolve()
+
+    for ignore_file in root_dir.rglob(filename):
+        prefix = ignore_file.parent.relative_to(root_dir).as_posix()
+        base_prefix = "" if prefix == "." else prefix
+
+        for pat in _read_ignore_file(ignore_file):
+            negate = pat.startswith("!")
+            pat_body = pat[1:] if negate else pat
+
+            # Leading slash means relative to the directory the ignore file is
+            # located in - remove it so we can prepend *prefix* below.
+            if pat_body.startswith("/"):
+                pat_body = pat_body.lstrip("/")
 
-    This function creates a tar archive of the specified directory in memory,
-    without writing to a temporary file on disk.
+            full_pattern = f"{base_prefix}/{pat_body}" if base_prefix else pat_body
+            if negate:
+                full_pattern = f"!{full_pattern}"
 
-    Args:
-        path: Path to the directory to convert
+            gathered.append(full_pattern)
 
-    Returns:
-        Bytes of the tar archive
+    return gathered
+
+
+def _compile_pathspec(
+    directory: Path,
+    *,
+    respect_gitignore: bool,
+    respect_dockerignore: bool,
+    respect_hudignore: bool,
+) -> PathSpec | None:
+    """Compile a ``PathSpec`` from all relevant ignore files under *directory*.
+
+    In addition to the standard ``.gitignore`` and ``.dockerignore`` files we now
+    recognise a project-specific ``.hudignore`` file that shares the same pattern
+    syntax. Each file can be toggled independently through the corresponding
+    ``respect_*`` keyword argument.
+    """
+    patterns: list[str] = []
+
+    if respect_gitignore:
+        patterns.extend(_gather_ignore_patterns(directory, ".gitignore"))
+    if respect_dockerignore:
+        patterns.extend(_gather_ignore_patterns(directory, ".dockerignore"))
+    if respect_hudignore:
+        patterns.extend(_gather_ignore_patterns(directory, ".hudignore"))
+
+    if not patterns:
+        return None
+
+    return PathSpec.from_lines("gitwildmatch", patterns)
+
+
+def _iter_files(
+    directory: Path,
+    *,
+    respect_gitignore: bool,
+    respect_dockerignore: bool,
+    respect_hudignore: bool,
+) -> Iterator[tuple[Path, Path]]:
+    """Yield ``(file_path, relative_path)`` while respecting ignore files."""
+    spec = _compile_pathspec(
+        directory,
+        respect_gitignore=respect_gitignore,
+        respect_dockerignore=respect_dockerignore,
+        respect_hudignore=respect_hudignore,
+    )
+
+    for file_path in directory.rglob("*"):
+        if not file_path.is_file():
+            continue
+        rel_path = file_path.relative_to(directory)
+        rel_str = rel_path.as_posix()
+        if spec and spec.match_file(rel_str):
+            continue
+        yield file_path, rel_path
+
+
+def directory_to_tar_bytes(
+    directory_path: Path,
+    *,
+    respect_gitignore: bool = False,
+    respect_dockerignore: bool = False,
+    respect_hudignore: bool = True,
+) -> bytes:
+    """
+    Converts a directory to a tar archive and returns it as bytes.
+
+    By default the archive respects ignore rules defined in ``.gitignore``,
+    ``.dockerignore`` and ``.hudignore`` (each can be disabled via kwargs).
     """
     output = io.BytesIO()
 
     with tarfile.open(fileobj=output, mode="w") as tar:
-        # Walk through the directory
-        for file_path in directory_path.rglob("*"):
-            if file_path.is_file():
-                # Calculate relative path for the archive
-                rel_path = file_path.relative_to(directory_path)
-                logger.debug("Adding %s to tar archive", rel_path)
-                tar.add(file_path, arcname=str(rel_path))
-
-    # Get the bytes from the BytesIO object
+        for file_path, rel_path in _iter_files(
+            directory_path,
+            respect_gitignore=respect_gitignore,
+            respect_dockerignore=respect_dockerignore,
+            respect_hudignore=respect_hudignore,
+        ):
+            logger.debug("Adding %s to tar archive", rel_path)
+            tar.add(file_path, arcname=str(rel_path))
+
     output.seek(0)
     return output.getvalue()
 
 
-def directory_to_zip_bytes(context_dir: Path) -> bytes:
-    """Zip a directory and return the zip archive as bytes."""
+def directory_to_zip_bytes(
+    context_dir: Path,
+    *,
+    respect_gitignore: bool = False,
+    respect_dockerignore: bool = False,
+    respect_hudignore: bool = True,
+) -> bytes:
+    """Zip *context_dir* and return the zip archive as bytes, respecting ignore rules."""
     output = io.BytesIO()
     with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as zipf:
-        for file_path in context_dir.rglob("*"):
-            if file_path.is_file():
-                rel_path = file_path.relative_to(context_dir)
-                logger.debug("Adding %s to zip archive", rel_path)
-                zipf.write(str(file_path), arcname=str(rel_path))
+        for file_path, rel_path in _iter_files(
+            context_dir,
+            respect_gitignore=respect_gitignore,
+            respect_dockerignore=respect_dockerignore,
+            respect_hudignore=respect_hudignore,
+        ):
+            logger.debug("Adding %s to zip archive", rel_path)
+            zipf.write(str(file_path), arcname=str(rel_path))
     return output.getvalue()
 
 

hud/utils/config.py

@@ -103,6 +103,17 @@ def expand_config(config: FunctionConfigs) -> list[FunctionConfig]:
 
         return [FunctionConfig(function=function_name, args=args)]
 
+    if isinstance(config, list):
+        result = []
+        for item in config:
+            if isinstance(item, tuple) and len(item) >= 1 and isinstance(item[0], str):
+                function_name = item[0]
+                args = list(item[1:]) if len(item) > 1 else []
+                result.append(FunctionConfig(function=function_name, args=args))
+            else:
+                raise ValueError(f"Invalid list item configuration: {item}")
+        return result
+
     # Unknown configuration type
     error_msg = f"Unknown configuration type: {type(config)}"
     logger.error(error_msg)

hud/utils/tests/test_common.py

@@ -50,3 +50,228 @@ async def test_get_gym_id(mocker: pytest_mock.MockerFixture):
     mocker.patch("hud.utils.common.make_request", return_value={"id": "test_gym_id"})
     gym_id = await get_gym_id("test_gym")
     assert gym_id == "test_gym_id"
+
+
+def test_function_config_stores_function_name_args_and_optional_id():
+    """FunctionConfig should store function name, args list, and optional id."""
+    from hud.utils.common import FunctionConfig
+
+    # Minimal config
+    minimal = FunctionConfig(function="test_func", args=[])
+    assert minimal.function == "test_func"
+    assert minimal.args == []
+    assert minimal.id is None
+
+    # With args
+    with_args = FunctionConfig(function="navigate", args=["https://example.com", {"wait": True}])
+    assert with_args.function == "navigate"
+    assert len(with_args.args) == 2
+    assert with_args.args[0] == "https://example.com"
+    assert with_args.args[1] == {"wait": True}
+
+    # With id
+    with_id = FunctionConfig(
+        function="complex_operation",
+        args=[42, "test", {"nested": {"key": "value"}}],
+        id="op_123",
+    )
+    assert with_id.function == "complex_operation"
+    assert len(with_id.args) == 3
+    assert with_id.id == "op_123"
+
+
+@pytest.mark.asyncio
+async def test_get_gym_id_fetches_id_from_api_response(
+    mocker: pytest_mock.MockerFixture,
+):
+    """get_gym_id should extract 'id' field from API response."""
+    # Arrange
+    api_response = {"id": "gym-123", "name": "Test Gym", "status": "active"}
+    mocker.patch("hud.utils.common.make_request", return_value=api_response)
+
+    # Act
+    gym_id = await get_gym_id("test_gym")
+
+    # Assert
+    assert gym_id == "gym-123"
+
+
+@pytest.mark.asyncio
+async def test_get_gym_id_propagates_network_errors(mocker: pytest_mock.MockerFixture):
+    """get_gym_id should propagate exceptions from make_request."""
+    # Arrange
+    mocker.patch("hud.utils.common.make_request", side_effect=ConnectionError("API unavailable"))
+
+    # Act & Assert
+    with pytest.raises(ConnectionError, match="API unavailable"):
+        await get_gym_id("test_gym")
+
+
+@pytest.mark.asyncio
+async def test_get_gym_id_raises_key_error_when_id_missing(
+    mocker: pytest_mock.MockerFixture,
+):
+    """get_gym_id should raise KeyError when response lacks 'id' field."""
+    # Arrange
+    incomplete_response = {"name": "Test Gym", "status": "active"}  # Missing 'id'
+    mocker.patch("hud.utils.common.make_request", return_value=incomplete_response)
+
+    # Act & Assert
+    with pytest.raises(KeyError):
+        await get_gym_id("test_gym")
+
+
+def test_directory_to_tar_bytes_creates_valid_tar_archive(
+    tmpdir_factory: pytest.TempdirFactory,
+):
+    """directory_to_tar_bytes should create a valid tar archive containing all files."""
+    # Arrange
+    temp_dir = tmpdir_factory.mktemp("test_archive")
+    temp_dir_path = Path(temp_dir)
+
+    # Create test structure
+    (temp_dir_path / "file1.txt").write_text("content1")
+    (temp_dir_path / "file2.py").write_text("import os\nprint('hello')")
+
+    subdir = temp_dir_path / "subdir"
+    subdir.mkdir()
+    (subdir / "nested.json").write_text('{"key": "value"}')
+
+    # Act
+    tar_bytes = directory_to_tar_bytes(temp_dir_path)
+
+    # Assert
+    assert isinstance(tar_bytes, bytes)
+    assert len(tar_bytes) > 0
+
+    # Verify contents
+    with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:*") as tar:
+        members = {m.name for m in tar.getmembers()}
+        assert "file1.txt" in members
+        assert "file2.py" in members
+        assert "subdir/nested.json" in members
+
+        # Verify file contents
+        content = tar.extractfile("file1.txt")
+        assert content is not None
+        assert content.read().decode() == "content1"
+
+
+def test_directory_to_tar_bytes_handles_empty_directory(
+    tmpdir_factory: pytest.TempdirFactory,
+):
+    """directory_to_tar_bytes should handle empty directories gracefully."""
+    # Arrange
+    empty_dir = tmpdir_factory.mktemp("empty")
+    empty_dir_path = Path(empty_dir)
+
+    # Act
+    tar_bytes = directory_to_tar_bytes(empty_dir_path)
+
+    # Assert
+    assert isinstance(tar_bytes, bytes)
+    assert len(tar_bytes) > 0  # Even empty tar has headers
+
+    with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:*") as tar:
+        members = tar.getmembers()
+        # May contain the directory itself or be completely empty
+        assert len(members) >= 0
+
+
+def test_directory_to_tar_bytes_preserves_directory_structure(
+    tmpdir_factory: pytest.TempdirFactory,
+):
+    """directory_to_tar_bytes should preserve nested directory structure."""
+    # Arrange
+    root = tmpdir_factory.mktemp("root")
+    root_path = Path(root)
+
+    # Create nested structure
+    (root_path / "a" / "b" / "c").mkdir(parents=True)
+    (root_path / "a" / "file1.txt").write_text("in a")
+    (root_path / "a" / "b" / "file2.txt").write_text("in b")
+    (root_path / "a" / "b" / "c" / "file3.txt").write_text("in c")
+
+    # Act
+    tar_bytes = directory_to_tar_bytes(root_path)
+
+    # Assert
+    with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:*") as tar:
+        members = {m.name for m in tar.getmembers()}
+        assert "a/file1.txt" in members
+        assert "a/b/file2.txt" in members
+        assert "a/b/c/file3.txt" in members
+
+
+def test_directory_to_tar_bytes_with_exclusions(tmpdir_factory: pytest.TempdirFactory):
+    """Test directory_to_tar_bytes with files to exclude."""
+    temp_dir = tmpdir_factory.mktemp("test_exclude_dir")
+    temp_dir_path = Path(temp_dir)
+
+    # Create various files
+    (temp_dir_path / "include_me.txt").write_text("include")
+    (temp_dir_path / ".git").mkdir()
+    (temp_dir_path / ".git" / "config").write_text("git config")
+    (temp_dir_path / "__pycache__").mkdir()
+    (temp_dir_path / "__pycache__" / "module.pyc").write_bytes(b"pyc content")
+    (temp_dir_path / "normal_dir").mkdir()
+    (temp_dir_path / "normal_dir" / "file.py").write_text("python code")
+
+    tar_bytes = directory_to_tar_bytes(temp_dir_path)
+
+    # Check contents
+    with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:*") as tar:
+        member_names = {m.name for m in tar.getmembers()}
+
+        # Should include regular files and directories
+        assert "include_me.txt" in member_names
+        assert "normal_dir/file.py" in member_names
+
+        # Implementation might exclude common patterns like .git and __pycache__
+        # This depends on the actual implementation
+
+
+def test_directory_to_tar_bytes_empty_directory(tmpdir_factory: pytest.TempdirFactory):
+    """Test directory_to_tar_bytes with empty directory."""
+    temp_dir = tmpdir_factory.mktemp("empty_dir")
+    temp_dir_path = Path(temp_dir)
+
+    tar_bytes = directory_to_tar_bytes(temp_dir_path)
+
+    # Should still create a valid tar even if empty
+    assert tar_bytes is not None
+    assert len(tar_bytes) > 0
+
+    with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:*") as tar:
+        members = tar.getmembers()
+        # Might be empty or contain just the root directory
+        assert len(members) >= 0
+
+
+def test_directory_to_tar_bytes_symlinks(tmpdir_factory: pytest.TempdirFactory):
+    """Test directory_to_tar_bytes with symbolic links."""
+    temp_dir = tmpdir_factory.mktemp("symlink_dir")
+    temp_dir_path = Path(temp_dir)
+
+    # Create a file and a symlink to it
+    target_file = temp_dir_path / "target.txt"
+    target_file.write_text("target content")
+
+    symlink = temp_dir_path / "link_to_target.txt"
+    try:
+        symlink.symlink_to(target_file)
+        has_symlink = True
+    except OSError:
+        # Symlinks might not be supported on all systems (e.g., Windows without admin)
+        has_symlink = False
+
+    tar_bytes = directory_to_tar_bytes(temp_dir_path)
+
+    with tarfile.open(fileobj=io.BytesIO(tar_bytes), mode="r:*") as tar:
+        members = {m.name: m for m in tar.getmembers()}
+
+        assert "target.txt" in members
+
+        if has_symlink:
+            # Check how symlinks are handled (might be followed or preserved)
+            assert "link_to_target.txt" in members

hud/utils/tests/test_version.py

@@ -5,4 +5,4 @@ def test_import():
     """Test that the package can be imported."""
     import hud
 
-    assert hud.__version__ == "0.2.6"
+    assert hud.__version__ == "0.2.8"

hud/version.py

@@ -4,4 +4,4 @@ Version information for the HUD SDK.
 
 from __future__ import annotations
 
-__version__ = "0.2.6"
+__version__ = "0.2.8"

{hud_python-0.2.6.dist-info → hud_python-0.2.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hud-python
-Version: 0.2.6
+Version: 0.2.8
 Summary: SDK for the HUD evaluation platform.
 Project-URL: Homepage, https://github.com/hud-evals/hud-sdk
 Project-URL: Bug Tracker, https://github.com/hud-evals/hud-sdk/issues
@@ -47,6 +47,7 @@ Requires-Dist: langchain-openai
 Requires-Dist: mcp
 Requires-Dist: numpy
 Requires-Dist: openai
+Requires-Dist: pathspec>=0.12.1
 Requires-Dist: pillow>=11.1.0
 Requires-Dist: pydantic-settings<3,>=2
 Requires-Dist: pydantic<3,>=2
@@ -61,7 +62,7 @@ Requires-Dist: ipython<9; extra == 'dev'
 Requires-Dist: jupyter-client; extra == 'dev'
 Requires-Dist: jupyter-core; extra == 'dev'
 Requires-Dist: openai; extra == 'dev'
-Requires-Dist: pyright==1.1.364; extra == 'dev'
+Requires-Dist: pyright==1.1.401; extra == 'dev'
 Requires-Dist: pytest-asyncio; extra == 'dev'
 Requires-Dist: pytest-cov; extra == 'dev'
 Requires-Dist: pytest-mock; extra == 'dev'
@@ -90,7 +91,7 @@ We're here to help with eval strategies, custom environments, or improving your
 
 ## ✨ What You Can Do
 
-**Evaluate Existing Benchmarks**
+**[Evaluate Existing Benchmarks](https://docs.hud.so/examples/benchmarking-agents)**
 ```python
 from hud import load_taskset, run_job, ClaudeAgent
 
@@ -98,7 +99,7 @@ taskset = await load_taskset("WebVoyager")  # or GAIA, OSWorld-Ubuntu, Mind2Web
 job = await run_job(ClaudeAgent, taskset, "my-evaluation")
 ```
 
-**Create Custom Tasks**
+**[Create Custom Tasks](https://docs.hud.so/task-creation)** 
 ```python
 from hud.task import Task
 
@@ -110,7 +111,7 @@ task = Task(
 )
 ```
 
-**Build Custom Environments**
+**[Build Custom Environments](https://docs.hud.so/environment-creation)**
 ```python
 from hud.types import CustomGym
 
@@ -123,7 +124,7 @@ custom_gym = CustomGym(
 # Or create complex Docker environments - see environments/ folder for examples
 ```
 
-**Trace Tool Calls Alongside HUD Environments (or Independently)**
+**[Trace Tool Calls Alongside HUD Environments (or Independently)](https://docs.hud.so/examples/mcp-agent-tracing)**
 ```python
 import hud
 
@@ -171,6 +172,7 @@ async def main():
         setup=("goto", "google.com"),
         evaluate=("contains_text", "capybara")
     )
+    print(f"Running task with prompt: {task.prompt}")
     
     # Create environment using the gym module
     env = await gym.make(task)
@@ -182,6 +184,7 @@ async def main():
     obs, _ = await env.reset() # Gets first observation
     for i in range(5):
         actions, done = await agent.predict(obs)
+        print(f"Agent action {i}: {actions}")
 
         obs, reward, terminated, info = await env.step(actions)
         if done or terminated: break