hud 0.6.18.dev1__tar.gz → 0.6.18.dev2__tar.gz

{hud-0.6.18.dev1 → hud-0.6.18.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: hud
-Version: 0.6.18.dev1
+Version: 0.6.18.dev2
 Summary: SDK for the HUD platform.
 Project-URL: Homepage, https://github.com/hud-evals/hud-python
 Project-URL: Bug Tracker, https://github.com/hud-evals/hud-python/issues

{hud-0.6.18.dev1 → hud-0.6.18.dev2}/hud/agents/gemini/agent.py

@@ -224,6 +224,9 @@ class GeminiAgent(ToolAgent[genai_types.Content, GeminiConfig]):
 
         if candidate.finish_reason is not None:
             result.finish_reason = candidate.finish_reason.name
+            if result.finish_reason == "MALFORMED_FUNCTION_CALL":
+                result.error = "Provider returned a malformed function call"
+                result.stop_reason = "malformed_tool_call"
 
         return result
 

{hud-0.6.18.dev1 → hud-0.6.18.dev2}/hud/agents/tests/test_gemini_agent.py

@@ -145,3 +145,19 @@ def test_grounding_citations() -> None:
     assert len(cites) == 1
     assert cites[0].source == "http://x"
     assert cites[0].type == "grounding"
+
+
+async def test_malformed_function_call_is_a_normalized_error() -> None:
+    response = _api_response(
+        SimpleNamespace(
+            content=None,
+            grounding_metadata=None,
+            finish_reason=SimpleNamespace(name="MALFORMED_FUNCTION_CALL"),
+        )
+    )
+    agent = _agent(response)
+    result = await agent.get_response(_state(agent))
+    assert result.error == "Provider returned a malformed function call"
+    assert result.stop_reason == "malformed_tool_call"
+    assert result.finish_reason == "MALFORMED_FUNCTION_CALL"
+    assert result.usage is not None

{hud-0.6.18.dev1 → hud-0.6.18.dev2}/hud/agents/tests/test_tool_agent.py

@@ -788,3 +788,20 @@ async def test_loop_stops_on_length_when_configured() -> None:
     assert run.trace.stop_reason == "length"
     assert run.trace.is_truncated is True
     assert all(not isinstance(step, ToolStep) for step in run.trace.steps)
+
+
+@pytest.mark.parametrize("auto_respond", [False, True])
+@pytest.mark.parametrize("reason", [None, "malformed_tool_call"])
+async def test_provider_error_stops_without_auto_response(
+    auto_respond: bool, reason: Literal["malformed_tool_call"] | None
+) -> None:
+    agent = DictAgent(
+        [AgentStep(error="provider failure", stop_reason=reason)],
+        auto_respond=auto_respond,
+    )
+    run = cast("Run", _FakeRun())
+    await agent._loop(run, RunState(), max_steps=3)
+    assert run.trace.status == "error"
+    assert run.trace.stop_reason == reason
+    assert len(run.trace.steps) == 1
+    assert run.trace.steps[0].error == "provider failure"

{hud-0.6.18.dev1 → hud-0.6.18.dev2}/hud/agents/tool_agent.py

@@ -253,6 +253,9 @@ class ToolAgent(Agent, Generic[MessageT, ConfigT]):
                 step.started_at = step.started_at or started_at
                 step.model = step.model or self.config.model
                 run.record(step)
+                if step.error:
+                    stopped = step.stop_reason
+                    break
 
                 if step.tool_calls:
                     logger.info("  → %s", ", ".join(c.name for c in step.tool_calls))
@@ -304,6 +307,8 @@ class ToolAgent(Agent, Generic[MessageT, ConfigT]):
             trace.status = "error" if step is not None and step.error else "completed"
             if stopped is not None:
                 trace.stop_reason = stopped
+            elif step is not None and step.error:
+                trace.stop_reason = None
             elif hit_max:
                 trace.stop_reason = "max_steps"
             elif step is not None and step.finish_reason in TRUNCATION_FINISH_REASONS:

{hud-0.6.18.dev1 → hud-0.6.18.dev2}/hud/agents/tools/ssh.py

@@ -65,7 +65,7 @@ class SSHTool(AgentTool[SSHClient]):
             await self.client.write_text(path, content)
         except asyncssh.ProcessError as e:
             return tool_err(_remote_error(e))
-        return tool_ok(f"wrote {len(content)} bytes to {path}")
+        return tool_ok(f"wrote {len(content.encode('utf-8'))} bytes to {path}")
 
     async def file_list(self, path: str = "/") -> MCPToolResult:
         """List directory entries through SSH exec."""

{hud-0.6.18.dev1 → hud-0.6.18.dev2}/hud/agents/types.py

@@ -288,6 +288,7 @@ class AgentStep(Step):
     #: No further tool calls expected — the loop's stop signal.
     done: bool = False
     finish_reason: str | None = None
+    stop_reason: StopCondition | None = None
     refusal: str | None = None
     citations: list[Citation] = Field(default_factory=list[Citation])
     raw: Any | None = None

{hud-0.6.18.dev1 → hud-0.6.18.dev2}/hud/cli/eval.py

@@ -102,7 +102,7 @@ class EvalConfig(BaseModel):
     group_size: int = 1
     gateway: bool = False
     #: ``LOCAL`` spawns each row's env (Docker for container rows, a subprocess
-    #: serving the bound env's source otherwise); other names hand rows to that
+    #: loading the task source otherwise); other names hand rows to that
     #: provider; a ``tcp://`` url attaches to an already-served env. ``None``
     #: infers from the source: a file on disk runs locally, a platform taskset hosted.
     runtime: Placement | TcpUrl | None = None
@@ -417,24 +417,33 @@ def eval_command(
         case AnyUrl():
             placement = Runtime(str(cfg.runtime))
         case Placement.LOCAL:
-            # Isolate each row: its container, or a subprocess serving the bound env's
-            # source (``Taskset.run`` alone would serve a live env in-process). Data
-            # rows (JSON/JSONL) only name their env; its source lives beside the file.
-            if not Path(source).exists():
+            rows = list(taskset)
+            rows.extend(
+                task.verifier
+                for task in taskset
+                if task.verifier is not None and not task.shares_verifier_runtime
+            )
+            if not Path(source).exists() and any(
+                not (
+                    task.runtime_config
+                    and (task.runtime_config.image or task.runtime_config.compose)
+                )
+                for task in rows
+            ):
                 raise ValueError(
                     f"{source} is a platform taskset, so there is no env source to spawn "
                     "locally. Run it with --remote, --runtime hud, or --runtime tcp://host:port."
                 )
             docker = DockerRuntime()
-            source_dir = Path(source).resolve()
-            beside = SubprocessRuntime(source_dir if source_dir.is_dir() else source_dir.parent)
+            source_path = Path(source).resolve()
+            beside = SubprocessRuntime(source_path if source_path.is_dir() else source_path.parent)
 
             def spawn(task: Task) -> AbstractAsyncContextManager[Runtime]:
                 config = task.runtime_config
                 if config and (config.image or config.compose):
                     return docker(task)
                 if task._env is not None:
-                    return SubprocessRuntime(task._env)(task)
+                    return SubprocessRuntime(source_path)(task)
                 return beside(task)
 
             placement = spawn

{hud-0.6.18.dev1 → hud-0.6.18.dev2}/hud/cli/tests/test_eval.py

@@ -5,12 +5,14 @@ from __future__ import annotations
 import json
 import sys
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import TYPE_CHECKING, Any
 from unittest.mock import MagicMock
 
 import pytest
 from typer.testing import CliRunner
 
+from hud.agents import OpenAIAgent
 from hud.agents.types import ClaudeConfig
 from hud.cli import eval as eval_mod
 from hud.cli.__main__ import app
@@ -34,8 +36,7 @@ from hud.utils.gateway import GatewayModelInfo
 from hud.utils.hud_console import HUDConsole
 
 if TYPE_CHECKING:
-    from collections.abc import Callable
-    from pathlib import Path
+    from collections.abc import Callable, Iterator
 
 _TASKS_PY = """\
 from hud import Environment
@@ -405,7 +406,7 @@ def test_local_placement_routes_each_row(eval_cli: _EvalCli, tmp_path: Path, mon
     assert eval_cli.taskset is not None
     bound = next(iter(eval_cli.taskset))
 
-    assert placement(bound) is subprocesses[bound._env].return_value
+    assert placement(bound) is subprocesses[(tmp_path / "mixed.py").resolve()].return_value
     assert placement(image) is docker.return_value
     # A row without a bound env is served from the tasks file's directory.
     assert placement(data_row) is subprocesses[tmp_path.resolve()].return_value
@@ -685,3 +686,268 @@ def test_result_payload_uses_job_metrics(eval_cli: _EvalCli) -> None:
     assert payload["error_count"] == 1
     assert [run["slug"] for run in payload["runs"]] == ["solve", None]
     assert [run["is_error"] for run in payload["runs"]] == [False, False]
+
+
+@pytest.fixture
+def local_eval(
+    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> Iterator[Callable[..., dict[str, Any]]]:
+    async def answer(self: OpenAIAgent, run: Run) -> None:
+        run.trace.content = "ok"
+
+    monkeypatch.setattr(OpenAIAgent, "__call__", answer)
+    monkeypatch.setattr(settings, "openai_api_key", "test-provider-key")
+    monkeypatch.setenv("HUD_API_KEY", "")
+    monkeypatch.setenv("HUD_TELEMETRY_ENABLED", "false")
+    monkeypatch.chdir(tmp_path)
+    original = dict(sys.modules)
+
+    def invoke(source: Path | str, *flags: str) -> dict[str, Any]:
+        result = CliRunner().invoke(
+            app, ["eval", str(source), "openai", "--all", "--yes", "--json", *flags]
+        )
+        assert result.exit_code == 0, result.output
+        payload = json.loads(result.stdout)
+        assert payload["error_count"] == 0, result.output
+        assert payload["mean_reward"] == 1.0, result.output
+        return payload
+
+    yield invoke
+    for name, module in list(sys.modules.items()):
+        file = getattr(module, "__file__", None)
+        if file and Path(file).is_relative_to(tmp_path):
+            if name in original:
+                sys.modules[name] = original[name]
+            else:
+                sys.modules.pop(name, None)
+
+
+@pytest.mark.parametrize(
+    "layout",
+    [
+        "single",
+        "standalone",
+        "split",
+        "hooks",
+        "hooks_source",
+        "assembled",
+        "assembled_source",
+        "json",
+        "jsonl",
+        "data_python",
+        "directory",
+        "assembled_directory",
+        "package",
+        "lazy_package",
+    ],
+)
+def test_local_eval_project_layouts(
+    local_eval: Callable[..., dict[str, Any]], tmp_path: Path, layout: str
+) -> None:
+    project = tmp_path / "project"
+    project.mkdir()
+    events = project / "events"
+    events.mkdir()
+    (project / "asset.txt").write_text("ok")
+    package = layout in {"package", "lazy_package"}
+    prefix = "." if package else ""
+    if package:
+        (project / "__init__.py").write_text("")
+    template = (
+        "from pathlib import Path\n"
+        '@env.template(id="solve")\n'
+        "async def solve():\n"
+        '    answer = yield "answer ok"\n'
+        '    yield 1.0 if answer == Path("asset.txt").read_text() else 0.0\n'
+    )
+    hooks = (
+        "import os\nfrom pathlib import Path\n"
+        "@env.initialize\nasync def start():\n"
+        '    Path(f"events/{os.getpid()}").write_text("started")\n'
+        "@env.shutdown\nasync def stop():\n"
+        '    Path(f"events/{os.getpid()}").write_text("stopped")\n'
+    )
+    if layout == "lazy_package":
+        (project / "expected.py").write_text(
+            'from pathlib import Path\nexpected = Path("asset.txt").read_text()\n'
+        )
+        template = template.replace(
+            '    answer = yield "answer ok"\n',
+            '    from .expected import expected\n    answer = yield "answer ok"\n',
+        ).replace('Path("asset.txt").read_text()', "expected")
+    if layout in {
+        "hooks",
+        "hooks_source",
+        "assembled",
+        "assembled_source",
+        "assembled_directory",
+        "package",
+    }:
+        (project / "local_core.py").write_text(
+            'from hud import Environment\nenv = Environment("local-test")\n'
+        )
+        (project / "local_templates.py").write_text(
+            f"from {prefix}local_core import env\n" + template
+        )
+        (project / "local_extra.py").write_text(
+            f"from {prefix}local_core import env\n"
+            '@env.template(id="extra")\nasync def extra():\n    yield "extra"\n    yield 1.0\n'
+        )
+        env_source = (
+            f"from {prefix}local_core import env\n"
+            f"from {prefix}local_templates import solve\n"
+            + (f"from {prefix}local_extra import extra\n" if not layout.startswith("hooks") else "")
+            + hooks
+        )
+    else:
+        env_source = (
+            'from hud import Environment\nenv = Environment("local-test")\n' + template + hooks
+        )
+    if layout in {"single", "standalone", "lazy_package", "hooks_source", "assembled_source"}:
+        env_source += "tasks = [solve()]\n"
+        source = project / "env.py"
+    else:
+        (project / "tasks.py").write_text(
+            f"from {prefix}env import env, solve\ntasks = [solve()]\n"
+        )
+        source = project / "tasks.py"
+    (project / "env.py").write_text(env_source)
+    if layout == "standalone":
+        source = project / "standalone.py"
+        source.write_text(env_source)
+        (project / "env.py").write_text(
+            'from hud import Environment\nenv = Environment("unrelated")\n'
+        )
+    if layout in {"json", "jsonl"}:
+        source = project / f"tasks.{layout}"
+        row = {"env": "local-test", "id": "solve"}
+        source.write_text(json.dumps([row] if layout == "json" else row))
+    if layout == "data_python":
+        source.write_text(
+            'from hud.eval import Task\ntasks = [Task(env="local-test", id="solve")]\n'
+        )
+    if layout in {"directory", "assembled_directory"}:
+        source = project
+    payload = local_eval(source, "--group", "2", "--max-concurrent", "2")
+    assert payload["run_count"] == 2
+    assert len(list(events.iterdir())) == 2
+    assert {event.read_text() for event in events.iterdir()} == {"stopped"}
+
+
+@pytest.mark.parametrize(
+    "source,same_name",
+    [
+        ("tasks.py", False),
+        ("tasks.py", True),
+        (".", False),
+        (".", True),
+        ("rows.json", False),
+        ("rows.json", True),
+        ("rows.jsonl", False),
+    ],
+)
+def test_local_eval_environment_selection(
+    local_eval: Callable[..., dict[str, Any]], tmp_path: Path, source: str, same_name: bool
+) -> None:
+    project = tmp_path / "project"
+    project.mkdir()
+    (project / "bound_env.py").write_text(
+        'from hud import Environment\nenv = Environment("bound")\n'
+        '@env.template(id="solve")\nasync def solve():\n'
+        '    answer = yield "answer ok"\n    yield float(answer == "ok")\n'
+    )
+    name = "bound" if same_name else "unrelated"
+    (project / "env.py").write_text(
+        f"from hud import Environment\nenv = Environment({name!r})\n"
+        '@env.template(id="solve")\nasync def solve():\n'
+        '    yield "unrelated"\n    yield 0.0\n'
+    )
+    (project / "tasks.py").write_text("from bound_env import solve\ntasks = [solve()]\n")
+    if source in {"rows.json", "rows.jsonl"}:
+        (project / source).write_text('{"env": "bound", "id": "solve"}\n')
+    if same_name and source != "tasks.py":
+        result = CliRunner().invoke(
+            app, ["eval", str(project / source), "openai", "--all", "--yes", "--json"]
+        )
+        assert result.exit_code == 0, result.output
+        assert json.loads(result.stdout)["error_count"] == 1
+        assert "multiple Environments" in result.output
+    else:
+        assert local_eval(project / source)["run_count"] == 1
+
+
+@pytest.mark.parametrize("shared", [True, False])
+def test_local_eval_places_nested_verifier(
+    local_eval: Callable[..., dict[str, Any]], tmp_path: Path, shared: bool
+) -> None:
+    project = tmp_path / "project"
+    project.mkdir()
+    (project / "local_core.py").write_text(
+        'from hud import Environment\nactor = Environment("actor")\n'
+        + ("judge = actor\n" if shared else 'judge = Environment("judge")\n')
+    )
+    (project / "local_actor.py").write_text(
+        "from local_core import actor\n"
+        '@actor.template(id="solve")\nasync def solve():\n'
+        '    answer = yield "answer ok"\n    yield {"score": 0.0, "answer": answer}\n'
+    )
+    (project / "local_judge.py").write_text(
+        "from local_core import judge\n"
+        '@judge.template(id="verify")\nasync def verify():\n'
+        '    result = yield ""\n    yield 1.0 if result["answer"] == "ok" else 0.0\n'
+    )
+    (project / "env.py").write_text(
+        "from local_core import actor, judge\n"
+        "from local_actor import solve\nfrom local_judge import verify\n"
+    )
+    (project / "tasks.py").write_text(
+        "from env import solve, verify\ntasks = [solve()]\ntasks[0].verifier = verify()\n"
+    )
+    assert local_eval(project / "tasks.py")["run_count"] == 1
+
+
+@pytest.mark.parametrize("container", ["image", "compose"])
+def test_platform_container_rows_can_use_local_placement(
+    eval_cli: _EvalCli, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, container: str
+) -> None:
+    compose = tmp_path / "compose.yaml"
+    compose.write_text("services:\n  main:\n    image: example:latest\n")
+    config = (
+        RuntimeConfig(image="example:latest")
+        if container == "image"
+        else RuntimeConfig.model_validate(
+            {"compose": {"document": str(compose), "root": str(tmp_path)}}
+        )
+    )
+    task = Task(env="demo", id="solve", runtime_config=config)
+    monkeypatch.setattr(Taskset, "from_api", classmethod(lambda cls, name: Taskset(name, [task])))
+    docker = MagicMock()
+    monkeypatch.setattr(eval_mod, "DockerRuntime", lambda: docker)
+    eval_cli.invoke("Platform Tasks", "openai", "--runtime", "local", "--yes")
+    assert eval_cli.kwargs["runtime"](task) is docker.return_value
+    docker.assert_called_once_with(task)
+
+
+@pytest.mark.parametrize("shared,configured", [(True, False), (False, True), (False, False)])
+def test_platform_local_placement_checks_separate_verifier(
+    eval_cli: _EvalCli, monkeypatch: pytest.MonkeyPatch, shared: bool, configured: bool
+) -> None:
+    config = RuntimeConfig(image="example:latest")
+    verifier = Task(
+        env="actor" if shared else "judge",
+        id="verify",
+        runtime_config=config if configured else None,
+    )
+    task = Task(env="actor", id="solve", runtime_config=config, verifier=verifier)
+    monkeypatch.setattr(Taskset, "from_api", classmethod(lambda cls, name: Taskset(name, [task])))
+    payload = eval_cli.invoke(
+        "Platform Tasks",
+        "openai",
+        "--runtime",
+        "local",
+        "--yes",
+        exit_code=0 if shared or configured else 2,
+    )
+    if not shared and not configured:
+        assert "no env source to spawn locally" in payload["message"]
+        assert eval_cli.taskset is None

{hud-0.6.18.dev1 → hud-0.6.18.dev2}/hud/environment/__init__.py

@@ -37,8 +37,9 @@ def load_environment(
     """Resolve an environment reference to the one :class:`Environment` it names.
 
     A source path (``.py`` file or directory) is scanned for ``Environment``
-    instances; *name* selects among several, matching the module attribute or
-    ``Environment.name``. A dotted importable module reads attribute *name*
+    instances and environments bound to exported tasks; *name* selects among
+    them, matching the module attribute or ``Environment.name``.
+    A dotted importable module reads attribute *name*
     (default ``env``): an ``Environment`` directly, or a factory called with
     *args* to build one — how programmatic envs (integrations, adapted
     images) are served without a source file. Raises when nothing resolves.
@@ -63,19 +64,32 @@ def load_environment(
         and (path / "__init__.py").is_file()
     )
     if path.exists() and not package_attribute:
+        # Avoid the environment -> eval -> runtime import cycle.
+        from hud.eval.taskset import Taskset
+
         if args:
             raise ValueError(f"args= applies to factory targets, not source path {target}")
-        matched = [
-            env
-            for module in iter_modules(path)
+        modules = list(iter_modules(path))
+        matched = {
+            id(env): env
+            for module in modules
             for attr, env in vars(module).items()
             if isinstance(env, Environment) and (name is None or name in (attr, env.name))
-        ]
+        }
+        for module in modules:
+            for task in Taskset._scan_tasks(module):
+                for row in (task, task.verifier):
+                    if (
+                        row is not None
+                        and (env := row._env) is not None
+                        and (name is None or name == env.name)
+                    ):
+                        matched[id(env)] = env
         if not matched:
             raise ValueError(f"no Environment{f' named {name!r}' if name else ''} found in {path}")
         if len(matched) > 1:
             raise ValueError(f"multiple Environments in {path}; select one by name")
-        return matched[0]
+        return next(iter(matched.values()))
 
     if path.is_file() or "/" in str(target):
         raise FileNotFoundError(f"no environment source at {target}")