hud-python 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl

hud/agent/claude_plays_pokemon.py

@@ -11,6 +11,7 @@ from anthropic.types.beta import (
     BetaImageBlockParam,
 )
 
+from hud.adapters.common.types import CLA, LogType
 from hud.agent import Agent
 from hud.adapters import Adapter
 from hud.settings import settings
@@ -128,7 +129,7 @@ def extract_json_from_response(response: str) -> str:
     return response.strip()
 
 
-class ClaudePlaysPokemon(Agent[AsyncAnthropic, None]):
+class ClaudePlaysPokemon(Agent[AsyncAnthropic, CLA]):
     """AI agent that plays Pokémon games using Claude."""
 
     def __init__(
@@ -191,7 +192,7 @@ class ClaudePlaysPokemon(Agent[AsyncAnthropic, None]):
             observation: The current game observation
 
         Returns:
-            tuple[list[dict[str, Any]], bool]: List of actions and whether the game is done
+            tuple[list[dict[str, Any]], bool, list[LogType] | None]: List of actions, whether the game is done, and a list of strings or dictionaries of logs.
 
         Raises:
             ValueError: If client is not initialized

hud/agent/langchain.py

@@ -24,6 +24,7 @@ from hud.adapters.common.types import (
     WaitAction,
     ResponseAction,
     CustomAction,
+    LogType,
     # Exclude ScreenshotFetch, PositionFetch as they are internal
 )
 
@@ -74,6 +75,7 @@ class LangchainAgent(Agent[LangchainModelOrRunnable, Any], Generic[LangchainMode
         langchain_model: LangchainModelOrRunnable,
         adapter: Optional[Adapter] = None,
         system_prompt: str | None = None,
+        name: str | None = None,
     ):
         """
         Initialize the LangchainAgent.
@@ -88,7 +90,9 @@ class LangchainAgent(Agent[LangchainModelOrRunnable, Any], Generic[LangchainMode
             system_prompt: An optional system prompt to guide the Langchain model.
                            If None, a default prompt encouraging single CLA output is used.
         """
-        super().__init__(client=langchain_model, adapter=adapter)  # Store model as 'client'
+        super().__init__(
+            client=langchain_model, adapter=adapter, name=name
+        )  # Store model as 'client'
         self.langchain_model = langchain_model  # Also store with specific name
 
         self.system_prompt_str = system_prompt or self._get_default_system_prompt()
@@ -137,7 +141,7 @@ class LangchainAgent(Agent[LangchainModelOrRunnable, Any], Generic[LangchainMode
         if not human_content:
             logger.warning("LangchainAgent received an observation with no text or screenshot.")
             # Decide how to handle empty observation - perhaps return no action?
-            return [], False  # Or raise an error?
+            return [], False
 
         current_human_message = HumanMessage(content=human_content)
 
@@ -202,7 +206,9 @@ class LangchainAgent(Agent[LangchainModelOrRunnable, Any], Generic[LangchainMode
         # TODO: Consider history truncation/summarization if it grows too long
 
         if actual_action:
+            actual_action = actual_action.model_dump()
             # Return the single action dictionary within a list
+            actual_action["logs"] = ai_message_content_for_history
             return [actual_action], is_done
         else:
             # Should ideally not happen if structure validation worked, but as a fallback

hud/agent/operator.py

@@ -19,6 +19,7 @@ from hud.adapters.operator import OperatorAdapter
 from hud.types import Gym
 from hud.utils.common import Observation
 from hud.settings import settings
+from hud.adapters.common.types import LogType
 
 logger = logging.getLogger(__name__)
 
@@ -37,9 +38,10 @@ class OperatorAgent(Agent[AsyncOpenAI, dict[str, Any]]):
         self,
         client: AsyncOpenAI | None = None,
         model: str = "computer-use-preview",
-        environment: Literal["windows", "mac", "linux", "browser"] = "linux",
+        environment: Literal["windows", "mac", "linux", "browser"] = "browser",
         adapter: Adapter | None = None,
         max_iterations: int = 8,
+        name: str | None = None,
     ):
         """
         Initialize the OperatorAgent.
@@ -50,6 +52,7 @@ class OperatorAgent(Agent[AsyncOpenAI, dict[str, Any]]):
             environment: The environment type (windows, mac, linux, browser)
             adapter: The adapter to use for preprocessing and postprocessing
             max_iterations: Maximum number of iterations for the agent
+            name: The name of the agent
         """
         # Initialize client if not provided
         if client is None:
@@ -65,7 +68,10 @@ class OperatorAgent(Agent[AsyncOpenAI, dict[str, Any]]):
 
         adapter = adapter or OperatorAdapter()
 
-        super().__init__(client=client, adapter=adapter)
+        if name is None:
+            name = f"openai-{model}"
+
+        super().__init__(client=client, adapter=adapter, name=name)
 
         self.model = model
         self.environment = environment
@@ -86,6 +92,8 @@ class OperatorAgent(Agent[AsyncOpenAI, dict[str, Any]]):
         self.initial_prompt = None
         self.pending_safety_checks = []
 
+        self.task_run_id = None
+
     async def fetch_response(self, observation: Observation) -> tuple[list[dict[str, Any]], bool]:
         """
         Fetch a response from the model based on the observation.
@@ -94,8 +102,8 @@ class OperatorAgent(Agent[AsyncOpenAI, dict[str, Any]]):
             observation: The preprocessed observation
 
         Returns:
-            tuple[list[dict[str, Any]], bool]: A tuple containing the list of raw actions and a
-                                             boolean indicating if the agent believes the task is complete
+            tuple[list[dict[str, Any]], bool, list[LogType] | None]: A tuple containing the list of raw actions,
+                                             boolean indicating if the agent believes the task is complete.
         """
         if not self.client:
             raise ValueError("Client is required")
@@ -112,7 +120,7 @@ class OperatorAgent(Agent[AsyncOpenAI, dict[str, Any]]):
         )
 
         # Process the observation based on whether it's the first one or a response to an action
-        if self.pending_call_id is None and self.last_response_id is None:
+        if self.pending_call_id is None:  # and self.last_response_id is None:
             # This is the first observation, store and send the prompt
             self.initial_prompt = observation.text
 
@@ -133,13 +141,15 @@ class OperatorAgent(Agent[AsyncOpenAI, dict[str, Any]]):
             # Structure the input correctly for the API using cast
             input_param = cast(ResponseInputParam, [{"role": "user", "content": input_content}])
 
-            # Call OpenAI API for the initial prompt (asynchronous call)
             response = await self.client.responses.create(
-                model=self.model, tools=[computer_tool], input=input_param, truncation="auto"
+                model=self.model,
+                tools=[computer_tool],
+                input=input_param,
+                truncation="auto",
+                reasoning={"summary": "auto"},
             )
 
         else:
-            # This is a response to a previous action
             if not observation.screenshot:
                 logger.warning("No screenshot provided for response to action")
                 return [], True
@@ -164,7 +174,6 @@ class OperatorAgent(Agent[AsyncOpenAI, dict[str, Any]]):
             )
             self.pending_safety_checks = []
 
-            # Call OpenAI API for follow-up (asynchronous call)
             response = await self.client.responses.create(
                 model=self.model,
                 previous_response_id=self.last_response_id,
@@ -181,6 +190,8 @@ class OperatorAgent(Agent[AsyncOpenAI, dict[str, Any]]):
         done = True  # Assume done unless a computer call is found
         final_text_response = ""
 
+        self.pending_call_id = None
+
         # Check for computer calls first
         computer_calls = [
             item
@@ -217,8 +228,22 @@ class OperatorAgent(Agent[AsyncOpenAI, dict[str, Any]]):
                 # No ResponseAgent logic here anymore - just return the response
                 actions = [{"type": "response", "text": final_text_response}]
                 done = True
-            # else:
-            #     logger.info("No computer calls and no final text message found.")
+            else:
+                logger.info("No computer calls and no final text message found.")
             # Keep done = True, actions remains empty
 
+        reasoning = ""
+        for item in response.output:
+            if item.type == "reasoning" and item.summary:
+                reasoning += f"Thinking: {item.summary[0].text}\n"
+            elif item.type == "message":
+                for content in item.content:
+                    if isinstance(content, ResponseOutputText):
+                        reasoning += f"{content.text}\n"
+
+        # add reasoning to the actions
+        for action in actions:
+            action["reasoning"] = reasoning
+            action["logs"] = response.model_dump()  # type: ignore[assignment]
+
         return actions, done

hud/agent/tests/test_base.py

@@ -22,9 +22,9 @@ class ConcreteAgent(Agent[Any, dict[str, Any]]):
     async def fetch_response(self, observation: Observation) -> tuple[list[dict[str, Any]], bool]:
         """Mock implementation that returns predefined responses."""
         if self.call_count < len(self.mock_responses):
-            response = self.mock_responses[self.call_count]
+            actions, done = self.mock_responses[self.call_count]
             self.call_count += 1
-            return response
+            return actions, done
         return [], True
 
 

hud/env/docker_client.py

@@ -12,7 +12,7 @@ import toml
 
 from hud.env.client import Client
 from hud.types import EnvironmentStatus
-from hud.utils.common import directory_to_tar_bytes
+from hud.utils.common import _compile_pathspec, directory_to_tar_bytes
 
 if TYPE_CHECKING:
     from hud.utils import ExecuteResult
@@ -151,15 +151,32 @@ class DockerClient(Client):
         if not self._source_path:
             return {}
 
-        file_mtimes = {}
+        # Build ignore spec (currently we only care about .hudignore but reuse
+        # the common helper for consistency).
+        spec = _compile_pathspec(
+            self._source_path,
+            respect_gitignore=False,
+            respect_dockerignore=False,
+            respect_hudignore=True,
+        )
+
+        file_mtimes: dict[str, float] = {}
+
         for root, _, files in os.walk(self._source_path):
             for file in files:
                 file_path = Path(root) / file
+                rel_path = file_path.relative_to(self._source_path).as_posix()
+
+                # Skip ignored files
+                if spec and spec.match_file(rel_path):
+                    continue
+
                 try:
                     file_mtimes[str(file_path)] = file_path.stat().st_mtime
                 except (FileNotFoundError, PermissionError):
                     # Skip files that can't be accessed
                     continue
+
         return file_mtimes
 
     async def needs_update(self) -> bool:
@@ -181,6 +198,11 @@ class DockerClient(Client):
         if not self._last_file_mtimes:
             return True
 
+        # Check for removed files
+        for file_path in self._last_file_mtimes:
+            if file_path not in current_mtimes:
+                return True
+
         # Check for new or modified files
         for file_path, mtime in current_mtimes.items():
             if file_path not in self._last_file_mtimes or mtime > self._last_file_mtimes[file_path]:

hud/env/environment.py

@@ -8,14 +8,14 @@ from typing import TYPE_CHECKING, Any
 from pydantic import BaseModel
 
 from hud.env.client import Client
-from hud.env.remote_client import RemoteClient
+from hud.env.remote_client import RemoteClient, SetupRequest
 from hud.task import Task
+from hud.utils.agent import format_agent_prompt
 from hud.utils.common import FunctionConfig, FunctionConfigs, Observation
 from hud.utils.config import (
     LOCAL_EVALUATORS,
     REMOTE_EVALUATE,
     REMOTE_FUNCTION_PREFIX,
-    REMOTE_SETUP,
     expand_config,
 )
 from hud.utils.telemetry import stream
@@ -41,9 +41,15 @@ class Environment(BaseModel):
     task: Task | None = None
     build_data: dict[str, Any]
 
+    # The task run id
+    task_run_id: str | None = None
+
     # final response
     final_response: str | None = None
 
+    # environment prompt information
+    environment_prompt: str | None = None
+
     async def _invoke_all(self, configs: FunctionConfigs) -> list[Any]:
         # Execute each config and collect results
         configs_all = [configs] if not isinstance(configs, list) else configs
@@ -69,24 +75,45 @@ class Environment(BaseModel):
     async def _setup(self, config: FunctionConfigs | None = None) -> None:
         """
         Setup the environment.
+        No-op if no config or task is provided.
 
         Args:
             config: The configuration to use for the setup
         """
         if isinstance(self.client, RemoteClient):
             await self.get_urls()
-            await self._invoke_all(create_remote_config(self, config, REMOTE_SETUP))
+
+            setup_request = SetupRequest()
+
+            if self.task:
+                setup_request.task_id = self.task.id
+                setup_request.config = self.task.config
+                setup_request.metadata = _format_task_metadata(self.task)
+                if self.task.setup:
+                    setup_request.setup = expand_config(self.task.setup)[0]
+            elif config:
+                setup_request.setup = expand_config(config)[0]
+            else:
+                raise ValueError("No task or config provided for remote environment")
+
+            result = await self.client.setup(setup_request)
+
+            if result and result.get("id"):
+                self.task_run_id = result.get("id")
+                logger.info("View the live trace at https://app.hud.so/trace/%s", self.task_run_id)
+            else:
+                logger.warning("No task run id found in the result")
         else:
             if config is not None:
                 await self._invoke_all(config)
             elif self.task and self.task.setup is not None:
                 await self._invoke_all(self.task.setup)
-            else:
-                raise ValueError(
-                    "No config, task or task setup function provided for local environment"
-                )
 
-    async def evaluate(self, config: FunctionConfigs | None = None) -> Any:
+    async def evaluate(
+        self,
+        config: FunctionConfigs | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> Any:
         """
         Evaluate the environment.
 
@@ -97,7 +124,9 @@ class Environment(BaseModel):
             Any: Result of the evaluation
         """
         if isinstance(self.client, RemoteClient):
-            results = await self._invoke_all(create_remote_config(self, config, REMOTE_EVALUATE))
+            results = await self._invoke_all(
+                create_remote_config(self, config, REMOTE_EVALUATE, metadata)
+            )
         else:
             if config is not None:
                 results = await self._invoke_all(config)
@@ -110,27 +139,32 @@ class Environment(BaseModel):
         else:
             return results
 
-    async def reset(
-        self, configs: FunctionConfigs | None = None
-    ) -> tuple[Observation, dict[str, Any]]:
+    async def reset(self) -> tuple[Observation, dict[str, Any]]:
         """
-        Reset the environment.
+        Reset the environment and return the first observation with the agent prompt.
 
         Args:
-            configs: The configuration to use for the reset
+            None
 
         Returns:
-            Observation: The first observation from the environment
+            Observation: The first observation from the environment with the agent prompt
             info: Dictionary of information about the environment
         """
         # await self._setup(configs)
         obs, _, _, info = await self.step()
-        if self.task and self.task.prompt:
-            obs.text = self.task.prompt
+
+        if self.build_data.get("environment_prompt"):
+            self.environment_prompt = self.build_data["environment_prompt"]
+
+        # Format the agent prompt with the environment prompt and the task prompt
+        obs.text = format_agent_prompt(self.environment_prompt, self.task)
+
         return obs, info
 
     async def step(
-        self, actions: CLA | list[CLA] | None = None
+        self,
+        actions: CLA | list[CLA] | None = None,
+        verbose: bool = False,
     ) -> tuple[Observation, float, bool, dict[str, Any]]:
         """Execute a step in the environment.
 
@@ -152,10 +186,11 @@ class Environment(BaseModel):
         result, stdout, stderr = await self.client.invoke(
             FunctionConfig(function="step", args=args)
         )
-        if stdout:
-            logger.info("Step produced stdout: %s", stdout.decode())
-        if stderr:
-            logger.warning("Step produced stderr: %s", stderr.decode())
+        if verbose:
+            if stdout:
+                logger.info("Step produced stdout: %s", stdout.decode())
+            if stderr:
+                logger.warning("Step produced stderr: %s", stderr.decode())
 
         observation = Observation.model_validate(result["observation"], strict=True)
 
@@ -199,12 +234,12 @@ class Environment(BaseModel):
         await self.client.close()
 
     async def stream(self) -> str | None:
-        urls = await self.get_urls()
-        if urls["live_url"] is None:
+        if not self.live_url:
+            await self.get_urls()
+        if self.live_url is None:
             logger.warning("No live URL found")
             return None
-        # Stream the live view
-        return stream(urls["live_url"])
+        return stream(self.live_url)
 
     async def run(self, agent: Agent, max_steps: int = 27, verbose: bool = True) -> Any:
         """Run an agent in the environment.
@@ -218,7 +253,11 @@ class Environment(BaseModel):
         for i in range(max_steps):
             action, done = await agent.predict(obs, verbose=verbose)
             if verbose:
-                logger.info("Step %d: Action: %s", i, action)
+                logger.info(
+                    "Step %d: Action: %s",
+                    i,
+                    [str(a) for a in action] if len(action) > 1 else str(action[0]),
+                )
             obs, reward, terminated, info = await self.step(action)
             if verbose:
                 logger.info("Step %d: Observation: %s", i, obs)
@@ -230,10 +269,21 @@ class Environment(BaseModel):
         return result
 
 
+def _format_task_metadata(task: Task) -> dict[str, Any]:
+    metadata = {}
+    if task.metadata:
+        for key, value in task.metadata.items():
+            metadata[str(key)] = value
+    if task.sensitive_data:
+        metadata["sensitive_data"] = task.sensitive_data
+    return metadata
+
+
 def create_remote_config(
     env: Environment | None = None,
     config: FunctionConfigs | None = None,
     function: str | None = None,
+    metadata: dict[str, Any] | None = None,
 ) -> list[FunctionConfig]:
     """
     Create a remote configuration for setup or evaluate, determining the final
@@ -317,6 +367,8 @@ def create_remote_config(
              `[FunctionConfig(function='evaluate', args=[])]`
     """
     # If no function provided, just expand the config and return it directly
+    if metadata is None:
+        metadata = {}
     if function is None:
         if config:
             return expand_config(config)
@@ -330,7 +382,7 @@ def create_remote_config(
             if not isinstance(expanded_configs[0].args, list):
                 expanded_configs[0].args = [expanded_configs[0].args]
             expanded_configs[0].args.append(env.final_response)  # for remote responses
-        return [FunctionConfig(function=function, args=expanded_configs)]
+        return [FunctionConfig(function=function, args=expanded_configs, metadata=metadata)]
 
     # Otherwise, use the environment's task
     task = env.task if env else None
@@ -339,6 +391,8 @@ def create_remote_config(
     if task is None:
         raise ValueError("Either task or config must be provided")
 
+    metadata = _format_task_metadata(task)
+
     # Case 2: Task has the specified function attribute
     task_config = getattr(task, function, None)
     if task_config:
@@ -350,11 +404,7 @@ def create_remote_config(
             if not isinstance(expanded_configs[0].args, list):
                 expanded_configs[0].args = [expanded_configs[0].args]
             expanded_configs[0].args.append(env.final_response)  # for remote responses
-        return [
-            FunctionConfig(
-                function=function, args=expanded_configs, metadata={"task": task.model_dump()}
-            )
-        ]
+        return [FunctionConfig(function=function, args=expanded_configs, metadata=metadata)]
 
     # Case 3: Check for task.config
     if hasattr(task, "config") and task.config:
@@ -369,11 +419,7 @@ def create_remote_config(
             if not isinstance(final_args["args"], list):
                 final_args["args"] = [final_args["args"]]
             final_args["args"].append(env.final_response)
-        return [
-            FunctionConfig(
-                function=function, args=[final_args], metadata={"task": task.model_dump()}
-            )
-        ]
+        return [FunctionConfig(function=function, args=[final_args], metadata=metadata)]
 
     # Case 4: Use task.id
     if task.id:
@@ -384,7 +430,7 @@ def create_remote_config(
             FunctionConfig(
                 function=f"{REMOTE_FUNCTION_PREFIX}{function}",
                 args=args_list,
-                metadata={"task": task.model_dump()},
+                metadata=metadata,
             )
         ]
 
@@ -392,4 +438,4 @@ def create_remote_config(
     args_list = []
     if env and env.final_response:
         args_list.append(env.final_response)
-    return [FunctionConfig(function=function, args=args_list, metadata={"task": task.model_dump()})]
+    return [FunctionConfig(function=function, args=args_list, metadata=metadata)]

hud/env/local_docker_client.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import io
 import logging
 import textwrap
@@ -34,6 +35,7 @@ class LocalDockerClient(DockerClient):
         """
         Build an image from a build context.
         """
+        logger.info("Building image from %s", build_context)
         # Create a unique image tag
         image_tag = f"hud-env-{uuid.uuid4().hex[:8]}"
 
@@ -67,6 +69,7 @@ class LocalDockerClient(DockerClient):
     async def create(
         cls,
         image: str,
+        host_config: dict[str, Any] | None = None,
     ) -> LocalDockerClient:
         """
         Creates a Docker environment client from a image.
@@ -81,20 +84,42 @@ class LocalDockerClient(DockerClient):
         # Initialize Docker client
         docker_client = aiodocker.Docker()
 
+        # Default host config
+        if host_config is None:
+            host_config = {
+                "PublishAllPorts": True,
+            }
+
         # Create and start the container
         container_config = {
             "Image": image,
             "Tty": True,
             "OpenStdin": True,
             "Cmd": None,
-            "HostConfig": {
-                "PublishAllPorts": True,
-            },
+            "HostConfig": host_config,
         }
 
         container = await docker_client.containers.create(config=container_config)
         await container.start()
 
+        # --------------------------------------------------
+        # Stream container logs while we wait for readiness
+        # --------------------------------------------------
+        async def _stream_logs() -> None:
+            try:
+                # .log() with follow=True -> async iterator of bytes/str
+                async for raw in container.log(stdout=True, stderr=True, follow=True):
+                    if isinstance(raw, bytes):
+                        raw = raw.decode(errors="replace")
+                    logger.info("container %s | %s", container.id[:12], raw.rstrip())
+            except asyncio.CancelledError:
+                # task cancelled during cleanup - silently exit
+                return
+            except Exception:
+                logger.exception("error while streaming logs from %s", container.id[:12])
+
+        log_task: asyncio.Task | None = asyncio.create_task(_stream_logs())
+
         inspection = await container.show()
         if health_check_config := inspection["Config"].get("Healthcheck"):
             # Using the interval as spinup deadline is a bit implicit - could
@@ -115,9 +140,21 @@ class LocalDockerClient(DockerClient):
                     raise TimeoutError(f"{container.id} not healthy after {window_secs}s")
                 await asyncio.sleep(1)
             logger.debug("Container %s is healthy", container.id)
+        else:
+            logger.debug("Container %s has no healthcheck, assuming ready", container.id)
+
+        # Stop the log stream now that the container is ready
+        if log_task is not None:
+            log_task.cancel()
+            with contextlib.suppress(Exception):
+                await log_task
+            log_task = None
 
         # Return the controller instance
-        return cls(docker_client, container.id)
+        client = cls(docker_client, container.id)
+        # store the task so close() can cancel if it is still running
+        client._log_task = log_task  # type: ignore[attr-defined]
+        return client
 
     def __init__(self, docker_conn: aiodocker.Docker, container_id: str) -> None:
         """
@@ -135,6 +172,9 @@ class LocalDockerClient(DockerClient):
         # Docker client will be initialized when needed
         self._docker = docker_conn
 
+        # Background task for streaming logs (may be None)
+        self._log_task: asyncio.Task | None = None
+
     @property
     def container_id(self) -> str:
         """Get the container ID."""
@@ -288,3 +328,9 @@ class LocalDockerClient(DockerClient):
             logger.warning("Error during Docker container cleanup: %s", e)
         finally:
             await self._docker.close()
+
+        # Cancel background log forwarding first (if still active)
+        if self._log_task is not None:
+            self._log_task.cancel()
+            with contextlib.suppress(Exception):
+                await self._log_task