hud-python 0.2.1__py3-none-any.whl → 0.2.3__py3-none-any.whl

hud/env/environment.py

@@ -10,25 +10,21 @@ from pydantic import BaseModel
 from hud.env.client import Client
 from hud.env.remote_client import RemoteClient
 from hud.task import Task
-from hud.utils.common import HudStyleConfig, HudStyleConfigs
-from hud.utils.config import REMOTE_EVALUATE, REMOTE_FUNCTION_PREFIX, REMOTE_SETUP, expand_config
+from hud.utils.common import FunctionConfig, FunctionConfigs, Observation
+from hud.utils.config import (
+    LOCAL_EVALUATORS,
+    REMOTE_EVALUATE,
+    REMOTE_FUNCTION_PREFIX,
+    REMOTE_SETUP,
+    expand_config,
+)
+from hud.utils.telemetry import stream
 
 logger = logging.getLogger("hud.environment")
 
 if TYPE_CHECKING:
     from hud.adapters.common import CLA
-
-class Observation(BaseModel):
-    """
-    Observation from the environment.
-
-    Attributes:
-        screenshot: Base64 encoded PNG string of the screen
-        text: Text observation, if available
-    """
-
-    screenshot: str | None = None  # base64 string png
-    text: str | None = None
+    from hud.agent import Agent
 
 
 class Environment(BaseModel):
@@ -48,7 +44,7 @@ class Environment(BaseModel):
     # final response
     final_response: str | None = None
 
-    async def _invoke_all(self, configs: HudStyleConfigs) -> list[Any]:
+    async def _invoke_all(self, configs: FunctionConfigs) -> list[Any]:
         # Execute each config and collect results
         configs_all = [configs] if not isinstance(configs, list) else configs
         results = []
@@ -69,8 +65,8 @@ class Environment(BaseModel):
                         stderr.decode(),
                     )
         return results
-    
-    async def _setup(self, config: HudStyleConfigs | None = None) -> None:
+
+    async def _setup(self, config: FunctionConfigs | None = None) -> None:
         """
         Setup the environment.
 
@@ -87,7 +83,7 @@ class Environment(BaseModel):
             else:
                 raise ValueError("No config or task provided for local environment")
 
-    async def evaluate(self, config: HudStyleConfigs | None = None) -> Any:
+    async def evaluate(self, config: FunctionConfigs | None = None) -> Any:
         """
         Evaluate the environment.
 
@@ -98,8 +94,7 @@ class Environment(BaseModel):
             Any: Result of the evaluation
         """
         if isinstance(self.client, RemoteClient):
-            results = await self._invoke_all(
-                create_remote_config(self, config, REMOTE_EVALUATE))
+            results = await self._invoke_all(create_remote_config(self, config, REMOTE_EVALUATE))
         else:
             if config is not None:
                 results = await self._invoke_all(config)
@@ -111,11 +106,10 @@ class Environment(BaseModel):
             return results[0]
         else:
             return results
-        
 
-    async def reset(self, configs: HudStyleConfigs | None = None) -> tuple[
-        Observation, dict[str, Any]
-    ]:
+    async def reset(
+        self, configs: FunctionConfigs | None = None
+    ) -> tuple[Observation, dict[str, Any]]:
         """
         Reset the environment.
 
@@ -126,15 +120,15 @@ class Environment(BaseModel):
             Observation: The first observation from the environment
             info: Dictionary of information about the environment
         """
-        #await self._setup(configs)
+        # await self._setup(configs)
         obs, _, _, info = await self.step()
         if self.task and self.task.prompt:
             obs.text = self.task.prompt
         return obs, info
 
-    async def step(self, actions: list[CLA] | None = None) -> tuple[
-        Observation, float, bool, dict[str, Any]
-    ]:
+    async def step(
+        self, actions: CLA | list[CLA] | None = None
+    ) -> tuple[Observation, float, bool, dict[str, Any]]:
         """Execute a step in the environment.
 
         Args:
@@ -143,6 +137,8 @@ class Environment(BaseModel):
         Returns:
             Any: Result of the step execution
         """
+        if not isinstance(actions, list) and actions is not None:
+            actions = [actions]
         if actions is None or len(actions) == 0:
             actions = []
         args = [[action.model_dump() for action in actions]]
@@ -150,20 +146,19 @@ class Environment(BaseModel):
         # TODO: Move this into the server side
         if self._maybe_store_response(actions):
             return Observation(text=self.final_response), 0, False, {}
-        
+
         result, stdout, stderr = await self.client.invoke(
-            HudStyleConfig(function="step", args=args)
+            FunctionConfig(function="step", args=args)
         )
         if stdout:
             logger.info("Step produced stdout: %s", stdout.decode())
         if stderr:
             logger.warning("Step produced stderr: %s", stderr.decode())
 
-
         observation = Observation.model_validate(result["observation"], strict=True)
 
         return observation, 0, False, {}
-    
+
     def _maybe_store_response(self, actions: list[CLA]) -> bool:
         """Store the final response into the environment.
 
@@ -178,14 +173,13 @@ class Environment(BaseModel):
             return True
         return False
 
-
     async def get_urls(self) -> dict[str, Any]:
         """Get URLs for the environment.
 
         Returns:
             dict: Dictionary of URLs for accessing the environment
         """
-        data, _, _ = await self.client.invoke(HudStyleConfig(function="get_urls", args=[]))
+        data, _, _ = await self.client.invoke(FunctionConfig(function="get_urls", args=[]))
 
         self.url = data.get("url")
         self.live_url = data.get("live_url")
@@ -202,11 +196,43 @@ class Environment(BaseModel):
         """
         await self.client.close()
 
+    async def stream(self) -> str | None:
+        urls = await self.get_urls()
+        if urls["live_url"] is None:
+            logger.warning("No live URL found")
+            return None
+        # Stream the live view
+        return stream(urls["live_url"])
+
+    async def run(self, agent: Agent, max_steps: int = 27, verbose: bool = True) -> Any:
+        """Run an agent in the environment.
+
+        Args:
+            agent: The agent to run
+        """
+        if verbose:
+            logger.info("[HUD] Running agent in environment...")
+        obs, _ = await self.reset()
+        for i in range(max_steps):
+            action, done = await agent.predict(obs)
+            if verbose:
+                logger.info("[HUD] Step %d: Action: %s", i, action)
+            obs, reward, terminated, info = await self.step(action)
+            if verbose:
+                logger.info("[HUD] Step %d: Observation: %s", i, obs)
+            if done or terminated:
+                break
+        result = await self.evaluate()
+        if verbose:
+            logger.info("[HUD] Evaluation result: %s", result)
+        return result
+
+
 def create_remote_config(
     env: Environment | None = None,
-    config: HudStyleConfigs | None = None,
+    config: FunctionConfigs | None = None,
     function: str | None = None,
-) -> list[HudStyleConfig]:
+) -> list[FunctionConfig]:
     """
     Create a remote configuration for setup or evaluate, determining the final
     function call structure based on the provided task or explicit config.
@@ -218,11 +244,11 @@ def create_remote_config(
         env: Environment object, potentially containing a task definition.
              Used to access `env.task` and `env.final_response`.
         config: Direct configuration override (e.g., passed to `env.evaluate(config=...)`).
-                Can be in various HudStyleConfigs formats.
+                Can be in various FunctionConfigs formats.
         function: The top-level function context, typically "setup" or "evaluate".
 
     Returns:
-        list[HudStyleConfig]: A list containing a single HudStyleConfig object
+        list[FunctionConfig]: A list containing a single FunctionConfig object
                               ready for remote invocation via `client.invoke`.
                               The specific function/arguments are chosen based on this priority:
                               1. Explicit `config` parameter (if provided).
@@ -242,8 +268,8 @@ def create_remote_config(
              `config=("contains_text", "Paris")`
              `function="evaluate"`
            - Example Output:
-             `[HudStyleConfig(function='evaluate', args=[
-                HudStyleConfig(function='contains_text', args=['Paris', 'Paris'])
+             `[FunctionConfig(function='evaluate', args=[
+                FunctionConfig(function='contains_text', args=['Paris', 'Paris'])
              ])]`
 
         2) No explicit `config`, Task has the attribute (e.g., `task.evaluate`):
@@ -255,7 +281,7 @@ def create_remote_config(
              `config=None`
              `function="evaluate"`
            - Example Output:
-             `[HudStyleConfig(function='evaluate', args=[HudStyleConfig(function='check_answer',
+             `[FunctionConfig(function='evaluate', args=[FunctionConfig(function='check_answer',
                 args=['42'], id='t1')])]`
 
         3) No explicit `config`, no specific Task attribute, Task has `task.config`:
@@ -267,7 +293,7 @@ def create_remote_config(
              `config=None`
              `function="evaluate"`
            - Example Output:
-             `[HudStyleConfig(function='evaluate', args=[{"expected": "val", "id": "t2"}])]`
+             `[FunctionConfig(function='evaluate', args=[{"expected": "val", "id": "t2"}])]`
 
         4) No explicit `config`, no specific Task attribute, no `task.config`, Task has `task.id`:
            Calls a private function (`private_<function>`) on the remote end, passing
@@ -277,7 +303,7 @@ def create_remote_config(
              `config=None`
              `function="evaluate"`
            - Example Output:
-             `[HudStyleConfig(function='private_evaluate', args=['t3'])]`
+             `[FunctionConfig(function='private_evaluate', args=['t3'])]`
 
         5) No explicit `config` and no relevant Task info:
            Calls the top-level `function` with empty args.
@@ -286,50 +312,50 @@ def create_remote_config(
              `config=None`
              `function="evaluate"`
            - Example Output:
-             `[HudStyleConfig(function='evaluate', args=[])]`
+             `[FunctionConfig(function='evaluate', args=[])]`
     """
     # If no function provided, just expand the config and return it directly
     if function is None:
         if config:
             return expand_config(config)
         raise ValueError("Either function or config must be provided")
-    
+
     # Case 1: Explicit config provided
     if config:
         expanded_configs = expand_config(config)
-        if env and env.final_response:
+        if env and env.final_response and expanded_configs[0].args[0] in LOCAL_EVALUATORS:
             # Ensure args is a list before appending
             if not isinstance(expanded_configs[0].args, list):
-                 expanded_configs[0].args = [expanded_configs[0].args]
-            expanded_configs[0].args.append(env.final_response) # for remote responses
-        return [HudStyleConfig(function=function, args=expanded_configs)]
-    
+                expanded_configs[0].args = [expanded_configs[0].args]
+            expanded_configs[0].args.append(env.final_response)  # for remote responses
+        return [FunctionConfig(function=function, args=expanded_configs)]
+
     # Otherwise, use the environment's task
     task = env.task if env else None
-    
+
     # Must have a task for the remaining cases
     if task is None:
         raise ValueError("Either task or config must be provided")
-    
+
     # Case 2: Task has the specified function attribute
     task_config = getattr(task, function, None)
     if task_config:
         expanded_configs = expand_config(task_config)
         if task.id:
-            expanded_configs[0].id = task.id # for remote IDs
-        elif env and env.final_response:
+            expanded_configs[0].id = task.id  # for remote IDs
+        if env and env.final_response and expanded_configs[0].args[0] in LOCAL_EVALUATORS:
             # Ensure args is a list before appending
             if not isinstance(expanded_configs[0].args, list):
-                 expanded_configs[0].args = [expanded_configs[0].args]
-            expanded_configs[0].args.append(env.final_response) # for remote responses
-        return [HudStyleConfig(function=function, args=expanded_configs)]
-    
+                expanded_configs[0].args = [expanded_configs[0].args]
+            expanded_configs[0].args.append(env.final_response)  # for remote responses
+        return [FunctionConfig(function=function, args=expanded_configs)]
+
     # Case 3: Check for task.config
     if hasattr(task, "config") and task.config:
         # Ensure task.config is a dictionary before adding id
         final_args = task.config.copy() if isinstance(task.config, dict) else {}
         if task.id:
-            final_args["id"] = task.id # for remote IDs
+            final_args["id"] = task.id  # for remote IDs
         if env and env.final_response:
             # Append response, ensuring args exists and is a list
             if "args" not in final_args:
@@ -337,18 +363,17 @@ def create_remote_config(
             if not isinstance(final_args["args"], list):
                 final_args["args"] = [final_args["args"]]
             final_args["args"].append(env.final_response)
-        return [HudStyleConfig(function=function, args=[final_args])]
-    
+        return [FunctionConfig(function=function, args=[final_args])]
+
     # Case 4: Use task.id
     if task.id:
         args_list = [task.id]
         if env and env.final_response:
-             args_list.append(env.final_response) # Append final response
-        return [HudStyleConfig(function=f"{REMOTE_FUNCTION_PREFIX}{function}", args=args_list)]
-    
+            args_list.append(env.final_response)  # Append final response
+        return [FunctionConfig(function=f"{REMOTE_FUNCTION_PREFIX}{function}", args=args_list)]
+
     # Case 5: No valid configuration found
     args_list = []
     if env and env.final_response:
         args_list.append(env.final_response)
-    return [HudStyleConfig(function=function, args=args_list)]
-
+    return [FunctionConfig(function=function, args=args_list)]

hud/env/local_docker_client.py

@@ -19,15 +19,16 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger("hud.env.docker_env_client")
 
+
 class LocalDockerClient(DockerClient):
     """
     Docker-based environment client implementation.
     """
 
     @classmethod
-    async def create(cls, dockerfile: str, ports: list[int] | None = None) -> tuple[
-            LocalDockerClient, dict[str, Any]
-        ]:
+    async def create(
+        cls, dockerfile: str, ports: list[int] | None = None
+    ) -> tuple[LocalDockerClient, dict[str, Any]]:
         """
         Creates a Docker environment client from a dockerfile.
 
@@ -86,9 +87,7 @@ class LocalDockerClient(DockerClient):
             "HostConfig": {
                 "PublishAllPorts": True,
             },
-            "ExposedPorts": {
-                f"{port}/tcp": {} for port in ports
-            },
+            "ExposedPorts": {f"{port}/tcp": {} for port in ports},
         }
 
         container = await docker_client.containers.create(config=container_config)
@@ -198,7 +197,6 @@ class LocalDockerClient(DockerClient):
             exit_code=0,
         )
 
-
     async def get_archive(self, path: str) -> bytes:
         """
         Get an archive of a path from the container.

hud/env/remote_client.py

@@ -5,23 +5,25 @@ from base64 import b64decode
 from typing import TYPE_CHECKING, Any
 
 from hud.env.client import Client
+from hud.exceptions import HudResponseError
 from hud.server import make_request
 from hud.settings import settings
 from hud.types import EnvironmentStatus
 from hud.utils import ExecuteResult
 
 if TYPE_CHECKING:
-    from hud.utils.config import HudStyleConfig
+    from hud.utils.config import FunctionConfig
 
 logger = logging.getLogger("hud.env.remote_env_client")
 
+
 class RemoteClient(Client):
     """
     Remote environment client implementation.
-    
+
     Uses the HUD API to manage a remote environment.
     """
-    
+
     @classmethod
     async def create(
         cls,
@@ -33,21 +35,23 @@ class RemoteClient(Client):
     ) -> tuple[RemoteClient, dict[str, Any]]:
         """
         Creates a remote environment client from a dockerfile or gym_id.
-        
+
         Args:
             dockerfile: The dockerfile content to build the environment
             gym_id: The gym_id of the environment to create
             metadata: Metadata to associate with the environment
-            
+
         Returns:
-            RemoteClient: An instance of the remote environment client
+            A tuple containing the remote environment client and the build metadata
+
+        Raises:
+            HudResponseError: If the environment creation is successful but the response is invalid.
         """
 
         # Validate arguments
         if metadata is None:
             metadata = {}
 
-        
         request_data = {
             # still named run_id for backwards compatibility
             "run_id": job_id,
@@ -63,29 +67,38 @@ class RemoteClient(Client):
             json=request_data,
             api_key=settings.api_key,
         )
-        
+
         # Get the environment ID from the response
         env_id = response.get("id")
         if not env_id:
-            raise ValueError("Failed to create remote environment: No ID returned")
-        
+            raise HudResponseError(
+                message="Failed to create remote environment: No ID returned in API response. "
+                "Please contact support if this issue persists.",
+                response_json=response,
+            )
+
         # Create the controller instance
         controller = cls(env_id)
-        
+
         build_data = response.get("metadata", {})
-        
+
+        if response.get("readme"):
+            logger.info(
+                "[HUD] %s gym created, see how to use it at %s", gym_id, response.get("readme")
+            )
+
         return controller, build_data
 
     def __init__(self, env_id: str) -> None:
         """
         Initialize the RemoteClient.
-        
+
         Args:
             env_id: ID of the remote environment to control
         """
         super().__init__()
         self._env_id = env_id
-        
+
     @property
     def env_id(self) -> str:
         """The ID of the remote environment."""
@@ -94,7 +107,7 @@ class RemoteClient(Client):
     async def get_status(self) -> EnvironmentStatus:
         """
         Get the current status of the remote environment.
-        
+
         Returns:
             EnvironmentStatus: The current status of the environment
         """
@@ -107,7 +120,7 @@ class RemoteClient(Client):
             logger.debug("Environment status response: %s", response)
 
             status = response.get("state", "").lower()
-            
+
             if status == "running":
                 return EnvironmentStatus.RUNNING
             elif status == "initializing" or status == "pending":
@@ -118,12 +131,12 @@ class RemoteClient(Client):
                 # Any other status is considered an error
                 logger.warning("Abnormal environment status response: %s", response)
                 return EnvironmentStatus.ERROR
-                
+
         except Exception:
             # If we can't connect to the API or there's any other error
             logger.info("(potentially transient) Error getting environment status")
             return EnvironmentStatus.ERROR
-    
+
     async def execute(
         self,
         command: list[str],
@@ -134,11 +147,11 @@ class RemoteClient(Client):
         """
         Execute a command in the environment.
         No-op in some environments (like browser use).
-        
+
         Args:
             command: Command to execute
             workdir: Working directory for the command (ignored for remote environments)
-            
+
         Returns:
             ExecuteResult: Result of the command execution
         """
@@ -146,21 +159,20 @@ class RemoteClient(Client):
             method="POST",
             url=f"{settings.base_url}/v2/environments/{self.env_id}/execute",
             json={
-               "command": command,
-               "workdir": workdir,
-               "timeout": timeout,
+                "command": command,
+                "workdir": workdir,
+                "timeout": timeout,
             },
             api_key=settings.api_key,
         )
-        
+
         return ExecuteResult(
             stdout=b64decode(data["stdout"]),
             stderr=b64decode(data["stderr"]),
-            exit_code=data["exit_code"]
+            exit_code=data["exit_code"],
         )
 
-    
-    async def invoke(self, config: HudStyleConfig) -> tuple[Any, bytes, bytes]:
+    async def invoke(self, config: FunctionConfig) -> tuple[Any, bytes, bytes]:
         """
         Invoke a function in the environment.
         """
@@ -170,9 +182,8 @@ class RemoteClient(Client):
             json=config.model_dump(),
             api_key=settings.api_key,
         )
-        
-        return data["result"], b64decode(data["stdout"]), b64decode(data["stderr"])
 
+        return data["result"], b64decode(data["stdout"]), b64decode(data["stderr"])
 
     async def close(self) -> None:
         """

hud/env/remote_docker_client.py

@@ -5,6 +5,7 @@ from base64 import b64decode, b64encode
 from typing import Any
 
 from hud.env.docker_client import DockerClient
+from hud.exceptions import HudResponseError
 from hud.server import make_request
 from hud.settings import settings
 from hud.types import EnvironmentStatus
@@ -39,7 +40,10 @@ class RemoteDockerClient(DockerClient):
             metadata: Metadata to associate with the environment
 
         Returns:
-            RemoteClient: An instance of the remote environment client
+            A tuple containing the remote environment client and the build metadata
+
+        Raises:
+            HudResponseError: If the environment creation fails.
         """
 
         # Validate arguments
@@ -48,7 +52,7 @@ class RemoteDockerClient(DockerClient):
 
         logger.info("Creating remote environment")
 
-        true_gym_id = await get_gym_id("local-docker")
+        true_gym_id = await get_gym_id("docker")
 
         # augment metadata with dockerfile
         if "environment_config" not in metadata:
@@ -73,7 +77,13 @@ class RemoteDockerClient(DockerClient):
         # Get the environment ID from the response
         env_id = response.get("id")
         if not env_id:
-            raise ValueError("Failed to create remote environment: No ID returned")
+            raise HudResponseError(
+                message=(
+                    "Failed to create remote environment: No ID returned in API response. "
+                    "Please contact support if this issue persists."
+                ),
+                response_json=response,
+            )
 
         # Create the controller instance
         controller = cls(env_id)

hud/evaluators/__init__.py

@@ -1,10 +1,9 @@
 """
 Evaluators for assessing task responses.
 """
+
 from __future__ import annotations
 
 from hud.evaluators.base import Evaluator
 
-__all__ = [
-    "Evaluator"
-]
+__all__ = ["Evaluator"]

hud/evaluators/base.py

@@ -11,21 +11,22 @@ if TYPE_CHECKING:
 
 class EvaluationResult(BaseModel):
     """Result of an evaluation.
-    
+
     Attributes:
         score: Float score between 0 and 1
         reason: Explanation of the evaluation
         mode: Mode used for matching, if applicable
     """
-    
+
     score: float
     reason: str
     mode: str | None = None
     criteria_scores: dict[str, float] | None = Field(default_factory=dict)
 
+
 class Evaluator(ABC):
     """Abstract base class for evaluators."""
-    
+
     @abstractmethod
     def evaluate(self, task: Task, response: str) -> EvaluationResult:
         """Evaluate a task and response."""

hud/evaluators/inspect.py

@@ -10,20 +10,15 @@ def inspect_evaluate(
     answer: Any,
 ) -> EvaluationResult:
     """Evaluate using Inspect-ai's evaluation models.
-    
+
     Args:
         response: The response to evaluate
         answer: The reference answer to compare against
         model_name: The Inspect model to use
         prompt: Optional custom prompt for evaluation
         metrics: Optional list of metrics to evaluate against
-        
+
     Returns:
         EvaluationResult with the evaluation results
     """
-    return EvaluationResult(
-        score=0.0,
-        reason="Inspect evaluation not implemented",
-        mode="inspect"
-    )
-
+    return EvaluationResult(score=0.0, reason="Inspect evaluation not implemented", mode="inspect")