hud-python 0.1.4__py3-none-any.whl → 0.2.0__py3-none-any.whl

hud/env/remote_docker_client.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+import logging
+from base64 import b64decode, b64encode
+from typing import Any
+
+from hud.env.docker_client import DockerClient
+from hud.server import make_request
+from hud.settings import settings
+from hud.types import EnvironmentStatus
+from hud.utils import ExecuteResult
+from hud.utils.common import get_gym_id
+
+logger = logging.getLogger("hud.env.remote_env_client")
+
+
+class RemoteDockerClient(DockerClient):
+    """
+    Remote environment client implementation.
+
+    Uses the HUD API to manage a remote environment.
+    """
+
+    @classmethod
+    async def create(
+        cls,
+        dockerfile: str,
+        *,
+        job_id: str | None = None,
+        task_id: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> tuple[RemoteDockerClient, dict[str, Any]]:
+        """
+        Creates a remote environment client from a dockerfile or gym_id.
+
+        Args:
+            dockerfile: The dockerfile content to build the environment
+            gym_id: The gym_id of the environment to create
+            metadata: Metadata to associate with the environment
+
+        Returns:
+            RemoteClient: An instance of the remote environment client
+        """
+
+        # Validate arguments
+        if metadata is None:
+            metadata = {}
+
+        logger.info("Creating remote environment")
+
+        true_gym_id = await get_gym_id("local-docker")
+
+        # augment metadata with dockerfile
+        if "environment_config" not in metadata:
+            metadata["environment_config"] = {}
+
+        metadata["environment_config"]["dockerfile"] = dockerfile
+
+        # Create a new environment via the HUD API
+        response = await make_request(
+            method="POST",
+            url=f"{settings.base_url}/v2/create_environment",
+            json={
+                # still named run_id for backwards compatibility
+                "run_id": job_id,
+                "metadata": metadata,
+                "gym_id": true_gym_id,
+                "task_id": task_id,
+            },
+            api_key=settings.api_key,
+        )
+
+        # Get the environment ID from the response
+        env_id = response.get("id")
+        if not env_id:
+            raise ValueError("Failed to create remote environment: No ID returned")
+
+        # Create the controller instance
+        controller = cls(env_id)
+
+        build_metadata = response.get("metadata", {})
+
+        return controller, build_metadata
+
+    def __init__(self, env_id: str) -> None:
+        """
+        Initialize the RemoteClient.
+
+        Args:
+            env_id: ID of the remote environment to control
+        """
+        super().__init__()
+        self._env_id = env_id
+
+    @property
+    def env_id(self) -> str:
+        """The ID of the remote environment."""
+        return self._env_id
+
+    async def get_status(self) -> EnvironmentStatus:
+        """
+        Get the current status of the remote environment.
+
+        Returns:
+            EnvironmentStatus: The current status of the environment
+        """
+        try:
+            response = await make_request(
+                method="GET",
+                url=f"{settings.base_url}/v2/environments/{self.env_id}/state",
+                api_key=settings.api_key,
+            )
+            logger.debug("Environment status response: %s", response)
+
+            status = response.get("state", "").lower()
+
+            if status == "running":
+                return EnvironmentStatus.RUNNING
+            elif status == "initializing" or status == "pending":
+                return EnvironmentStatus.INITIALIZING
+            elif status == "completed" or status == "terminated":
+                return EnvironmentStatus.COMPLETED
+            else:
+                # Any other status is considered an error
+                logger.warning("Abnormal environment status response: %s", response)
+                return EnvironmentStatus.ERROR
+
+        except Exception:
+            # If we can't connect to the API or there's any other error
+            logger.info("(potentially transient) Error getting environment status")
+            return EnvironmentStatus.ERROR
+
+    async def execute(
+        self,
+        command: list[str],
+        *,
+        workdir: str | None = None,
+        timeout: float | None = None,
+    ) -> ExecuteResult:
+        """
+        Execute a command in the environment.
+        No-op in some environments (like browser use).
+
+        Args:
+            command: Command to execute
+            workdir: Working directory for the command (ignored for remote environments)
+
+        Returns:
+            ExecuteResult: Result of the command execution
+        """
+        data = await make_request(
+            method="POST",
+            url=f"{settings.base_url}/v2/environments/{self.env_id}/execute",
+            json={
+                "command": command,
+                "workdir": workdir,
+                "timeout": timeout,
+            },
+            api_key=settings.api_key,
+        )
+
+        return ExecuteResult(
+            stdout=b64decode(data["stdout"]),
+            stderr=b64decode(data["stderr"]),
+            exit_code=data["exit_code"],
+        )
+
+    async def get_archive(self, path: str) -> bytes:
+        """
+        Get an archive of a path from the environment.
+        May not be supported for all environments.
+
+        Args:
+            path: Path in the environment to archive
+
+        Returns:
+            bytes: Content of the file or archive
+        """
+        data = await make_request(
+            method="POST",
+            url=f"{settings.base_url}/v2/environments/{self.env_id}/get_archive",
+            json={"path": path},
+            api_key=settings.api_key,
+        )
+
+        # Return the content decoded from base64
+        return b64decode(data["content"])
+
+    async def put_archive(self, path: str, data: bytes) -> bool:
+        """
+        Put an archive of data at a path in the environment.
+        May not be supported for all environments.
+
+        Args:
+            path: Path in the environment to extract the archive to
+            data: Bytes of the data to send
+
+        Returns:
+            bool: True if successful
+        """
+        await make_request(
+            method="POST",
+            url=f"{settings.base_url}/v2/environments/{self.env_id}/put_archive",
+            json={
+                "path": path,
+                "content": b64encode(data).decode("utf-8"),
+            },
+            api_key=settings.api_key,
+        )
+
+        return True
+
+    async def close(self) -> None:
+        """
+        Close the remote environment by making a request to the server.
+        """
+        await make_request(
+            method="POST",
+            url=f"{settings.base_url}/v2/environments/{self.env_id}/close",
+            api_key=settings.api_key,
+        )

hud/evaluators/__init__.py

@@ -0,0 +1,10 @@
+"""
+Evaluators for assessing task responses.
+"""
+from __future__ import annotations
+
+from hud.evaluators.base import Evaluator
+
+__all__ = [
+    "Evaluator"
+]

hud/evaluators/base.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel, Field
+
+if TYPE_CHECKING:
+    from hud.task import Task
+
+
+class EvaluationResult(BaseModel):
+    """Result of an evaluation.
+    
+    Attributes:
+        score: Float score between 0 and 1
+        reason: Explanation of the evaluation
+        mode: Mode used for matching, if applicable
+    """
+    
+    score: float
+    reason: str
+    mode: str | None = None
+    criteria_scores: dict[str, float] | None = Field(default_factory=dict)
+
+class Evaluator(ABC):
+    """Abstract base class for evaluators."""
+    
+    @abstractmethod
+    def evaluate(self, task: Task, response: str) -> EvaluationResult:
+        """Evaluate a task and response."""

hud/evaluators/inspect.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import Any
+
+from hud.evaluators.base import EvaluationResult
+
+
+def inspect_evaluate(
+    response: Any,
+    answer: Any,
+) -> EvaluationResult:
+    """Evaluate using Inspect-ai's evaluation models.
+    
+    Args:
+        response: The response to evaluate
+        answer: The reference answer to compare against
+        model_name: The Inspect model to use
+        prompt: Optional custom prompt for evaluation
+        metrics: Optional list of metrics to evaluate against
+        
+    Returns:
+        EvaluationResult with the evaluation results
+    """
+    return EvaluationResult(
+        score=0.0,
+        reason="Inspect evaluation not implemented",
+        mode="inspect"
+    )
+

hud/evaluators/judge.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+import asyncio
+import base64
+from typing import Any, Protocol, TypedDict
+
+from hud.evaluators.base import EvaluationResult
+from hud.server import make_request
+from hud.settings import settings
+
+
+class LLM(Protocol):
+    """Protocol for LLM interfaces that can be used for evaluation."""
+    async def ainvoke(self, prompt: str) -> str: ...
+
+
+class Criterion(TypedDict, total=False):
+    """Criterion for judge-based evaluation."""
+    
+    description: str
+    weight: float
+
+
+async def _call_eval_endpoint(
+    response: Any,
+    answer: Any,
+    criteria: list[Any],
+    mode: str
+) -> dict[str, Any]:
+    """Call the run_eval endpoint to evaluate the response."""
+    try:
+        result = await make_request(
+            method="POST",
+            url=f"{settings.base_url}/evaluations/run_eval",
+            json={
+                "response": response,
+                "answer": answer,
+                "criteria": criteria,
+                "mode": mode
+            },
+            api_key=settings.api_key,
+        )
+        return result
+    except Exception as e:
+        # Fallback to local evaluation if remote call fails
+        return {
+            "score": -1.0,
+            "reason": f"Remote evaluation failed: {e!s}. Fallback to default score.",
+            "criteria_scores": {}
+        }
+
+
+def _determine_mode(answer: Any) -> str:
+    """Determine the evaluation mode based on answer type."""
+    if isinstance(answer, bytes) or _is_base64_image(answer):
+        return "VLM"
+    return "LLM"
+
+
+def _process_input(data: Any) -> Any:
+    """Process input data, detecting and handling base64 images."""
+    if isinstance(data, bytes):
+        # Convert bytes to base64 string
+        return base64.b64encode(data).decode("utf-8")
+    
+    if isinstance(data, str) and _is_base64_image(data):
+        # It's already a base64 string, just return it
+        return data
+        
+    if isinstance(data, list) and all(isinstance(item, str) for item in data):
+        # Process list of strings
+        return data
+        
+    # For other types, convert to string
+    return str(data) if not isinstance(data, str | dict) else data
+
+
+def _is_base64_image(data: Any) -> bool:
+    """Check if a string is a base64 encoded image."""
+    if not isinstance(data, str):
+        return False
+        
+    # Check for common image data URI pattern
+    if data.startswith(("data:image/", "data:application/octet-stream")):
+        return True
+        
+    # Check if it's a base64 encoded string with image header
+    try:
+        # First, validate it's base64 decodable
+        padding_needed = len(data) % 4
+        if padding_needed:
+            data += "=" * (4 - padding_needed)
+
+        # Try to decode the first few bytes to check for image signatures
+        sample = base64.b64decode(data[:30])
+
+        # Check for common image format signatures
+        return (
+            sample.startswith((b"\xff\xd8\xff", b"\x89PNG\r\n\x1a\n", b"GIF8", b"RIFF"))
+        )
+    except Exception:
+        return False
+
+
+def judge(
+    response: Any,
+    answer: Any,
+    llm: LLM | None = None,
+    criteria: list[str] | list[dict] | None = None,
+) -> EvaluationResult:
+    """Judge a response against an answer using an LLM.
+    
+    Args:
+        response: The response to evaluate
+        answer: The reference answer to compare against
+        llm: Optional langchain LLM to use for evaluation
+        criteria: Evaluation criteria as strings or dictionaries
+        
+    Returns:
+        EvaluationResult with evaluation results
+    """
+    # Process inputs
+    processed_response = _process_input(response)
+    processed_answer = _process_input(answer)
+    
+    # If LLM is provided, use it for evaluation
+    if llm:
+        return _evaluate_with_llm(processed_response, processed_answer, llm, criteria)
+    
+    # Otherwise, use the remote evaluation service
+    mode = "LLM"
+    if isinstance(answer, bytes) or _is_base64_image(answer):
+        mode = "VLM"
+    
+    # Call the eval endpoint synchronously
+    result = asyncio.run(_call_eval_endpoint(
+        response=processed_response,
+        answer=processed_answer,
+        criteria=criteria or [],
+        mode=mode
+    ))
+    
+    return EvaluationResult(
+        score=result.get("score", -1.0),
+        reason=result.get("reason", "Response evaluated"),
+        mode=mode,
+        criteria_scores=result.get("criteria_scores", {})
+    )
+
+
+def _evaluate_with_llm(
+    response: Any,
+    answer: Any,
+    llm: LLM,
+    criteria: list[str] | list[dict] | None = None
+) -> EvaluationResult:
+    """Evaluate a response against an answer using a provided LLM."""
+    criteria_text = ""
+    if criteria:
+        criteria_text = "Use the following criteria:\n"
+        for c in criteria:
+            if isinstance(c, dict) and "description" in c:
+                criteria_text += f"- {c['description']}\n"
+            elif isinstance(c, str):
+                criteria_text += f"- {c}\n"
+    
+    prompt = f"""Evaluate the quality of a response given a reference answer.
+
+REFERENCE ANSWER:
+{answer}
+
+RESPONSE TO EVALUATE:
+{response}
+
+{criteria_text}
+Rate the response on a scale from 0.0 to 1.0, where 1.0 is perfect.
+Provide a brief explanation for your rating.
+Format your answer as a JSON object with 'score' (float) and 'reason' (string) fields.
+"""
+
+    try:
+        # Run the evaluation asynchronously
+        result_text = asyncio.run(llm.ainvoke(prompt))
+        
+        # Attempt to parse JSON response
+        import json
+        import re
+        
+        # Try to extract JSON if wrapped in other text
+        json_match = re.search(r"\{.*?\}", result_text, re.DOTALL)
+        if json_match:
+            json_str = json_match.group(0)
+            result = json.loads(json_str)
+            
+            return EvaluationResult(
+                score=float(result.get("score", 0.5)),
+                reason=result.get("reason", "Evaluated with custom LLM"),
+                mode="custom_llm"
+            )
+        
+        # If can't parse as JSON, use default values
+        return EvaluationResult(
+            score=0.5,
+            reason=f"Unable to parse LLM response as JSON. Raw response: {result_text[:100]}...",
+            mode="custom_llm"
+        )
+        
+    except Exception as e:
+        return EvaluationResult(
+            score=0.0,
+            reason=f"LLM evaluation error: {e!s}",
+            mode="custom_llm"
+        )

hud/evaluators/match.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+import re
+from difflib import SequenceMatcher
+from typing import Any
+
+from textdistance import levenshtein
+
+from hud.evaluators.base import EvaluationResult
+
+
+def match_single(response: Any, answer: Any) -> EvaluationResult:
+    """Check if the answer is present within the response.
+    
+    Args:
+        response: The response to evaluate
+        answer: The expected answer
+        
+    Returns:
+        EvaluationResult with score=1.0 if match, 0.0 otherwise
+    """
+    passed = str(answer).lower().strip() in str(response).lower().strip()
+    return EvaluationResult(
+        score=1.0 if passed else 0.0,
+        reason="Exact match" if passed else "No exact match found",
+        mode="single"
+    )
+
+
+def match_all(response: Any, answers: list) -> EvaluationResult:
+    """Count how many expected answers are in the response.
+    
+    Args:
+        response: The response to evaluate
+        answers: List of expected answers
+        
+    Returns:
+        EvaluationResult with score=proportion of matches (0.0-1.0)
+    """
+    response_str = str(response).lower()
+    matches = 0
+    
+    for answer in answers:
+        if str(answer).lower() in response_str:
+            matches += 1
+            
+    score = matches / len(answers) if answers else 0.0
+    
+    if matches == len(answers):
+        reason = f"All {matches} expected items found"
+    else:
+        reason = f"Only {matches} of {len(answers)} expected items found"
+        
+    return EvaluationResult(
+        score=score,
+        reason=reason,
+        mode="all"
+    )
+
+
+def match_fuzzy(response: Any, answer: Any) -> EvaluationResult:
+    """Calculate similarity using Levenshtein distance.
+    
+    Args:
+        response: The response to evaluate
+        answer: The expected answer
+        
+    Returns:
+        EvaluationResult with score=similarity (0.0-1.0)
+    """
+    s1 = str(response).lower()
+    s2 = str(answer).lower()
+    
+    if s1 == s2:
+        score = 1.0
+    elif len(s1) == 0 or len(s2) == 0:
+        score = 0.0
+    else:
+        # Use Levenshtein distance
+        distance = levenshtein.distance(s1, s2)
+        max_len = max(len(s1), len(s2))
+        score = 1.0 - (distance / max_len)
+        
+    return EvaluationResult(
+        score=score,
+        reason=f"Fuzzy match with {score:.1%} similarity",
+        mode="fuzz"
+    )
+
+
+def match_regex(response: Any, pattern: str) -> EvaluationResult:
+    """Check if response matches regex pattern.
+    
+    Args:
+        response: The response to evaluate
+        pattern: Regular expression pattern to match
+        
+    Returns:
+        EvaluationResult with score=1.0 if match, 0.0 otherwise
+    """
+    try:
+        regex = re.compile(pattern, re.DOTALL)
+        passed = bool(regex.search(str(response)))
+        return EvaluationResult(
+            score=1.0 if passed else 0.0,
+            reason="Regex pattern matched" if passed else "Regex pattern did not match",
+            mode="regex"
+        )
+    except re.error:
+        return EvaluationResult(
+            score=0.0,
+            reason="Invalid regex pattern",
+            mode="regex"
+        )
+
+
+def match_diff(response: Any, answer: Any) -> EvaluationResult:
+    """Compare difference between response and answer.
+    
+    Args:
+        response: The response to evaluate
+        answer: The expected answer
+        
+    Returns:
+        EvaluationResult with score=similarity (0.0-1.0)
+    """
+    if isinstance(response, int | float) and isinstance(answer, int | float):
+        score = _match_numeric_diff(response, answer)
+        reason = f"Numeric difference: {abs(response - answer)}"
+    else:
+        score = _match_string_diff(response, answer)
+        reason = f"String difference with {score:.1%} similarity"
+            
+    return EvaluationResult(
+        score=score,
+        reason=reason,
+        mode="diff"
+    )
+
+
+def _match_string_diff(response: Any, answer: Any) -> float:
+    """Compare difference between response and answer strings."""
+    matcher = SequenceMatcher(None, str(response), str(answer))
+    return matcher.ratio()
+    
+
+def _match_numeric_diff(response: float, answer: float) -> float:
+    """Calculate normalized difference between numeric values.
+    
+    Returns a value between 0 and 1, where 1 means identical and 0 means maximum difference.
+    """
+    if response == answer:
+        return 1.0
+        
+    # Simple absolute difference normalized to a 0-1 scale
+    diff = abs(response - answer)
+    max_val = max(abs(response), abs(answer))
+    
+    if max_val == 0:
+        return 1.0  # Both are zero
+        
+    # Normalize and invert so 1.0 means identical
+    return max(0.0, 1.0 - min(1.0, diff / max_val))