hud-python 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl

hud/evaluators/tests/test_remote.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import pytest
+
+from hud.evaluators.base import EvaluationResult
+from hud.evaluators.remote import _remote_eval_call, remote_evaluate
+
+
+@pytest.mark.asyncio
+async def test_remote_eval_call_success(mocker):
+    mock_response = {
+        "score": 0.85,
+        "reason": "Good match",
+        "details": {"relevance": 0.9, "correctness": 0.8},
+    }
+    mock_make_request = mocker.patch(
+        "hud.evaluators.remote.make_request", return_value=mock_response
+    )
+
+    result = await _remote_eval_call(
+        response="test response", answer="test answer", eval_type="match"
+    )
+
+    assert result == mock_response
+    mock_make_request.assert_called_once()
+    call_args = mock_make_request.call_args[1]
+    assert call_args["method"] == "POST"
+    assert "evaluations/evaluate" in call_args["url"]
+    assert call_args["json"]["response"] == "test response"
+    assert call_args["json"]["answer"] == "test answer"
+    assert call_args["json"]["type"] == "match"
+
+
+@pytest.mark.asyncio
+async def test_remote_eval_call_with_config(mocker):
+    mock_response = {"score": 0.75, "reason": "Good", "details": {}}
+    mock_make_request = mocker.patch(
+        "hud.evaluators.remote.make_request", return_value=mock_response
+    )
+
+    config = {"threshold": 0.8, "strict": True}
+    result = await _remote_eval_call(
+        response="test response", answer="test answer", eval_type="judge", config=config
+    )
+
+    assert result == mock_response
+    mock_make_request.assert_called_once()
+    call_args = mock_make_request.call_args[1]
+    assert call_args["json"]["config"] == config
+
+
+@pytest.mark.asyncio
+async def test_remote_eval_call_failure(mocker):
+    mocker.patch("hud.evaluators.remote.make_request", side_effect=Exception("API error"))
+
+    result = await _remote_eval_call(
+        response="test response", answer="test answer", eval_type="match"
+    )
+
+    assert result["score"] == -1.0
+    assert "Remote evaluation failed" in result["reason"]
+    assert "API error" in result["reason"]
+    assert result["details"] == {}
+
+
+def test_remote_evaluate(mocker):
+    mock_result = {"score": 0.9, "reason": "Excellent match", "details": {"similarity": 0.95}}
+
+    async def mock_remote_call(*args, **kwargs):
+        return mock_result
+
+    mocker.patch("hud.evaluators.remote._remote_eval_call", side_effect=mock_remote_call)
+
+    result = remote_evaluate(
+        response="test response", answer="test answer", eval_type="custom_eval"
+    )
+
+    assert isinstance(result, EvaluationResult)
+    assert result.score == 0.9
+    assert result.reason == "Excellent match"
+    assert result.mode == "custom_eval"
+    assert result.criteria_scores == {"similarity": 0.95}
+
+
+def test_remote_evaluate_missing_fields(mocker):
+    mock_result = {"score": 0.8}  # Missing reason and details
+
+    async def mock_remote_call(*args, **kwargs):
+        return mock_result
+
+    mocker.patch("hud.evaluators.remote._remote_eval_call", side_effect=mock_remote_call)
+
+    result = remote_evaluate(response="test response", answer="test answer")
+
+    assert result.score == 0.8
+    assert result.reason == "Remote evaluation completed"
+    assert result.mode == "default"
+    assert result.criteria_scores == {}

hud/exceptions.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import httpx
+    from typing_extensions import Self
+
+logger = logging.getLogger(__name__)
+
+
+class HudException(Exception):
+    """Base exception class for all HUD SDK errors.
+
+    This is the parent class for all exceptions raised by the HUD SDK.
+    Consumers should be able to catch this exception to handle any HUD-related error.
+    """
+
+
+class HudRequestError(Exception):
+    """Any request to the HUD API can raise this exception."""
+
+    def __init__(
+        self,
+        message: str,
+        status_code: int | None = None,
+        response_text: str | None = None,
+        response_json: dict[str, Any] | None = None,
+        response_headers: dict[str, str] | None = None,
+    ) -> None:
+        self.message = message
+        self.status_code = status_code
+        self.response_text = response_text
+        self.response_json = response_json
+        self.response_headers = response_headers
+        super().__init__(message)
+
+    def __str__(self) -> str:
+        parts = [self.message]
+
+        if self.status_code:
+            parts.append(f"Status: {self.status_code}")
+
+        if self.response_text:
+            parts.append(f"Response Text: {self.response_text}")
+
+        if self.response_json:
+            parts.append(f"Response JSON: {self.response_json}")
+
+        if self.response_headers:
+            parts.append(f"Headers: {self.response_headers}")
+
+        return " | ".join(parts)
+
+    @classmethod
+    def from_httpx_error(cls, error: httpx.HTTPStatusError, context: str = "") -> Self:
+        """Create a RequestError from an HTTPx error response.
+
+        Args:
+            error: The HTTPx error response.
+            context: Additional context to include in the error message.
+
+        Returns:
+            A RequestError instance.
+        """
+        response = error.response
+        status_code = response.status_code
+        response_text = response.text
+        response_headers = dict(response.headers)
+
+        # Try to get detailed error info from JSON if available
+        response_json = None
+        try:
+            response_json = response.json()
+            detail = response_json.get("detail")
+            if detail:
+                message = f"Request failed: {detail}"
+            else:
+                # If no detail field but we have JSON, include a summary
+                message = f"Request failed with status {status_code}"
+                if len(response_json) <= 5:  # If it's a small object, include it in the message
+                    message += f" - JSON response: {response_json}"
+        except Exception:
+            # Fallback to simple message if JSON parsing fails
+            message = f"Request failed with status {status_code}"
+
+        # Add context if provided
+        if context:
+            message = f"{context}: {message}"
+
+        # Log the error details
+        logger.error(
+            "HTTP error from HUD SDK: %s | URL: %s | Status: %s | Response: %s%s",
+            message,
+            response.url,
+            status_code,
+            response_text[:500],
+            "..." if len(response_text) > 500 else "",
+        )
+        return cls(
+            message=message,
+            status_code=status_code,
+            response_text=response_text,
+            response_json=response_json,
+            response_headers=response_headers,
+        )
+
+
+class HudResponseError(HudException):
+    """Raised when an API response is invalid or missing required data.
+
+    This exception is raised when we receive a successful response (e.g. 200)
+    but the response data is invalid, missing required fields, or otherwise
+    cannot be processed.
+
+    Attributes:
+        message: A human-readable error message
+        response_json: The invalid response data
+    """
+
+    def __init__(
+        self,
+        message: str,
+        response_json: dict[str, Any] | None = None,
+    ) -> None:
+        self.message = message
+        self.response_json = response_json
+        super().__init__(message)
+
+    def __str__(self) -> str:
+        parts = [self.message]
+        if self.response_json:
+            parts.append(f"Response: {self.response_json}")
+        return " | ".join(parts)
+
+
+class HudAuthenticationError(HudException):
+    """Raised when authentication with the HUD API fails.
+
+    This exception is raised when an API key is missing, invalid, or
+    has insufficient permissions for the requested operation.
+    """
+
+
+class HudRateLimitError(HudException):
+    """Raised when the rate limit for the HUD API is exceeded.
+
+    This exception is raised when too many requests are made in a
+    short period of time.
+    """
+
+
+class HudTimeoutError(HudException):
+    """Raised when a request to the HUD API times out.
+
+    This exception is raised when a request takes longer than the
+    configured timeout period.
+    """
+
+
+class HudNetworkError(HudException):
+    """Raised when there is a network-related error.
+
+    This exception is raised when there are issues with the network
+    connection, DNS resolution, or other network-related problems.
+    """

hud/gym.py

@@ -17,6 +17,7 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger("hud.gym")
 
+
 async def make(
     env_src: Gym | Task,
     *,
@@ -26,7 +27,7 @@ async def make(
 ) -> Environment:
     """
     Create an environment from an environment ID or a Task object.
-    
+
     Args:
         env_src: Environment ID or Task object
         job: Job object to associate with this environment
@@ -35,7 +36,7 @@ async def make(
     """
     if metadata is None:
         metadata = {}
-    
+
     # Handle job parameter
     effective_job_id = None
     if job is not None:
@@ -45,13 +46,14 @@ async def make(
     else:
         # Try to get an active job from the decorator context
         try:
-            from hud.job import get_active_job
-            active_job = get_active_job()
+            import hud.job
+
+            active_job = hud.job.get_active_job()
             if active_job:
                 effective_job_id = active_job.id
         except ImportError:
             pass  # Module not available, skip
-    
+
     gym = None
     task = None
     if isinstance(env_src, str | CustomGym):
@@ -77,7 +79,7 @@ async def make(
             )
         else:
             raise ValueError(f"Invalid environment location: {gym.location}")
-            
+
         # Set up the environment with a source path
         if gym.controller_source_dir:
             logger.info("Setting source path")
@@ -101,7 +103,7 @@ async def make(
 
     # Create the environment itself
     environment = Environment(client=client, metadata=metadata, task=task, build_data=build_data)
-    
+
     if task:
         await environment._setup()