hud-python 0.4.55__tar.gz → 0.4.57__tar.gz

{hud_python-0.4.55 → hud_python-0.4.57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hud-python
-Version: 0.4.55
+Version: 0.4.57
 Summary: SDK for the HUD platform.
 Project-URL: Homepage, https://github.com/hud-evals/hud-python
 Project-URL: Bug Tracker, https://github.com/hud-evals/hud-python/issues

hud_python-0.4.57/environments/rubrics/README.md

@@ -0,0 +1,182 @@
+# Rubrics Environment
+
+Web research environment powered by Exa API for searching and fetching content, with rubric-based evaluation for structured grading.
+See [docs](https://docs.hud.so/build-environments) for the complete environment design workflow.
+
+## Architecture
+
+**`environment/`** - Manages Exa API integration and state
+- Holds the Exa API key server-side
+- Exposes HTTP endpoints `/search`, `/fetch`, `/answer`, `/evaluate` for research workflows
+- Implements exponential backoff for rate limiting
+
+**`server/`** - Wraps data in MCP tools
+- Provides `search()`, `fetch()`, `answer()`, `evaluate()` tools for agents
+- Agents and tasks interact only with these tools
+
+**Why separate?** Edit tools for the agent or tasks without restarting the environment backend.
+
+## Tools
+
+- **`search(query: str)`** - Search the web using Exa API, returns list of results with titles and URLs
+- **`fetch(url: str)`** - Fetch full content from a URL, returns summary, highlights, and text
+- **`answer(final_answer: str)`** - Submit the final research answer
+- **`evaluate(rubric: list[dict])`** - Evaluate submitted answer using a structured rubric with weighted requirements
+
+### Rubric-Based Evaluation
+
+The `evaluate` tool uses The LLM Data Company's [rubric](https://github.com/The-LLM-Data-Company/rubric/) package to grade answers against structured criteria with autograders.
+
+## Setup
+
+### Requirements
+- Exa API key (get one at [exa.ai](https://exa.ai))
+
+### Environment Variables
+```bash
+export EXA_API_KEY="your_exa_api_key_here"
+```
+
+## Development
+
+```bash
+# Terminal 1 - Environment backend
+cd environment
+export EXA_API_KEY="your_key"
+uv run uvicorn server:app --reload
+
+# Terminal 2 - MCP server
+cd server
+uv run hud dev
+```
+
+The environment includes exponential backoff for rate limiting, so API calls will automatically retry on 429 errors.
+
+In general, we recommend starting work on the environment backend first, then developing the MCP server to expose the right things to the agent.
+
+For complex environments that require many dependencies, we recommend running `hud dev` in the environment root:
+```bash
+cd ..
+export EXA_API_KEY="your_key"
+hud dev
+```
+
+## Tasks & Evaluation
+
+```bash
+# Build first in the global folder with the Dockerfile (creates rubrics:0.1.0)
+hud build
+```
+
+Your `tasks.json` uses `docker run` to launch the environment:
+
+```json
+{
+  "prompt": "Research and answer: What is the capital of France?",
+  "mcp_config": {
+    "local": {
+      "command": "docker",
+      "args": ["run", "--rm", "-i", "-e", "EXA_API_KEY", "rubrics:latest"]
+    }
+  },
+  "evaluate_tool": {
+    "name": "evaluate",
+    "arguments": {
+      "rubric": [
+        {
+          "requirement": "Correctly identifies Paris as the capital of France",
+          "weight": 5
+        },
+        {
+          "requirement": "Provides additional context about Paris (population, history, or geography)",
+          "weight": 10
+        }
+      ]
+    }
+  }
+}
+```
+
+**Note:** The `-e EXA_API_KEY` flag passes your local API key to the container.
+
+**Commands:**
+```bash
+# Build first
+hud build
+
+# Test task locally
+export EXA_API_KEY="your_key"
+hud eval tasks.json
+
+# Push environment for remote running
+hud push
+
+# Production RL training
+hud rl tasks.json  # Auto-converts docker→remote, builds & pushes if needed
+```
+
+## Publishing Your Environment
+
+Once your environment is ready, you can share it with the community:
+
+### 1. Push to Registry
+```bash
+# Build and push your environment (requires docker hub login and hud api key)
+hud build
+hud push
+```
+
+### 2. Create a Dataset
+
+Create a dataset on HuggingFace with your tasks:
+
+**Option A: Upload manually**
+1. Upload your `tasks.json` to HuggingFace
+2. Make sure it's **public** to appear on leaderboards
+
+**Option B: Use the SDK**
+```python
+from hud.datasets import save_tasks
+import json
+
+# Load your tasks
+with open("tasks.json") as f:
+    tasks = json.load(f)
+
+# Push to HuggingFace
+save_tasks(tasks, repo_id="your-org/your-dataset")
+```
+
+### 3. Run and Track Performance
+
+```bash
+# Run Claude on your benchmark
+hud eval "your-org/your-dataset" --agent claude
+
+# View results at:
+# hud.so/leaderboards/your-org/your-dataset
+```
+
+**Note**: Only public HuggingFace datasets appear as leaderboards!
+
+📚 Learn more: [Creating Benchmarks](https://docs.hud.so/evaluate-agents/create-benchmarks) | [Leaderboards](https://docs.hud.so/evaluate-agents/leaderboards)
+
+## Example Research Workflow
+
+```python
+# Agent searches for information
+results = search("latest AI developments 2024")
+
+# Agent fetches detailed content from top result
+content = fetch(results[0]["url"])
+
+# Agent submits final answer
+answer("Based on research, AI developments in 2024 include...")
+
+# Evaluate answer using rubric
+result = evaluate(rubric=[
+    {"requirement": "Mentions at least 3 specific AI developments", "weight": 15},
+    {"requirement": "Includes dates or timeframes for developments", "weight": 5},
+])
+# Returns: {"reward": float, "info": {"report": [...]}, "done": True}
+```

hud_python-0.4.57/environments/rubrics/environment/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "rubrics-environment"
+version = "0.1.0"
+description = "Backend service for Rubrics environment"
+requires-python = ">=3.11"
+dependencies = [
+    "fastapi>=0.104.1",
+    "uvicorn[standard]>=0.24.0",
+    "httpx>=0.24.0",
+    "rubric>=1.1.7",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["environment"]

hud_python-0.4.57/environments/rubrics/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "rubrics"
+version = "0.1.0"
+description = "Rubrics HUD environment with HTTP backend (EXA on server)"
+requires-python = ">=3.11"
+dependencies = [ "hud-python==0.4.42", "fastapi>=0.104.1", "uvicorn[standard]>=0.24.0", "httpx>=0.24.0",]
+
+[build-system]
+requires = [ "hatchling",]
+build-backend = "hatchling.build"
+
+[tool.hud]
+image = "rubrics:dev"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.build.targets.wheel]
+packages = [ "controller", "environment",]

hud_python-0.4.57/environments/rubrics/server/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "rubrics-mcp"
+version = "0.1.0"
+description = "MCP server for Rubrics environment"
+requires-python = ">=3.11"
+dependencies = [
+    "hud-python>=0.4.54",
+    "httpx>=0.24.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["mcp"]

{hud_python-0.4.55 → hud_python-0.4.57}/hud/cli/__init__.py

@@ -746,7 +746,7 @@ def init(
         None,
         "--preset",
         "-p",
-        help="Preset to use: blank, deep-research, browser. If omitted, you'll choose interactively.",  # noqa: E501
+        help="Preset to use: blank, deep-research, browser, rubrics. If omitted, you'll choose interactively.",  # noqa: E501
     ),
     directory: str = typer.Option(".", "--dir", "-d", help="Target directory"),
     force: bool = typer.Option(False, "--force", "-f", help="Overwrite existing files"),

{hud_python-0.4.55 → hud_python-0.4.57}/hud/cli/dev.py

@@ -237,7 +237,7 @@ async def run_mcp_module(
 
                 from hud.cli.flows.dev import create_dynamic_trace
 
-                live_trace_url = await create_dynamic_trace(
+                _, live_trace_url = await create_dynamic_trace(
                     mcp_config=local_mcp_config,
                     build_status=False,
                     environment_name=mcp_server.name or "mcp-server",
@@ -608,7 +608,7 @@ def run_docker_dev_server(
                     "headers": {},
                 }
             }
-            live_trace_url = _asy.run(
+            _, live_trace_url = _asy.run(
                 create_dynamic_trace(
                     mcp_config=local_mcp_config,
                     build_status=True,
@@ -661,13 +661,33 @@ def run_docker_dev_server(
     # Create and run proxy with HUD helpers
     async def run_proxy() -> None:
         from fastmcp import FastMCP
+        from fastmcp.server.proxy import ProxyClient
 
-        # Create FastMCP proxy to Docker stdio
-        fastmcp_proxy = FastMCP.as_proxy(mcp_config, name="HUD Docker Dev Proxy")
+        # Create ProxyClient without custom log handler since we capture Docker logs directly
+        proxy_client = ProxyClient(mcp_config, name="HUD Docker Dev Proxy")
+
+        # Extract container name from docker args and store for logs endpoint
+        docker_cmd = mcp_config["docker"]["args"]
+        container_name = None
+        for i, arg in enumerate(docker_cmd):
+            if arg == "--name" and i + 1 < len(docker_cmd):
+                container_name = docker_cmd[i + 1]
+                break
+
+        if container_name:
+            # Store container name for logs endpoint to use
+            os.environ["_HUD_DEV_DOCKER_CONTAINER"] = container_name
+            hud_console.debug(f"Docker container: {container_name}")
+
+        # Create FastMCP proxy using the ProxyClient
+        fastmcp_proxy = FastMCP.as_proxy(proxy_client)
 
         # Wrap in MCPServer to get /docs and REST wrappers
         proxy = MCPServer(name="HUD Docker Dev Proxy")
 
+        # Enable logs endpoint on HTTP server
+        os.environ["_HUD_DEV_LOGS_PROVIDER"] = "enabled"
+
         # Import all tools from the FastMCP proxy
         await proxy.import_server(fastmcp_proxy)
 

{hud_python-0.4.55 → hud_python-0.4.57}/hud/cli/flows/dev.py

@@ -18,7 +18,7 @@ async def create_dynamic_trace(
     mcp_config: dict[str, dict[str, Any]],
     build_status: bool,
     environment_name: str,
-) -> str | None:
+) -> tuple[str | None, str | None]:
     """
     Create a dynamic trace for HUD dev sessions when running in HTTP mode.
 
@@ -43,27 +43,16 @@ async def create_dynamic_trace(
     api_key = settings.api_key
     if not api_key:
         logger.warning("Skipping dynamic trace creation; missing HUD_API_KEY")
-        return None
+        return None, None
 
     try:
         resp = await make_request("POST", url=url, json=payload, api_key=api_key)
         # New API returns an id; construct the URL as https://hud.so/trace/{id}
-        trace_id = None
-        if isinstance(resp, dict):
-            trace_id = resp.get("id")
-            if trace_id is None:
-                data = resp.get("data", {}) or {}
-                if isinstance(data, dict):
-                    trace_id = data.get("id")
-            # Backcompat: if url is provided directly
-            if not trace_id:
-                direct_url = resp.get("url") or (resp.get("data", {}) or {}).get("url")
-                if isinstance(direct_url, str) and direct_url:
-                    return direct_url
+        trace_id = resp.get("id")
 
         if isinstance(trace_id, str) and trace_id:
-            return f"https://hud.so/trace/{trace_id}"
-        return None
+            return trace_id, f"https://hud.so/trace/{trace_id}"
+        return None, None
     except Exception as e:
         # Do not interrupt dev flow
         try:
@@ -71,7 +60,7 @@ async def create_dynamic_trace(
             logger.warning("Failed to create dynamic dev trace: %s | payload=%s", e, preview)
         except Exception:
             logger.warning("Failed to create dynamic dev trace: %s", e)
-        return None
+        return None, None
 
 
 def show_dev_ui(

{hud_python-0.4.55 → hud_python-0.4.57}/hud/cli/init.py

@@ -23,6 +23,7 @@ PRESET_MAP: dict[str, str | None] = {
     "blank": "blank",
     "deep-research": "deepresearch",
     "browser": "browser",
+    "rubrics": "rubrics",
 }
 
 SKIP_DIR_NAMES = {"node_modules", "__pycache__", "dist", "build", ".next", ".git"}
@@ -91,6 +92,7 @@ def _prompt_for_preset() -> str:
             {"name": "blank", "message": "blank"},
             {"name": "deep-research", "message": "deep-research"},
             {"name": "browser", "message": "browser"},
+            {"name": "rubrics", "message": "rubrics"},
         ]
         display_choices = [c["message"] for c in choices]
         selected = questionary.select(
@@ -194,7 +196,7 @@ def create_environment(
     if preset_normalized not in PRESET_MAP:
         hud_console.warning(
             f"Unknown preset '{preset_normalized}', defaulting to 'blank' "
-            "(available: blank, deep-research, browser)"
+            "(available: blank, deep-research, browser, rubrics)"
         )
         preset_normalized = "blank"