hud-python 0.4.55__tar.gz → 0.4.56__tar.gz

{hud_python-0.4.55 → hud_python-0.4.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hud-python
-Version: 0.4.55
+Version: 0.4.56
 Summary: SDK for the HUD platform.
 Project-URL: Homepage, https://github.com/hud-evals/hud-python
 Project-URL: Bug Tracker, https://github.com/hud-evals/hud-python/issues

hud_python-0.4.56/environments/rubrics/README.md

@@ -0,0 +1,182 @@
+# Rubrics Environment
+
+Web research environment powered by Exa API for searching and fetching content, with rubric-based evaluation for structured grading.
+See [docs](https://docs.hud.so/build-environments) for the complete environment design workflow.
+
+## Architecture
+
+**`environment/`** - Manages Exa API integration and state
+- Holds the Exa API key server-side
+- Exposes HTTP endpoints `/search`, `/fetch`, `/answer`, `/evaluate` for research workflows
+- Implements exponential backoff for rate limiting
+
+**`server/`** - Wraps data in MCP tools
+- Provides `search()`, `fetch()`, `answer()`, `evaluate()` tools for agents
+- Agents and tasks interact only with these tools
+
+**Why separate?** Edit tools for the agent or tasks without restarting the environment backend.
+
+## Tools
+
+- **`search(query: str)`** - Search the web using Exa API, returns list of results with titles and URLs
+- **`fetch(url: str)`** - Fetch full content from a URL, returns summary, highlights, and text
+- **`answer(final_answer: str)`** - Submit the final research answer
+- **`evaluate(rubric: list[dict])`** - Evaluate submitted answer using a structured rubric with weighted requirements
+
+### Rubric-Based Evaluation
+
+The `evaluate` tool uses The LLM Data Company's [rubric](https://github.com/The-LLM-Data-Company/rubric/) package to grade answers against structured criteria with autograders.
+
+## Setup
+
+### Requirements
+- Exa API key (get one at [exa.ai](https://exa.ai))
+
+### Environment Variables
+```bash
+export EXA_API_KEY="your_exa_api_key_here"
+```
+
+## Development
+
+```bash
+# Terminal 1 - Environment backend
+cd environment
+export EXA_API_KEY="your_key"
+uv run uvicorn server:app --reload
+
+# Terminal 2 - MCP server
+cd server
+uv run hud dev
+```
+
+The environment includes exponential backoff for rate limiting, so API calls will automatically retry on 429 errors.
+
+In general, we recommend starting work on the environment backend first, then developing the MCP server to expose the right things to the agent.
+
+For complex environments that require many dependencies, we recommend running `hud dev` in the environment root:
+```bash
+cd ..
+export EXA_API_KEY="your_key"
+hud dev
+```
+
+## Tasks & Evaluation
+
+```bash
+# Build first in the global folder with the Dockerfile (creates rubrics:0.1.0)
+hud build
+```
+
+Your `tasks.json` uses `docker run` to launch the environment:
+
+```json
+{
+  "prompt": "Research and answer: What is the capital of France?",
+  "mcp_config": {
+    "local": {
+      "command": "docker",
+      "args": ["run", "--rm", "-i", "-e", "EXA_API_KEY", "rubrics:latest"]
+    }
+  },
+  "evaluate_tool": {
+    "name": "evaluate",
+    "arguments": {
+      "rubric": [
+        {
+          "requirement": "Correctly identifies Paris as the capital of France",
+          "weight": 5
+        },
+        {
+          "requirement": "Provides additional context about Paris (population, history, or geography)",
+          "weight": 10
+        }
+      ]
+    }
+  }
+}
+```
+
+**Note:** The `-e EXA_API_KEY` flag passes your local API key to the container.
+
+**Commands:**
+```bash
+# Build first
+hud build
+
+# Test task locally
+export EXA_API_KEY="your_key"
+hud eval tasks.json
+
+# Push environment for remote running
+hud push
+
+# Production RL training
+hud rl tasks.json  # Auto-converts docker→remote, builds & pushes if needed
+```
+
+## Publishing Your Environment
+
+Once your environment is ready, you can share it with the community:
+
+### 1. Push to Registry
+```bash
+# Build and push your environment (requires docker hub login and hud api key)
+hud build
+hud push
+```
+
+### 2. Create a Dataset
+
+Create a dataset on HuggingFace with your tasks:
+
+**Option A: Upload manually**
+1. Upload your `tasks.json` to HuggingFace
+2. Make sure it's **public** to appear on leaderboards
+
+**Option B: Use the SDK**
+```python
+from hud.datasets import save_tasks
+import json
+
+# Load your tasks
+with open("tasks.json") as f:
+    tasks = json.load(f)
+
+# Push to HuggingFace
+save_tasks(tasks, repo_id="your-org/your-dataset")
+```
+
+### 3. Run and Track Performance
+
+```bash
+# Run Claude on your benchmark
+hud eval "your-org/your-dataset" --agent claude
+
+# View results at:
+# hud.so/leaderboards/your-org/your-dataset
+```
+
+**Note**: Only public HuggingFace datasets appear as leaderboards!
+
+📚 Learn more: [Creating Benchmarks](https://docs.hud.so/evaluate-agents/create-benchmarks) | [Leaderboards](https://docs.hud.so/evaluate-agents/leaderboards)
+
+## Example Research Workflow
+
+```python
+# Agent searches for information
+results = search("latest AI developments 2024")
+
+# Agent fetches detailed content from top result
+content = fetch(results[0]["url"])
+
+# Agent submits final answer
+answer("Based on research, AI developments in 2024 include...")
+
+# Evaluate answer using rubric
+result = evaluate(rubric=[
+    {"requirement": "Mentions at least 3 specific AI developments", "weight": 15},
+    {"requirement": "Includes dates or timeframes for developments", "weight": 5},
+])
+# Returns: {"reward": float, "info": {"report": [...]}, "done": True}
+```

hud_python-0.4.56/environments/rubrics/environment/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "rubrics-environment"
+version = "0.1.0"
+description = "Backend service for Rubrics environment"
+requires-python = ">=3.11"
+dependencies = [
+    "fastapi>=0.104.1",
+    "uvicorn[standard]>=0.24.0",
+    "httpx>=0.24.0",
+    "rubric>=1.1.7",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["environment"]

hud_python-0.4.56/environments/rubrics/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "rubrics"
+version = "0.1.0"
+description = "Rubrics HUD environment with HTTP backend (EXA on server)"
+requires-python = ">=3.11"
+dependencies = [ "hud-python==0.4.42", "fastapi>=0.104.1", "uvicorn[standard]>=0.24.0", "httpx>=0.24.0",]
+
+[build-system]
+requires = [ "hatchling",]
+build-backend = "hatchling.build"
+
+[tool.hud]
+image = "rubrics:dev"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.build.targets.wheel]
+packages = [ "controller", "environment",]

hud_python-0.4.56/environments/rubrics/server/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "rubrics-mcp"
+version = "0.1.0"
+description = "MCP server for Rubrics environment"
+requires-python = ">=3.11"
+dependencies = [
+    "hud-python>=0.4.54",
+    "httpx>=0.24.0",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["mcp"]

{hud_python-0.4.55 → hud_python-0.4.56}/hud/cli/__init__.py

@@ -746,7 +746,7 @@ def init(
         None,
         "--preset",
         "-p",
-        help="Preset to use: blank, deep-research, browser. If omitted, you'll choose interactively.",  # noqa: E501
+        help="Preset to use: blank, deep-research, browser, rubrics. If omitted, you'll choose interactively.",  # noqa: E501
     ),
     directory: str = typer.Option(".", "--dir", "-d", help="Target directory"),
     force: bool = typer.Option(False, "--force", "-f", help="Overwrite existing files"),

{hud_python-0.4.55 → hud_python-0.4.56}/hud/cli/init.py

@@ -23,6 +23,7 @@ PRESET_MAP: dict[str, str | None] = {
     "blank": "blank",
     "deep-research": "deepresearch",
     "browser": "browser",
+    "rubrics": "rubrics",
 }
 
 SKIP_DIR_NAMES = {"node_modules", "__pycache__", "dist", "build", ".next", ".git"}
@@ -91,6 +92,7 @@ def _prompt_for_preset() -> str:
             {"name": "blank", "message": "blank"},
             {"name": "deep-research", "message": "deep-research"},
             {"name": "browser", "message": "browser"},
+            {"name": "rubrics", "message": "rubrics"},
         ]
         display_choices = [c["message"] for c in choices]
         selected = questionary.select(
@@ -194,7 +196,7 @@ def create_environment(
     if preset_normalized not in PRESET_MAP:
         hud_console.warning(
             f"Unknown preset '{preset_normalized}', defaulting to 'blank' "
-            "(available: blank, deep-research, browser)"
+            "(available: blank, deep-research, browser, rubrics)"
         )
         preset_normalized = "blank"
 

{hud_python-0.4.55 → hud_python-0.4.56}/hud/utils/tests/test_version.py

@@ -5,4 +5,4 @@ def test_import():
     """Test that the package can be imported."""
     import hud
 
-    assert hud.__version__ == "0.4.55"
+    assert hud.__version__ == "0.4.56"

{hud_python-0.4.55 → hud_python-0.4.56}/hud/version.py

@@ -4,4 +4,4 @@ Version information for the HUD SDK.
 
 from __future__ import annotations
 
-__version__ = "0.4.55"
+__version__ = "0.4.56"

{hud_python-0.4.55 → hud_python-0.4.56}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hud-python"
-version = "0.4.55"
+version = "0.4.56"
 description = "SDK for the HUD platform."
 readme = "README.md"
 requires-python = ">=3.11, <3.13"