hud-python 0.5.1__py3-none-any.whl → 0.5.13__py3-none-any.whl

hud/cli/dev.py

@@ -250,6 +250,15 @@ async def run_mcp_module(
     elif hasattr(module, "__dict__") and attr_name in module.__dict__:
         mcp_server = module.__dict__[attr_name]
 
+    # If default 'mcp' not found, try 'env' as fallback
+    if mcp_server is None and attr_name == "mcp":
+        for fallback in ["env", "environment", "server"]:
+            if hasattr(module, fallback):
+                mcp_server = getattr(module, fallback)
+                if verbose:
+                    hud_console.info(f"Found '{fallback}' instead of 'mcp'")
+                break
+
     if mcp_server is None:
         hud_console.error(f"Module '{module_name}' does not have '{attr_name}' defined")
         hud_console.info("")
@@ -258,8 +267,8 @@ async def run_mcp_module(
         hud_console.info("")
         hud_console.info("[bold cyan]Expected structure:[/bold cyan]")
         hud_console.info("  from hud.environment import Environment")
-        hud_console.info(f"  {attr_name} = Environment('my-env')")
-        raise AttributeError(f"Module '{module_name}' must define '{attr_name}'")
+        hud_console.info("  env = Environment('my-env')  # or mcp = ...")
+        raise AttributeError(f"Module '{module_name}' must define 'mcp', 'env', or 'environment'")
 
     # Only show full header on first run, brief message on reload
     if is_reload:

hud/cli/eval.py

@@ -164,6 +164,7 @@ class EvalConfig(BaseModel):
         "auto_respond",
         "quiet",
         "gateway",
+        "taskset",
     }
     # Fields loaded from [agent] section
     _AGENT_FIELDS: ClassVar[set[str]] = {"allowed_tools", "disallowed_tools"}
@@ -184,6 +185,7 @@ class EvalConfig(BaseModel):
     remote: bool = False
     quiet: bool = False  # Suppress opening browser for eval links
     gateway: bool = False  # Use HUD Gateway for LLM API calls
+    taskset: str | None = None  # Taskset slug to associate job with
 
     # Base agent config (these merge with task's agent_config)
     allowed_tools: list[str] | None = None
@@ -338,47 +340,27 @@ class EvalConfig(BaseModel):
 
         # Configure gateway mode - route LLM API calls through HUD gateway
         if self.gateway:
-            hud_api_key = settings.api_key
-            if not hud_api_key:
+            if not settings.api_key:
                 raise typer.Exit(1)  # Already validated in validate_api_keys()
 
-            if self.agent_type == AgentType.CLAUDE:
-                from anthropic import AsyncAnthropic
+            from hud.agents.gateway import build_gateway_client
 
-                kwargs["model_client"] = AsyncAnthropic(
-                    api_key=hud_api_key,
-                    base_url=settings.hud_gateway_url,
-                )
-                hud_console.info("🌐 Using HUD Gateway for Claude API")
-            elif self.agent_type in (AgentType.OPENAI, AgentType.OPERATOR):
-                from openai import AsyncOpenAI
+            # Map AgentType to provider
+            agent_to_provider = {
+                AgentType.CLAUDE: "anthropic",
+                AgentType.OPENAI: "openai",
+                AgentType.OPERATOR: "openai",
+                AgentType.GEMINI: "gemini",
+                AgentType.GEMINI_CUA: "gemini",
+                AgentType.OPENAI_COMPATIBLE: "openai",
+            }
+            provider = agent_to_provider.get(self.agent_type, "openai")
+            client = build_gateway_client(provider)
 
-                kwargs["model_client"] = AsyncOpenAI(
-                    api_key=hud_api_key,
-                    base_url=settings.hud_gateway_url,
-                )
-                hud_console.info("🌐 Using HUD Gateway for OpenAI API")
-            elif self.agent_type == AgentType.OPENAI_COMPATIBLE:
-                from openai import AsyncOpenAI
-
-                kwargs["openai_client"] = AsyncOpenAI(
-                    api_key=hud_api_key,
-                    base_url=settings.hud_gateway_url,
-                )
-                hud_console.info("🌐 Using HUD Gateway for OpenAI-compatible API")
-            elif self.agent_type in (AgentType.GEMINI, AgentType.GEMINI_CUA):
-                from google import genai
-                from google.genai.types import HttpOptions
-
-                kwargs["model_client"] = genai.Client(
-                    api_key="PLACEHOLDER",
-                    http_options=HttpOptions(
-                        api_version="v1beta",
-                        base_url=settings.hud_gateway_url,
-                        headers={"Authorization": f"Bearer {hud_api_key}"},
-                    ),
-                )
-                hud_console.info("🌐 Using HUD Gateway for Gemini API")
+            # OpenAI-compatible uses openai_client key
+            is_oai_compat = self.agent_type == AgentType.OPENAI_COMPATIBLE
+            kwargs["openai_client" if is_oai_compat else "model_client"] = client
+            hud_console.info(f"🌐 Using HUD Gateway for {provider} API")
 
         return kwargs
 
@@ -584,7 +566,7 @@ class EvalConfig(BaseModel):
             table.add_row("", "")
             table.add_row(f"[dim]{self.agent_type.value} config[/dim]", "")
 
-            config_cls = self.agent_type.cls.config_cls
+            config_cls = self.agent_type.config_cls
             defaults = config_cls()
             overrides = self.agent_config.get(self.agent_type.value, {})
             skip = {
@@ -675,16 +657,41 @@ async def _run_evaluation(cfg: EvalConfig) -> tuple[list[Any], list[Any]]:
         agent_kwargs = {
             k: v for k, v in agent_kwargs.items() if k not in ("api_key", "model_client")
         }
-        # Create a job ID for tracking
         import uuid
 
         from hud.datasets.utils import submit_rollouts
+        from hud.eval.manager import _send_job_enter
 
         job_id = str(uuid.uuid4())
         hud_console.info(
             f"Submitting {len(tasks)} task(s) for remote execution (job_id: {job_id})…"
         )
 
+        if cfg.taskset:
+            tasks_to_create = [t for t in tasks if not t.id]
+            tasks_data = (
+                [t.model_dump(mode="json", exclude_none=True) for t in tasks_to_create]
+                if tasks_to_create
+                else None
+            )
+            ids = await _send_job_enter(
+                job_id=job_id,
+                name=f"eval ({cfg.source})" if cfg.source else "eval",
+                variants=None,
+                group=cfg.group_size,
+                api_key=None,
+                taskset=cfg.taskset,
+                tasks=tasks_data,
+            )
+            if ids:
+                if len(ids) != len(tasks_to_create):
+                    hud_console.warning(
+                        f"Task count mismatch: sent {len(tasks_to_create)} tasks, "
+                        f"received {len(ids)} IDs. Some tasks may not be linked."
+                    )
+                for task_obj, task_version_id in zip(tasks_to_create, ids, strict=False):
+                    task_obj.id = task_version_id
+
         await submit_rollouts(
             tasks=tasks,
             job_id=job_id,
@@ -721,6 +728,7 @@ async def _run_evaluation(cfg: EvalConfig) -> tuple[list[Any], list[Any]]:
         max_concurrent=cfg.max_concurrent,
         group_size=cfg.group_size,
         quiet=cfg.quiet,
+        taskset=cfg.taskset,
     )
 
     # Show reward for single task
@@ -787,6 +795,9 @@ def eval_command(
     gateway: bool = typer.Option(
         False, "--gateway", "-g", help="Route LLM API calls through HUD Gateway"
     ),
+    taskset: str | None = typer.Option(
+        None, "--taskset", "-t", help="Taskset slug to associate job with"
+    ),
 ) -> None:
     """🚀 Run evaluation on datasets or individual tasks with agents.
 
@@ -821,6 +832,7 @@ def eval_command(
         byok=byok,
         quiet=quiet,
         gateway=gateway,
+        taskset=taskset,
     )
 
     # Find source if not provided

hud/cli/flows/init.py

@@ -102,7 +102,7 @@ def smart_init(
         hud_console.info("  hud set HUD_API_KEY=your-key-here")
         hud_console.info("  Or: export HUD_API_KEY=your-key")
         hud_console.info("")
-        hud_console.info("Get your key at: https://hud.ai/settings/api-keys")
+        hud_console.info("Get your key at: https://hud.ai/project/api-keys")
         return
 
     target = Path(directory).resolve()

hud/cli/pull.py

@@ -63,7 +63,7 @@ def fetch_lock_from_registry(reference: str) -> dict | None:
 
         # URL-encode the path segments to handle special characters in tags
         url_safe_path = "/".join(quote(part, safe="") for part in reference.split("/"))
-        registry_url = f"{settings.hud_telemetry_url.rstrip('/')}/registry/envs/{url_safe_path}"
+        registry_url = f"{settings.hud_api_url.rstrip('/')}/registry/envs/{url_safe_path}"
 
         headers = {}
         if settings.api_key:

hud/cli/push.py

@@ -420,13 +420,20 @@ def push_environment(
 
         # URL-encode the path segments to handle special characters in tags
         url_safe_path = "/".join(quote(part, safe="") for part in name_with_tag.split("/"))
-        registry_url = f"{settings.hud_telemetry_url.rstrip('/')}/registry/envs/{url_safe_path}"
+        registry_url = f"{settings.hud_api_url.rstrip('/')}/registry/envs/{url_safe_path}"
+
+        # Detect git remote URL for matching existing GitHub-connected registries
+        from hud.cli.utils.git import get_git_remote_url
+
+        github_url = get_git_remote_url(Path(directory))
 
         # Prepare the payload
-        payload = {
+        payload: dict[str, str | None] = {
             "lock": yaml.dump(lock_data, default_flow_style=False, sort_keys=False),
             "digest": pushed_digest.split("@")[-1] if "@" in pushed_digest else None,
         }
+        if github_url:
+            payload["github_url"] = github_url
 
         headers = {"Authorization": f"Bearer {settings.api_key}"}
 

hud/cli/tests/test_build.py

@@ -60,12 +60,12 @@ class TestIncrementVersion:
     def test_increment_minor(self):
         """Test incrementing minor version."""
         assert increment_version("1.2.3", "minor") == "1.3.0"
-        assert increment_version("0.5.10", "minor") == "0.6.0"
+        assert increment_version("0.5.13", "minor") == "0.6.0"
 
     def test_increment_major(self):
         """Test incrementing major version."""
         assert increment_version("1.2.3", "major") == "2.0.0"
-        assert increment_version("0.5.10", "major") == "1.0.0"
+        assert increment_version("0.5.13", "major") == "1.0.0"
 
     def test_increment_with_v_prefix(self):
         """Test incrementing version with v prefix."""

hud/cli/tests/test_push.py

@@ -160,7 +160,7 @@ class TestPushEnvironment:
         mock_hud_console = mock.Mock()
         mock_hud_console_class.return_value = mock_hud_console
         mock_settings.api_key = "test-key"
-        mock_settings.hud_telemetry_url = "https://api.hud.test"
+        mock_settings.hud_api_url = "https://api.hud.test"
         mock_get_username.return_value = "testuser"
 
         # Create lock file

hud/cli/utils/metadata.py

@@ -32,7 +32,7 @@ def fetch_lock_from_registry(reference: str) -> dict | None:
 
         # URL-encode the path segments to handle special characters in tags
         url_safe_path = "/".join(quote(part, safe="") for part in reference.split("/"))
-        registry_url = f"{settings.hud_telemetry_url.rstrip('/')}/registry/envs/{url_safe_path}"
+        registry_url = f"{settings.hud_api_url.rstrip('/')}/registry/envs/{url_safe_path}"
 
         headers = {}
         if settings.api_key:

hud/cli/utils/tests/test_metadata.py

@@ -18,7 +18,7 @@ if TYPE_CHECKING:
 @patch("hud.cli.utils.metadata.settings")
 @patch("requests.get")
 def test_fetch_lock_from_registry_success(mock_get, mock_settings):
-    mock_settings.hud_telemetry_url = "https://api.example.com"
+    mock_settings.hud_api_url = "https://api.example.com"
     mock_settings.api_key = None
     resp = MagicMock(status_code=200)
     resp.json.return_value = {"lock": "image: img\n"}

hud/clients/mcp_use.py

@@ -64,9 +64,14 @@ class MCPUseHUDClient(BaseHUDClient):
             return
 
         # Use configurable timeout for SSE read operations to support long-running tool calls.
+        max_request_timeout = 840
         for server_cfg in mcp_config.values():
             if "sse_read_timeout" not in server_cfg:
-                server_cfg["sse_read_timeout"] = settings.client_timeout
+                server_cfg["sse_read_timeout"] = (
+                    min(settings.client_timeout, max_request_timeout)
+                    if settings.client_timeout > 0
+                    else max_request_timeout
+                )
 
         # If a server target matches HUD's MCP host and no auth is provided,
         # inject the HUD API key as a Bearer token to avoid OAuth browser flow.

hud/datasets/loader.py

@@ -14,6 +14,10 @@ import warnings
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, overload
 
+import httpx
+
+from hud.settings import settings
+
 if TYPE_CHECKING:
     from hud.eval.task import Task
 
@@ -63,7 +67,8 @@ def _load_from_file(path: Path) -> list[Task]:
     from hud.eval.task import Task
 
     raw_items = _load_raw_from_file(path)
-    return [Task(**item) for item in raw_items]
+    # Default args to {} for runnable tasks (None = template)
+    return [Task(**{**item, "args": item.get("args") or {}}) for item in raw_items]
 
 
 def _load_raw_from_huggingface(dataset_name: str) -> list[dict[str, Any]]:
@@ -99,15 +104,12 @@ def _load_from_huggingface(dataset_name: str) -> list[Task]:
     raw_items = _load_raw_from_huggingface(dataset_name)
     from hud.eval.task import Task
 
-    return [Task(**item) for item in raw_items]
+    # Default args to {} for runnable tasks (None = template)
+    return [Task(**{**item, "args": item.get("args") or {}}) for item in raw_items]
 
 
 def _load_raw_from_api(dataset_name: str) -> list[dict[str, Any]]:
     """Load raw task dicts from HUD API."""
-    import httpx
-
-    from hud.settings import settings
-
     headers = {}
     if settings.api_key:
         headers["Authorization"] = f"Bearer {settings.api_key}"
@@ -138,7 +140,8 @@ def _load_from_api(dataset_name: str) -> list[Task]:
     from hud.eval.task import Task
 
     raw_items = _load_raw_from_api(dataset_name)
-    return [Task(**item) for item in raw_items]
+    # Default args to {} for runnable tasks (None = template)
+    return [Task(**{**item, "args": item.get("args") or {}}) for item in raw_items]
 
 
 @overload
@@ -234,15 +237,15 @@ def save_tasks(
 ) -> str:
     """Save tasks to the HUD API.
 
-    Creates or updates an evalset with the given tasks.
+    Creates or updates a taskset with the given tasks.
 
     Args:
-        name: Evalset name/slug (e.g., "my-evals/benchmark-v1").
+        name: Taskset name/slug (e.g., "my-evals/benchmark-v1").
             If no org prefix, uses user's default org.
         tasks: List of Task objects (v5 format) to save.
 
     Returns:
-        The evalset ID of the created/updated evalset.
+        The taskset ID of the created/updated taskset.
 
     Example:
         ```python
@@ -258,7 +261,7 @@ def save_tasks(
         ]
 
         # Save to HUD API
-        evalset_id = save_tasks("my-evals/benchmark-v1", tasks)
+        taskset_id = save_tasks("my-evals/benchmark-v1", tasks)
 
         # Later, load them back
         loaded = load_tasks("my-evals/benchmark-v1")
@@ -268,10 +271,6 @@ def save_tasks(
         TypeError: If any task is not a v5 Task object (must have 'scenario')
         ValueError: If API key is not set or save fails
     """
-    import httpx
-
-    from hud.settings import settings
-
     if not settings.api_key:
         raise ValueError("HUD_API_KEY is required to save tasks")
 
@@ -303,9 +302,9 @@ def save_tasks(
             )
             response.raise_for_status()
             data = response.json()
-            evalset_id = data.get("evalset_id") or data.get("id") or name
-            logger.info("Saved %d tasks to evalset: %s", len(tasks), evalset_id)
-            return evalset_id
+            taskset_id = data.get("evalset_id") or data.get("id") or name
+            logger.info("Saved %d tasks to taskset: %s", len(tasks), taskset_id)
+            return taskset_id
     except httpx.HTTPStatusError as e:
         raise ValueError(f"Failed to save tasks: {e.response.text}") from e
     except Exception as e:

hud/datasets/runner.py

@@ -29,6 +29,7 @@ async def run_dataset(
     max_concurrent: int = 30,
     group_size: int = 1,
     quiet: bool = True,
+    taskset: str | None = None,
 ) -> list[EvalContext]:
     """Run an agent on a dataset of tasks.
 
@@ -40,7 +41,7 @@ async def run_dataset(
             - A source string (file path, API slug) - loaded via load_tasks()
             - A single TaskInput (Task, LegacyTask, or dict)
             - A list of TaskInput objects
-        agent_type: Type of agent to create (e.g., "claude", "openai", AgentType.CLAUDE).
+        agent_type: Agent type (e.g., "claude", "openai", AgentType.CLAUDE).
         agent_params: Parameters to pass to agent.create().
         max_steps: Maximum steps per task.
         max_concurrent: Maximum concurrent tasks (for parallel execution).
@@ -70,6 +71,10 @@ async def run_dataset(
     from hud.datasets.loader import load_tasks
     from hud.eval.task import Task
 
+    # Normalize agent_type to AgentType enum
+    if isinstance(agent_type, str):
+        agent_type = AgentType(agent_type)
+
     # Normalize tasks to list[Task]
     task_list: list[Task]
     if isinstance(tasks, str):
@@ -86,19 +91,21 @@ async def run_dataset(
     if not task_list:
         raise ValueError("No tasks to run")
 
-    # Resolve agent class
-    agent_type_enum = agent_type if isinstance(agent_type, AgentType) else AgentType(agent_type)
-    agent_cls = agent_type_enum.cls
-
     # Use hud.eval() for both single and parallel execution
     async with hud.eval(
         task_list,
         group=group_size,
         max_concurrent=max_concurrent,
         quiet=quiet,
+        taskset=taskset,
     ) as ctx:
-        # Create agent fresh for each context (ensures correct tool initialization)
-        agent = agent_cls.create(**(agent_params or {}))
+        # Build agent params - use system_prompt from ctx (set from task.agent_config)
+        final_agent_params = dict(agent_params or {})
+        if ctx.system_prompt and "system_prompt" not in final_agent_params:
+            final_agent_params["system_prompt"] = ctx.system_prompt
+
+        # Create agent using AgentType.cls.create()
+        agent = agent_type.cls.create(**final_agent_params)
         await agent.run(ctx, max_steps=max_steps)
         # Reward is computed by EvalContext.__aexit__ from evaluate tools
 
@@ -198,9 +205,8 @@ async def run_single_task(
         if ctx.system_prompt and "system_prompt" not in final_agent_params:
             final_agent_params["system_prompt"] = ctx.system_prompt
 
-        # Create agent inside ctx so it has access to context-derived values
-        agent_cls = agent_type.cls
-        agent = agent_cls.create(**final_agent_params)
+        # Create agent using AgentType.cls.create()
+        agent = agent_type.cls.create(**final_agent_params)
 
         # Store metadata if provided
         if metadata:

hud/datasets/tests/test_loader.py

@@ -12,8 +12,8 @@ from hud.datasets.loader import load_tasks
 class TestLoadTasks:
     """Tests for load_tasks() function."""
 
-    @patch("httpx.Client")
-    @patch("hud.settings.settings")
+    @patch("hud.datasets.loader.httpx.Client")
+    @patch("hud.datasets.loader.settings")
     def test_load_tasks_success(
         self, mock_settings: MagicMock, mock_client_class: MagicMock
     ) -> None:
@@ -22,7 +22,7 @@ class TestLoadTasks:
         mock_settings.api_key = "test_key"
 
         mock_response = MagicMock()
-        # New EvalsetTasksResponse format: tasks keyed by task ID
+        # EvalsetTasksResponse format: tasks keyed by task ID
         mock_response.json.return_value = {
             "evalset_id": "evalset-123",
             "evalset_name": "test-dataset",
@@ -62,8 +62,8 @@ class TestLoadTasks:
             params={"all": "true"},
         )
 
-    @patch("httpx.Client")
-    @patch("hud.settings.settings")
+    @patch("hud.datasets.loader.httpx.Client")
+    @patch("hud.datasets.loader.settings")
     def test_load_tasks_single_task(
         self, mock_settings: MagicMock, mock_client_class: MagicMock
     ) -> None:
@@ -97,8 +97,8 @@ class TestLoadTasks:
         assert tasks[0].scenario == "checkout"
         assert tasks[0].id == "task-1"
 
-    @patch("httpx.Client")
-    @patch("hud.settings.settings")
+    @patch("hud.datasets.loader.httpx.Client")
+    @patch("hud.datasets.loader.settings")
     def test_load_tasks_no_api_key(
         self, mock_settings: MagicMock, mock_client_class: MagicMock
     ) -> None:
@@ -129,8 +129,8 @@ class TestLoadTasks:
             params={"all": "true"},
         )
 
-    @patch("httpx.Client")
-    @patch("hud.settings.settings")
+    @patch("hud.datasets.loader.httpx.Client")
+    @patch("hud.datasets.loader.settings")
     def test_load_tasks_http_error(
         self, mock_settings: MagicMock, mock_client_class: MagicMock
     ) -> None:
@@ -149,8 +149,8 @@ class TestLoadTasks:
         with pytest.raises(ValueError, match="Failed to load tasks"):
             load_tasks("test-org/test-dataset")
 
-    @patch("httpx.Client")
-    @patch("hud.settings.settings")
+    @patch("hud.datasets.loader.httpx.Client")
+    @patch("hud.datasets.loader.settings")
     def test_load_tasks_json_error(
         self, mock_settings: MagicMock, mock_client_class: MagicMock
     ) -> None:
@@ -171,8 +171,8 @@ class TestLoadTasks:
         with pytest.raises(ValueError, match="Failed to load tasks"):
             load_tasks("test-org/test-dataset")
 
-    @patch("httpx.Client")
-    @patch("hud.settings.settings")
+    @patch("hud.datasets.loader.httpx.Client")
+    @patch("hud.datasets.loader.settings")
     def test_load_tasks_empty(self, mock_settings: MagicMock, mock_client_class: MagicMock) -> None:
         """load_tasks() handles empty dataset."""
         mock_settings.hud_api_url = "https://api.hud.ai"
@@ -192,8 +192,8 @@ class TestLoadTasks:
 
         assert len(tasks) == 0
 
-    @patch("httpx.Client")
-    @patch("hud.settings.settings")
+    @patch("hud.datasets.loader.httpx.Client")
+    @patch("hud.datasets.loader.settings")
     def test_load_tasks_missing_fields(
         self, mock_settings: MagicMock, mock_client_class: MagicMock
     ) -> None:

hud/environment/__init__.py

@@ -27,8 +27,8 @@ Usage:
 from hud.environment.connection import ConnectionConfig, ConnectionType, Connector
 from hud.environment.environment import Environment
 from hud.environment.mock import MockMixin, generate_mock_value
-from hud.environment.router import ConflictResolution, ToolRouter
-from hud.environment.scenarios import ScenarioMixin
+from hud.environment.router import ConflictResolution, MCPRouter, ToolRouter
+from hud.environment.scenarios import ScenarioMixin, ScenarioSession
 from hud.environment.types import EnvConfig
 from hud.environment.utils import ToolFormat, format_result, parse_tool_call, parse_tool_calls
 
@@ -39,10 +39,12 @@ __all__ = [
     "Connector",
     "EnvConfig",
     "Environment",
+    "MCPRouter",
     "MockMixin",
     "ScenarioMixin",
+    "ScenarioSession",
     "ToolFormat",
-    "ToolRouter",
+    "ToolRouter",  # Backwards compat alias for MCPRouter
     "format_result",
     "generate_mock_value",
     "parse_tool_call",