hud-python 0.5.12__tar.gz → 0.5.13__tar.gz

{hud_python-0.5.12 → hud_python-0.5.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hud-python
-Version: 0.5.12
+Version: 0.5.13
 Summary: SDK for the HUD platform.
 Project-URL: Homepage, https://github.com/hud-evals/hud-python
 Project-URL: Bug Tracker, https://github.com/hud-evals/hud-python/issues
@@ -91,7 +91,7 @@ Requires-Dist: pyright==1.1.407; extra == 'dev'
 Requires-Dist: pytest-asyncio; extra == 'dev'
 Requires-Dist: pytest-cov; extra == 'dev'
 Requires-Dist: pytest-mock; extra == 'dev'
-Requires-Dist: pytest<9,>=8.1.1; extra == 'dev'
+Requires-Dist: pytest>=8.1.1; extra == 'dev'
 Requires-Dist: ruff>=0.11.8; extra == 'dev'
 Requires-Dist: tornado>=6.5.2; extra == 'dev'
 Description-Content-Type: text/markdown

{hud_python-0.5.12 → hud_python-0.5.13}/hud/cli/__init__.py

@@ -741,6 +741,11 @@ def build(
     platform: str | None = typer.Option(
         None, "--platform", help="Set Docker target platform (e.g., linux/amd64)"
     ),
+    secrets: list[str] | None = typer.Option(  # noqa: B008
+        None,
+        "--secret",
+        help=("Docker build secret (repeatable), e.g. --secret id=GITHUB_TOKEN,env=GITHUB_TOKEN"),
+    ),
     remote_cache: str | None = typer.Option(
         None, "--remote-cache", help="Enable remote cache using Amazon ECR with specified repo name"
     ),
@@ -812,7 +817,15 @@ def build(
             i += 1
 
     build_command(
-        directory, tag, no_cache, verbose, env_vars, platform, remote_cache, build_args or None
+        directory,
+        tag,
+        no_cache,
+        verbose,
+        env_vars,
+        platform,
+        secrets,
+        remote_cache,
+        build_args or None,
     )
 
 

{hud_python-0.5.12 → hud_python-0.5.13}/hud/cli/build.py

@@ -555,11 +555,13 @@ def build_docker_image(
     verbose: bool = False,
     build_args: dict[str, str] | None = None,
     platform: str | None = None,
+    secrets: list[str] | None = None,
     remote_cache: str | None = None,
 ) -> bool:
     """Build a Docker image from a directory."""
     hud_console = HUDConsole()
     build_args = build_args or {}
+    secrets = secrets or []
 
     # Check if Dockerfile exists (prefer Dockerfile.hud)
     dockerfile = find_dockerfile(directory)
@@ -629,6 +631,10 @@ def build_docker_image(
     for key, value in build_args.items():
         cmd.extend(["--build-arg", f"{key}={value}"])
 
+    # Add secrets
+    for secret in secrets:
+        cmd.extend(["--secret", secret])
+
     cmd.append(str(directory))
 
     # Always show build output
@@ -636,7 +642,10 @@ def build_docker_image(
 
     try:
         # Use Docker's native output formatting - no capture, let Docker handle display
-        result = subprocess.run(cmd, check=False)  # noqa: S603
+        env = os.environ.copy()
+        if secrets:
+            env["DOCKER_BUILDKIT"] = "1"
+        result = subprocess.run(cmd, check=False, env=env)  # noqa: S603
         return result.returncode == 0
     except Exception as e:
         hud_console.error(f"Build error: {e}")
@@ -650,6 +659,7 @@ def build_environment(
     verbose: bool = False,
     env_vars: dict[str, str] | None = None,
     platform: str | None = None,
+    secrets: list[str] | None = None,
     remote_cache: str | None = None,
     build_args: dict[str, str] | None = None,
 ) -> None:
@@ -725,6 +735,7 @@ def build_environment(
         verbose,
         build_args=build_args or None,
         platform=platform,
+        secrets=secrets,
         remote_cache=remote_cache,
     ):
         hud_console.error("Docker build failed")
@@ -1008,16 +1019,23 @@ def build_environment(
     for key, value in build_args.items():
         label_cmd.extend(["--build-arg", f"{key}={value}"])
 
+    # Add secrets to final image build (same as initial build)
+    for secret in secrets or []:
+        label_cmd.extend(["--secret", secret])
+
     label_cmd.append(str(env_dir))
 
     # Run rebuild using Docker's native output formatting
+    env = os.environ.copy()
+    if secrets:
+        env["DOCKER_BUILDKIT"] = "1"
     if verbose:
         # Show Docker's native output when verbose
-        result = subprocess.run(label_cmd, check=False)  # noqa: S603
+        result = subprocess.run(label_cmd, check=False, env=env)  # noqa: S603
     else:
         # Capture output for error reporting, but don't show unless it fails
         result = subprocess.run(  # noqa: S603
-            label_cmd, capture_output=True, text=True, check=False
+            label_cmd, capture_output=True, text=True, check=False, env=env
         )
 
     if result.returncode != 0:
@@ -1111,10 +1129,15 @@ def build_command(
     verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed output"),
     env_vars: dict[str, str] | None = None,
     platform: str | None = None,
+    secrets: list[str] | None = typer.Option(  # noqa: B008
+        None,
+        "--secret",
+        help=("Docker build secret (repeatable), e.g. --secret id=GITHUB_TOKEN,env=GITHUB_TOKEN"),
+    ),
     remote_cache: str | None = None,
     build_args: dict[str, str] | None = None,
 ) -> None:
     """Build a HUD environment and generate lock file."""
     build_environment(
-        directory, tag, no_cache, verbose, env_vars, platform, remote_cache, build_args
+        directory, tag, no_cache, verbose, env_vars, platform, secrets, remote_cache, build_args
     )

{hud_python-0.5.12 → hud_python-0.5.13}/hud/cli/eval.py

@@ -164,6 +164,7 @@ class EvalConfig(BaseModel):
         "auto_respond",
         "quiet",
         "gateway",
+        "taskset",
     }
     # Fields loaded from [agent] section
     _AGENT_FIELDS: ClassVar[set[str]] = {"allowed_tools", "disallowed_tools"}
@@ -184,6 +185,7 @@ class EvalConfig(BaseModel):
     remote: bool = False
     quiet: bool = False  # Suppress opening browser for eval links
     gateway: bool = False  # Use HUD Gateway for LLM API calls
+    taskset: str | None = None  # Taskset slug to associate job with
 
     # Base agent config (these merge with task's agent_config)
     allowed_tools: list[str] | None = None
@@ -655,16 +657,41 @@ async def _run_evaluation(cfg: EvalConfig) -> tuple[list[Any], list[Any]]:
         agent_kwargs = {
             k: v for k, v in agent_kwargs.items() if k not in ("api_key", "model_client")
         }
-        # Create a job ID for tracking
         import uuid
 
         from hud.datasets.utils import submit_rollouts
+        from hud.eval.manager import _send_job_enter
 
         job_id = str(uuid.uuid4())
         hud_console.info(
             f"Submitting {len(tasks)} task(s) for remote execution (job_id: {job_id})…"
         )
 
+        if cfg.taskset:
+            tasks_to_create = [t for t in tasks if not t.id]
+            tasks_data = (
+                [t.model_dump(mode="json", exclude_none=True) for t in tasks_to_create]
+                if tasks_to_create
+                else None
+            )
+            ids = await _send_job_enter(
+                job_id=job_id,
+                name=f"eval ({cfg.source})" if cfg.source else "eval",
+                variants=None,
+                group=cfg.group_size,
+                api_key=None,
+                taskset=cfg.taskset,
+                tasks=tasks_data,
+            )
+            if ids:
+                if len(ids) != len(tasks_to_create):
+                    hud_console.warning(
+                        f"Task count mismatch: sent {len(tasks_to_create)} tasks, "
+                        f"received {len(ids)} IDs. Some tasks may not be linked."
+                    )
+                for task_obj, task_version_id in zip(tasks_to_create, ids, strict=False):
+                    task_obj.id = task_version_id
+
         await submit_rollouts(
             tasks=tasks,
             job_id=job_id,
@@ -701,6 +728,7 @@ async def _run_evaluation(cfg: EvalConfig) -> tuple[list[Any], list[Any]]:
         max_concurrent=cfg.max_concurrent,
         group_size=cfg.group_size,
         quiet=cfg.quiet,
+        taskset=cfg.taskset,
     )
 
     # Show reward for single task
@@ -767,6 +795,9 @@ def eval_command(
     gateway: bool = typer.Option(
         False, "--gateway", "-g", help="Route LLM API calls through HUD Gateway"
     ),
+    taskset: str | None = typer.Option(
+        None, "--taskset", "-t", help="Taskset slug to associate job with"
+    ),
 ) -> None:
     """🚀 Run evaluation on datasets or individual tasks with agents.
 
@@ -801,6 +832,7 @@ def eval_command(
         byok=byok,
         quiet=quiet,
         gateway=gateway,
+        taskset=taskset,
     )
 
     # Find source if not provided

{hud_python-0.5.12 → hud_python-0.5.13}/hud/cli/tests/test_build.py

@@ -60,12 +60,12 @@ class TestIncrementVersion:
     def test_increment_minor(self):
         """Test incrementing minor version."""
         assert increment_version("1.2.3", "minor") == "1.3.0"
-        assert increment_version("0.5.12", "minor") == "0.6.0"
+        assert increment_version("0.5.13", "minor") == "0.6.0"
 
     def test_increment_major(self):
         """Test incrementing major version."""
         assert increment_version("1.2.3", "major") == "2.0.0"
-        assert increment_version("0.5.12", "major") == "1.0.0"
+        assert increment_version("0.5.13", "major") == "1.0.0"
 
     def test_increment_with_v_prefix(self):
         """Test incrementing version with v prefix."""

{hud_python-0.5.12 → hud_python-0.5.13}/hud/datasets/runner.py

@@ -29,6 +29,7 @@ async def run_dataset(
     max_concurrent: int = 30,
     group_size: int = 1,
     quiet: bool = True,
+    taskset: str | None = None,
 ) -> list[EvalContext]:
     """Run an agent on a dataset of tasks.
 
@@ -96,6 +97,7 @@ async def run_dataset(
         group=group_size,
         max_concurrent=max_concurrent,
         quiet=quiet,
+        taskset=taskset,
     ) as ctx:
         # Build agent params - use system_prompt from ctx (set from task.agent_config)
         final_agent_params = dict(agent_params or {})

{hud_python-0.5.12 → hud_python-0.5.13}/hud/environment/scenarios.py

@@ -382,6 +382,7 @@ class ScenarioMixin:
         self,
         name: str | None = None,
         description: str | None = None,
+        required_env_vars: list[str] | None = None,
     ) -> Callable[
         [Callable[..., AsyncGenerator[Any, None]]],
         Callable[..., AsyncGenerator[Any, None]],
@@ -396,19 +397,21 @@ class ScenarioMixin:
         Args:
             name: Optional name for the scenario (defaults to function name)
             description: Optional description of what the scenario does
+            required_env_vars: Optional list of environment variable names this scenario requires.
+                These are used by the HUD platform to check if users have configured the
+                necessary API keys/credentials before running this specific scenario.
 
         Example:
-            @env.scenario()
-            async def search_cats(url: str):
-                await env.call_tool("navigate", url=url)
-                yield "Find cat images"
-                result = await env.call_tool("count_cats")
-                yield float(result > 0)
+            @env.scenario(required_env_vars=["OPENAI_API_KEY"])
+            async def chat(query: str):
+                yield f"Answer this question: {query}"
+                # ... evaluate
+                yield 1.0
 
             # MCP client usage:
-            # 1. get_prompt("{env_name}:search_cats", {url: "..."}) -> prompt messages
+            # 1. get_prompt("{env_name}:chat", {query: "..."}) -> prompt messages
             # 2. agent runs...
-            # 3. read_resource("{env_name}:search_cats") -> {"reward": 0.95}
+            # 3. read_resource("{env_name}:chat") -> {"reward": 0.95}
         """
 
         def decorator(
@@ -568,6 +571,8 @@ class ScenarioMixin:
                 scenario_meta["code"] = source_code
             if prompt_args:
                 scenario_meta["arguments"] = prompt_args
+            if required_env_vars:
+                scenario_meta["required_env_vars"] = required_env_vars
 
             prompt = FunctionPrompt(
                 name=scenario_id,

{hud_python-0.5.12 → hud_python-0.5.13}/hud/eval/manager.py

@@ -56,14 +56,16 @@ def _get_eval_name(tasks: list[Task] | None = None) -> str:
     return "eval"
 
 
-def _send_job_enter(
+async def _send_job_enter(
     job_id: str,
     name: str,
     variants: dict[str, Any] | None,
     group: int,
     api_key: str | None,
-) -> None:
-    """Send job enter payload (sync request before traces start)."""
+    taskset: str | None = None,
+    tasks: list[dict[str, Any]] | None = None,
+) -> list[str] | None:
+    """Send job enter payload (async request before traces start)."""
     import httpx
 
     from hud.eval.types import JobEnterPayload
@@ -71,23 +73,35 @@ def _send_job_enter(
 
     api_key = api_key or settings.api_key
     if not settings.telemetry_enabled or not api_key:
-        return
+        return None
 
     payload = JobEnterPayload(
         name=name,
         variants=variants,
         group=group,
+        taskset=taskset,
+        tasks=tasks if taskset else None,  # only send tasks if taskset specified
     )
 
     try:
-        httpx.post(
-            f"{settings.hud_api_url}/trace/job/{job_id}/enter",
-            json=payload.model_dump(exclude_none=True),
-            headers={"Authorization": f"Bearer {api_key}"},
-            timeout=10.0,
-        )
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            resp = await client.post(
+                f"{settings.hud_api_url}/trace/job/{job_id}/enter",
+                json=payload.model_dump(exclude_none=True),
+                headers={"Authorization": f"Bearer {api_key}"},
+            )
+        if resp.is_success:
+            try:
+                data = resp.json()
+            except Exception:
+                return None
+            if isinstance(data, dict):
+                ids = data.get("task_version_ids")
+                if isinstance(ids, list) and all(isinstance(x, str) for x in ids):
+                    return ids
     except Exception as e:
         logger.warning("Failed to send job enter: %s", e)
+    return None
 
 
 @asynccontextmanager
@@ -105,6 +119,7 @@ async def run_eval(
     max_concurrent: int | None = None,
     trace: bool = True,
     quiet: bool = False,
+    taskset: str | None = None,
 ) -> AsyncGenerator[EvalContext, None]:
     """Standalone eval context manager.
 
@@ -235,13 +250,37 @@ async def run_eval(
 
     if total_evals == 1:
         if tasks:
+            # Even for single-task evals, --taskset requires a job_enter call so the run
+            # and task are linked to the taskset (via job_id + task_version_id).
+            job_id_for_run = job_id
+            if taskset:
+                eval_name = _get_eval_name(tasks=tasks)
+                if job_id_for_run is None:
+                    job_id_for_run = str(uuid.uuid4())
+
+                task_data = None
+                if not tasks[0].id:
+                    task_data = [tasks[0].model_dump(mode="json", exclude_none=True)]
+
+                created_task_version_ids = await _send_job_enter(
+                    job_id=job_id_for_run,
+                    name=eval_name,
+                    variants=variants,
+                    group=group,
+                    api_key=api_key,
+                    taskset=taskset,
+                    tasks=task_data,
+                )
+                if created_task_version_ids and not tasks[0].id:
+                    tasks[0].id = created_task_version_ids[0]
+
             # Single task - use EvalContext.from_task()
             ctx = EvalContext.from_task(
                 tasks[0],
                 name=name,
                 trace_id=trace_id,
                 api_key=api_key,
-                job_id=job_id,
+                job_id=job_id_for_run,
                 group_id=group_id,
                 variants=variant_combos[0],
                 code_snippet=code_snippet,
@@ -273,13 +312,41 @@ async def run_eval(
         job_url = f"https://hud.ai/jobs/{implicit_job_id}"
 
         # Send job enter (sync request before traces start)
-        _send_job_enter(
+        # Serialize tasks for auto-add to taskset (only tasks without existing backend id).
+        # For v5 scenario tasks, the backend task_version_id is carried in Task.id.
+        tasks_data = None
+        tasks_to_create: list[Task] = []
+        if taskset and tasks:
+            tasks_to_create = [t for t in tasks if not t.id]
+            tasks_data = (
+                [t.model_dump(mode="json", exclude_none=True) for t in tasks_to_create]
+                if tasks_to_create
+                else None
+            )
+        created_task_version_ids = await _send_job_enter(
             job_id=implicit_job_id,
             name=eval_name,
             variants=variants,
             group=group,
             api_key=api_key,
+            taskset=taskset,
+            tasks=tasks_data,
         )
+        if created_task_version_ids and tasks_to_create:
+            # Assign backend IDs back onto the in-memory tasks so trace enter includes
+            # task_version_id.
+            # Platform guarantees ordered one-to-one mapping, but warn if counts differ.
+            if len(created_task_version_ids) != len(tasks_to_create):
+                logger.warning(
+                    "Task count mismatch: sent %d tasks, received %d IDs. "
+                    "Some tasks may not be linked to the taskset.",
+                    len(tasks_to_create),
+                    len(created_task_version_ids),
+                )
+            for task_obj, task_version_id in zip(
+                tasks_to_create, created_task_version_ids, strict=False
+            ):
+                task_obj.id = task_version_id
 
         # Print job URL (not individual trace URLs)
         if not quiet:

{hud_python-0.5.12 → hud_python-0.5.13}/hud/eval/types.py

@@ -53,6 +53,8 @@ class JobEnterPayload(BaseModel):
     name: str | None = None
     variants: dict[str, Any] | None = None  # Full variant config
     group: int | None = None
+    taskset: str | None = None  # taskset slug to associate job with
+    tasks: list[dict[str, Any]] | None = None  # task definitions to add to taskset
 
 
 __all__ = [

{hud_python-0.5.12 → hud_python-0.5.13}/hud/utils/tests/test_version.py

@@ -5,4 +5,4 @@ def test_import():
     """Test that the package can be imported."""
     import hud
 
-    assert hud.__version__ == "0.5.12"
+    assert hud.__version__ == "0.5.13"

{hud_python-0.5.12 → hud_python-0.5.13}/hud/version.py

@@ -4,4 +4,4 @@ Version information for the HUD SDK.
 
 from __future__ import annotations
 
-__version__ = "0.5.12"
+__version__ = "0.5.13"

{hud_python-0.5.12 → hud_python-0.5.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "hud-python"
-version = "0.5.12"
+version = "0.5.13"
 description = "SDK for the HUD platform."
 readme = "README.md"
 requires-python = ">=3.11, <3.13"
@@ -140,7 +140,7 @@ dev = [
     "dotenv>=0.9.9",
     # Testing and linting
     "ruff >=0.11.8",
-    "pytest >=8.1.1,<9",
+    "pytest >=8.1.1",
     "pytest-asyncio",
     "pytest-mock",
     "pytest-cov",