hud-python 0.4.45__py3-none-any.whl → 0.5.1__py3-none-any.whl

hud/cli/flows/tasks.py

@@ -4,21 +4,17 @@ import json
 import logging
 import re
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 import typer
 import yaml
 
 from hud.cli.push import push_environment
 from hud.cli.utils.docker import require_docker_running
-from hud.cli.utils.env_check import ensure_built, find_environment_dir
+from hud.cli.utils.env_check import find_environment_dir
 from hud.cli.utils.registry import extract_name_and_tag
+from hud.datasets import load_tasks
 from hud.utils.hud_console import hud_console
-from hud.utils.tasks import load_tasks
-
-if TYPE_CHECKING:
-    from hud.types import Task
-
 
 logger = logging.getLogger(__name__)
 
@@ -29,11 +25,11 @@ def _is_remote_url(url: str) -> bool:
     return bool(re.match(r"^(https?:\/\/)?(www\.)?[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,}(\/\S*)?$", url))
 
 
-def _validate_tasks(tasks: list[Task]) -> bool:
+def _validate_tasks(tasks: list[dict[str, Any]]) -> bool:
     """Validate the tasks file: return True if tasks already reference a remote MCP URL.
 
     A task is considered remote if any "url" field anywhere inside mcp_config
-    is a valid remote URL (e.g., https://mcp.hud.so/v3/mcp).
+    is a valid remote URL (e.g., https://mcp.hud.ai/v3/mcp).
     """
 
     def _has_remote_url(obj: Any) -> bool:
@@ -50,13 +46,15 @@ def _validate_tasks(tasks: list[Task]) -> bool:
         return False
 
     for task in tasks:
-        cfg = task.mcp_config or {}
+        cfg = task.get("mcp_config") or {}
         if not _has_remote_url(cfg):
             return False
     return True
 
 
-def _ensure_pushed(env_dir: Path, lock_data: dict[str, Any]) -> dict[str, Any]:
+def _ensure_pushed(
+    env_dir: Path, lock_data: dict[str, Any], check_docker: bool = True
+) -> dict[str, Any]:
     """Ensure the environment is pushed to a registry; return updated lock data."""
     pushed = bool(lock_data.get("push"))
     if not pushed:
@@ -64,7 +62,8 @@ def _ensure_pushed(env_dir: Path, lock_data: dict[str, Any]) -> dict[str, Any]:
         if not hud_console.confirm("Push to a registry now (runs 'hud push')?", default=True):
             raise typer.Exit(1)
         # Check Docker availability before attempting a push
-        require_docker_running()
+        if check_docker:
+            require_docker_running()
 
         # If Docker or login is not configured, the push function will fail and halt.
         push_environment(str(env_dir), yes=True)
@@ -78,29 +77,41 @@ def _ensure_pushed(env_dir: Path, lock_data: dict[str, Any]) -> dict[str, Any]:
 
 
 def _derive_remote_image(lock_data: dict[str, Any]) -> str:
-    """Derive org/name:tag from lock file for MCP header.
+    """Derive org/name:tag from lock file for remote MCP header.
 
-    Preference order:
-    1) lock_data["push"]["image_with_tag"] if present
-    2) Derive from lock_data["image"] (may be a digest; falls back to latest)
+    Preference order (new lock first, then legacy):
+    1) lock_data["push"]["image_with_tag"] (exact org/name:tag that was pushed)
+    2) lock_data["images"]["local"] (base name with internal version)
+    3) lock_data["image"] (legacy field; may contain tag or digest)
     """
-    push_info = lock_data.get("push", {}) if isinstance(lock_data, dict) else {}
+    if not isinstance(lock_data, dict):  # Defensive
+        raise typer.Exit(1)
 
-    # 1) Exact image_with_tag if present
-    pushed_with_tag = str(push_info.get("image_with_tag", "")).strip()
+    # 1) Prefer the exact image that was pushed (org/name:tag)
+    push_info = lock_data.get("push") or {}
+    pushed_with_tag = str(push_info.get("image_with_tag") or "").strip()
     if pushed_with_tag:
         name, tag = extract_name_and_tag(pushed_with_tag)
         return f"{name}:{tag}"
 
-    # Base name always comes from lock_data.image to preserve org/repo
-    image_ref = str(lock_data.get("image", "")).strip()
-    if not image_ref:
-        raise typer.Exit(1)
-    name, tag = extract_name_and_tag(image_ref)
-    return f"{name}:{tag}"
+    # 2) Fall back to the local tag recorded in the new lock schema
+    images = lock_data.get("images") or {}
+    local_image = str(images.get("local") or "").strip()
+    if local_image:
+        name, tag = extract_name_and_tag(local_image)
+        return f"{name}:{tag}"
 
+    # 3) Legacy top-level image field
+    legacy_image = str(lock_data.get("image") or "").strip()
+    if legacy_image:
+        name, tag = extract_name_and_tag(legacy_image)
+        return f"{name}:{tag}"
+
+    # If none of the above exist, we cannot derive an image
+    raise typer.Exit(1)
 
-def _extract_existing_images(tasks: list[Task]) -> set[str]:
+
+def _extract_existing_images(tasks: list[dict[str, Any]]) -> set[str]:
     """Extract all Mcp-Image references from tasks."""
     images = set()
 
@@ -119,8 +130,9 @@ def _extract_existing_images(tasks: list[Task]) -> set[str]:
                 _extract_from_obj(item)
 
     for task in tasks:
-        if task.mcp_config:
-            _extract_from_obj(task.mcp_config)
+        mcp_config = task.get("mcp_config")
+        if mcp_config:
+            _extract_from_obj(mcp_config)
 
     return images
 
@@ -183,6 +195,63 @@ def _extract_dotenv_api_key_vars(env_dir: Path) -> set[str]:
     return detected
 
 
+def _extract_env_vars_from_docker_args(args: list[str]) -> set[str]:
+    """Extract environment variable names from docker run arguments.
+
+    Parses args like: ["run", "--rm", "-i", "-e", "API_KEY=value", "-e", "TOKEN", "image:tag"]
+    Returns set of env var names (not values).
+    """
+    env_vars: set[str] = set()
+    i = 0
+    while i < len(args):
+        arg = args[i]
+
+        # Check for -e or --env flags
+        if arg in ("-e", "--env"):
+            if i + 1 < len(args):
+                env_spec = args[i + 1]
+                # Could be "KEY=value" or just "KEY"
+                var_name = env_spec.split("=", 1)[0].strip()
+                if var_name:
+                    env_vars.add(var_name)
+                i += 2
+                continue
+        # Check for --env=KEY=value format
+        elif arg.startswith("--env="):
+            env_spec = arg[6:]  # Remove "--env=" prefix
+            var_name = env_spec.split("=", 1)[0].strip()
+            if var_name:
+                env_vars.add(var_name)
+
+        i += 1
+
+    env_vars.discard("HUD_API_KEY")
+    return env_vars
+
+
+def _extract_vars_from_task_configs(raw_tasks: list[dict[str, Any]]) -> set[str]:
+    """Extract environment variable names from docker run commands in task mcp_configs."""
+    all_env_vars: set[str] = set()
+
+    for task in raw_tasks:
+        mcp_config = task.get("mcp_config", {})
+
+        # Iterate through all server configs
+        for server_config in mcp_config.values():
+            if not isinstance(server_config, dict):
+                continue
+
+            command = server_config.get("command", "")
+            args = server_config.get("args", [])
+
+            # Only process docker run commands
+            if command == "docker" and "run" in args:
+                env_vars = _extract_env_vars_from_docker_args(args)
+                all_env_vars.update(env_vars)
+
+    return all_env_vars
+
+
 def convert_tasks_to_remote(tasks_file: str) -> str:
     """Convert a local tasks file to remote MCP tasks and return new filename.
 
@@ -190,17 +259,18 @@ def convert_tasks_to_remote(tasks_file: str) -> str:
     1) Find env dir; ensure built (hud.lock.yaml), otherwise build
     2) Ensure pushed to registry, otherwise push
     3) Check for outdated images in existing task configurations
-    4) Create remote_[tasks].json with mcp_config pointing to mcp.hud.so and Mcp-Image
+    4) Create remote_[tasks].json with mcp_config pointing to mcp.hud.ai and Mcp-Image
     5) Return the new tasks file path
     """
     tasks_path = Path(tasks_file).resolve()
 
-    # Load validated tasks for decision-making (may resolve env vars)
-    tasks: list[Task] = load_tasks(str(tasks_path))  # type: ignore[assignment]
-
-    # Load raw tasks to preserve placeholders when writing back to disk
+    # Load raw tasks - we work with dicts directly to preserve placeholders
+    # when writing back to disk (e.g., ${HUD_API_KEY})
     raw_tasks: list[dict[str, Any]] = load_tasks(str(tasks_path), raw=True)  # type: ignore[assignment]
 
+    # Use the same raw tasks for validation (they have mcp_config structure)
+    tasks = raw_tasks
+
     # Ensure HUD_API_KEY is available: prefer process env, else load from env_dir/.env
     from hud.settings import settings
 
@@ -224,9 +294,24 @@ def convert_tasks_to_remote(tasks_file: str) -> str:
         hud_console.hint("Ensure you're in or near your environment folder before running 'hud rl'")
         raise typer.Exit(1)
 
-    # Ensure built and pushed
-    lock_data = ensure_built(env_dir, interactive=True)
-    lock_data = _ensure_pushed(env_dir, lock_data)
+    # For convert command, we don't need Docker running - just check for lock file
+    # This avoids showing Docker-related messages during conversion
+    lock_path = env_dir / "hud.lock.yaml"
+    if not lock_path.exists():
+        hud_console.error("No hud.lock.yaml found. The environment needs to be built first.")
+        hud_console.info("Run 'hud build' in the environment directory to build it.")
+        raise typer.Exit(1)
+
+    # Load lock data directly
+    try:
+        with open(lock_path) as f:
+            lock_data: dict[str, Any] = yaml.safe_load(f) or {}
+    except Exception as e:
+        hud_console.error(f"Failed to read hud.lock.yaml: {e}")
+        raise typer.Exit(1) from e
+
+    # Check if pushed - don't check Docker for convert command
+    lock_data = _ensure_pushed(env_dir, lock_data, check_docker=False)
 
     # Derive remote image name org/name:tag
     remote_image = _derive_remote_image(lock_data)
@@ -297,20 +382,35 @@ def convert_tasks_to_remote(tasks_file: str) -> str:
         hud_console.success(f"Updated {tasks_path.name} with latest image: {remote_image}")
         return str(tasks_path)
 
-    # Extract additional API key headers from lock and suggest from .env
+    # Extract environment variables from multiple sources:
+    # 1. Lock file (authoritative for required env vars)
     provided_keys = _extract_api_key_vars(lock_data)
+
+    # 2. Task configs (docker run -e flags)
+    task_env_vars = _extract_vars_from_task_configs(raw_tasks)
+
+    # 3. .env file (detect API-like vars)
     dotenv_keys = _extract_dotenv_api_key_vars(env_dir)
 
-    # If .env contains API-like vars not in lock, offer to include them
-    missing = sorted(dotenv_keys - provided_keys)
+    # Combine: lock file vars + task config vars, then check for missing from .env
+    all_detected = provided_keys | task_env_vars
+
+    # If .env contains API-like vars not yet included, offer to add them
+    missing = sorted(dotenv_keys - all_detected)
     if missing:
         names_preview = ", ".join(missing)
         prompt = (
             f"Detected env vars in .env that look like API keys: {names_preview}.\n"
             "Include them as remote headers (values will be ${VAR} placeholders)?"
         )
-        if hud_console.confirm(prompt, default=True):
-            provided_keys.update(missing)
+        if not hud_console.confirm(prompt, default=True):
+            # User cancelled - exit without creating the file
+            hud_console.info("Conversion cancelled by user")
+            raise typer.Exit(0)
+        all_detected.update(missing)
+
+    # Final set of env vars to convert to headers
+    provided_keys = all_detected
 
     extra_api_key_headers: dict[str, str] = {}
     for var_name in provided_keys:
@@ -344,10 +444,10 @@ def convert_tasks_to_remote(tasks_file: str) -> str:
     tasks_payload: list[dict[str, Any]] = []
     for t in tasks:
         item: dict[str, Any] = {
-            "prompt": t.prompt,
+            "prompt": t.get("prompt"),
             "mcp_config": {
                 "hud": {
-                    "url": "https://mcp.hud.so/v3/mcp",
+                    "url": settings.hud_mcp_url,
                     "headers": {
                         "Authorization": "Bearer ${HUD_API_KEY}",
                         "Mcp-Image": remote_image,
@@ -360,18 +460,16 @@ def convert_tasks_to_remote(tasks_file: str) -> str:
         item["mcp_config"]["hud"]["headers"].update(extra_api_key_headers)
 
         # Optional fields, omit Nones
-        if t.setup_tool is not None:
-            item["setup_tool"] = _simplify_tool_call(t.setup_tool)
-        if t.evaluate_tool is not None:
-            item["evaluate_tool"] = _simplify_tool_call(t.evaluate_tool)
-        if t.agent_tools is not None:
-            item["agent_tools"] = t.agent_tools
-        if t.system_prompt is not None:
-            item["system_prompt"] = t.system_prompt
-        if t.metadata:
-            item["metadata"] = t.metadata
-        if t.id is not None:
-            item["id"] = t.id
+        if t.get("setup_tool") is not None:
+            item["setup_tool"] = _simplify_tool_call(t["setup_tool"])
+        if t.get("evaluate_tool") is not None:
+            item["evaluate_tool"] = _simplify_tool_call(t["evaluate_tool"])
+        if t.get("agent_config") is not None:
+            item["agent_config"] = t["agent_config"]
+        if t.get("metadata"):
+            item["metadata"] = t["metadata"]
+        if t.get("id") is not None:
+            item["id"] = t["id"]
 
         tasks_payload.append(item)
 
@@ -382,6 +480,5 @@ def convert_tasks_to_remote(tasks_file: str) -> str:
         f.write("\n")
 
     hud_console.success(f"Created remote tasks file: {remote_path.name}")
-    hud_console.hint("Proceeding with RL training on the remote environment")
 
     return str(remote_path)

hud/cli/flows/templates.py

@@ -0,0 +1,151 @@
+"""Templates for hud init command."""
+
+DOCKERFILE_HUD = """\
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends curl \\
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY pyproject.toml uv.lock* ./
+RUN pip install uv && uv sync --frozen --no-dev 2>/dev/null || uv sync --no-dev
+COPY . .
+
+# Most of the time this command should not change, except if you change your env path
+# or launch some other service before running the environment
+CMD ["uv", "run", "python", "-m", "hud", "dev", "env:env", "--stdio"]
+"""
+
+# fmt: off
+ENV_PY = '''\
+"""{env_name} - HUD Environment"""
+
+import asyncio
+
+import hud
+from hud.settings import settings
+from openai import AsyncOpenAI, Omit
+from hud.environment import Environment
+
+env = Environment("{env_name}")
+
+
+# =============================================================================
+# 1. TOOLS - Functions the agent can call
+# =============================================================================
+
+@env.tool()
+def count_letter(text: str, letter: str) -> int:
+    """Count occurrences of a letter in text."""
+    return text.lower().count(letter.lower())
+
+
+# =============================================================================
+# 2. SCRIPTS - Define prompts and evaluation logic
+# =============================================================================
+
+@env.scenario("count")
+async def count_script(sentence: str, letter: str, fmt: str = "integer"):
+    """Agent must count a letter. We check if they got it right."""
+    # Yield the prompt, receive the agent's final answer
+    answer = yield f"How many times does '{{letter}}' appear in: '{{sentence}}'? Format: {{fmt}}."
+
+    # Score: 1.0 if correct, 0.0 otherwise
+    correct = str(sentence.lower().count(letter.lower()))
+    yield correct in answer
+
+
+# =============================================================================
+# 3. CONNECT EXISTING SERVERS (optional)
+# =============================================================================
+
+# --- FastAPI app ---
+# from my_app import app
+# env.connect_fastapi(app)
+
+# --- FastMCP / MCPServer ---
+# from my_server import mcp
+# env.connect_server(mcp)
+
+# --- OpenAPI spec (URL or file path) ---
+# env.connect_openapi("https://api.example.com/openapi.json")
+
+# --- MCP config (stdio or SSE) ---
+# env.connect_mcp_config({{
+#     "my-server": {{"command": "uvx", "args": ["some-mcp-server"]}}
+# }})
+
+# --- HUD hub (requires deployment, see below) ---
+# env.connect_hub("my-org/my-env", prefix="remote")
+
+
+# =============================================================================
+# TEST - Run with: python env.py
+# =============================================================================
+
+async def test():
+    client = AsyncOpenAI(
+        base_url=settings.hud_gateway_url,
+        api_key=settings.api_key,
+    )
+
+    # Create a task from the scenario
+    task = env("count", sentence="Strawberry world", letter="r")
+
+    # Test with and without tools
+    async with hud.eval(task, variants={{"tools": [True, False]}}) as ctx:
+        response = await client.chat.completions.create(
+            model="gpt-4o-mini",
+            messages=[{{"role": "user", "content": ctx.prompt}}],
+            tools=ctx.as_openai_chat_tools() if ctx.variants["tools"] else Omit(),
+        )
+
+        # Handle tool calls if present
+        message = response.choices[0].message
+        if message.tool_calls:
+            result = await ctx.call_tool(message.tool_calls[0])
+            answer = str(result["content"])
+        else:
+            answer = message.content
+
+        await ctx.submit(answer or "")
+
+
+if __name__ == "__main__":
+    asyncio.run(test())
+
+
+# =============================================================================
+# DEPLOYMENT
+# =============================================================================
+# To deploy this environment on HUD:
+#
+# 1. Push this repo to GitHub
+# 2. Go to hud.ai -> New -> Environment
+# 3. Choose "From GitHub URL" and paste your repo URL
+# 4. This deploys the environment for remote connection
+#
+# Once deployed, connect to it from other environments:
+#   env.connect_hub("{env_name}")
+#
+# Remote deployment enables:
+# - Parallelized evaluations (run many agents simultaneously)
+# - Training data collection at scale
+# - Shared environments across team members
+#
+# Note: The test() function above is just for local testing.
+# It's not required for the deployed environment.
+'''
+# fmt: on
+
+PYPROJECT_TOML = """\
+[project]
+name = "{name}"
+version = "0.1.0"
+requires-python = ">=3.10"
+dependencies = ["hud-python", "openai"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+"""

hud/cli/flows/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for CLI flows."""

hud/cli/flows/tests/test_dev.py

@@ -0,0 +1,126 @@
+"""Tests for CLI flows dev module."""
+
+from __future__ import annotations
+
+import base64
+import json
+from unittest import mock
+
+import pytest
+
+from hud.cli.flows.dev import generate_cursor_deeplink
+
+
+class TestGenerateCursorDeeplink:
+    """Test Cursor deeplink generation."""
+
+    def test_generate_deeplink_basic(self):
+        """Test basic deeplink generation."""
+        result = generate_cursor_deeplink("my-server", 8000)
+
+        assert result.startswith("cursor://anysphere.cursor-deeplink/mcp/install?")
+        assert "name=my-server" in result
+        assert "config=" in result
+
+    def test_generate_deeplink_config_content(self):
+        """Test that config contains correct URL."""
+        result = generate_cursor_deeplink("test-server", 9999)
+
+        # Extract and decode the config
+        config_part = result.split("config=")[1]
+        decoded = base64.b64decode(config_part).decode()
+        config = json.loads(decoded)
+
+        assert config["url"] == "http://localhost:9999/mcp"
+
+    def test_generate_deeplink_different_ports(self):
+        """Test deeplink generation with different ports."""
+        result_8000 = generate_cursor_deeplink("server", 8000)
+        result_3000 = generate_cursor_deeplink("server", 3000)
+
+        # Decode configs
+        config_8000 = json.loads(base64.b64decode(result_8000.split("config=")[1]))
+        config_3000 = json.loads(base64.b64decode(result_3000.split("config=")[1]))
+
+        assert "8000" in config_8000["url"]
+        assert "3000" in config_3000["url"]
+
+    def test_generate_deeplink_special_characters_in_name(self):
+        """Test deeplink with special characters in server name."""
+        # Server name with special characters should still work
+        result = generate_cursor_deeplink("my-cool_server.v2", 8000)
+
+        assert "name=my-cool_server.v2" in result
+
+
+class TestCreateDynamicTrace:
+    """Test dynamic trace creation."""
+
+    @pytest.mark.asyncio
+    @mock.patch("hud.cli.flows.dev.make_request")
+    @mock.patch("hud.cli.utils.git.get_git_info")
+    @mock.patch("hud.cli.flows.dev.settings")
+    async def test_create_dynamic_trace_success(self, mock_settings, mock_git, mock_request):
+        """Test successful trace creation."""
+        from hud.cli.flows.dev import create_dynamic_trace
+
+        mock_settings.hud_api_url = "https://api.hud.ai"
+        mock_settings.api_key = "test-key"
+        mock_git.return_value = {"remote_url": "https://github.com/user/repo"}
+        mock_request.return_value = {"id": "trace-123"}
+
+        trace_id, url = await create_dynamic_trace(
+            mcp_config={"server": {"url": "http://localhost:8000"}},
+            build_status=True,
+            environment_name="test-env",
+        )
+
+        assert trace_id == "trace-123"
+        assert url == "https://hud.ai/trace/trace-123"
+        mock_request.assert_called_once()
+
+    @pytest.mark.asyncio
+    @mock.patch("hud.cli.flows.dev.make_request")
+    @mock.patch("hud.cli.utils.git.get_git_info")
+    @mock.patch("hud.cli.flows.dev.settings")
+    async def test_create_dynamic_trace_no_git(self, mock_settings, mock_git, mock_request):
+        """Test trace creation without git info."""
+        from hud.cli.flows.dev import create_dynamic_trace
+
+        mock_settings.hud_api_url = "https://api.hud.ai"
+        mock_settings.api_key = "test-key"
+        mock_git.return_value = {}  # No remote_url
+        mock_request.return_value = {"id": "trace-456"}
+
+        trace_id, _ = await create_dynamic_trace(
+            mcp_config={"server": {"url": "http://localhost:8000"}},
+            build_status=False,
+            environment_name="test-env",
+        )
+
+        assert trace_id == "trace-456"
+        # Verify git_info was not included in payload
+        call_args = mock_request.call_args
+        assert "git_info" not in call_args.kwargs.get("json", {})
+
+    @pytest.mark.asyncio
+    @mock.patch("hud.cli.flows.dev.make_request")
+    @mock.patch("hud.cli.utils.git.get_git_info")
+    @mock.patch("hud.cli.flows.dev.settings")
+    async def test_create_dynamic_trace_api_error(self, mock_settings, mock_git, mock_request):
+        """Test trace creation when API fails."""
+        from hud.cli.flows.dev import create_dynamic_trace
+
+        mock_settings.hud_api_url = "https://api.hud.ai"
+        mock_settings.api_key = "test-key"
+        mock_git.return_value = {}
+        mock_request.side_effect = Exception("API Error")
+
+        trace_id, url = await create_dynamic_trace(
+            mcp_config={"server": {}},
+            build_status=True,
+            environment_name="test-env",
+        )
+
+        assert trace_id is None
+        assert url is None