hud-python 0.4.35__py3-none-any.whl → 0.4.37__py3-none-any.whl

hud/cli/eval.py

@@ -5,15 +5,18 @@ from __future__ import annotations
 import asyncio
 import logging
 from pathlib import Path
-from typing import Any, Literal
+from typing import TYPE_CHECKING, Any, Literal
 
 import typer
 
 import hud
+from hud.cli.utils.env_check import ensure_built, find_environment_dir
 from hud.settings import settings
 from hud.utils.group_eval import display_group_statistics, run_tasks_grouped
 from hud.utils.hud_console import HUDConsole
 
+if TYPE_CHECKING:
+    from hud.types import Task
 logger = logging.getLogger(__name__)
 hud_console = HUDConsole()
 
@@ -27,7 +30,7 @@ def get_available_models() -> list[dict[str, str | None]]:
     try:
         from hud.cli.rl import rl_api
 
-        hud_console.info("Fetching your models from https://app.hud.so/models")
+        hud_console.info("Fetching your models from https://hud.so/models")
         models = rl_api.list_models()
 
         # Filter for ready models only and sort by recency
@@ -66,7 +69,7 @@ def get_available_models() -> list[dict[str, str | None]]:
 
 
 def build_agent(
-    agent_type: Literal["claude", "openai", "vllm"],
+    agent_type: Literal["claude", "openai", "vllm", "litellm"],
     *,
     model: str | None = None,
     allowed_tools: list[str] | None = None,
@@ -138,6 +141,22 @@ def build_agent(
         else:
             return OperatorAgent(verbose=verbose)
 
+    elif agent_type == "litellm":
+        try:
+            from hud.agents.lite_llm import LiteAgent
+        except ImportError as e:
+            hud_console.error(
+                "LiteLLM agent dependencies are not installed. "
+                "Please install with: pip install 'hud-python[agent]'"
+            )
+            raise typer.Exit(1) from e
+
+        return LiteAgent(
+            model_name=model or "gpt-4o-mini",
+            allowed_tools=allowed_tools,
+            verbose=verbose,
+        )
+
     # Fallback Claude agent (Anthropic)
     try:
         from hud.agents import ClaudeAgent
@@ -166,7 +185,7 @@ def build_agent(
 async def run_single_task(
     source: str,
     *,
-    agent_type: Literal["claude", "openai", "vllm"] = "claude",
+    agent_type: Literal["claude", "openai", "vllm", "litellm"] = "claude",
     model: str | None = None,
     allowed_tools: list[str] | None = None,
     max_steps: int = 10,
@@ -192,7 +211,16 @@ async def run_single_task(
         hud_console.info("📊 Loading task file…")
 
         # Use unified loader for both JSON and JSONL
-        tasks = load_tasks(str(path))
+        tasks: list[Task] = load_tasks(str(path))  # type: ignore[assignment]
+
+        # If tasks reference a local environment (nearby), ensure it's built/up-to-date.
+        try:
+            env_dir = find_environment_dir(path)
+            if env_dir is not None:
+                # Non-interactive for eval; warn but don't block
+                ensure_built(env_dir, interactive=True)
+        except Exception as e:
+            hud_console.debug(f"Eval preflight env check skipped: {e}")
 
         # Single task - use the first (and only) task
         task = tasks[0]
@@ -200,7 +228,7 @@ async def run_single_task(
     else:
         # Load from HuggingFace dataset or non-file source
         hud_console.info(f"📊 Loading tasks from: {source}…")
-        tasks = load_tasks(source)
+        tasks: list[Task] = load_tasks(source)  # type: ignore[assignment]
 
         if not tasks:
             hud_console.error(f"No tasks found in: {source}")
@@ -248,6 +276,16 @@ async def run_single_task(
             agent_config = {"verbose": verbose}
             if allowed_tools:
                 agent_config["allowed_tools"] = allowed_tools
+        elif agent_type == "litellm":
+            from hud.agents.lite_llm import LiteAgent
+
+            agent_class = LiteAgent
+            agent_config = {
+                "model_name": model or "gpt-4o-mini",
+                "verbose": verbose,
+            }
+            if allowed_tools:
+                agent_config["allowed_tools"] = allowed_tools
         else:
             from hud.agents import ClaudeAgent
 
@@ -292,7 +330,7 @@ async def run_single_task(
 async def run_full_dataset(
     source: str,
     *,
-    agent_type: Literal["claude", "openai", "vllm"] = "claude",
+    agent_type: Literal["claude", "openai", "vllm", "litellm"] = "claude",
     model: str | None = None,
     allowed_tools: list[str] | None = None,
     max_concurrent: int = 30,
@@ -322,7 +360,7 @@ async def run_full_dataset(
 
     # Load tasks using unified loader
     hud_console.info(f"📊 Loading tasks from: {source}…")
-    tasks = load_tasks(source)
+    tasks: list[Task] = load_tasks(source)  # type: ignore[assignment]
 
     if not tasks:
         hud_console.error(f"No tasks found in: {source}")
@@ -385,6 +423,25 @@ async def run_full_dataset(
         if allowed_tools:
             agent_config["allowed_tools"] = allowed_tools
 
+    elif agent_type == "litellm":
+        try:
+            from hud.agents.lite_llm import LiteAgent
+
+            agent_class = LiteAgent
+        except ImportError as e:
+            hud_console.error(
+                "LiteLLM agent dependencies are not installed. "
+                "Please install with: pip install 'hud-python[agent]'"
+            )
+            raise typer.Exit(1) from e
+
+        agent_config = {
+            "model_name": model or "gpt-4o-mini",
+            "verbose": verbose,
+        }
+        if allowed_tools:
+            agent_config["allowed_tools"] = allowed_tools
+
     else:
         try:
             from hud.agents import ClaudeAgent
@@ -501,10 +558,10 @@ def eval_command(
         "--full",
         help="Run the entire dataset (omit for single-task debug mode)",
     ),
-    agent: Literal["claude", "openai", "vllm"] = typer.Option(
+    agent: Literal["claude", "openai", "vllm", "litellm"] = typer.Option(
         "claude",
         "--agent",
-        help="Agent backend to use (claude, openai, or vllm for local server)",
+        help="Agent backend to use (claude, openai, vllm for local server, or litellm)",
     ),
     model: str | None = typer.Option(
         None,
@@ -546,6 +603,12 @@ def eval_command(
         "--verbose",
         help="Enable verbose output from the agent",
     ),
+    very_verbose: bool = typer.Option(
+        False,
+        "--very-verbose",
+        "-vv",
+        help="Enable debug-level logs for maximum visibility",
+    ),
     vllm_base_url: str | None = typer.Option(
         None,
         "--vllm-base-url",
@@ -595,17 +658,34 @@ def eval_command(
     """
     from hud.settings import settings
 
+    if very_verbose:
+        logging.basicConfig(
+            level=logging.DEBUG,
+            format="%(asctime)s - %(name)s - %(message)s",
+            datefmt="%H:%M:%S",
+        )
+        logging.getLogger("hud.agents").setLevel(logging.DEBUG)
+        logging.getLogger("hud.agents.base").setLevel(logging.DEBUG)
+    elif verbose:
+        logging.basicConfig(
+            level=logging.INFO,
+            format="%(asctime)s - %(name)s - %(message)s",
+            datefmt="%H:%M:%S",
+        )
+        logging.getLogger("hud.agents").setLevel(logging.INFO)
+        logging.getLogger("hud.agents.base").setLevel(logging.INFO)
+
     # Check for required API keys
     if agent == "claude":
         if not settings.anthropic_api_key:
             hud_console.error("ANTHROPIC_API_KEY is required for Claude agent")
             hud_console.info(
-                "Set it in your environment or .env file: ANTHROPIC_API_KEY=your-key-here"
+                "Set it in your environment or run: hud set ANTHROPIC_API_KEY=your-key-here"
             )
             raise typer.Exit(1)
     elif agent == "openai" and not settings.openai_api_key:
         hud_console.error("OPENAI_API_KEY is required for OpenAI agent")
-        hud_console.info("Set it in your environment or .env file: OPENAI_API_KEY=your-key-here")
+        hud_console.info("Set it in your environment or run: hud set OPENAI_API_KEY=your-key-here")
         raise typer.Exit(1)
     elif agent == "vllm":
         if model:
@@ -617,7 +697,8 @@ def eval_command(
     # Check for HUD_API_KEY if using HUD services
     if not settings.api_key:
         hud_console.warning("HUD_API_KEY not set. Some features may be limited.")
-        hud_console.info("Get your API key at: https://app.hud.so")
+        hud_console.info("Get your API key at: https://hud.so")
+        hud_console.info("Set it in your environment or run: hud set HUD_API_KEY=your-key-here")
 
     # Parse allowed tools
     allowed_tools_list = (
@@ -641,7 +722,7 @@ def eval_command(
                 parallel=parallel,
                 max_workers=max_workers,
                 max_concurrent_per_worker=max_concurrent_per_worker,
-                verbose=verbose,
+                verbose=very_verbose or verbose,
                 vllm_base_url=vllm_base_url,
                 group_size=group_size,
             )
@@ -654,7 +735,7 @@ def eval_command(
                 model=model,
                 allowed_tools=allowed_tools_list,
                 max_steps=max_steps,
-                verbose=verbose,
+                verbose=very_verbose or verbose,
                 vllm_base_url=vllm_base_url,
                 group_size=group_size,
             )

hud/cli/flows/tasks.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import logging
 import re
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
@@ -8,10 +9,9 @@ from typing import TYPE_CHECKING, Any
 import typer
 import yaml
 
-from hud.cli.build import build_environment
 from hud.cli.push import push_environment
 from hud.cli.utils.docker import require_docker_running
-from hud.cli.utils.environment import is_environment_directory
+from hud.cli.utils.env_check import ensure_built, find_environment_dir
 from hud.cli.utils.registry import extract_name_and_tag
 from hud.utils.hud_console import hud_console
 from hud.utils.tasks import load_tasks
@@ -20,6 +20,9 @@ if TYPE_CHECKING:
     from hud.types import Task
 
 
+logger = logging.getLogger(__name__)
+
+
 def _is_remote_url(url: str) -> bool:
     """Match the remote url."""
     # See if a url is a remote url
@@ -53,62 +56,6 @@ def _validate_tasks(tasks: list[Task]) -> bool:
     return True
 
 
-def _find_environment_dir(tasks_path: Path) -> Path | None:
-    """Find the environment directory related to a tasks file.
-
-    Strategy:
-    - Prefer a directory containing hud.lock.yaml
-    - Fallback to a directory that looks like an environment (Dockerfile + pyproject.toml)
-    - Search the tasks file directory, CWD, and a couple of parents
-    """
-    candidates: list[Path] = []
-    cwd = Path.cwd()
-    candidates.extend([tasks_path.parent, cwd])
-
-    # Add parents (up to 2 levels for each)
-    for base in list(candidates):
-        p = base
-        for _ in range(2):
-            p = p.parent
-            if p not in candidates:
-                candidates.append(p)
-
-    # Prefer those with hud.lock.yaml
-    for d in candidates:
-        if (d / "hud.lock.yaml").exists():
-            return d
-
-    # Otherwise, find a plausible environment dir
-    for d in candidates:
-        try:
-            if is_environment_directory(d):
-                return d
-        except Exception as e:
-            hud_console.debug(f"Skipping path {d}: {e}")
-            continue
-
-    return None
-
-
-def _ensure_built(env_dir: Path) -> dict[str, Any]:
-    """Ensure the environment is built and a lock file exists; return lock data."""
-    lock_path = env_dir / "hud.lock.yaml"
-    if not lock_path.exists():
-        hud_console.warning("No hud.lock.yaml found. The environment hasn't been built.")
-        if not hud_console.confirm("Build the environment now (runs 'hud build')?", default=True):
-            raise typer.Exit(1)
-        # Check Docker availability before attempting a build
-        require_docker_running()
-        # Run build (non-interactive). If Docker isn't running, this will raise and stop the flow.
-        # Force linux/amd64 platform to ensure compatibility during RL flows.
-        build_environment(str(env_dir), platform="linux/amd64")
-
-    # Load lock file
-    with open(lock_path) as f:
-        lock_data = yaml.safe_load(f) or {}
-    return lock_data
-
-
 def _ensure_pushed(env_dir: Path, lock_data: dict[str, Any]) -> dict[str, Any]:
     """Ensure the environment is pushed to a registry; return updated lock data."""
     pushed = bool(lock_data.get("push"))
@@ -153,44 +100,228 @@ def _derive_remote_image(lock_data: dict[str, Any]) -> str:
     return f"{name}:{tag}"
 
 
+def _extract_existing_images(tasks: list[Task]) -> set[str]:
+    """Extract all Mcp-Image references from tasks."""
+    images = set()
+
+    def _extract_from_obj(obj: Any) -> None:
+        if isinstance(obj, dict):
+            # Check for Mcp-Image in headers
+            if "headers" in obj and isinstance(obj["headers"], dict):
+                mcp_image = obj["headers"].get("Mcp-Image")
+                if mcp_image:
+                    images.add(mcp_image)
+            # Recursively check nested objects
+            for v in obj.values():
+                _extract_from_obj(v)
+        elif isinstance(obj, list):
+            for item in obj:
+                _extract_from_obj(item)
+
+    for task in tasks:
+        if task.mcp_config:
+            _extract_from_obj(task.mcp_config)
+
+    return images
+
+
+def _env_var_to_header_key(var_name: str) -> str:
+    """Convert ENV_VAR style to Env-Env-Var header style.
+
+    Example: OPENAI_API_KEY -> Env-Openai-Api-Key
+    """
+    parts = str(var_name).split("_")
+    return f"Env-{'-'.join(part.capitalize() for part in parts)}"
+
+
+def _extract_api_key_vars(lock_data: dict[str, Any]) -> set[str]:
+    """Extract env var names from lock file's provided section (authoritative source).
+
+    We only use keys listed under environment.variables.provided, and exclude HUD_API_KEY
+    because Authorization already carries it.
+    """
+    provided_keys: set[str] = set()
+    if not isinstance(lock_data, dict):
+        return provided_keys
+    try:
+        env_section = (lock_data.get("environment") or {}).get("variables") or {}
+        provided = env_section.get("provided") or {}
+        for name in provided:
+            provided_keys.add(str(name))
+    except Exception as e:
+        logger.debug("Failed to parse provided env vars from lock data: %s", e)
+    provided_keys.discard("HUD_API_KEY")
+    return provided_keys
+
+
+def _extract_dotenv_api_key_vars(env_dir: Path) -> set[str]:
+    """Parse .env for API-like variables to suggest as headers.
+
+    We intentionally include only keys that look like secrets to avoid noise:
+    any key containing one of: api, key, token, secret, password (case-insensitive).
+    """
+    dotenv_path = env_dir / ".env"
+    detected: set[str] = set()
+    if not dotenv_path.exists():
+        return detected
+    try:
+        for line in dotenv_path.read_text(encoding="utf-8").splitlines():
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            if "=" not in line:
+                continue
+            name, _ = line.split("=", 1)
+            name = name.strip()
+            lowered = name.lower()
+            if any(s in lowered for s in ("api", "key", "token", "secret", "password")):
+                detected.add(name)
+    except Exception:
+        # Best-effort only
+        return detected
+    detected.discard("HUD_API_KEY")
+    return detected
+
+
 def convert_tasks_to_remote(tasks_file: str) -> str:
     """Convert a local tasks file to remote MCP tasks and return new filename.
 
     Steps:
     1) Find env dir; ensure built (hud.lock.yaml), otherwise build
     2) Ensure pushed to registry, otherwise push
-    3) Create remote_[tasks].json with mcp_config pointing to mcp.hud.so and Mcp-Image
-    4) Return the new tasks file path
+    3) Check for outdated images in existing task configurations
+    4) Create remote_[tasks].json with mcp_config pointing to mcp.hud.so and Mcp-Image
+    5) Return the new tasks file path
     """
     tasks_path = Path(tasks_file).resolve()
 
-    tasks = load_tasks(str(tasks_path))
+    # Load validated tasks for decision-making (may resolve env vars)
+    tasks: list[Task] = load_tasks(str(tasks_path))  # type: ignore[assignment]
+
+    # Load raw tasks to preserve placeholders when writing back to disk
+    raw_tasks: list[dict[str, Any]] = load_tasks(str(tasks_path), raw=True)  # type: ignore[assignment]
 
     # Ensure HUD_API_KEY is available: prefer process env, else load from env_dir/.env
     from hud.settings import settings
 
     if not settings.api_key or not settings.api_key.strip():
         hud_console.error("HUD_API_KEY is not set")
+        hud_console.info("Set it in your environment or run: hud set HUD_API_KEY=your-key-here")
         raise typer.Exit(1)
 
+    # Check if tasks already have remote URLs
+    already_remote = _validate_tasks(tasks)
+
+    # Extract existing images from tasks
+    existing_images = _extract_existing_images(tasks)
+
     # Load tasks (supports .json and .jsonl)
-    if _validate_tasks(tasks):
+    if already_remote and not existing_images:
+        # Tasks are remote but have no image references - just return as-is
         return str(tasks_path)
 
     # Locate environment
-    env_dir = _find_environment_dir(tasks_path)
+    env_dir = find_environment_dir(tasks_path)
     if not env_dir:
         hud_console.error("Could not locate an environment directory (Dockerfile + pyproject.toml)")
         hud_console.hint("Ensure you're in or near your environment folder before running 'hud rl'")
         raise typer.Exit(1)
 
     # Ensure built and pushed
-    lock_data = _ensure_built(env_dir)
+    lock_data = ensure_built(env_dir, interactive=True)
     lock_data = _ensure_pushed(env_dir, lock_data)
 
     # Derive remote image name org/name:tag
     remote_image = _derive_remote_image(lock_data)
 
+    # Check if existing images are outdated
+    needs_update = False
+    should_update_image = False
+    if existing_images:
+        # Check if any existing image differs from the latest
+        for existing_img in existing_images:
+            if existing_img != remote_image:
+                hud_console.warning(f"Detected outdated image reference: {existing_img}")
+                hud_console.info(f"Latest pushed image: {remote_image}")
+                needs_update = True
+                break
+
+        if needs_update:
+            confirm_msg = "Update task configuration with the latest image?"
+            if hud_console.confirm(confirm_msg, default=True):
+                hud_console.info("Updating task configuration with latest image...")
+                should_update_image = True
+            else:
+                # If user doesn't want to update, just return the original file
+                if already_remote:
+                    return str(tasks_path)
+                # Otherwise, continue with conversion but keep old images
+                remote_image = next(iter(existing_images))  # Use the first existing image
+
+    # If tasks are already remote and up-to-date (no update needed), return original file
+    if already_remote and not needs_update:
+        return str(tasks_path)
+
+    # If tasks are already remote and we just need to update the image
+    if already_remote and should_update_image:
+        # Update image references in-place on RAW tasks (preserve placeholders)
+        def _update_image_refs_raw(obj: Any) -> Any:
+            if isinstance(obj, dict):
+                new_obj = {}
+                for k, v in obj.items():
+                    if k == "Mcp-Image" and isinstance(v, str) and v in existing_images:
+                        new_obj[k] = remote_image
+                    else:
+                        new_obj[k] = _update_image_refs_raw(v)
+                return new_obj
+            elif isinstance(obj, list):
+                return [_update_image_refs_raw(item) for item in obj]
+            else:
+                return obj
+
+        updated_raw_tasks: list[dict[str, Any]] = []
+        for t in raw_tasks:
+            td = dict(t)
+            if "mcp_config" in td:
+                td["mcp_config"] = _update_image_refs_raw(td["mcp_config"])
+            updated_raw_tasks.append(td)
+
+        # Write updated file (preserve original format - check if it's .jsonl)
+        if tasks_path.suffix == ".jsonl":
+            with open(tasks_path, "w", encoding="utf-8") as f:
+                for task in updated_raw_tasks:
+                    json.dump(task, f, ensure_ascii=False)
+                    f.write("\n")
+        else:
+            with open(tasks_path, "w", encoding="utf-8") as f:
+                json.dump(updated_raw_tasks, f, ensure_ascii=False, indent=2)
+                f.write("\n")
+
+        hud_console.success(f"Updated {tasks_path.name} with latest image: {remote_image}")
+        return str(tasks_path)
+
+    # Extract additional API key headers from lock and suggest from .env
+    provided_keys = _extract_api_key_vars(lock_data)
+    dotenv_keys = _extract_dotenv_api_key_vars(env_dir)
+
+    # If .env contains API-like vars not in lock, offer to include them
+    missing = sorted(dotenv_keys - provided_keys)
+    if missing:
+        names_preview = ", ".join(missing)
+        prompt = (
+            f"Detected env vars in .env that look like API keys: {names_preview}.\n"
+            "Include them as remote headers (values will be ${VAR} placeholders)?"
+        )
+        if hud_console.confirm(prompt, default=True):
+            provided_keys.update(missing)
+
+    extra_api_key_headers: dict[str, str] = {}
+    for var_name in provided_keys:
+        if str(var_name).upper() == "HUD_API_KEY":
+            continue
+        header_key = _env_var_to_header_key(var_name)
+        extra_api_key_headers[header_key] = f"${{{var_name}}}"
+
     # Helper to strip extra fields from tool calls
     def _simplify_tool_call(tool: Any) -> Any:
         def _one(x: Any) -> dict[str, Any]:
@@ -228,6 +359,9 @@ def convert_tasks_to_remote(tasks_file: str) -> str:
             },
         }
 
+        # Merge additional API key headers
+        item["mcp_config"]["hud"]["headers"].update(extra_api_key_headers)
+
         # Optional fields, omit Nones
         if t.setup_tool is not None:
             item["setup_tool"] = _simplify_tool_call(t.setup_tool)
@@ -242,7 +376,6 @@ def convert_tasks_to_remote(tasks_file: str) -> str:
 
         tasks_payload.append(item)
 
-    # Write new file: remote_<name>.json (always JSON array)
     remote_name = f"remote_{tasks_path.stem}.json"
     remote_path = tasks_path.parent / remote_name
     with open(remote_path, "w", encoding="utf-8") as f: