hud-python 0.4.45__py3-none-any.whl → 0.5.1__py3-none-any.whl

hud/cli/init.py

@@ -14,24 +14,29 @@ import typer
 
 from hud.utils.hud_console import HUDConsole
 
-# Presets mapping to environment folders in public SDK repo
+# Presets mapping to public GitHub repositories under hud-evals org
 GITHUB_OWNER = "hud-evals"
-GITHUB_REPO = "hud-python"
 GITHUB_BRANCH = "main"
 
 PRESET_MAP: dict[str, str | None] = {
-    "blank": "blank",
-    "deep-research": "deepresearch",
-    "browser": "browser",
+    "blank": "hud-blank",
+    "deep-research": "hud-deepresearch",
+    "browser": "hud-browser",
+    "rubrics": "hud-rubrics",
+    "verilog-coding-template": "verilog-coding-template",
+    "data-science-template": "data-science-template",
 }
 
 SKIP_DIR_NAMES = {"node_modules", "__pycache__", "dist", "build", ".next", ".git"}
 
 # Files that need placeholder replacement
 PLACEHOLDER_FILES = {
-    "pyproject.toml",
+    "server/pyproject.toml",
+    "environment/pyproject.toml",
+    "server/main.py",
+    "server/README.md",
+    "environment/README.md",
     "tasks.json",
-    "src/controller/server.py",
     "test_env.ipynb",
     "README.md",
 }
@@ -48,7 +53,7 @@ def _replace_placeholders(target_dir: Path, env_name: str) -> list[str]:
         List of files that were modified
     """
     modified_files = []
-    placeholder = "test_test"
+    placeholder = "blank"  # Placeholder used in blank environment template
 
     # Normalize environment name for use in code/configs
     # Replace spaces and special chars with underscores for Python identifiers
@@ -86,8 +91,11 @@ def _prompt_for_preset() -> str:
     try:
         choices = [
             {"name": "blank", "message": "blank"},
-            {"name": "deep-research", "message": "deep-research"},
             {"name": "browser", "message": "browser"},
+            {"name": "deep-research", "message": "deep-research"},
+            {"name": "rubrics", "message": "rubrics"},
+            {"name": "verilog-coding-template", "message": "verilog-coding-template"},
+            {"name": "data-science-template", "message": "data-science-template"},
         ]
         display_choices = [c["message"] for c in choices]
         selected = questionary.select(
@@ -103,10 +111,10 @@ def _prompt_for_preset() -> str:
         return "blank"
 
 
-def _download_tarball_subdir(
-    owner: str, repo: str, ref: str, subdir: str, dest_dir: Path, files_created: list[str]
+def _download_tarball_repo(
+    owner: str, repo: str, ref: str, dest_dir: Path, files_created: list[str]
 ) -> None:
-    """Download a GitHub tarball and extract only a subdirectory."""
+    """Download a GitHub tarball and extract the entire repository."""
     tarball_url = f"https://codeload.github.com/{owner}/{repo}/tar.gz/{ref}"
 
     token = os.getenv("GITHUB_TOKEN")
@@ -135,16 +143,17 @@ def _download_tarball_subdir(
             if not members:
                 return
             top = members[0].name.split("/", 1)[0]
-            target_prefix = f"{top}/environments/{subdir.strip('/')}"
 
             for member in members:
                 name = member.name
-                if not (name == target_prefix or name.startswith(target_prefix + "/")):
+                if name == top:
+                    continue
+
+                if not name.startswith(top + "/"):
                     continue
 
-                rel_path = name[len(target_prefix) :].lstrip("/")
+                rel_path = name[len(top) + 1 :]
                 if not rel_path:
-                    dest_dir.mkdir(parents=True, exist_ok=True)
                     continue
 
                 out_path = (dest_dir / rel_path).resolve()
@@ -177,21 +186,21 @@ def create_environment(
 
     hud_console = HUDConsole()
 
-    # Determine environment name/target directory
-    if name is None:
-        current_dir = Path.cwd()
-        name = current_dir.name
-        target_dir = current_dir
-        hud_console.info(f"Using current directory name: {name}")
-    else:
-        target_dir = Path(directory) / name
-
     # Choose preset
     preset_normalized = (preset or "").strip().lower() if preset else _prompt_for_preset()
+
+    # If no name is provided, use the preset name as the environment name
+    if name is None:
+        name = preset_normalized
+        hud_console.info(f"Using preset name as environment name: {name}")
+
+    # Always create a new directory based on the name
+    target_dir = Path.cwd() / name if directory == "." else Path(directory) / name
+
     if preset_normalized not in PRESET_MAP:
+        available = ", ".join(sorted(PRESET_MAP.keys()))
         hud_console.warning(
-            f"Unknown preset '{preset_normalized}', defaulting to 'blank' "
-            "(available: blank, deep-research, browser)"
+            f"Unknown preset '{preset_normalized}', defaulting to 'blank' (available: {available})"
         )
         preset_normalized = "blank"
 
@@ -205,17 +214,14 @@ def create_environment(
             hud_console.warning(f"Overwriting existing files in {target_dir}")
 
     # Download preset from GitHub
-    env_folder = PRESET_MAP[preset_normalized]
-    if env_folder is None:
-        hud_console.error("Internal error: preset mapping missing folder name")
+    repo_name = PRESET_MAP[preset_normalized]
+    if repo_name is None:
+        hud_console.error("Internal error: preset mapping missing repo name")
         raise typer.Exit(1)
 
     hud_console.header(f"Initializing HUD Environment: {name} (preset: {preset_normalized})")
-    hud_console.section_title("Downloading template from public SDK")
-    source_url = (
-        f"https://github.com/{GITHUB_OWNER}/{GITHUB_REPO}/tree/"
-        f"{GITHUB_BRANCH}/environments/{env_folder}"
-    )
+    hud_console.section_title("Downloading template from GitHub")
+    source_url = f"https://github.com/{GITHUB_OWNER}/{repo_name}"
     hud_console.info("Source: " + source_url)
 
     target_dir.mkdir(parents=True, exist_ok=True)
@@ -223,11 +229,10 @@ def create_environment(
     started = time.time()
     files_created_dl: list[str] = []
     try:
-        _download_tarball_subdir(
+        _download_tarball_repo(
             owner=GITHUB_OWNER,
-            repo=GITHUB_REPO,
+            repo=repo_name,
             ref=GITHUB_BRANCH,
-            subdir=env_folder,
             dest_dir=target_dir,
             files_created=files_created_dl,
         )
@@ -240,31 +245,28 @@ def create_environment(
         f"Downloaded {len(files_created_dl)} files in {duration_ms} ms into {target_dir}"
     )
 
-    # Replace placeholders in template files
-    hud_console.section_title("Customizing template files")
-    modified_files = _replace_placeholders(target_dir, name)
-    if modified_files:
-        hud_console.success(f"Replaced placeholders in {len(modified_files)} files:")
-        for file in modified_files[:5]:  # Show first 5 files
-            hud_console.status_item(file, "updated")
-        if len(modified_files) > 5:
-            hud_console.info(f"... and {len(modified_files) - 5} more files")
-    else:
-        hud_console.info("No placeholder replacements needed")
+    # Replace placeholders in template files (only for blank preset)
+    if preset_normalized == "blank":
+        hud_console.section_title("Customizing template files")
+        modified_files = _replace_placeholders(target_dir, name)
+        if modified_files:
+            hud_console.success(f"Replaced placeholders in {len(modified_files)} files:")
+            for file in modified_files[:5]:  # Show first 5 files
+                hud_console.status_item(file, "updated")
+            if len(modified_files) > 5:
+                hud_console.info(f"... and {len(modified_files) - 5} more files")
+        else:
+            hud_console.info("No placeholder replacements needed")
 
     hud_console.section_title("Top-level files and folders")
     for entry in sorted(os.listdir(target_dir)):
         hud_console.status_item(entry, "added")
 
     hud_console.section_title("Next steps")
-    if target_dir == Path.cwd():
-        hud_console.info("1. Start development server (with MCP inspector):")
-        hud_console.command_example("hud dev --inspector")
-    else:
-        hud_console.info("1. Enter the directory:")
-        hud_console.command_example(f"cd {target_dir}")
-        hud_console.info("\n2. Start development server (with MCP inspector):")
-        hud_console.command_example("hud dev --inspector")
-
+    # Since we now almost always create a new directory, show cd command
+    hud_console.info("1. Enter the directory:")
+    hud_console.command_example(f"cd {target_dir.name}")
+    hud_console.info("\n2. Start development server (with MCP inspector):")
+    hud_console.command_example("hud dev --inspector")
     hud_console.info("\n3. Review the README in this preset for specific instructions.")
     hud_console.info("\n4. Customize as needed.")

hud/cli/push.py

@@ -152,7 +152,7 @@ def push_environment(
         hud_console.error("No HUD API key found")
         hud_console.warning("A HUD API key is required to push environments.")
         hud_console.info("\nTo get started:")
-        hud_console.info("1. Get your API key at: https://hud.so/settings")
+        hud_console.info("1. Get your API key at: https://hud.ai/settings")
         hud_console.info("Set it in your environment or run: hud set HUD_API_KEY=your-key-here")
         hud_console.command_example("hud push", "Try again")
         hud_console.info("")
@@ -163,10 +163,7 @@ def push_environment(
         lock_data = yaml.safe_load(f)
 
     # Handle both old and new lock file formats
-    local_image = lock_data.get("image", "")
-    if not local_image and "build" in lock_data:
-        # New format might have image elsewhere
-        local_image = lock_data.get("image", "")
+    local_image = lock_data.get("images", {}).get("local") or lock_data.get("image", "")
 
     # Get internal version from lock file
     internal_version = lock_data.get("build", {}).get("version", None)
@@ -293,7 +290,7 @@ def push_environment(
     # Push the image
     hud_console.progress_message(f"Pushing {image} to registry...")
 
-    # Show push output
+    # Show push output (filtered for cleaner display)
     process = subprocess.Popen(  # noqa: S603
         ["docker", "push", image],  # noqa: S607
         stdout=subprocess.PIPE,
@@ -303,8 +300,27 @@ def push_environment(
         errors="replace",
     )
 
+    # Filter output to only show meaningful progress
+    layers_pushed = 0
     for line in process.stdout or []:
-        hud_console.info(line.rstrip())
+        line = line.rstrip()
+        # Only show: digest, pushed, mounted, or error lines
+        if any(
+            keyword in line.lower()
+            for keyword in ["digest:", "pushed", "mounted", "error", "denied"]
+        ):
+            if "pushed" in line.lower():
+                layers_pushed += 1
+            if (
+                verbose
+                or "error" in line.lower()
+                or "denied" in line.lower()
+                or "digest:" in line.lower()
+            ):
+                hud_console.info(line)
+
+    if layers_pushed > 0 and not verbose:
+        hud_console.info(f"Pushed {layers_pushed} layer(s)")
 
     process.wait()
 
@@ -331,8 +347,10 @@ def push_environment(
     hud_console.section_title("Pushed Image")
     hud_console.status_item("Registry", pushed_digest, primary=True)
 
-    # Update the lock file with registry information
-    lock_data["image"] = pushed_digest
+    # Update the lock file with pushed image reference
+    if "images" not in lock_data:
+        lock_data["images"] = {}
+    lock_data["images"]["pushed"] = image
 
     # Add push information
     from datetime import UTC, datetime
@@ -348,7 +366,7 @@ def push_environment(
     with open(lock_path, "w") as f:
         yaml.dump(lock_data, f, default_flow_style=False, sort_keys=False)
 
-    hud_console.success("Updated lock file with registry image")
+    hud_console.success("Updated lock file with pushed image reference")
 
     # Upload lock file to HUD registry
     try:
@@ -422,7 +440,7 @@ def push_environment(
         elif response.status_code == 401:
             hud_console.error("Authentication failed")
             hud_console.info("Check your HUD_API_KEY is valid")
-            hud_console.info("Get a new key at: https://hud.so/settings")
+            hud_console.info("Get a new key at: https://hud.ai/settings")
             hud_console.info("Set it in your environment or run: hud set HUD_API_KEY=your-key-here")
         elif response.status_code == 403:
             hud_console.error("Permission denied")

hud/cli/rft.py

@@ -0,0 +1,311 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+import httpx
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from hud.datasets import load_tasks
+from hud.settings import settings
+from hud.utils.hud_console import HUDConsole
+
+logger = logging.getLogger(__name__)
+console = Console()
+hud_console = HUDConsole()
+
+
+def _patch_mcp_urls_to_staging(tasks: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Recursively patch all mcp.hud.so URLs to https://orcstaging.hud.so in task configs."""
+
+    def patch_value(obj: Any) -> Any:
+        if isinstance(obj, dict):
+            return {k: patch_value(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [patch_value(item) for item in obj]
+        elif isinstance(obj, str):
+            # Replace any occurrence of mcp.hud.so with orcstaging.hud.so
+            # Handle various URL formats
+            if "mcp.hud.so" in obj:
+                # Replace the domain while preserving the protocol and path
+                return obj.replace("mcp.hud.so", "orcstaging.hud.so")
+            elif "mcp.hud.ai" in obj:
+                # Also handle mcp.hud.ai URLs
+                return obj.replace("mcp.hud.ai", "orcstaging.hud.so")
+            return obj
+        else:
+            return obj
+
+    return [patch_value(task) for task in tasks]
+
+
+def _fetch_models() -> list[dict[str, Any]]:
+    """Fetch trainable models from the HUD API for the user's team."""
+    url = f"{settings.hud_api_url}/models/"
+    headers = {
+        "Authorization": f"Bearer {settings.api_key}",
+        "x-api-key": settings.api_key or "",
+    }
+    params = {"team_only": "true", "limit": 200}
+
+    try:
+        with httpx.Client(timeout=30.0) as client:
+            resp = client.get(url, headers=headers, params=params)
+            resp.raise_for_status()
+            data = resp.json()
+            return data.get("models", [])
+    except httpx.HTTPStatusError as e:
+        hud_console.error(f"Failed to fetch models: {e.response.status_code}")
+        if e.response.status_code == 401:
+            hud_console.hint("Check that your HUD_API_KEY is valid")
+        raise typer.Exit(1) from e
+    except httpx.RequestError as e:
+        hud_console.error(f"Connection error while fetching models: {e}")
+        raise typer.Exit(1) from e
+
+
+def _select_model(models: list[dict[str, Any]]) -> dict[str, Any]:
+    """Display models and let user select one for training."""
+    # Filter to only trainable models that are ready
+    trainable_models = [
+        m
+        for m in models
+        if m.get("is_trainable", False)
+        and m.get("status") == "ready"
+        and not m.get("public", False)
+        and m.get("model_name") is not None
+    ]
+
+    if not trainable_models:
+        hud_console.error("No trainable models found in your team.")
+        hud_console.hint("Fork a trainable model at https://api.hud.so/models to start training.")
+        raise typer.Exit(1)
+
+    # Display models in a table
+    hud_console.section_title("Available Trainable Models")
+    table = Table(show_header=True, header_style="bold")
+    table.add_column("#", style="dim", width=4)
+    table.add_column("Name", style="bold")
+    table.add_column("Status")
+    table.add_column("Provider")
+
+    for i, model in enumerate(trainable_models, 1):
+        provider_name = (
+            model.get("provider", {}).get("name", "unknown") if model.get("provider") else "unknown"
+        )
+        table.add_row(
+            str(i),
+            model.get("name", "unnamed"),
+            model.get("status", "unknown"),
+            provider_name,
+        )
+
+    hud_console.console.print(table)
+    hud_console.print("")
+
+    # Build choices for selection
+    choices = [
+        {"name": f"{m.get('name', 'unnamed')} ({m.get('base_model', 'unknown')})", "value": m}
+        for m in trainable_models
+    ]
+
+    selected: dict[str, Any] = hud_console.select("Select a model to train:", choices)  # type: ignore[assignment]
+    return selected
+
+
+def rft_command(
+    tasks_file: str,
+    reasoning_effort: str = "medium",
+    verbose: bool = False,
+    yes: bool = False,
+    model_id: str | None = None,
+) -> None:
+    """
+    Run Reinforcement Fine-Tuning (RFT) via the HUD RL service.
+    """
+    hud_console.header("HUD RFT (Reinforcement Fine-Tuning)")
+
+    # Preflight check: API key
+    if not settings.api_key:
+        hud_console.error("HUD_API_KEY not found in environment.")
+        hud_console.info("Run 'hud set HUD_API_KEY=...' or export it.")
+        raise typer.Exit(1)
+
+    # Model selection
+    selected_model_id: str
+    if model_id:
+        # Use provided model_id directly
+        selected_model_id = model_id
+        hud_console.info(f"Using provided model ID: {selected_model_id}")
+    else:
+        # Fetch and let user select a model
+        hud_console.section_title("Fetching available models")
+        hud_console.info("Loading models from your team...")
+        models = _fetch_models()
+
+        if yes:
+            # Auto-select first trainable model in non-interactive mode
+            trainable_models = [
+                m
+                for m in models
+                if m.get("is_trainable", False)
+                and m.get("status") == "ready"
+                and not m.get("public", False)
+                and m.get("model_name") is not None
+            ]
+            if not trainable_models:
+                hud_console.error("No trainable models found in your team.")
+                hud_console.hint(
+                    "Fork a trainable model at https://api.hud.so/models to start training."
+                )
+                raise typer.Exit(1)
+            selected_model = trainable_models[0]
+            hud_console.info(
+                f"Auto-selected first trainable model (--yes mode): "
+                f"{selected_model.get('name', 'unnamed')}"
+            )
+        else:
+            selected_model = _select_model(models)
+
+        selected_model_id = selected_model["id"]
+        hud_console.success(
+            f"Selected model: {selected_model.get('name', 'unnamed')} (ID: {selected_model_id})"
+        )
+
+    # Preflight check: Convert tasks to remote if needed
+    hud_console.section_title("Preparing tasks for remote training")
+    try:
+        from hud.cli.flows.tasks import convert_tasks_to_remote
+
+        hud_console.info("Checking task configuration...")
+        tasks_file = convert_tasks_to_remote(tasks_file)
+        hud_console.success("Tasks are ready for remote training")
+    except typer.Exit:
+        raise
+    except Exception as e:
+        hud_console.error(f"Tasks file is not valid for remote training: {e!s}")
+        hud_console.hint("Either ensure the tasks file has remote urls")
+        hud_console.hint("Or run 'hud rft' within an environment directory")
+        raise typer.Exit(1) from e
+
+    # Load and validate tasks
+    try:
+        # Load tasks as raw dicts for patching and serialization
+        tasks: list[dict[str, Any]] = load_tasks(tasks_file, raw=True)  # type: ignore[assignment]
+        if not tasks:
+            hud_console.error(f"No tasks found in {tasks_file}")
+            raise typer.Exit(1)
+
+        # Preflight check: Minimum task count
+        task_count = len(tasks)
+        if task_count < 10:
+            hud_console.error(
+                f"Insufficient tasks for RFT training: found {task_count}, need at least 10"
+            )
+            hud_console.hint("RFT requires a minimum of 10 tasks for effective training")
+            raise typer.Exit(1)
+
+        hud_console.info(f"Loaded {task_count} tasks from {tasks_file}")
+
+        # Preflight check: Vision support
+        hud_console.section_title("Vision Support Check")
+        hud_console.warning(
+            "RFT does not currently support environments that require vision capabilities."
+        )
+        hud_console.info(
+            "Vision support includes: screenshots, image analysis, visual UI interaction, etc."
+        )
+
+        if not yes:
+            if hud_console.confirm("Does your environment require vision support?", default=False):
+                hud_console.error("RFT does not support vision-based environments at this time.")
+                hud_console.hint(
+                    "Please use environments that rely on text-based interactions only."
+                )
+                raise typer.Exit(1)
+        else:
+            hud_console.info("Skipping vision support check (--yes mode)")
+
+        # Patch all mcp.hud.so URLs to orcstaging.hud.so
+        hud_console.info("Patching MCP URLs for staging environment...")
+        tasks = _patch_mcp_urls_to_staging(tasks)
+
+        # Show task preview
+        if tasks:
+            if yes:
+                # Skip interactive preview in auto-accept mode
+                hud_console.info("Skipping task preview in auto-accept mode (--yes)")
+            else:
+                try:
+                    from hud.cli.utils.viewer import show_json_interactive
+
+                    hud_console.section_title("Task Preview")
+                    show_json_interactive(
+                        tasks[0], title="Example Task from Dataset", initial_expanded=False
+                    )
+                    hud_console.info("This is how your task will be sent to the RFT service.")
+
+                    # Ask for confirmation
+                    if not hud_console.confirm(
+                        "\nProceed with RFT training on this dataset?", default=True
+                    ):
+                        hud_console.error("RFT training cancelled")
+                        raise typer.Exit(0)
+                except typer.Exit:
+                    raise  # Re-raise typer.Exit to properly exit on cancellation
+                except Exception as e:
+                    hud_console.warning(f"Could not display task preview: {e}")
+
+    except typer.Exit:
+        raise  # Re-raise typer.Exit to properly exit
+    except Exception as e:
+        hud_console.error(f"Failed to load tasks file: {e}")
+        raise typer.Exit(1) from e
+
+    # Prepare payload
+    payload = {
+        "model_id": selected_model_id,
+        "dataset": {"tasks": tasks},
+        "config": {"parameters": {"reasoning_effort": reasoning_effort}},
+    }
+
+    # Send request to service
+    hud_console.section_title("Submitting RFT job")
+
+    base_url = settings.hud_rl_url
+    url = f"{base_url}/training/jobs"
+
+    headers = {"Authorization": f"Bearer {settings.api_key}", "Content-Type": "application/json"}
+
+    hud_console.info(
+        f"Submitting job to {url}... (this may take a few minutes to run all safety checks)"
+    )
+
+    try:
+        with httpx.Client(timeout=300.0) as client:
+            resp = client.post(url, json=payload, headers=headers)
+
+            if resp.status_code >= 400:
+                try:
+                    detail = resp.json()
+                except Exception as e:
+                    detail = f"{resp.text} - {e}"
+                hud_console.error(f"Request failed ({resp.status_code}): {detail}")
+                raise typer.Exit(1)
+
+            data = resp.json()
+            job_id = data.get("job_id")
+            model_id = data.get("model", {}).get("id")
+
+            hud_console.success(f"Job launched successfully! ID: {job_id}")
+            hud_console.info(f"Model ID: {model_id}")
+
+            # Provide helpful next steps
+            hud_console.info(f"To check job status, run: hud rft status {model_id}")
+
+    except httpx.RequestError as e:
+        hud_console.error(f"Connection error: {e}")
+        hud_console.info("Is the RL service running?")
+        raise typer.Exit(1) from e