hud-python 0.4.45__py3-none-any.whl → 0.5.13__py3-none-any.whl

hud/cli/build.py

@@ -5,9 +5,12 @@ from __future__ import annotations
 import asyncio
 import contextlib
 import hashlib
+import json
+import os
+import re
 import subprocess
 import time
-from datetime import datetime
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any
 
@@ -15,13 +18,36 @@ import typer
 import yaml
 
 from hud.cli.utils.source_hash import compute_source_hash, list_source_files
-from hud.clients import MCPClient
 from hud.utils.hud_console import HUDConsole
 from hud.version import __version__ as hud_version
 
 from .utils.registry import save_to_registry
 
 
+def find_dockerfile(directory: Path) -> Path | None:
+    """Find the Dockerfile in a directory, preferring Dockerfile.hud.
+
+    Checks for Dockerfile.hud first (HUD-specific), then falls back to Dockerfile.
+
+    Args:
+        directory: Directory to search in
+
+    Returns:
+        Path to the Dockerfile if found, None otherwise
+    """
+    # Prefer Dockerfile.hud for HUD environments
+    hud_dockerfile = directory / "Dockerfile.hud"
+    if hud_dockerfile.exists():
+        return hud_dockerfile
+
+    # Fall back to standard Dockerfile
+    standard_dockerfile = directory / "Dockerfile"
+    if standard_dockerfile.exists():
+        return standard_dockerfile
+
+    return None
+
+
 def parse_version(version_str: str) -> tuple[int, int, int]:
     """Parse version string like '1.0.0' or '1.0' into tuple of integers."""
     # Remove 'v' prefix if present
@@ -50,6 +76,140 @@ def increment_version(version_str: str, increment_type: str = "patch") -> str:
         return f"{major}.{minor}.{patch + 1}"
 
 
+def find_task_files_in_env(env_dir: Path) -> list[Path]:
+    """Find all task files in an environment directory.
+
+    This looks for .json and .jsonl files that contain task definitions,
+    excluding config files and lock files.
+
+    Args:
+        env_dir: Environment directory to search
+
+    Returns:
+        List of task file paths
+    """
+    task_files: list[Path] = []
+
+    # Find all .json and .jsonl files
+    json_files = list(env_dir.glob("*.json")) + list(env_dir.glob("*.jsonl"))
+
+    # Filter out config files and lock files
+    for file in json_files:
+        # Skip hidden files, config files, and lock files
+        if (
+            file.name.startswith(".")
+            or file.name == "package.json"
+            or file.name == "tsconfig.json"
+            or file.name == "gcp.json"
+            or file.name.endswith(".lock.json")
+        ):
+            continue
+
+        # Check if it's a task file by looking for mcp_config
+        try:
+            with open(file, encoding="utf-8") as f:
+                content = json.load(f)
+
+            # It's a task file if it's a list with mcp_config entries
+            if (
+                isinstance(content, list)
+                and len(content) > 0
+                and any(isinstance(item, dict) and "mcp_config" in item for item in content)
+            ):
+                task_files.append(file)
+        except (json.JSONDecodeError, Exception):  # noqa: S112
+            continue
+
+    return task_files
+
+
+def update_tasks_json_versions(
+    env_dir: Path, base_name: str, old_version: str | None, new_version: str
+) -> list[Path]:
+    """Update image references in tasks.json files to use the new version.
+
+    Args:
+        env_dir: Environment directory
+        base_name: Base image name (without version)
+        old_version: Previous version (if any)
+        new_version: New version to use
+
+    Returns:
+        List of updated task files
+    """
+    hud_console = HUDConsole()
+    updated_files: list[Path] = []
+
+    for task_file in find_task_files_in_env(env_dir):
+        try:
+            with open(task_file, encoding="utf-8") as f:
+                tasks = json.load(f)
+            if not isinstance(tasks, list):
+                continue
+
+            modified = False
+
+            # Process each task
+            for task in tasks:
+                if not isinstance(task, dict) or "mcp_config" not in task:
+                    continue
+
+                mcp_config = task["mcp_config"]
+
+                # Handle local Docker format
+                if "local" in mcp_config and isinstance(mcp_config["local"], dict):
+                    local_config = mcp_config["local"]
+
+                    # Check for docker run args
+                    if "args" in local_config and isinstance(local_config["args"], list):
+                        for i, arg in enumerate(local_config["args"]):
+                            # Match image references
+                            if isinstance(arg, str) and (
+                                arg == f"{base_name}:latest"
+                                or (old_version and arg == f"{base_name}:{old_version}")
+                                or re.match(rf"^{re.escape(base_name)}:\d+\.\d+\.\d+$", arg)
+                            ):
+                                # Update to new version
+                                local_config["args"][i] = f"{base_name}:{new_version}"
+                                modified = True
+
+                # Handle HUD API format (remote MCP)
+                elif "hud" in mcp_config and isinstance(mcp_config["hud"], dict):
+                    hud_config = mcp_config["hud"]
+
+                    # Check headers for Mcp-Image
+                    if "headers" in hud_config and isinstance(hud_config["headers"], dict):
+                        headers = hud_config["headers"]
+
+                        if "Mcp-Image" in headers:
+                            image_ref = headers["Mcp-Image"]
+
+                            # Match various image formats
+                            if isinstance(image_ref, str) and ":" in image_ref:
+                                # Split into image name and tag
+                                image_name, _ = image_ref.rsplit(":", 1)
+
+                                if (
+                                    image_name == base_name  # Exact match
+                                    or image_name.endswith(f"/{base_name}")  # With prefix
+                                ):
+                                    # Update to new version, preserving the full image path
+                                    headers["Mcp-Image"] = f"{image_name}:{new_version}"
+                                    modified = True
+
+            # Save the file if modified
+            if modified:
+                with open(task_file, "w") as f:
+                    json.dump(tasks, f, indent=2)
+                updated_files.append(task_file)
+                hud_console.success(f"Updated {task_file.name} with version {new_version}")
+
+        except Exception as e:
+            hud_console.warning(f"Could not update {task_file.name}: {e}")
+
+    return updated_files
+
+
 def get_existing_version(lock_path: Path) -> str | None:
     """Get the internal version from existing lock file if it exists."""
     if not lock_path.exists():
@@ -154,6 +314,121 @@ def extract_env_vars_from_dockerfile(dockerfile_path: Path) -> tuple[list[str],
     return required, optional
 
 
+def parse_base_image(dockerfile_path: Path) -> str | None:
+    """Extract the base image from the first FROM directive in Dockerfile.
+
+    For multi-stage builds, returns the image from the first FROM. Strips any
+    trailing AS <stage> segment.
+    """
+    try:
+        if not dockerfile_path.exists():
+            return None
+        for raw_line in dockerfile_path.read_text().splitlines():
+            line = raw_line.strip()
+            if not line or line.startswith("#"):
+                continue
+            if line.upper().startswith("FROM "):
+                rest = line[5:].strip()
+                # Remove stage alias if present
+                lower = rest.lower()
+                if " as " in lower:
+                    # Split using the original case string at the index of lower-case match
+                    idx = lower.index(" as ")
+                    rest = rest[:idx]
+                return rest.strip()
+    except Exception:
+        return None
+    return None
+
+
+def collect_runtime_metadata(image: str, *, verbose: bool = False) -> dict[str, str | None]:
+    """Probe container to capture Python/CUDA/cuDNN/PyTorch versions.
+
+    Runs a tiny Python snippet inside the built image using docker run.
+    """
+    hud_console = HUDConsole()
+
+    runtime_script = (
+        "import json, platform\n"
+        "info = {'python': platform.python_version()}\n"
+        "try:\n"
+        "    import torch\n"
+        "    info['pytorch'] = getattr(torch, '__version__', None)\n"
+        "    cuda_version = None\n"
+        "    try:\n"
+        "        cuda_version = getattr(getattr(torch, 'version', None), 'cuda', None)\n"
+        "    except Exception:\n"
+        "        cuda_version = None\n"
+        "    if cuda_version:\n"
+        "        info['cuda'] = cuda_version\n"
+        "    try:\n"
+        "        cudnn_version = torch.backends.cudnn.version()\n"
+        "    except Exception:\n"
+        "        cudnn_version = None\n"
+        "    if cudnn_version:\n"
+        "        info['cudnn'] = str(cudnn_version)\n"
+        "except Exception:\n"
+        "    pass\n"
+        "info.setdefault('pytorch', None)\n"
+        "info.setdefault('cuda', None)\n"
+        "info.setdefault('cudnn', None)\n"
+        "print(json.dumps(info))\n"
+    )
+
+    for binary in ("python", "python3"):
+        cmd = [
+            "docker",
+            "run",
+            "--rm",
+            image,
+            binary,
+            "-c",
+            runtime_script,
+        ]
+        try:
+            result = subprocess.run(  # noqa: S603
+                cmd, capture_output=True, text=True, check=False
+            )
+        except FileNotFoundError:
+            return {}
+
+        if result.returncode != 0:
+            if verbose:
+                hud_console.debug(
+                    f"Runtime probe failed with {binary}: {result.stderr.strip() or 'no stderr'}"
+                )
+            continue
+
+        output = (result.stdout or "").strip()
+        if not output:
+            return {}
+
+        try:
+            data = json.loads(output.splitlines()[-1])
+        except json.JSONDecodeError:
+            if verbose:
+                hud_console.debug(
+                    "Runtime probe returned non-JSON output; skipping metadata capture"
+                )
+            return {}
+
+        if not isinstance(data, dict):
+            if verbose:
+                hud_console.debug(
+                    "Runtime probe returned JSON that is not an object; skipping metadata capture"
+                )
+            return {}
+
+        return {
+            "python": data.get("python"),
+            "cuda": data.get("cuda"),
+            "cudnn": data.get("cudnn"),
+            "pytorch": data.get("pytorch"),
+        }
+
+    return {}
+
+
 async def analyze_mcp_environment(
     image: str, verbose: bool = False, env_vars: dict[str, str] | None = None
 ) -> dict[str, Any]:
@@ -161,50 +436,103 @@ async def analyze_mcp_environment(
     hud_console = HUDConsole()
     env_vars = env_vars or {}
 
-    # Build Docker command to run the image
-    docker_cmd = ["docker", "run", "--rm", "-i"]
+    # Build Docker command to run the image, injecting any provided env vars
+    from hud.cli.utils.docker import build_env_flags
 
-    # Add environment variables
-    for key, value in env_vars.items():
-        docker_cmd.extend(["-e", f"{key}={value}"])
+    docker_cmd = ["docker", "run", "--rm", "-i", *build_env_flags(env_vars), image]
 
-    docker_cmd.append(image)
+    # Show full docker command being used for analysis
+    hud_console.dim_info("Command:", " ".join(docker_cmd))
 
-    # Create MCP config
-    config = {
-        "server": {"command": docker_cmd[0], "args": docker_cmd[1:] if len(docker_cmd) > 1 else []}
-    }
+    # Create MCP config consistently with analyze helpers
+    from hud.cli.analyze import parse_docker_command
+
+    mcp_config = parse_docker_command(docker_cmd)
 
     # Initialize client and measure timing
+    # Use FastMCP client directly - no mcp_use deprecation warnings
+    from hud.clients.fastmcp import FastMCPHUDClient
+
     start_time = time.time()
-    client = MCPClient(mcp_config=config, verbose=verbose, auto_trace=False)
+    client = FastMCPHUDClient(mcp_config=mcp_config, verbose=verbose)
     initialized = False
 
     try:
         if verbose:
-            hud_console.info(f"Initializing MCP client with command: {' '.join(docker_cmd)}")
+            hud_console.info("Initializing MCP client...")
 
-        await client.initialize()
+        # Add timeout to fail fast instead of hanging (60 seconds)
+        await asyncio.wait_for(client.initialize(), timeout=60.0)
         initialized = True
         initialize_ms = int((time.time() - start_time) * 1000)
 
-        # Get tools
-        tools = await client.list_tools()
-
-        # Extract tool information
-        tool_info = []
-        for tool in tools:
-            tool_dict = {"name": tool.name, "description": tool.description}
-            if hasattr(tool, "inputSchema") and tool.inputSchema:
-                tool_dict["inputSchema"] = tool.inputSchema
-            tool_info.append(tool_dict)
-
-        return {
+        # Delegate to standard analysis helper
+        full_analysis = await client.analyze_environment()
+
+        # Normalize and enrich with internalTools if a hub map is present
+        tools_list = full_analysis.get("tools", [])
+        hub_map = full_analysis.get("hub_tools", {}) or full_analysis.get("hubTools", {})
+
+        normalized_tools: list[dict[str, Any]] = []
+        internal_total = 0
+        for t in tools_list:
+            # Extract core fields (support object or dict forms)
+            if hasattr(t, "name"):
+                name = getattr(t, "name", None)
+                description = getattr(t, "description", None)
+                input_schema = getattr(t, "inputSchema", None)
+                existing_internal = getattr(t, "internalTools", None)
+            else:
+                name = t.get("name")
+                description = t.get("description")
+                # accept either inputSchema or input_schema
+                input_schema = t.get("inputSchema") or t.get("input_schema")
+                # accept either internalTools or internal_tools
+                existing_internal = t.get("internalTools") or t.get("internal_tools")
+
+            tool_entry: dict[str, Any] = {"name": name}
+            if description:
+                tool_entry["description"] = description
+            if input_schema:
+                tool_entry["inputSchema"] = input_schema
+
+            # Merge internal tools: preserve any existing declaration and add hub_map[name]
+            merged_internal: list[str] = []
+            if isinstance(existing_internal, list):
+                merged_internal.extend([str(x) for x in existing_internal])
+            if isinstance(hub_map, dict) and name in hub_map and isinstance(hub_map[name], list):
+                merged_internal.extend([str(x) for x in hub_map[name]])
+            if merged_internal:
+                # Deduplicate while preserving order
+                merged_internal = list(dict.fromkeys(merged_internal))
+                tool_entry["internalTools"] = merged_internal
+                internal_total += len(merged_internal)
+
+            normalized_tools.append(tool_entry)
+
+        result = {
             "initializeMs": initialize_ms,
-            "toolCount": len(tools),
-            "tools": tool_info,
+            "toolCount": len(tools_list),
+            "internalToolCount": internal_total,
+            "tools": normalized_tools,
             "success": True,
         }
+        if hub_map:
+            result["hub_tools"] = hub_map
+        # Include prompts and resources from analysis
+        if full_analysis.get("prompts"):
+            result["prompts"] = full_analysis["prompts"]
+        if full_analysis.get("resources"):
+            result["resources"] = full_analysis["resources"]
+        return result
+    except TimeoutError:
+        from hud.shared.exceptions import HudException
+
+        hud_console.error("MCP server initialization timed out after 60 seconds")
+        hud_console.info(
+            "The server likely crashed during startup - check stderr logs with 'hud debug'"
+        )
+        raise HudException("MCP server initialization timeout") from None
     except Exception as e:
         from hud.shared.exceptions import HudException
 
@@ -227,32 +555,86 @@ def build_docker_image(
     verbose: bool = False,
     build_args: dict[str, str] | None = None,
     platform: str | None = None,
+    secrets: list[str] | None = None,
+    remote_cache: str | None = None,
 ) -> bool:
     """Build a Docker image from a directory."""
     hud_console = HUDConsole()
     build_args = build_args or {}
+    secrets = secrets or []
 
-    # Check if Dockerfile exists
-    dockerfile = directory / "Dockerfile"
-    if not dockerfile.exists():
+    # Check if Dockerfile exists (prefer Dockerfile.hud)
+    dockerfile = find_dockerfile(directory)
+    if dockerfile is None:
         hud_console.error(f"No Dockerfile found in {directory}")
+        hud_console.info("Expected: Dockerfile.hud or Dockerfile")
         return False
 
-    # Default platform to match RL pipeline unless explicitly overridden
+    # Build command - use buildx when remote cache is enabled
     effective_platform = platform if platform is not None else "linux/amd64"
+    cmd = ["docker", "buildx", "build"] if remote_cache else ["docker", "build"]
+
+    # Specify dockerfile explicitly if not the default name
+    if dockerfile.name != "Dockerfile":
+        cmd.extend(["-f", str(dockerfile)])
 
-    # Build command
-    cmd = ["docker", "build"]
     if effective_platform:
         cmd.extend(["--platform", effective_platform])
     cmd.extend(["-t", tag])
     if no_cache:
         cmd.append("--no-cache")
 
+    # Add remote cache support for ECR
+    if remote_cache:
+        try:
+            # Validate ECR repo name
+            if not re.match(r"^[a-z0-9]([a-z0-9\-_/]*[a-z0-9])?$", remote_cache):
+                hud_console.error(f"Invalid ECR repo name: {remote_cache}")
+                hud_console.info(
+                    "ECR repo names must contain only lowercase letters, numbers, hyphens, underscores, and forward slashes"  # noqa: E501
+                )
+                return False
+
+            # Get required environment variables
+            aws_account_id = os.getenv("AWS_ACCOUNT_ID")
+            aws_region = os.getenv("AWS_DEFAULT_REGION", "us-east-1")
+
+            if not aws_account_id:
+                hud_console.error("AWS_ACCOUNT_ID environment variable not set")
+                return False
+
+            # ECR cache image reference
+            cache_image = (
+                f"{aws_account_id}.dkr.ecr.{aws_region}.amazonaws.com/{remote_cache}:cache"
+            )
+
+            # Add cache arguments with proper ECR format
+            cmd.extend(
+                [
+                    "--cache-from",
+                    f"type=registry,ref={cache_image}",
+                    "--cache-to",
+                    f"mode=max,image-manifest=true,oci-mediatypes=true,type=registry,ref={cache_image}",
+                    "--load",  # Load image to local Docker after build
+                ]
+            )
+
+            hud_console.success(f"Remote cache configured: {cache_image}")
+
+        except typer.Exit:
+            raise
+        except Exception as e:
+            hud_console.error(f"Remote cache setup error: {e}")
+            return False
+
     # Add build args
     for key, value in build_args.items():
         cmd.extend(["--build-arg", f"{key}={value}"])
 
+    # Add secrets
+    for secret in secrets:
+        cmd.extend(["--secret", secret])
+
     cmd.append(str(directory))
 
     # Always show build output
@@ -260,7 +642,10 @@ def build_docker_image(
 
     try:
         # Use Docker's native output formatting - no capture, let Docker handle display
-        result = subprocess.run(cmd, check=False)  # noqa: S603
+        env = os.environ.copy()
+        if secrets:
+            env["DOCKER_BUILDKIT"] = "1"
+        result = subprocess.run(cmd, check=False, env=env)  # noqa: S603
         return result.returncode == 0
     except Exception as e:
         hud_console.error(f"Build error: {e}")
@@ -274,10 +659,14 @@ def build_environment(
     verbose: bool = False,
     env_vars: dict[str, str] | None = None,
     platform: str | None = None,
+    secrets: list[str] | None = None,
+    remote_cache: str | None = None,
+    build_args: dict[str, str] | None = None,
 ) -> None:
     """Build a HUD environment and generate lock file."""
     hud_console = HUDConsole()
     env_vars = env_vars or {}
+    build_args = build_args or {}
     hud_console.header("HUD Environment Build")
 
     # Resolve directory
@@ -286,26 +675,52 @@ def build_environment(
         hud_console.error(f"Directory not found: {directory}")
         raise typer.Exit(1)
 
-    # Check for pyproject.toml
-    pyproject_path = env_dir / "pyproject.toml"
-    if not pyproject_path.exists():
-        hud_console.error(f"No pyproject.toml found in {directory}")
-        raise typer.Exit(1)
+    from hud.cli.utils.docker import require_docker_running
 
-    # Read pyproject.toml to get image name
-    try:
-        import toml
+    require_docker_running()
 
-        pyproject = toml.load(pyproject_path)
-        default_image = pyproject.get("tool", {}).get("hud", {}).get("image", None)
-        if not default_image:
-            # Generate default from directory name
-            default_image = f"{env_dir.name}:dev"
-    except Exception:
-        default_image = f"{env_dir.name}:dev"
-
-    # Determine final image tag to use
-    image_tag: str = tag if tag else default_image
+    # Step 1: Check for hud.lock.yaml (previous build)
+    lock_path = env_dir / "hud.lock.yaml"
+    base_name = None
+
+    if lock_path.exists():
+        try:
+            with open(lock_path) as f:
+                lock_data = yaml.safe_load(f)
+            # Get base name from lock file (strip version/digest)
+            lock_image = lock_data.get("images", {}).get("local") or lock_data.get("image", "")
+            if lock_image:
+                # Remove @sha256:... digest if present
+                if "@" in lock_image:
+                    lock_image = lock_image.split("@")[0]
+                # Extract base name (remove :version tag)
+                base_name = lock_image.split(":")[0] if ":" in lock_image else lock_image
+                hud_console.info(f"Using base name from lock file: {base_name}")
+        except Exception as e:
+            hud_console.warning(f"Could not read lock file: {e}")
+
+    # Step 2: If no lock, check for Dockerfile
+    if not base_name:
+        dockerfile_path = find_dockerfile(env_dir)
+        if dockerfile_path is None:
+            hud_console.error(f"Not a valid environment directory: {directory}")
+            hud_console.info("Expected: Dockerfile.hud, Dockerfile, or hud.lock.yaml")
+            raise typer.Exit(1)
+
+        # First build - use directory name
+        base_name = env_dir.name
+        hud_console.info(f"First build - using base name: {base_name}")
+        if dockerfile_path.name == "Dockerfile.hud":
+            hud_console.info("Using Dockerfile.hud")
+
+    # If user provides --tag, respect it; otherwise use base name only (version added later)
+    if tag:
+        # User explicitly provided a tag
+        image_tag = tag
+        base_name = image_tag.split(":")[0] if ":" in image_tag else image_tag
+    else:
+        # No tag provided - we'll add version later
+        image_tag = None
 
     # Build temporary image first
     temp_tag = f"hud-build-temp:{int(time.time())}"
@@ -318,52 +733,80 @@ def build_environment(
         temp_tag,
         no_cache,
         verbose,
-        build_args=None,
+        build_args=build_args or None,
         platform=platform,
+        secrets=secrets,
+        remote_cache=remote_cache,
     ):
         hud_console.error("Docker build failed")
         raise typer.Exit(1)
 
     hud_console.success(f"Built temporary image: {temp_tag}")
 
-    # Analyze the environment
+    # Analyze the environment (merge folder .env if present)
     hud_console.progress_message("Analyzing MCP environment...")
 
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
     try:
-        analysis = loop.run_until_complete(analyze_mcp_environment(temp_tag, verbose, env_vars))
+        # Merge .env from env_dir for analysis only
+        try:
+            from hud.cli.utils.docker import load_env_vars_for_dir
+
+            env_from_file = load_env_vars_for_dir(env_dir)
+        except Exception:
+            env_from_file = {}
+        merged_env_for_analysis = {**env_from_file, **(env_vars or {})}
+
+        analysis = loop.run_until_complete(
+            analyze_mcp_environment(temp_tag, verbose, merged_env_for_analysis)
+        )
+    except Exception as e:
+        hud_console.error(f"Failed to analyze MCP environment: {e}")
+        hud_console.info("")
+        hud_console.info("To debug this issue, run:")
+        hud_console.command_example(f"hud debug {temp_tag}")
+        hud_console.info("")
+        raise typer.Exit(1) from e
     finally:
         loop.close()
 
-    hud_console.success(f"Analyzed environment: {analysis['toolCount']} tools found")
+    # Show analysis results including hub tools, prompts, resources
+    tool_count = analysis["toolCount"]
+    prompt_count = len(analysis.get("prompts") or [])
+    resource_count = len(analysis.get("resources") or [])
+
+    parts = [f"{tool_count} tools"]
+    if prompt_count:
+        parts.append(f"{prompt_count} prompts")
+    if resource_count:
+        parts.append(f"{resource_count} resources")
+
+    tool_msg = f"Analyzed environment: {', '.join(parts)} found"
+    hud_console.success(tool_msg)
 
     # Extract environment variables from Dockerfile
-    dockerfile_path = env_dir / "Dockerfile"
+    dockerfile_path = find_dockerfile(env_dir) or env_dir / "Dockerfile"
     required_env, optional_env = extract_env_vars_from_dockerfile(dockerfile_path)
 
-    # Merge user-provided env vars with detected ones
-    provided_env_vars: dict[str, str] = {}
-    missing_required = []
-    if env_vars:
-        # Use placeholders in lock file for any provided values to avoid storing secrets
-        provided_env_vars = {k: f"${{{k}}}" for k in env_vars}
-        # Track which required vars are still missing
-        missing_required = [e for e in required_env if e not in env_vars]
-
-        # Show what env vars were provided
-        hud_console.success(f"Using provided environment variables: {', '.join(env_vars.keys())}")
-    else:
-        missing_required = required_env[:]
+    # Show env vars detected from .env file
+    if env_from_file:
+        hud_console.info(
+            f"Detected environment variables from .env file: {', '.join(sorted(env_from_file.keys()))}"  # noqa: E501
+        )
 
-    # Warn about missing required variables
-    if missing_required:
+    # Create a complete set of all required variables for warning
+    all_required_for_warning = set(required_env)
+    all_required_for_warning.update(env_from_file.keys())
+
+    # Find which ones are missing (not provided via -e flags)
+    all_missing = all_required_for_warning - set(env_vars.keys() if env_vars else [])
+
+    if all_missing:
         hud_console.warning(
-            f"Missing required environment variables: {', '.join(missing_required)}"
-        )
-        hud_console.info(
-            "These can be added to the lock file after build or provided with -e flags"
+            f"Environment variables not provided via -e flags: {', '.join(sorted(all_missing))}"
         )
+        hud_console.info("These will be added to the required list in the lock file")
 
     # Check for existing version and increment
     lock_path = env_dir / "hud.lock.yaml"
@@ -378,17 +821,32 @@ def build_environment(
         new_version = "0.1.0"
         hud_console.info(f"Setting initial version: {new_version}")
 
-    # Create lock file content - minimal and useful
+    # Determine base name for image references
+    if image_tag:
+        base_name = image_tag.split(":")[0] if ":" in image_tag else image_tag
+
+    # Collect runtime metadata and compute base image/platform
+    runtime_info = collect_runtime_metadata(temp_tag, verbose=verbose)
+    base_image = parse_base_image(dockerfile_path)
+    effective_platform = platform if platform is not None else "linux/amd64"
+
+    # Create lock file content with images subsection at top
     lock_content = {
-        "version": "1.0",  # Lock file format version
-        "image": tag,  # Will be updated with ID/digest later
+        "version": "1.3",  # Lock file format version
+        "images": {
+            "local": f"{base_name}:{new_version}",  # Local tag with version
+            "full": None,  # Will be set with digest after build
+            "pushed": None,  # Will be set by hud push
+        },
         "build": {
-            "generatedAt": datetime.utcnow().isoformat() + "Z",
+            "generatedAt": datetime.now(UTC).isoformat() + "Z",
             "hudVersion": hud_version,
             "directory": str(env_dir.name),
-            "version": new_version,  # Internal environment version
+            "version": new_version,
             # Fast source fingerprint for change detection
             "sourceHash": compute_source_hash(env_dir),
+            "baseImage": base_image,
+            "platform": effective_platform,
         },
         "environment": {
             "initializeMs": analysis["initializeMs"],
@@ -396,8 +854,19 @@ def build_environment(
         },
     }
 
+    if runtime_info:
+        lock_content["environment"]["runtime"] = runtime_info
+    internal_count = int(analysis.get("internalToolCount", 0) or 0)
+    lock_content["environment"]["internalToolCount"] = internal_count
+
     # Add environment variables section if any exist
-    if missing_required or optional_env or provided_env_vars:
+    # Include env vars from .env file as well
+    env_vars_from_file = set(env_from_file.keys()) if env_from_file else set()
+
+    # Check if we have any env vars to document
+    has_env_vars = bool(required_env or optional_env or env_vars or env_vars_from_file)
+
+    if has_env_vars:
         lock_content["environment"]["variables"] = {}
 
         # Add note about editing environment variables
@@ -406,23 +875,53 @@ def build_environment(
             "Provided variables will be used when running the environment."
         )
 
-        if provided_env_vars:
-            lock_content["environment"]["variables"]["provided"] = provided_env_vars
-        if missing_required:
-            lock_content["environment"]["variables"]["required"] = missing_required
+        # Combine all required variables: from Dockerfile, .env file, and provided vars
+        all_required = set(required_env)
+
+        # Add all env vars from .env file to required
+        all_required.update(env_vars_from_file)
+
+        # Add all provided env vars to required
+        if env_vars:
+            all_required.update(env_vars.keys())
+
+        # Remove any that are optional - they stay in optional
+        all_required = all_required - set(optional_env)
+
+        if all_required:
+            lock_content["environment"]["variables"]["required"] = sorted(list(all_required))
         if optional_env:
             lock_content["environment"]["variables"]["optional"] = optional_env
 
     # Add tools with full schemas for RL config generation
     if analysis["tools"]:
-        lock_content["tools"] = [
-            {
+        tools_serialized: list[dict[str, Any]] = []
+        for tool in analysis["tools"]:
+            entry: dict[str, Any] = {
                 "name": tool["name"],
+                # Preserve legacy shape: always include description/inputSchema
                 "description": tool.get("description", ""),
                 "inputSchema": tool.get("inputSchema", {}),
             }
-            for tool in analysis["tools"]
-        ]
+            if tool.get("internalTools"):
+                entry["internalTools"] = tool.get("internalTools")
+            tools_serialized.append(entry)
+        lock_content["tools"] = tools_serialized
+
+    # Add hub tools if present (analyze_environment returns hub_tools with snake_case)
+    hub_tools = analysis.get("hub_tools") or analysis.get("hubTools")
+    if hub_tools:
+        lock_content["hubTools"] = hub_tools
+
+    # Add prompts if present
+    prompts = analysis.get("prompts")
+    if prompts:
+        lock_content["prompts"] = prompts
+
+    # Add resources if present
+    resources = analysis.get("resources")
+    if resources:
+        lock_content["resources"] = resources
 
     # Write lock file
     lock_path = env_dir / "hud.lock.yaml"
@@ -450,15 +949,55 @@ def build_environment(
     hud_console.progress_message("Rebuilding with lock file metadata...")
 
     # Build final image with label (uses cache from first build)
-    # Also tag with version
-    base_name = image_tag.split(":")[0] if ":" in image_tag else image_tag
+    # Create tags: versioned and latest (and custom tag if provided)
     version_tag = f"{base_name}:{new_version}"
+    latest_tag = f"{base_name}:latest"
+
+    # Build command - use buildx when remote cache is enabled
+    label_cmd = ["docker", "buildx", "build"] if remote_cache else ["docker", "build"]
+
+    # Specify dockerfile explicitly if not the default name
+    if dockerfile_path and dockerfile_path.name != "Dockerfile":
+        label_cmd.extend(["-f", str(dockerfile_path)])
 
-    label_cmd = ["docker", "build"]
     # Use same defaulting for the second build step
     label_platform = platform if platform is not None else "linux/amd64"
     if label_platform:
         label_cmd.extend(["--platform", label_platform])
+
+    # Add remote cache support for final build
+    if remote_cache:
+        try:
+            if not re.match(r"^[a-z0-9]([a-z0-9\-_/]*[a-z0-9])?$", remote_cache):
+                hud_console.error(f"Invalid ECR repo name: {remote_cache}")
+                raise typer.Exit(1)
+
+            aws_account_id = os.getenv("AWS_ACCOUNT_ID")
+            aws_region = os.getenv("AWS_DEFAULT_REGION", "us-east-1")
+
+            if not aws_account_id:
+                hud_console.error("AWS_ACCOUNT_ID environment variable not set")
+                raise typer.Exit(1)
+
+            cache_image = (
+                f"{aws_account_id}.dkr.ecr.{aws_region}.amazonaws.com/{remote_cache}:cache"
+            )
+
+            label_cmd.extend(
+                [
+                    "--cache-from",
+                    f"type=registry,ref={cache_image}",
+                    "--cache-to",
+                    f"mode=max,image-manifest=true,oci-mediatypes=true,type=registry,ref={cache_image}",
+                    "--load",  # Load image to local Docker after build
+                ]
+            )
+        except typer.Exit:
+            raise
+        except Exception as e:
+            hud_console.error(f"Remote cache setup error: {e}")
+            raise typer.Exit(1) from e
+
     label_cmd.extend(
         [
             "--label",
@@ -466,47 +1005,68 @@ def build_environment(
             "--label",
             f"org.hud.version={new_version}",
             "-t",
-            image_tag,
+            version_tag,  # Always tag with new version
             "-t",
-            version_tag,
+            latest_tag,  # Always tag with latest
         ]
     )
 
+    # Add custom tag if user provided one
+    if image_tag and image_tag not in [version_tag, latest_tag]:
+        label_cmd.extend(["-t", image_tag])
+
+    # Add build args to final image build (same as initial build)
+    for key, value in build_args.items():
+        label_cmd.extend(["--build-arg", f"{key}={value}"])
+
+    # Add secrets to final image build (same as initial build)
+    for secret in secrets or []:
+        label_cmd.extend(["--secret", secret])
+
     label_cmd.append(str(env_dir))
 
     # Run rebuild using Docker's native output formatting
+    env = os.environ.copy()
+    if secrets:
+        env["DOCKER_BUILDKIT"] = "1"
     if verbose:
         # Show Docker's native output when verbose
-        result = subprocess.run(label_cmd, check=False)  # noqa: S603
+        result = subprocess.run(label_cmd, check=False, env=env)  # noqa: S603
     else:
-        # Hide output when not verbose
+        # Capture output for error reporting, but don't show unless it fails
         result = subprocess.run(  # noqa: S603
-            label_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False
+            label_cmd, capture_output=True, text=True, check=False, env=env
         )
 
     if result.returncode != 0:
         hud_console.error("Failed to rebuild with label")
+        if not verbose and result.stderr:
+            hud_console.info("Error output:")
+            hud_console.info(str(result.stderr))
+        if not verbose:
+            hud_console.info("")
+            hud_console.info("Run with --verbose to see full build output:")
+            hud_console.command_example("hud build --verbose")
         raise typer.Exit(1)
 
     hud_console.success("Built final image with lock file metadata")
 
     # NOW get the image ID after the final build
-    image_id = get_docker_image_id(image_tag)
+    image_id = get_docker_image_id(version_tag)
     if image_id:
-        # For local builds, store the image ID
-        # Docker IDs come as sha256:hash, we want tag@sha256:hash
+        # Store full reference with digest
         if image_id.startswith("sha256:"):
-            lock_content["image"] = f"{image_tag}@{image_id}"
+            lock_content["images"]["full"] = f"{version_tag}@{image_id}"
         else:
-            lock_content["image"] = f"{image_tag}@sha256:{image_id}"
+            lock_content["images"]["full"] = f"{version_tag}@sha256:{image_id}"
 
-        # Update the lock file with the new image reference
+        # Update the lock file with the full image reference
         with open(lock_path, "w") as f:
             yaml.dump(lock_content, f, default_flow_style=False, sort_keys=False)
 
-        hud_console.success("Updated lock file with image ID")
+        hud_console.success("Updated lock file with image digest")
     else:
-        hud_console.warning("Could not retrieve image ID for lock file")
+        hud_console.warning("Could not retrieve image digest")
 
     # Remove temp image after we're done
     subprocess.run(["docker", "rmi", "-f", temp_tag], capture_output=True)  # noqa: S603, S607
@@ -514,15 +1074,32 @@ def build_environment(
     # Add to local registry
     if image_id:
         # Save to local registry using the helper
-        save_to_registry(lock_content, lock_content.get("image", tag), verbose)
+        local_ref = lock_content.get("images", {}).get("local", version_tag)
+        save_to_registry(lock_content, local_ref, verbose)
+
+    # Update tasks.json files with new version
+    hud_console.progress_message("Updating task files with new version...")
+    updated_task_files = update_tasks_json_versions(
+        env_dir, base_name, existing_version, new_version
+    )
+
+    if updated_task_files:
+        hud_console.success(f"Updated {len(updated_task_files)} task file(s)")
+    else:
+        hud_console.dim_info("No task files found or updated", value="")
 
     # Print summary
     hud_console.section_title("Build Complete")
 
     # Show the version tag as primary since that's what will be pushed
     hud_console.status_item("Built image", version_tag, primary=True)
-    if image_tag:
-        hud_console.status_item("Also tagged", image_tag)
+
+    # Show additional tags
+    additional_tags = [latest_tag]
+    if image_tag and image_tag not in [version_tag, latest_tag]:
+        additional_tags.append(image_tag)
+    hud_console.status_item("Also tagged", ", ".join(additional_tags))
+
     hud_console.status_item("Version", new_version)
     hud_console.status_item("Lock file", "hud.lock.yaml")
     hud_console.status_item("Tools found", str(analysis["toolCount"]))
@@ -534,7 +1111,7 @@ def build_environment(
     hud_console.section_title("Next Steps")
     hud_console.info("Test locally:")
     hud_console.command_example("hud dev", "Hot-reload development")
-    hud_console.command_example(f"hud run {image_tag}", "Run the built image")
+    hud_console.command_example(f"hud run {version_tag}", "Run the built image")
     hud_console.info("")
     hud_console.info("Publish to registry:")
     hud_console.command_example("hud push", f"Push as {version_tag}")
@@ -552,6 +1129,15 @@ def build_command(
     verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed output"),
     env_vars: dict[str, str] | None = None,
     platform: str | None = None,
+    secrets: list[str] | None = typer.Option(  # noqa: B008
+        None,
+        "--secret",
+        help=("Docker build secret (repeatable), e.g. --secret id=GITHUB_TOKEN,env=GITHUB_TOKEN"),
+    ),
+    remote_cache: str | None = None,
+    build_args: dict[str, str] | None = None,
 ) -> None:
     """Build a HUD environment and generate lock file."""
-    build_environment(directory, tag, no_cache, verbose, env_vars, platform)
+    build_environment(
+        directory, tag, no_cache, verbose, env_vars, platform, secrets, remote_cache, build_args
+    )