hud-python 0.4.11__py3-none-any.whl → 0.4.12__py3-none-any.whl

hud/cli/eval.py

@@ -6,18 +6,13 @@ import asyncio
 import json
 import logging
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal
+from typing import Any, Literal
 
 import typer
 
 import hud
 from hud.utils.design import HUDDesign
 
-if TYPE_CHECKING:
-    from datasets import Dataset
-    from hud.agents import ClaudeAgent, OperatorAgent
-    from hud.agents.misc.response_agent import ResponseAgent
-
 logger = logging.getLogger(__name__)
 design = HUDDesign()
 
@@ -29,7 +24,7 @@ def build_agent(
     allowed_tools: list[str] | None = None,
 ) -> Any:
     """Create and return the requested agent type."""
-    
+
     # Import agents lazily to avoid dependency issues
     try:
         from hud.agents.misc.response_agent import ResponseAgent
@@ -39,7 +34,7 @@ def build_agent(
             "Please install with: pip install 'hud-python[agent]'"
         )
         raise typer.Exit(1) from e
-    
+
     if agent_type == "openai":
         try:
             from hud.agents import OperatorAgent
@@ -49,14 +44,14 @@ def build_agent(
                 "Please install with: pip install 'hud-python[agent]'"
             )
             raise typer.Exit(1) from e
-            
+
         allowed_tools = allowed_tools or ["openai_computer"]
-        
+
         return OperatorAgent(
             allowed_tools=allowed_tools,
             response_agent=ResponseAgent(),
         )
-    
+
     # Fallback Claude agent (Anthropic)
     try:
         from hud.agents import ClaudeAgent
@@ -66,10 +61,10 @@ def build_agent(
             "Please install with: pip install 'hud-python[agent]'"
         )
         raise typer.Exit(1) from e
-        
+
     model = model or "claude-sonnet-4-20250514"
     allowed_tools = allowed_tools or ["anthropic_computer"]
-    
+
     return ClaudeAgent(
         model=model,
         allowed_tools=allowed_tools,
@@ -85,26 +80,82 @@ async def run_single_task(
     allowed_tools: list[str] | None = None,
     max_steps: int = 10,
 ) -> None:
-    """Load one task and execute it."""
-    
+    """Load one task and execute it, or detect if JSON contains a list and run as dataset."""
+
     design.info("📊 Loading dataset…")
-    
-    # Import Task lazily
+
+    # Import Task and run_dataset lazily
     try:
-        from hud.datasets import Task
+        from hud.datasets import Task, run_dataset
     except ImportError as e:
         design.error(
             "Dataset dependencies are not installed. "
             "Please install with: pip install 'hud-python[agent]'"
         )
         raise typer.Exit(1) from e
-    
-    # Check if it's a single task JSON file
+
+    # Check if it's a JSON file
     path = Path(source)
     if path.exists() and path.suffix == ".json":
-        with open(path, "r") as f:
-            task_data = json.load(f)
-        task = Task(**task_data)
+        with open(path) as f:  # noqa: ASYNC230
+            json_data = json.load(f)
+
+        # Check if JSON contains a list of tasks
+        if isinstance(json_data, list):
+            design.info(f"Found {len(json_data)} tasks in JSON file, running as dataset…")
+
+            # Build agent class and config for run_dataset
+            if agent_type == "openai":
+                try:
+                    from hud.agents import OperatorAgent
+
+                    agent_class = OperatorAgent
+                except ImportError as e:
+                    design.error(
+                        "OpenAI agent dependencies are not installed. "
+                        "Please install with: pip install 'hud-python[agent]'"
+                    )
+                    raise typer.Exit(1) from e
+
+                agent_config: dict[str, Any] = {
+                    "allowed_tools": allowed_tools or ["openai_computer"],
+                }
+            else:
+                try:
+                    from hud.agents import ClaudeAgent
+
+                    agent_class = ClaudeAgent
+                except ImportError as e:
+                    design.error(
+                        "Claude agent dependencies are not installed. "
+                        "Please install with: pip install 'hud-python[agent]'"
+                    )
+                    raise typer.Exit(1) from e
+
+                agent_config = {
+                    "model": model or "claude-sonnet-4-20250514",
+                    "allowed_tools": allowed_tools or ["anthropic_computer"],
+                }
+
+            # Run as dataset with single-task concurrency to maintain debug behavior
+            results = await run_dataset(
+                name=f"JSON Dataset: {path.name}",
+                dataset=json_data,  # Pass the list directly
+                agent_class=agent_class,
+                agent_config=agent_config,
+                max_concurrent=1,  # Run sequentially for debug mode
+                metadata={"source": str(path)},
+                max_steps=max_steps,
+                auto_respond=True,
+            )
+
+            # Display summary
+            successful = sum(1 for r in results if getattr(r, "reward", 0) > 0)
+            design.success(f"Completed {len(results)} tasks: {successful} successful")
+            return
+
+        # Single task JSON
+        task = Task(**json_data)
     else:
         # Load from HuggingFace dataset
         try:
@@ -115,15 +166,15 @@ async def run_single_task(
                 "Please install with: pip install 'hud-python[agent]'"
             )
             raise typer.Exit(1) from e
-            
+
         dataset = load_dataset(source, split="train")
-        
+
         # Get first task from dataset
         sample_task = dataset[0]  # type: ignore[index]
         task = Task(**sample_task)  # type: ignore[arg-type]
-    
+
     task_prompt = task.prompt[:50] + "..." if len(task.prompt) > 50 else task.prompt
-    
+
     with hud.trace(name=task_prompt):
         agent = build_agent(
             agent_type,
@@ -145,7 +196,7 @@ async def run_full_dataset(
     max_steps: int = 50,
 ) -> list[Any]:
     """Run evaluation across the entire dataset using hud.datasets.run_dataset."""
-    
+
     # Import run_dataset lazily
     try:
         from hud.datasets import run_dataset
@@ -155,11 +206,29 @@ async def run_full_dataset(
             "Please install with: pip install 'hud-python[agent]'"
         )
         raise typer.Exit(1) from e
-    
+
+    # Check if source is a JSON file with list of tasks
+    path = Path(source)
+    dataset_or_tasks = source
+    dataset_name = source.split("/")[-1]
+
+    if path.exists() and path.suffix == ".json":
+        with open(path) as f:  # noqa: ASYNC230
+            json_data = json.load(f)
+
+        if isinstance(json_data, list):
+            dataset_or_tasks = json_data
+            dataset_name = f"JSON Dataset: {path.name}"
+            design.info(f"Found {len(json_data)} tasks in JSON file")
+        else:
+            design.error("JSON file must contain a list of tasks when using --full flag")
+            raise typer.Exit(1)
+
     # Build agent class + config for run_dataset
     if agent_type == "openai":
         try:
             from hud.agents import OperatorAgent
+
             agent_class = OperatorAgent
         except ImportError as e:
             design.error(
@@ -167,13 +236,14 @@ async def run_full_dataset(
                 "Please install with: pip install 'hud-python[agent]'"
             )
             raise typer.Exit(1) from e
-            
+
         agent_config: dict[str, Any] = {
             "allowed_tools": allowed_tools or ["openai_computer"],
         }
     else:
         try:
             from hud.agents import ClaudeAgent
+
             agent_class = ClaudeAgent
         except ImportError as e:
             design.error(
@@ -181,16 +251,16 @@ async def run_full_dataset(
                 "Please install with: pip install 'hud-python[agent]'"
             )
             raise typer.Exit(1) from e
-            
+
         agent_config = {
             "model": model or "claude-sonnet-4-20250514",
             "allowed_tools": allowed_tools or ["anthropic_computer"],
         }
-    
+
     design.info("🚀 Running evaluation…")
     return await run_dataset(
-        name=f"Evaluation {source.split('/')[-1]}",
-        dataset=source,
+        name=f"Evaluation {dataset_name}",
+        dataset=dataset_or_tasks,
         agent_class=agent_class,
         agent_config=agent_config,
         max_concurrent=max_concurrent,
@@ -203,7 +273,7 @@ async def run_full_dataset(
 def eval_command(
     source: str = typer.Argument(
         ...,
-        help="HuggingFace dataset identifier (e.g. 'hud-evals/SheetBench-50') or task JSON file",
+        help="HuggingFace dataset identifier (e.g. 'hud-evals/SheetBench-50'), single task JSON file, or JSON file with list of tasks",  # noqa: E501
     ),
     full: bool = typer.Option(
         False,
@@ -237,66 +307,76 @@ def eval_command(
     ),
 ) -> None:
     """🚀 Run evaluation on datasets or individual tasks with agents.
-    
+
     Examples:
         # Evaluate a single task from SheetBench
         hud eval hud-evals/SheetBench-50
-        
+
         # Evaluate the FULL SheetBench dataset with Claude
         hud eval hud-evals/SheetBench-50 --full --agent claude
-        
+
         # Run a single task from a JSON file
         hud eval task.json
-        
+
+        # Run multiple tasks from a JSON file (auto-detects list)
+        hud eval tasks.json  # If tasks.json contains a list, runs all tasks
+
+        # Run JSON list with full dataset mode and concurrency
+        hud eval tasks.json --full --max-concurrent 10
+
         # Run with OpenAI Operator agent
         hud eval hud-evals/OSWorld-Gold-Beta --agent openai
     """
-    from hud.settings import settings
     import os
-    
+
+    from hud.settings import settings
+
     # Check for required API keys
     if agent == "claude":
         if not settings.anthropic_api_key or not os.environ.get("ANTHROPIC_API_KEY"):
             design.error("ANTHROPIC_API_KEY is required for Claude agent")
             design.info("Set it in your environment or .env file: ANTHROPIC_API_KEY=your-key-here")
             raise typer.Exit(1)
-    elif agent == "openai":
-        if not settings.openai_api_key or not os.environ.get("OPENAI_API_KEY"):
-            design.error("OPENAI_API_KEY is required for OpenAI agent")
-            design.info("Set it in your environment or .env file: OPENAI_API_KEY=your-key-here")
-            raise typer.Exit(1)
-    
+    elif agent == "openai" and (
+        not settings.openai_api_key or not os.environ.get("OPENAI_API_KEY")
+    ):
+        design.error("OPENAI_API_KEY is required for OpenAI agent")
+        design.info("Set it in your environment or .env file: OPENAI_API_KEY=your-key-here")
+        raise typer.Exit(1)
+
     # Check for HUD_API_KEY if using HUD services
     if not settings.api_key or not os.environ.get("HUD_API_KEY"):
         design.warning("HUD_API_KEY not set. Some features may be limited.")
         design.info("Get your API key at: https://app.hud.so")
-    
+
     # Parse allowed tools
     allowed_tools_list = (
-        [t.strip() for t in allowed_tools.split(",") if t.strip()]
-        if allowed_tools
-        else None
+        [t.strip() for t in allowed_tools.split(",") if t.strip()] if allowed_tools else None
     )
-    
+
     # Set default max_steps if not provided
     if max_steps is None:
         max_steps = 50 if full else 10
-    
+
     # Run evaluation
     if full:
-        asyncio.run(run_full_dataset(
-            source,
-            agent_type=agent,
-            model=model,
-            allowed_tools=allowed_tools_list,
-            max_concurrent=max_concurrent,
-            max_steps=max_steps,
-        ))
+        asyncio.run(
+            run_full_dataset(
+                source,
+                agent_type=agent,
+                model=model,
+                allowed_tools=allowed_tools_list,
+                max_concurrent=max_concurrent,
+                max_steps=max_steps,
+            )
+        )
     else:
-        asyncio.run(run_single_task(
-            source,
-            agent_type=agent,
-            model=model,
-            allowed_tools=allowed_tools_list,
-            max_steps=max_steps,
-        ))
+        asyncio.run(
+            run_single_task(
+                source,
+                agent_type=agent,
+                model=model,
+                allowed_tools=allowed_tools_list,
+                max_steps=max_steps,
+            )
+        )

hud/cli/init.py

@@ -173,7 +173,6 @@ def sanitize_name(name: str) -> str:
 
 def create_environment(name: str | None, directory: str, force: bool) -> None:
     """Create a new HUD environment from templates."""
-    from hud.utils.design import HUDDesign
 
     design = HUDDesign()
 

hud/cli/list_func.py

@@ -2,9 +2,7 @@
 
 from __future__ import annotations
 
-import re
 from datetime import datetime
-from pathlib import Path
 
 import typer
 import yaml
@@ -12,30 +10,33 @@ from rich.table import Table
 
 from hud.utils.design import HUDDesign
 
-from .registry import get_registry_dir, list_registry_entries, extract_name_and_tag
+from .utils.registry import extract_name_and_tag, get_registry_dir, list_registry_entries
 
 
 def format_timestamp(timestamp: float | None) -> str:
     """Format timestamp to human-readable relative time."""
     if not timestamp:
         return "unknown"
-    
+
     dt = datetime.fromtimestamp(timestamp)
     now = datetime.now()
     delta = now - dt
-    
-    if delta.days > 365:
-        return f"{delta.days // 365}y ago"
-    elif delta.days > 30:
-        return f"{delta.days // 30}mo ago"
-    elif delta.days > 0:
+
+    # Get total seconds to handle edge cases properly
+    total_seconds = delta.total_seconds()
+
+    if total_seconds < 60:
+        return "just now"
+    elif total_seconds < 3600:
+        return f"{int(total_seconds // 60)}m ago"
+    elif total_seconds < 86400:  # Less than 24 hours
+        return f"{int(total_seconds // 3600)}h ago"
+    elif delta.days < 30:
         return f"{delta.days}d ago"
-    elif delta.seconds > 3600:
-        return f"{delta.seconds // 3600}h ago"
-    elif delta.seconds > 60:
-        return f"{delta.seconds // 60}m ago"
+    elif delta.days < 365:
+        return f"{delta.days // 30}mo ago"
     else:
-        return "just now"
+        return f"{delta.days // 365}y ago"
 
 
 def list_environments(
@@ -46,71 +47,73 @@ def list_environments(
 ) -> None:
     """List all HUD environments in the local registry."""
     design = HUDDesign()
-    
+
     if not json_output:
         design.header("HUD Environment Registry")
-    
+
     # Check for environment directory
     env_dir = get_registry_dir()
     if not env_dir.exists():
         if json_output:
-            print("[]")
+            print("[]")  # noqa: T201
         else:
             design.info("No environments found in local registry.")
             design.info("")
             design.info("Pull environments with: [cyan]hud pull <org/name:tag>[/cyan]")
             design.info("Build environments with: [cyan]hud build[/cyan]")
         return
-    
+
     # Collect all environments using the registry helper
     environments = []
-    
+
     for digest, lock_file in list_registry_entries():
         try:
             # Read lock file
             with open(lock_file) as f:
                 lock_data = yaml.safe_load(f)
-            
+
             # Extract metadata
             image = lock_data.get("image", "unknown")
             name, tag = extract_name_and_tag(image)
-            
+
             # Apply filter if specified
             if filter_name and filter_name.lower() not in name.lower():
                 continue
-            
+
             # Get additional metadata
             metadata = lock_data.get("metadata", {})
             description = metadata.get("description", "")
             tools_count = len(metadata.get("tools", []))
-            
+
             # Get file modification time as pulled time
             pulled_time = lock_file.stat().st_mtime
-            
-            environments.append({
-                "name": name,
-                "tag": tag,
-                "digest": digest,
-                "description": description,
-                "tools_count": tools_count,
-                "pulled_time": pulled_time,
-                "image": image,
-                "path": str(lock_file),
-            })
-            
+
+            environments.append(
+                {
+                    "name": name,
+                    "tag": tag,
+                    "digest": digest,
+                    "description": description,
+                    "tools_count": tools_count,
+                    "pulled_time": pulled_time,
+                    "image": image,
+                    "path": str(lock_file),
+                }
+            )
+
         except Exception as e:
             if verbose:
                 design.warning(f"Failed to read {lock_file}: {e}")
-    
+
     # Sort by pulled time (newest first)
     environments.sort(key=lambda x: x["pulled_time"], reverse=True)
-    
+
     if json_output:
         # Output as JSON
         import json
-        json_data = []
-        for env in environments:
-            json_data.append({
+
+        json_data = [
+            {
                 "name": env["name"],
                 "tag": env["tag"],
                 "digest": env["digest"],
@@ -118,67 +121,71 @@ def list_environments(
                 "tools_count": env["tools_count"],
                 "pulled_time": env["pulled_time"],
                 "image": env["image"],
-                "path": env["path"],
-            })
-        print(json.dumps(json_data, indent=2))
+                "path": str(env["path"]).replace("\\", "/"),  # Normalize path separators for JSON
+            }
+            for env in environments
+        ]
+        print(json.dumps(json_data, indent=2))  # noqa: T201
         return
-    
+
     if not environments:
         design.info("No environments found matching criteria.")
         design.info("")
         design.info("Pull environments with: [cyan]hud pull <org/name:tag>[/cyan]")
         design.info("Build environments with: [cyan]hud build[/cyan]")
         return
-    
+
     # Create table
-    table = Table(title=f"Found {len(environments)} environment{'s' if len(environments) != 1 else ''}")
+    table = Table(
+        title=f"Found {len(environments)} environment{'s' if len(environments) != 1 else ''}"
+    )
     table.add_column("Environment", style="cyan", no_wrap=True)
     table.add_column("Description", style="white")
     table.add_column("Tools", justify="right", style="yellow")
     table.add_column("Pulled", style="dim")
-    
+
     if show_all or verbose:
         table.add_column("Digest", style="dim")
-    
+
     # Add rows
     for env in environments:
         # Truncate description if too long
         desc = env["description"]
         if desc and len(desc) > 50 and not verbose:
             desc = desc[:47] + "..."
-        
+
         # Combine name and tag for easy copying
         full_ref = f"{env['name']}:{env['tag']}"
-        
+
         row = [
             full_ref,
             desc or "[dim]No description[/dim]",
             str(env["tools_count"]),
             format_timestamp(env["pulled_time"]),
         ]
-        
+
         if show_all or verbose:
             row.append(env["digest"][:12])
-        
+
         table.add_row(*row)
-    
-    design.print(table)
+
+    design.print(str(table))
     design.info("")
-    
+
     # Show usage hints
     design.section_title("Usage")
     if environments:
         # Use the most recently pulled environment as example
         example_env = environments[0]
         example_ref = f"{example_env['name']}:{example_env['tag']}"
-        
+
         design.info(f"Run an environment: [cyan]hud run {example_ref}[/cyan]")
         design.info(f"Analyze tools: [cyan]hud analyze {example_ref}[/cyan]")
         design.info(f"Debug server: [cyan]hud debug {example_ref}[/cyan]")
-    
+
     design.info("Pull more environments: [cyan]hud pull <org/name:tag>[/cyan]")
     design.info("Build new environments: [cyan]hud build[/cyan]")
-    
+
     if verbose:
         design.info("")
         design.info(f"[dim]Registry location: {env_dir}[/dim]")
@@ -188,21 +195,15 @@ def list_command(
     filter_name: str | None = typer.Option(
         None, "--filter", "-f", help="Filter environments by name (case-insensitive)"
     ),
-    json_output: bool = typer.Option(
-        False, "--json", help="Output as JSON"
-    ),
-    show_all: bool = typer.Option(
-        False, "--all", "-a", help="Show all columns including digest"
-    ),
-    verbose: bool = typer.Option(
-        False, "--verbose", "-v", help="Show detailed output"
-    ),
+    json_output: bool = typer.Option(False, "--json", help="Output as JSON"),
+    show_all: bool = typer.Option(False, "--all", "-a", help="Show all columns including digest"),
+    verbose: bool = typer.Option(False, "--verbose", "-v", help="Show detailed output"),
 ) -> None:
     """📋 List all HUD environments in local registry.
-    
+
     Shows environments pulled with 'hud pull' or built with 'hud build',
     stored in ~/.hud/envs/
-    
+
     Examples:
         hud list                    # List all environments
         hud list --filter text      # Filter by name
@@ -210,4 +211,4 @@ def list_command(
         hud list --all             # Show digest column
         hud list --verbose         # Show full descriptions
     """
-    list_environments(filter_name, json_output, show_all, verbose)
+    list_environments(filter_name, json_output, show_all, verbose)

hud/cli/pull.py

@@ -6,7 +6,6 @@ import subprocess
 from pathlib import Path
 from urllib.parse import quote
 
-import click
 import requests
 import typer
 import yaml
@@ -15,7 +14,7 @@ from rich.table import Table
 from hud.settings import settings
 from hud.utils.design import HUDDesign
 
-from .registry import save_to_registry
+from .utils.registry import save_to_registry
 
 
 def get_docker_manifest(image: str) -> dict | None: