hud-python 0.4.45__py3-none-any.whl → 0.5.1__py3-none-any.whl

hud/utils/agent_factories.py

@@ -1,86 +0,0 @@
-"""Factory functions for creating agents compatible with run_dataset."""
-
-from __future__ import annotations
-
-from typing import Any
-
-from openai import AsyncOpenAI
-
-from hud.agents.grounded_openai import GroundedOpenAIChatAgent
-from hud.agents.openai_chat_generic import GenericOpenAIChatAgent
-from hud.tools.grounding import GrounderConfig
-
-
-def create_openai_agent(**kwargs: Any) -> GenericOpenAIChatAgent:
-    """Factory for GenericOpenAIChatAgent with run_dataset compatibility.
-
-    Args:
-        api_key: OpenAI API key
-        base_url: Optional custom API endpoint
-        model_name: Model to use (e.g., "gpt-4o-mini")
-        **kwargs: Additional arguments passed to GenericOpenAIChatAgent
-
-    Returns:
-        Configured GenericOpenAIChatAgent instance
-
-    Example:
-        >>> from hud.datasets import run_dataset
-        >>> from hud.utils.agent_factories import create_openai_agent
-        >>> results = await run_dataset(
-        ...     "My Eval",
-        ...     "hud-evals/SheetBench-50",
-        ...     create_openai_agent,
-        ...     {"api_key": "your-key", "model_name": "gpt-4o-mini"},
-        ... )
-    """
-    api_key = kwargs.pop("api_key", None)
-    base_url = kwargs.pop("base_url", None)
-
-    openai_client = AsyncOpenAI(api_key=api_key, base_url=base_url)
-
-    return GenericOpenAIChatAgent(openai_client=openai_client, **kwargs)
-
-
-def create_grounded_agent(**kwargs: Any) -> GroundedOpenAIChatAgent:
-    """Factory for GroundedOpenAIChatAgent with run_dataset compatibility.
-
-    Args:
-        api_key: OpenAI API key for planning model
-        base_url: Optional custom API endpoint for planning model
-        model_name: Planning model to use (e.g., "gpt-4o-mini")
-        grounder_api_key: API key for grounding model
-        grounder_api_base: API base URL for grounding model (default: OpenRouter)
-        grounder_model: Grounding model to use (default: qwen/qwen-2.5-vl-7b-instruct)
-        **kwargs: Additional arguments passed to GroundedOpenAIChatAgent
-
-    Returns:
-        Configured GroundedOpenAIChatAgent instance
-
-    Example:
-        >>> from hud.datasets import run_dataset
-        >>> from hud.utils.agent_factories import create_grounded_agent
-        >>> results = await run_dataset(
-        ...     "Grounded Eval",
-        ...     dataset,
-        ...     create_grounded_agent,
-        ...     {
-        ...         "api_key": "openai-key",
-        ...         "grounder_api_key": "openrouter-key",
-        ...         "model_name": "gpt-4o-mini",
-        ...     },
-        ... )
-    """
-    api_key = kwargs.pop("api_key", None)
-    base_url = kwargs.pop("base_url", None)
-    grounder_api_key = kwargs.pop("grounder_api_key", None)
-    grounder_api_base = kwargs.pop("grounder_api_base", "https://openrouter.ai/api/v1")
-    grounder_model = kwargs.pop("grounder_model", "qwen/qwen-2.5-vl-7b-instruct")
-
-    openai_client = AsyncOpenAI(api_key=api_key, base_url=base_url)
-    grounder_config = GrounderConfig(
-        api_base=grounder_api_base, model=grounder_model, api_key=grounder_api_key
-    )
-
-    return GroundedOpenAIChatAgent(
-        openai_client=openai_client, grounder_config=grounder_config, **kwargs
-    )

hud/utils/async_utils.py

@@ -1,65 +0,0 @@
-"""Async utilities for HUD SDK.
-
-This module provides utilities for running async code in various environments,
-including Jupyter notebooks and synchronous contexts.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-import threading
-from typing import TYPE_CHECKING, Any
-
-if TYPE_CHECKING:
-    from collections.abc import Coroutine
-
-logger = logging.getLogger(__name__)
-
-
-def fire_and_forget(coro: Coroutine[Any, Any, Any], description: str = "task") -> None:
-    """Execute a coroutine in a fire-and-forget manner.
-
-    This function handles running async code in various contexts:
-    - When an event loop is already running (normal async context)
-    - When no event loop exists (sync context, some Jupyter setups)
-    - Gracefully handles interpreter shutdown
-
-    Args:
-        coro: The coroutine to execute
-        description: Description of the task for logging (e.g., "update job status")
-
-    Example:
-        fire_and_forget(
-            some_async_function(),
-            description="update status"
-        )
-    """
-    try:
-        # Try to get current event loop
-        loop = asyncio.get_running_loop()
-        # Schedule the coroutine
-        task = loop.create_task(coro)
-        # Add error handler to prevent unhandled exceptions
-        task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
-    except RuntimeError:
-        # No running event loop (e.g., Jupyter without %autoawait, sync context)
-        try:
-            # Try to run in a thread as a fallback
-            def run_in_thread() -> None:
-                loop = asyncio.new_event_loop()
-                asyncio.set_event_loop(loop)
-                try:
-                    loop.run_until_complete(coro)
-                except Exception as e:
-                    # Suppress warnings about interpreter shutdown
-                    if "interpreter shutdown" not in str(e):
-                        logger.debug("Error in threaded %s: %s", description, e)
-
-            thread = threading.Thread(target=run_in_thread, daemon=True)
-            thread.start()
-        except Exception as e:
-            # If that fails too, just log and continue
-            # Special case: suppress "cannot schedule new futures after interpreter shutdown"
-            if "interpreter shutdown" not in str(e):
-                logger.debug("Could not %s - no event loop available: %s", description, e)

hud/utils/group_eval.py

@@ -1,223 +0,0 @@
-"""Utilities for grouped evaluation of tasks, following the RL pattern."""
-
-from __future__ import annotations
-
-import asyncio
-from statistics import mean, stdev
-from typing import Any
-
-import numpy as np
-
-import hud
-from hud.datasets import Task
-from hud.types import Trace
-from hud.utils.hud_console import HUDConsole
-
-hud_console = HUDConsole()
-
-
-async def run_tasks_grouped(
-    tasks: list[Any],
-    agent_class: type | Any,
-    agent_config: dict[str, Any] | None = None,
-    group_size: int = 1,
-    max_parallel_episodes: int = 48,
-    max_steps: int = 10,
-    verbose: bool = False,
-    job_id: str | None = None,
-) -> list[dict[str, Any]]:
-    """
-    Run tasks with grouping, following the RL Actor pattern.
-
-    Args:
-        tasks: List of tasks to run
-        agent_class: Agent class or instance to use
-        agent_config: Configuration for agent instantiation
-        group_size: Number of times to run each task
-        max_parallel_episodes: Maximum parallel episodes to run
-        max_steps: Maximum steps per episode
-        verbose: Whether to show progress
-        job_id: Optional job ID for tracking
-
-    Returns:
-        List of statistics for each task group
-    """
-    agent_config = agent_config or {}
-
-    # Duplicate tasks according to group_size, exactly like RL
-    grouped_tasks = []
-    task_mapping = []  # Track which group each result belongs to
-
-    for i, task in enumerate(tasks):
-        for _ in range(group_size):
-            grouped_tasks.append(task)
-            task_mapping.append(i)
-
-    hud_console.info(
-        f"Running {len(tasks)} tasks with group_size={group_size} ({len(grouped_tasks)} total runs)"
-    )
-
-    # Run all episodes, respecting max_parallel_episodes
-    all_traces = []
-
-    for batch_start in range(0, len(grouped_tasks), max_parallel_episodes):
-        batch_end = min(batch_start + max_parallel_episodes, len(grouped_tasks))
-        batch = grouped_tasks[batch_start:batch_end]
-
-        # Run batch in parallel
-        async def run_single_episode(task_data: dict[str, Any] | Task, idx: int) -> Trace:
-            """Run a single episode."""
-            try:
-                # Create task if needed
-                task = Task(**task_data) if isinstance(task_data, dict) else task_data
-
-                # Create fresh agent instance
-                if isinstance(agent_class, type):
-                    agent = agent_class(**agent_config)
-                else:
-                    # Agent is already instantiated
-                    agent = agent_class
-
-                # Run the task
-                trace_name = f"Eval | {task.id if hasattr(task, 'id') else 'Task'} | Group {task_mapping[idx]}"  # noqa: E501
-                with hud.trace(trace_name, job_id=job_id):
-                    result = await agent.run(task, max_steps=max_steps)
-                    return result
-
-            except Exception as e:
-                hud_console.warning_log(f"Episode failed: {e}")
-                return Trace(isError=True, content=str(e), reward=0.0, done=True)
-
-        # Run batch
-        batch_results = await asyncio.gather(
-            *[run_single_episode(t, batch_start + i) for i, t in enumerate(batch)],
-            return_exceptions=True,
-        )
-
-        # Normalize exceptions to error traces
-        for res in batch_results:
-            if isinstance(res, Exception):
-                hud_console.warning_log(f"Episode error: {res}")
-                all_traces.append(Trace(isError=True, content=str(res), reward=0.0, done=True))
-            else:
-                all_traces.append(res)
-
-        if verbose:
-            hud_console.info(f"Completed batch: {len(all_traces)}/{len(grouped_tasks)} episodes")
-
-    # Group results back by original task and calculate statistics
-    return calculate_group_statistics(tasks, all_traces, task_mapping, group_size)
-
-
-def calculate_group_statistics(
-    original_tasks: list[Any],
-    traces: list[Trace],
-    task_mapping: list[int],
-    group_size: int,
-) -> list[dict[str, Any]]:
-    """
-    Calculate statistics for each group, similar to preprocess_advantages.
-
-    Args:
-        original_tasks: Original task list
-        traces: All traces from grouped runs
-        task_mapping: Mapping of trace index to task index
-        group_size: Number of runs per task
-
-    Returns:
-        List of statistics for each task
-    """
-    stats = []
-
-    # Process each original task
-    for task_idx, task in enumerate(original_tasks):
-        # Get all traces for this task
-        task_traces = [
-            traces[i] for i, mapping_idx in enumerate(task_mapping) if mapping_idx == task_idx
-        ]
-
-        # Extract rewards
-        rewards = np.array([t.reward for t in task_traces])
-        errors = [t for t in task_traces if t.isError]
-
-        # Calculate statistics
-        task_stats = {
-            "task_id": task.id
-            if isinstance(task, Task) and hasattr(task, "id")
-            else f"task_{task_idx}",
-            "prompt": task.prompt if isinstance(task, Task) else task.get("prompt", ""),
-            "group_size": group_size,
-            "rewards": rewards.tolist(),
-            "mean_reward": float(np.mean(rewards)),
-            "std_reward": float(np.std(rewards)) if len(rewards) > 1 else 0.0,
-            "min_reward": float(np.min(rewards)),
-            "max_reward": float(np.max(rewards)),
-            "success_rate": float(np.sum(rewards > 0) / len(rewards)) if len(rewards) > 0 else 0.0,
-            "error_rate": len(errors) / len(task_traces) if len(task_traces) > 0 else 0.0,
-            "traces": task_traces,  # Keep full traces for detailed analysis
-        }
-
-        # Add variance info like RL does
-        if task_stats["std_reward"] > 1e-6:
-            task_stats["normalized_rewards"] = [
-                (r - task_stats["mean_reward"]) / task_stats["std_reward"] for r in rewards
-            ]
-        else:
-            task_stats["normalized_rewards"] = [0.0] * len(rewards)
-
-        stats.append(task_stats)
-
-    return stats
-
-
-def display_group_statistics(stats: list[dict[str, Any]], show_details: bool = True) -> None:
-    """Display statistics from grouped evaluation."""
-    from rich.console import Console
-    from rich.table import Table
-
-    console = Console()
-
-    # Overall statistics
-    all_means = [s["mean_reward"] for s in stats]
-    overall_mean = mean(all_means) if all_means else 0.0
-    overall_std = stdev(all_means) if len(all_means) > 1 else 0.0
-
-    hud_console.success("\n📊 Evaluation Summary")
-    hud_console.info(f"Tasks evaluated: {len(stats)}")
-    hud_console.info(f"Episodes per task: {stats[0]['group_size'] if stats else 0}")
-    hud_console.info(f"Total episodes: {sum(len(s['rewards']) for s in stats)}")
-    hud_console.info(f"Overall mean reward: {overall_mean:.3f} ± {overall_std:.3f}")
-
-    # Detailed table
-    if show_details and len(stats) <= 50:  # Only show for reasonable dataset sizes
-        table = Table(title="\nPer-Task Performance Distribution")
-        table.add_column("Task", style="cyan", no_wrap=True)
-        table.add_column("Mean±Std", justify="right", style="green")
-        table.add_column("Min/Max", justify="right")
-        table.add_column("Success%", justify="right", style="yellow")
-        table.add_column("Rewards", style="dim")
-
-        for stat in stats:
-            task_name = stat["prompt"][:30] + "..." if len(stat["prompt"]) > 30 else stat["prompt"]
-            rewards_str = " ".join([f"{r:.2f}" for r in stat["rewards"][:5]])
-            if len(stat["rewards"]) > 5:
-                rewards_str += " ..."
-
-            table.add_row(
-                task_name,
-                f"{stat['mean_reward']:.3f}±{stat['std_reward']:.3f}",
-                f"{stat['min_reward']:.2f}/{stat['max_reward']:.2f}",
-                f"{stat['success_rate'] * 100:.0f}%",
-                rewards_str,
-            )
-
-        console.print(table)
-
-    # High variance tasks
-    high_variance_tasks = [s for s in stats if s["std_reward"] > 0.3 and s["group_size"] > 1]
-    if high_variance_tasks:
-        hud_console.warning(f"\n{len(high_variance_tasks)} tasks show high variance (std > 0.3)")
-        for task in high_variance_tasks[:3]:
-            hud_console.info(
-                f"  • {task['task_id']}: μ={task['mean_reward']:.3f}, σ={task['std_reward']:.3f}"  # noqa: RUF001
-            )

hud/utils/progress.py

@@ -1,149 +0,0 @@
-from __future__ import annotations
-
-import time
-from collections import defaultdict
-
-
-class StepProgressTracker:
-    """
-    Tracks progress across potentially parallel async tasks based on steps completed.
-    Provides estimates assuming tasks run up to max_steps_per_task.
-    """
-
-    def __init__(self, total_tasks: int, max_steps_per_task: int) -> None:
-        """
-        Initialize the StepProgressTracker.
-
-        Args:
-            total_tasks: The total number of tasks to track.
-            max_steps_per_task: The maximum number of steps per task.
-
-        Raises:
-            ValueError: If total_tasks or max_steps_per_task is not positive.
-        """
-        if total_tasks <= 0:
-            raise ValueError("total_tasks must be positive")
-        if max_steps_per_task <= 0:
-            raise ValueError("max_steps_per_task must be positive")
-
-        self.total_tasks = total_tasks
-        self.max_steps_per_task = max_steps_per_task
-        self.total_potential_steps = total_tasks * max_steps_per_task
-
-        # Use asyncio.Lock for potentially concurrent updates/reads if needed,
-        # but start without for simplicity in single-threaded asyncio.
-        # self._lock = asyncio.Lock()
-        self._task_steps: dict[str, int] = defaultdict(int)
-        self._finished_tasks: dict[str, bool] = defaultdict(bool)
-        self._tasks_started = 0
-        self._tasks_finished = 0
-
-        self.start_time: float | None = None
-        self.current_total_steps = 0
-
-    def start_task(self, task_id: str) -> None:
-        # async with self._lock: # If using lock
-        if self.start_time is None:
-            self.start_time = time.monotonic()
-        self._task_steps[task_id] = 0
-        self._finished_tasks[task_id] = False
-        self._tasks_started += 1
-
-    def increment_step(self, task_id: str) -> None:
-        # async with self._lock:
-        if (
-            not self._finished_tasks[task_id]
-            and self._task_steps[task_id] < self.max_steps_per_task
-        ):
-            self._task_steps[task_id] += 1
-            # Update overall progress immediately
-            self._update_total_steps()
-
-    def finish_task(self, task_id: str) -> None:
-        # async with self._lock:
-        if not self._finished_tasks[task_id]:
-            # For calculation, consider a finished task as having completed max steps
-            self._task_steps[task_id] = self.max_steps_per_task
-            self._finished_tasks[task_id] = True
-            self._tasks_finished += 1
-            # Update overall progress
-            self._update_total_steps()
-
-    def _update_total_steps(self) -> None:
-        # This could be expensive if called extremely frequently.
-        # Called after increment or finish.
-        # async with self._lock:
-        self.current_total_steps = sum(self._task_steps.values())
-
-    def get_progress(self) -> tuple[int, int, float]:
-        """Returns (current_steps, total_potential_steps, percentage)."""
-        # async with self._lock:
-        # Recalculate here for safety, though _update_total_steps should keep it current
-        # current_steps = sum(self._task_steps.values())
-        current_steps = self.current_total_steps
-
-        percentage = 0.0
-        if self.total_potential_steps > 0:
-            percentage = (current_steps / self.total_potential_steps) * 100
-        return current_steps, self.total_potential_steps, percentage
-
-    def get_stats(self) -> tuple[float, float | None]:
-        """Returns (rate_steps_per_minute, eta_seconds_upper_bound)."""
-        # async with self._lock:
-        if self.start_time is None or self._tasks_started == 0:
-            return 0.0, None  # No rate or ETA yet
-
-        elapsed_time = time.monotonic() - self.start_time
-        current_steps = self.current_total_steps
-
-        rate_sec = 0.0
-        if elapsed_time > 0:
-            rate_sec = current_steps / elapsed_time
-
-        rate_min = rate_sec * 60  # Convert rate to steps per minute
-
-        eta = None
-        # ETA calculation still uses rate_sec (steps/second) for time estimation in seconds
-        if rate_sec > 0:
-            remaining_steps = self.total_potential_steps - current_steps
-            eta = remaining_steps / rate_sec if remaining_steps > 0 else 0.0
-
-        return rate_min, eta  # Return rate in steps/min
-
-    def is_finished(self) -> bool:
-        # async with self._lock:
-        return self._tasks_finished >= self.total_tasks
-
-    def display(self, bar_length: int = 40) -> str:
-        """Generates a progress string similar to tqdm."""
-        current_steps, total_steps, percentage = self.get_progress()
-        rate_min, eta = self.get_stats()  # Rate is now per minute
-
-        # Ensure valid values for display
-        current_steps = min(current_steps, total_steps)
-        percentage = max(0.0, min(100.0, percentage))
-
-        filled_length = int(bar_length * current_steps // total_steps) if total_steps else 0
-        bar = "█" * filled_length + "-" * (bar_length - filled_length)
-
-        # Format time
-        elapsed_str = "0:00"
-        eta_str = "??:??"
-        if self.start_time:
-            elapsed_seconds = int(time.monotonic() - self.start_time)
-            elapsed_str = f"{elapsed_seconds // 60}:{elapsed_seconds % 60:02d}"
-            if eta is not None:
-                eta_seconds = int(eta)
-                eta_str = f"{eta_seconds // 60}:{eta_seconds % 60:02d}"
-            elif self.is_finished():
-                eta_str = "0:00"
-
-        # Update rate string format
-        rate_str = f"{rate_min:.1f} steps/min" if rate_min > 0 else "?? steps/min"
-
-        # Format steps - use K/M for large numbers if desired, keep simple for now
-        steps_str = f"{current_steps}/{total_steps}"
-
-        # tasks_str = f" {self._tasks_finished}/{self.total_tasks} tasks" # Optional tasks counter
-
-        return f"{percentage:3.0f}%|{bar}| {steps_str} [{elapsed_str}<{eta_str}, {rate_str}]"

hud/utils/tasks.py

@@ -1,127 +0,0 @@
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-from hud.types import Task
-from hud.utils.hud_console import HUDConsole
-
-hud_console = HUDConsole()
-
-
-def load_tasks(tasks_input: str | list[dict], *, raw: bool = False) -> list[Task] | list[dict]:
-    """Load tasks from various sources.
-
-    Args:
-        tasks_input: Either:
-            - Path to a JSON file (array of tasks)
-            - Path to a JSONL file (one task per line)
-            - HuggingFace dataset name (format: "username/dataset" or "username/dataset:split")
-            - List of task dictionaries
-        raw: If True, return raw dicts without validation or env substitution
-
-    Returns:
-        - If raw=False (default): list[Task]
-        - If raw=True: list[dict]
-    """
-    tasks: list[Task] | list[dict] = []
-
-    if isinstance(tasks_input, list):
-        # Direct list of task dicts
-        hud_console.info(f"Loading {len(tasks_input)} tasks from provided list")
-        if raw:
-            return [item for item in tasks_input if isinstance(item, dict)]
-        for item in tasks_input:
-            task = Task(**item)
-            tasks.append(task)
-
-    elif isinstance(tasks_input, str):
-        # Check if it's a file path
-        if Path(tasks_input).exists():
-            file_path = Path(tasks_input)
-
-            with open(file_path, encoding="utf-8") as f:
-                # Handle JSON files (array of tasks)
-                if file_path.suffix.lower() == ".json":
-                    data = json.load(f)
-                    if not isinstance(data, list):
-                        raise ValueError(
-                            f"JSON file must contain an array of tasks, got {type(data)}"
-                        )
-                    if raw:
-                        return [item for item in data if isinstance(item, dict)]
-                    for item in data:
-                        task = Task(**item)
-                        tasks.append(task)
-
-                # Handle JSONL files (one task per line)
-                else:
-                    raw_items: list[dict] = []
-                    for line in f:
-                        line = line.strip()
-                        if not line:
-                            continue
-                        item = json.loads(line)
-                        if isinstance(item, list):
-                            raw_items.extend([it for it in item if isinstance(it, dict)])
-                        elif isinstance(item, dict):
-                            raw_items.append(item)
-                        else:
-                            raise ValueError(
-                                f"Invalid JSONL format: expected dict or list of dicts, got {type(item)}"  # noqa: E501
-                            )
-                    if raw:
-                        return raw_items
-                    for it in raw_items:
-                        task = Task(**it)
-                        tasks.append(task)
-
-        # Check if it's a HuggingFace dataset
-        elif "/" in tasks_input:
-            hud_console.info(f"Loading tasks from HuggingFace dataset: {tasks_input}")
-            try:
-                from datasets import load_dataset
-
-                # Parse dataset name and optional split
-                if ":" in tasks_input:
-                    dataset_name, split = tasks_input.split(":", 1)
-                else:
-                    dataset_name = tasks_input
-                    split = "train"  # Default split
-
-                dataset = load_dataset(dataset_name, split=split)
-
-                # Convert dataset rows to Task objects
-                raw_rows: list[dict] = []
-                for item in dataset:
-                    if not isinstance(item, dict):
-                        raise ValueError(
-                            f"Invalid HuggingFace dataset: expected dict, got {type(item)}"
-                        )
-                    if not item["mcp_config"] or not item["prompt"]:
-                        raise ValueError(
-                            f"Invalid HuggingFace dataset: expected mcp_config and prompt, got {item}"  # noqa: E501
-                        )
-                    raw_rows.append(item)
-                if raw:
-                    return raw_rows
-                for row in raw_rows:
-                    task = Task(**row)
-                    tasks.append(task)
-
-            except ImportError as e:
-                raise ImportError(
-                    "Please install 'datasets' to load from HuggingFace: uv pip install datasets"
-                ) from e
-            except Exception as e:
-                raise ValueError(f"Failed to load HuggingFace dataset '{tasks_input}': {e}") from e
-
-        else:
-            raise ValueError(
-                f"Invalid tasks input: '{tasks_input}' is neither a file path nor a HuggingFace dataset"  # noqa: E501
-            )
-
-    else:
-        raise TypeError(f"tasks_input must be str or list, got {type(tasks_input)}")
-
-    return tasks