hud-python 0.4.45__py3-none-any.whl → 0.5.1__py3-none-any.whl

hud/rl/tests/test_learner.py

@@ -1,186 +0,0 @@
-from __future__ import annotations
-
-import pytest
-import torch
-
-from hud.rl.config import Config
-from hud.rl.learner import GRPOLearner
-from hud.rl.types import TrainingSample
-
-
-@pytest.fixture()
-def learner_stub(monkeypatch):
-    cfg = Config()
-    # Speed up: tiny settings
-    cfg.training.epochs = 1
-    cfg.training.group_size = 1
-    cfg.training.mini_batch_size = 1
-    cfg.training.use_8bit_optimizer = False
-
-    # Stub _load_models to avoid heavy model init
-    def _stub_load_models(self):
-        class DummyPolicy(torch.nn.Module):
-            def __init__(self):
-                super().__init__()
-                self.w = torch.nn.Parameter(torch.zeros(1))
-
-        dummy_policy = DummyPolicy()
-        dummy_opt = torch.optim.SGD(dummy_policy.parameters(), lr=0.1)
-        return None, dummy_policy, None, dummy_opt
-
-    monkeypatch.setattr(GRPOLearner, "_load_models", _stub_load_models, raising=True)
-    return GRPOLearner(cfg)
-
-
-def make_sample(
-    pol_logp_tok: torch.Tensor,
-    old_logp_tok: torch.Tensor,
-    ref_logp_tok: torch.Tensor,
-    advantage: float,
-):
-    # Minimal-but-correct object for GRPOLearner.compute_loss.
-    # Needs assistant_mask (T-1) and attention_mask (T) for sanity_check().
-    Tm1 = pol_logp_tok.size(-1)
-    inputs = {
-        "input_ids": torch.zeros(1, Tm1 + 1, dtype=torch.long),
-        "attention_mask": torch.ones(1, Tm1 + 1, dtype=torch.long),
-        "assistant_mask": torch.ones(1, Tm1, dtype=torch.bool),
-    }
-    return TrainingSample(
-        inputs=inputs,
-        old_logprobs=old_logp_tok,
-        ref_logprobs=ref_logp_tok,
-        # advantage must be 1D so .view(-1,1) works in compute_loss
-        advantage=torch.tensor([advantage], dtype=torch.float32),
-    )
-
-
-def patch_compute_logprobs(
-    monkeypatch, learner: GRPOLearner, pol_logp_tok: torch.Tensor, pol_entropy_tok: torch.Tensor
-):
-    # Return (pol_logp, pol_entropy) as expected by compute_loss
-    def _stub_compute_logprobs(self, model, inputs):
-        return pol_logp_tok.to(inputs["input_ids"].device), pol_entropy_tok.to(
-            inputs["input_ids"].device
-        )
-
-    monkeypatch.setattr(GRPOLearner, "compute_logprobs", _stub_compute_logprobs, raising=True)
-
-
-def test_per_token_mean_vs_sum(monkeypatch, learner_stub: GRPOLearner):
-    # Setup
-    _, Tm1 = 1, 4
-    pol = torch.tensor([[-1.0, -1.0, -1.0, -1.0]], dtype=torch.float32)  # logp
-    old = torch.tensor([[-1.2, -0.8, -1.0, -1.1]], dtype=torch.float32)
-    ref = torch.tensor([[-1.0, -1.0, -1.0, -1.0]], dtype=torch.float32)
-    ent = torch.zeros_like(pol)
-    patch_compute_logprobs(monkeypatch, learner_stub, pol, ent)
-
-    # Common config
-    learner_stub.config.training.kl_beta = 0.0
-    learner_stub.config.training.entropy_beta = 0.0
-    learner_stub.config.training.top_eps = 0.2
-    learner_stub.config.training.bottom_eps = 0.1
-
-    sample = make_sample(pol, old, ref, advantage=1.0)
-
-    # token_agg=mean
-    learner_stub.config.training.ppo_mode = "per_token"
-    learner_stub.config.training.token_agg = "mean"
-    loss_mean = learner_stub.compute_loss(sample).item()
-
-    # token_agg=sum
-    learner_stub.config.training.token_agg = "sum"
-    loss_sum = learner_stub.compute_loss(sample).item()
-
-    # Expect sum ≈ mean * num_tokens
-    assert pytest.approx(loss_sum, rel=1e-5) == loss_mean * Tm1
-
-
-def test_per_trace_vs_per_token(monkeypatch, learner_stub: GRPOLearner):
-    # Equal per-token deltas -> per_trace matches per_token(mean)
-    pol = torch.tensor([[-1.0, -1.0, -1.0]], dtype=torch.float32)
-    old = torch.tensor([[-1.2, -1.2, -1.2]], dtype=torch.float32)
-    ref = torch.tensor([[-1.1, -1.1, -1.1]], dtype=torch.float32)
-    ent = torch.zeros_like(pol)
-    patch_compute_logprobs(monkeypatch, learner_stub, pol, ent)
-
-    learner_stub.config.training.kl_beta = 0.0
-    learner_stub.config.training.entropy_beta = 0.0
-    learner_stub.config.training.top_eps = 0.2
-    learner_stub.config.training.bottom_eps = 0.1
-
-    sample = make_sample(pol, old, ref, advantage=1.0)
-
-    learner_stub.config.training.ppo_mode = "per_token"
-    learner_stub.config.training.token_agg = "mean"
-    ltok = learner_stub.compute_loss(sample).item()
-
-    learner_stub.config.training.ppo_mode = "per_trace"
-    ltraj = learner_stub.compute_loss(sample).item()
-
-    assert pytest.approx(ltraj, rel=1e-6) == ltok
-
-
-def test_entropy_beta_effect(monkeypatch, learner_stub: GRPOLearner):
-    pol = torch.tensor([[-1.0, -1.1]], dtype=torch.float32)
-    old = torch.tensor([[-1.0, -1.1]], dtype=torch.float32)
-    ref = torch.tensor([[-1.0, -1.1]], dtype=torch.float32)
-    ent = torch.tensor([[0.5, 1.5]], dtype=torch.float32)
-    patch_compute_logprobs(monkeypatch, learner_stub, pol, ent)
-
-    # No policy/kl effect, only entropy
-    learner_stub.config.training.ppo_mode = "per_token"
-    learner_stub.config.training.token_agg = "mean"
-    learner_stub.config.training.kl_beta = 0.0
-
-    sample = make_sample(pol, old, ref, advantage=0.0)
-
-    learner_stub.config.training.entropy_beta = 0.0
-    l0 = learner_stub.compute_loss(sample).item()
-
-    learner_stub.config.training.entropy_beta = 2.0
-    l1 = learner_stub.compute_loss(sample).item()
-
-    # Mean entropy = (0.5+1.5)/2 = 1.0, scaled by beta=2.0 -> +2.0
-    assert pytest.approx(l1 - l0, rel=1e-6) == 2.0
-
-
-def test_skip_update_when_zero_adv(monkeypatch, learner_stub: GRPOLearner):
-    # Patch prepare_groups to yield a single group with a minibatch-like object
-    class MiniBatch:
-        def __init__(self):
-            self.advantage = torch.zeros(1)
-
-        def to_device(self, device: torch.device) -> MiniBatch:
-            return self
-
-    def _stub_prepare_groups(self, samples: list[TrainingSample]) -> list[list[MiniBatch]]:
-        return [[MiniBatch(), MiniBatch()]]
-
-    monkeypatch.setattr(GRPOLearner, "prepare_groups", _stub_prepare_groups, raising=True)
-
-    # Return a zero scalar loss that *depends* on params so backward works,
-    # but has zero gradients (no update signal).
-    def _zero_loss(self, sample) -> torch.Tensor:
-        return sum(p.sum() for p in self.policy.parameters()) * 0.0  # type: ignore
-
-    monkeypatch.setattr(GRPOLearner, "compute_loss", _zero_loss, raising=True)
-
-    # Count optimizer.step calls
-    steps = {"n": 0}
-    # orig_step = learner_stub.optimizer.step
-
-    def _count_step():
-        steps["n"] += 1
-
-    monkeypatch.setattr(learner_stub.optimizer, "step", _count_step, raising=False)
-
-    # Ensure dummy backward can touch a parameter
-    assert any(p.requires_grad for p in learner_stub.policy.parameters())
-
-    learner_stub.update([])
-    # With the current learner implementation we still call optimizer.step()
-    # even if the per-minibatch "advantage" is zero (the step is a no-op
-    # because the gradients are zero). So we expect exactly one step here.
-    assert steps["n"] == 1

hud/rl/train.py

@@ -1,382 +0,0 @@
-"""Main training loop for GRPO RL."""
-
-from __future__ import annotations
-
-import os
-
-# Disable tokenizer parallelism warnings
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
-os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
-import argparse
-import asyncio
-import json
-import logging
-from pathlib import Path
-from typing import TYPE_CHECKING, cast
-
-import hud
-from hud.rl.actor import Actor
-from hud.rl.buffer import DatasetBuffer, ReplayBuffer
-from hud.rl.config import Config
-from hud.rl.distributed import (
-    broadcast_object,
-    cleanup_distributed,
-    get_global_rank,
-    get_world_size,
-    is_main_process,
-    scatter_object,
-    setup_distributed,
-    synchronize,
-)
-from hud.rl.learner import GRPOLearner
-from hud.rl.utils import (
-    aggregate_metrics_across_ranks,
-    ensure_dir,
-    preprocess_advantages,
-    set_seed,
-)
-from hud.rl.vllm_adapter import VLLMAdapter
-from hud.utils.hud_console import HUDConsole
-from hud.utils.tasks import load_tasks
-
-if TYPE_CHECKING:
-    from hud.types import Task
-hud_console = HUDConsole(logging.getLogger(__name__))
-
-
-async def train(config: Config, tasks: list[Task]) -> None:
-    """Main training loop."""
-    # Setup distributed environment
-    setup_distributed()
-
-    # Initialize components
-    set_seed(config.seed + get_global_rank())  # Different seed per rank
-    ensure_dir(config.out_dir)
-    if config.verbose:
-        logging.basicConfig(level=logging.INFO)
-        # Remove httpx logger
-        logging.getLogger("httpx").setLevel(logging.WARNING)
-    if config.very_verbose:
-        logging.basicConfig(level=logging.DEBUG)
-        # Remove httpx logger
-        logging.getLogger("httpx").setLevel(logging.INFO)
-
-    if is_main_process():
-        hud_console.header("Starting GRPO Training")
-        hud_console.section_title(
-            f"\n[1/3] Initializing components (world_size={get_world_size()})..."
-        )
-
-    num_gpus = get_world_size()
-
-    # Actor is responsible for running tasks and collecting episodes
-    actor = Actor(config) if is_main_process() else None
-
-    # Learner is responsible for updating the policy
-    learner = GRPOLearner(config)
-
-    # Dataset buffer is responsible for storing tasks
-    dataset_buffer = DatasetBuffer(tasks, config)
-    if is_main_process():
-        hud_console.key_value_table(dataset_buffer.info)
-
-    if dataset_buffer.groups_per_batch % num_gpus != 0:
-        hud_console.warning(
-            f"Groups per batch {dataset_buffer.groups_per_batch} is not divisible by number of GPUs {num_gpus}"  # noqa: E501
-        )
-        exit(1)
-
-    # Replay buffer is responsible for storing episodes for training
-    trace_buffer = ReplayBuffer(config)
-
-    # VLLM adapter is responsible for loading and unloading adapters (only on main process)
-    vllm = (
-        VLLMAdapter(config.actor.vllm_base_url, config.actor.vllm_api_key)
-        if is_main_process()
-        else None
-    )
-
-    # Training state
-    step = 0
-    last_metrics = None  # Store last successful metrics for error recovery
-
-    if is_main_process():
-        hud_console.section_title("\n[2/3] Running training loop...")
-
-    # Create job on main process and distribute ID across GPUs
-    if is_main_process():
-        hud_console.info(f"Creating job with config.job_id: {config.job_id}")
-        job_obj = hud.create_job(
-            job_id=config.job_id,
-            name=config.job_name,
-            metadata={"config": config.to_dict(), "agent_class": config.model.base_model},
-        )
-        hud_console.info(f"Created job with job_obj.id: {job_obj.id}")
-        job_obj.update_status_sync("running")
-        job_id = job_obj.id
-    else:
-        job_obj = None
-        job_id = None
-
-    # Broadcast job ID to all ranks
-    job_id = broadcast_object(job_id, src=0)
-
-    try:
-        while len(dataset_buffer) > 0:
-            if is_main_process():
-                hud_console.section_title(f"Step {step + 1}/{dataset_buffer.training_steps}")
-                hud_console.info(f"{len(dataset_buffer)} tasks remaining")
-            # Get batch of tasks (all ranks need same tasks)
-            tasks = dataset_buffer.get_tasks()
-
-            # Initialize variables on all ranks
-            global_reward_stats = None
-            global_advantage_stats = None
-
-            # Step-state gate: ensure all ranks branch coherently
-            state = {"ok": False, "err": None, "num_samples": 0}
-            rank_samples = None
-            episode_time_value = None
-
-            # Only rank 0 runs tasks and prepares distribution
-            if is_main_process() and actor is not None:
-                import time
-
-                try:
-                    episode_start_time = time.time()
-                    traces = await actor.run_tasks(tasks, job_id=job_id)
-                    episode_time = time.time() - episode_start_time
-                    hud_console.info(f"Sampled {len(traces)} traces in {episode_time:.1f}s")
-                    trace_buffer.add(traces)
-                    global_reward_stats = [trace.reward for trace in traces]
-
-                    # Get all traces from buffer for distribution
-                    all_traces = trace_buffer.sample_traces()
-
-                    # Preprocess traces to training samples
-                    preprocessed_traces = preprocess_advantages(all_traces, config)
-
-                    # Store these for later use in metrics
-                    global_advantage_stats = [sample.advantage for sample in preprocessed_traces]
-
-                    # Distribute preprocessed samples in groups across ranks via scatter
-                    # Ensure list length is a multiple of num_gpus by allowing empty per-rank slices
-                    gpu_batch_size = max(1, (len(preprocessed_traces) + num_gpus - 1) // num_gpus)
-                    rank_samples = [
-                        preprocessed_traces[i : i + gpu_batch_size]
-                        for i in range(0, len(preprocessed_traces), gpu_batch_size)
-                    ]
-                    # Pad rank_samples to exactly num_gpus entries
-                    if len(rank_samples) < num_gpus:
-                        rank_samples.extend([[] for _ in range(num_gpus - len(rank_samples))])
-
-                    # Log distribution info
-                    dist_msg = (
-                        f"Distributing {len(preprocessed_traces)} samples as {gpu_batch_size} "
-                        f"sized batches across {num_gpus} GPUs"
-                    )
-                    hud_console.info(dist_msg)
-                    for rank in range(num_gpus):
-                        n_samples = len(rank_samples[rank]) if rank < len(rank_samples) else 0
-                        hud_console.info(f"  Rank {rank}: {n_samples} samples")
-
-                    hud_console.section_title(f"Training on {len(all_traces)} traces")
-                    episode_time_value = episode_time
-
-                    state.update({"ok": True, "num_samples": len(preprocessed_traces)})
-                except Exception as e:
-                    state.update({"ok": False, "err": str(e)})
-
-            # Broadcast step-state to keep ranks in lockstep
-            state = broadcast_object(state, src=0)
-            if not state.get("ok", False):
-                hud_console.warning("Step failed on rank 0; skipping this step coherently")
-                synchronize()
-                continue
-
-            # Scatter per-rank samples; each rank receives only its slice
-            my_samples = scatter_object(rank_samples if is_main_process() else None, src=0)
-            # Broadcast the episode time (small object)
-            episode_time_value = broadcast_object(episode_time_value, src=0)
-
-            # Process only assigned samples
-            last_metrics = learner.update(my_samples)
-
-            # Add episode time (same for all ranks since episodes run on rank 0)
-            if episode_time_value is not None:
-                last_metrics.update(
-                    {
-                        "episode_time": episode_time_value,
-                    }
-                )
-
-            # Aggregate metrics across all GPUs for proper statistics
-            aggregate_metrics_across_ranks(last_metrics)
-
-            if is_main_process() and job_obj is not None:
-                # Use the global statistics we collected before distribution
-                if global_reward_stats is not None and global_advantage_stats is not None:
-                    last_metrics.update(
-                        {
-                            "advantage": global_advantage_stats,
-                            "reward": global_reward_stats,
-                        }
-                    )
-                else:
-                    # Fallback: use only this rank's data
-                    hud_console.warning("Global statistics not available, using partial data")
-                    last_metrics.update(
-                        {
-                            "advantage": [sample.advantage for sample in my_samples]
-                            if my_samples
-                            else [],
-                            "reward": [sample.reward for sample in my_samples]
-                            if my_samples
-                            else [],
-                        }
-                    )
-
-                job_obj.log_sync(last_metrics.to_dict())
-
-                if step % config.stats_interval == 0:
-                    hud_console.key_value_table(last_metrics.to_dict())
-
-            # Increment step counter on all processes
-            step += 1
-
-            # Save checkpoint and update vLLM (only on main process)
-            if step % config.training.save_every_batches == 0:
-                if is_main_process() and vllm is not None and actor is not None:
-                    hud_console.section_title("Saving checkpoint and updating vLLM")
-                    checkpoint_path = Path(config.out_dir) / f"{config.adapter_prefix}-{step}"
-                    learner.save(str(checkpoint_path))
-
-                    # Wait for 6 seconds to ensure the checkpoint is saved
-                    await asyncio.sleep(6)
-
-                    # If there is a previous adapter, unload it
-                    current_adapter = vllm.get_current()
-                    if current_adapter is not None:
-                        vllm.unload_adapter(current_adapter)
-
-                    adapter_name = f"{config.adapter_prefix}-{step}"
-                    if vllm.load_adapter(adapter_name, str(checkpoint_path)):
-                        actor.update_adapter(adapter_name)
-                        hud_console.info(f"✓ Checkpoint saved and loaded: {adapter_name}")
-                    else:
-                        hud_console.warning(f"Failed to hot-load adapter {adapter_name}")
-
-                # Ensure all processes wait for checkpoint operations to complete
-                synchronize()
-
-        if is_main_process():
-            hud_console.section_title("\n[3/3] Training completed!")
-            # Update job status to completed
-            if job_obj:
-                job_obj.update_status_sync("completed")
-    except Exception as e:
-        # Log error and any available metrics before failing
-        hud_console.error(f"Training failed on rank {get_global_rank()}: {e}")
-
-        if is_main_process():
-            # Log final metrics if we have any
-            if last_metrics and job_obj:
-                try:
-                    job_obj.log_sync(last_metrics.to_dict())
-                except Exception:
-                    hud_console.warning("Failed to log final metrics")
-
-            # Update job status to failed
-            if job_obj:
-                job_obj.update_status_sync("failed")
-
-        # Don't re-raise immediately to allow cleanup
-        raise
-
-    finally:
-        # Try to sync one last time, but don't fail if it doesn't work
-        try:
-            synchronize()
-        except Exception:
-            hud_console.warning("Failed to synchronize during cleanup")
-
-        # Clean up distributed environment
-        cleanup_distributed()
-
-
-async def main() -> None:
-    parser = argparse.ArgumentParser(description="GRPO RL Training")
-    parser.add_argument("--config", type=str, help="Path to config JSON file")
-    parser.add_argument("--test", action="store_true", help="Run in test mode")
-    parser.add_argument("--debug", action="store_true", help="Enable debug mode")
-    parser.add_argument("--verbose", action="store_true", help="Enable verbose mode")
-    # Task input arguments
-    parser.add_argument(
-        "--tasks", type=str, help="Path to tasks JSONL file or HuggingFace dataset name"
-    )
-    parser.add_argument("--tasks-json", type=json.loads, help="Tasks as JSON list string")
-
-    args = parser.parse_args()
-
-    # Load config
-    if args.config:
-        with open(args.config, encoding="utf-8") as f:  # noqa: ASYNC230
-            config_dict = json.load(f)
-        config = Config.from_dict(config_dict)
-    else:
-        config = Config()
-
-    # Apply test mode settings
-    if args.test:
-        hud_console.info("[TEST MODE] Using minimal configuration")
-        eps = 6
-        config.training.batch_size = eps
-        config.actor.max_parallel_episodes = 12
-        config.training.group_size = eps
-        config.training.mini_batch_size = 3
-        config.training.training_steps = 4
-        config.actor.max_steps_per_episode = 4
-
-    # Calculate the memory usage
-    INITIAL_MEMORY = 8.0
-    SCALING_FACTOR = 4 / (28 * 28 * 256 * 1024)
-    token_estimate = (
-        config.training.mini_batch_size
-        * config.actor.max_steps_per_episode
-        * config.actor.max_new_tokens
-    )
-    hud_console.info(f"Estimated tokens per forward pass: {token_estimate}")
-    image_estimate = config.model.max_pixels
-    total_memory = INITIAL_MEMORY + SCALING_FACTOR * token_estimate * image_estimate
-    hud_console.info(f"Estimated memory peak: {total_memory:.2f} GB")
-    if total_memory > 75.0:
-        hud_console.warning(
-            "Potential memory usage is too high, decrease either training steps or mini batch size"
-        )
-        exit(1)
-
-    # Load tasks
-    if args.tasks_json:
-        # Tasks provided as JSON list via command line
-        tasks = load_tasks(args.tasks_json)
-    elif args.tasks:
-        # Tasks provided as file path or HuggingFace dataset
-        tasks = load_tasks(args.tasks)
-    else:
-        # Default to browser_2048_tasks.jsonl if it exists
-        default_tasks_path = "browser_2048_tasks.jsonl"
-        if Path(default_tasks_path).exists():
-            hud_console.info(f"No tasks specified, using default: {default_tasks_path}")
-            tasks = load_tasks(default_tasks_path)
-        else:
-            raise ValueError(
-                "No tasks specified. Use --tasks, --tasks-json, or specify tasks_file in config"
-            )
-
-    # Run training
-    tasks_typed = cast("list[Task]", tasks)
-    await train(config, tasks_typed)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

hud/rl/types.py

@@ -1,101 +0,0 @@
-"""Shared types for RL training."""
-
-from __future__ import annotations
-
-import math
-from typing import Any
-
-from pydantic import ConfigDict, Field
-from pydantic.dataclasses import dataclass
-
-from hud.types import Trace
-
-try:
-    import torch
-except ImportError:
-    raise ImportError("uv tool install hud-python[rl] to use this module") from None
-
-
-class TrainingSample(Trace):
-    """A single training sample for GRPO."""
-
-    model_config = ConfigDict(arbitrary_types_allowed=True)
-    # Tokenized inputs to the model (model.forward(*inputs))
-    # This includes the input tokens, logit mask, etc.
-    inputs: dict[str, torch.Tensor] = Field(default_factory=dict)
-    old_logprobs: torch.Tensor | None = Field(default=None)
-    ref_logprobs: torch.Tensor | None = Field(default=None)
-
-    # Weighted advantage of group calculation
-    advantage: torch.Tensor | None = Field(default=None)
-
-    def to_device(self, device: torch.device) -> TrainingSample:
-        """Move sample to device."""
-        self.inputs = {
-            k: (t.to(device, non_blocking=True) if hasattr(t, "to") else t)
-            for k, t in self.inputs.items()
-        }
-        self.advantage = self.advantage.to(device) if self.advantage is not None else None
-        self.old_logprobs = self.old_logprobs.to(device) if self.old_logprobs is not None else None
-        self.ref_logprobs = self.ref_logprobs.to(device) if self.ref_logprobs is not None else None
-        return self
-
-
-@dataclass
-class Metric:
-    """A tuple for metrics."""
-
-    name: str = Field(default="")
-    mean: float = Field(default=0.0)
-    std: float = Field(default=0.0)
-    values: list[float] = Field(default_factory=list)
-
-    def update(
-        self, value: float | torch.Tensor | list[float] | list[int] | list[torch.Tensor]
-    ) -> None:
-        """Update metric."""
-        if isinstance(value, list):
-            self.values.extend(value.item() if isinstance(value, torch.Tensor) else value)  # type: ignore
-        else:
-            self.values.append(value.item() if isinstance(value, torch.Tensor) else value)  # type: ignore
-        mean_val = sum(self.values) / len(self.values)
-        self.mean = mean_val.item() if isinstance(mean_val, torch.Tensor) else float(mean_val)  # type: ignore
-        variance = sum((x - self.mean) ** 2 for x in self.values) / len(self.values)
-        variance_val = variance.item() if isinstance(variance, torch.Tensor) else float(variance)  # type: ignore
-        self.std = math.sqrt(variance_val)
-
-
-@dataclass
-class TrainingMetrics:
-    """Metrics for GRPO training (per training step)."""
-
-    # Learner metrics
-    grad_norm: Metric = Field(default=Metric())
-    loss: Metric = Field(default=Metric())
-    kl: Metric = Field(default=Metric())
-    reward: Metric = Field(default=Metric())
-    advantage: Metric = Field(default=Metric())
-    policy_ratio: Metric = Field(default=Metric())
-    tokens: Metric = Field(default=Metric())
-    entropy: Metric = Field(default=Metric())
-
-    # Computation metrics
-    gpu_util: Metric = Field(default=Metric())  # GPU utilization percentage
-    gpu_memory: Metric = Field(default=Metric())  # GPU memory usage in GB
-    episode_time: Metric = Field(default=Metric())  # Time to run episodes (actor)
-    training_time: Metric = Field(default=Metric())  # Time for gradient updates (learner)
-    samples_per_second: Metric = Field(default=Metric())  # Training throughput
-
-    def update(self, metrics: dict[str, Any]) -> None:
-        """Update metrics."""
-        for key, value in metrics.items():
-            if key in self.__dataclass_fields__:
-                getattr(self, key).update(value)
-
-    def to_dict(self) -> dict[str, Any]:
-        """Convert metrics to dictionary."""
-        final_metrics = {}
-        for key in self.__dataclass_fields__:
-            final_metrics[f"{key}_mean"] = getattr(self, key).mean
-            final_metrics[f"{key}_std"] = getattr(self, key).std
-        return final_metrics

hud/rl/utils/start_vllm_server.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-# Start vLLM server with OpenAI-compatible API
-
-echo "Starting vLLM server for Qwen2.5-VL-3B-Instruct..."
-
-# Enable runtime LoRA adapter loading
-export VLLM_ALLOW_RUNTIME_LORA_UPDATING=True
-
-export TOKENIZERS_PARALLELISM=false
-export VLLM_LOGGING_LEVEL=DEBUG
-export CUDA_LAUNCH_BLOCKING=1  # Better error messages for CUDA errors
-
-# Common vLLM server command
-# Using CUDA_VISIBLE_DEVICES to put vLLM on GPU 1
-CUDA_VISIBLE_DEVICES=1 uv run vllm serve \
-    Qwen/Qwen2.5-VL-3B-Instruct \
-    --api-key token-abc123 \
-    --host 0.0.0.0 \
-    --port 8000 \
-    --tensor-parallel-size 1 \
-    --trust-remote-code \
-    --max-model-len 16384 \
-    --enable-lora \
-    --max-lora-rank 64 \
-    --max-cpu-loras 4 \
-    --enable-auto-tool-choice \
-    --tool-call-parser hermes \
-    --chat-template chat_template.jinja \
-    --enable-log-requests \
-    --uvicorn-log-level=debug 2>&1 | tee vllm_debug.log