hud-python 0.4.28__py3-none-any.whl → 0.4.30__py3-none-any.whl

hud/rl/buffer.py

@@ -0,0 +1,371 @@
+"""Replay buffer for storing and sampling episodes."""
+
+from __future__ import annotations
+
+import logging
+import random
+from collections import deque
+from typing import TYPE_CHECKING, Generic, TypeVar
+
+from hud.types import Task, Trace
+from hud.utils.hud_console import HUDConsole
+
+logger = logging.getLogger(__name__)
+hud_console = HUDConsole(logger=logger)
+
+T = TypeVar("T")
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from hud.rl.config import Config
+
+
+class Buffer(Generic[T]):
+    """Simple buffer for a list of tasks, traces or episodes."""
+
+    def __init__(self, max_size: int = 10000) -> None:
+        self.max_size = max_size
+        self.buffer: deque[T] = deque(maxlen=max_size)
+
+    def add(self, items: list[T] | T, shuffle: bool = False) -> None:
+        """Add items to buffer."""
+        if isinstance(items, list):
+            for item in items:
+                self.buffer.append(item)
+        else:
+            self.buffer.append(items)
+        if shuffle:
+            random.shuffle(self.buffer)
+
+    def add_fill(self, items: list[T] | T, target_size: int, shuffle: bool = False) -> None:
+        """Add items to buffer until the buffer is at least the target size."""
+        while len(self.buffer) < target_size:
+            self.add(items, shuffle)
+
+    def get(self, n: int = 0) -> list[T]:
+        """Get items from the buffer."""
+        if n == 0:
+            return list(self.buffer)
+        if n > len(self.buffer):
+            raise ValueError("Not enough items in buffer")
+        return list(self.buffer)[-n:]
+
+    def consume(self, n: int = 0) -> list[T]:
+        """Consume items from the buffer."""
+        if n == 0:
+            return list(self.buffer)
+        if n > len(self.buffer):
+            raise ValueError("Not enough items in buffer")
+
+        return [self.buffer.pop() for _ in range(n)]
+
+    def get_filtered(
+        self, n: int = 0, filter_fn: Callable[[T], bool] | None = None, consume: bool = False
+    ) -> list[T]:
+        """Filter the buffer by a filter function."""
+        filtered = (
+            [item for item in self.buffer if filter_fn(item)] if filter_fn else list(self.buffer)
+        )
+        if n == 0:
+            return filtered
+        return self.consume(n) if consume else self.get(n)
+
+    def sample(
+        self,
+        batch_size: int,
+        n: int = 0,
+        filter_fn: Callable[[T], bool] | None = None,
+        consume: bool = False,
+    ) -> list[T]:
+        """Sample a batch of items with optional filtering."""
+        items = self.get_filtered(n, filter_fn, consume)
+
+        if len(items) < batch_size:
+            hud_console.warning(f"Buffer has {len(items)} items, requested {batch_size}")
+            return items
+
+        return random.sample(items, batch_size)
+
+    def clear(self) -> None:
+        """Clear the buffer."""
+        self.buffer.clear()
+
+    def __len__(self) -> int:
+        """Use len() directly on Buffer instances."""
+        return len(self.buffer)
+
+
+class DatasetBuffer(Buffer[Task]):
+    """
+    Buffer for a dataset.
+    Loads in individual tasks that will be trained for a specified number of training steps.
+    """
+
+    def __init__(
+        self,
+        dataset: list[Task] | Task,
+        config: Config,
+    ) -> None:
+        self.config = config
+
+        self.group_size = config.training.group_size
+        self.batch_size = config.training.batch_size
+        self.training_steps = config.training.training_steps
+
+        if self.group_size > self.batch_size:
+            raise ValueError(
+                f"Group size is greater than batch size, {self.group_size} > {self.batch_size}"
+            )
+
+        if self.batch_size % self.group_size != 0:
+            raise ValueError(
+                f"A batch cannot have irregular groups, {self.group_size} % {self.batch_size} != 0"
+            )
+
+        if self.group_size % config.training.mini_batch_size != 0:
+            raise ValueError(
+                f"Group size is not a multiple of mini batch size, {self.group_size} % {config.training.mini_batch_size} != 0"  # noqa: E501
+            )
+
+        self.groups_per_batch = self.batch_size // self.group_size
+        self.number_of_tasks = self.training_steps * self.groups_per_batch
+
+        super().__init__(self.number_of_tasks)
+
+        dataset = dataset if isinstance(dataset, list) else [dataset]
+        tasks = self._validate_tasks(dataset)
+        if config.training.shuffle_dataset:
+            random.shuffle(tasks)
+        if len(tasks) > self.number_of_tasks:
+            leftovers = len(tasks) - self.number_of_tasks
+            hud_console.warning(
+                f"Training steps ({self.training_steps}) will lead to {leftovers} tasks not being trained"  # noqa: E501
+            )
+            tasks = tasks[: self.number_of_tasks]
+
+        # Check if the dataset is imbalanced
+        self.dataset_size = len(tasks)
+        if self.training_steps % self.dataset_size != 0:
+            leftovers = self.number_of_tasks % self.dataset_size
+            hud_console.warning(
+                f"Dataset imbalanced ({leftovers} tasks will be trained 1 more time)"
+            )
+            hud_console.warning(
+                f"This is because the number of training steps ({self.training_steps}) is not a multiple of the dataset size ({self.dataset_size})"  # noqa: E501
+            )
+
+        self.add_fill(tasks, self.number_of_tasks, config.training.shuffle_dataset)
+
+    def _validate_tasks(self, tasks: list[Task]) -> list[Task]:
+        """Validate that all tasks are proper HUD Task objects."""
+        if not tasks:
+            raise ValueError("No tasks provided to DatasetBuffer")
+
+        validated_tasks = []
+        for i, task in enumerate(tasks):
+            if not isinstance(task, Task):
+                raise TypeError(f"Task at index {i} is not a HUD Task object, got {type(task)}")
+            validated_tasks.append(task)
+
+        return validated_tasks
+
+    @property
+    def info(self) -> dict[str, int | float | str]:
+        """Get the info of the buffer."""
+        return {
+            "total_items": len(self),
+            "total_traces": self.number_of_tasks * self.group_size,
+            "total_batches": self.training_steps,
+            "task_repeats": self.number_of_tasks // self.dataset_size,
+            "dataset_size": self.dataset_size,
+            "group_size": self.group_size,
+            "batch_size": self.batch_size,
+        }
+
+    def get_tasks(self, consume: bool = True) -> list[Task]:
+        """Get tasks for a batch."""
+        tasks = self.consume(self.groups_per_batch) if consume else self.get(self.groups_per_batch)
+        # Create groups where each group contains group_size copies of the same task
+        result = []
+        for task in tasks:
+            result.extend([task] * self.group_size)
+        return result
+
+
+class ReplayBuffer(Buffer[Trace]):
+    """Buffer for traces."""
+
+    def __init__(self, config: Config) -> None:
+        self.config = config
+
+        self.buffer_steps = config.training.buffer_steps
+        self.select_strategy = config.training.select_strategy
+        self.group_size = config.training.group_size
+        self.batch_size = config.training.batch_size
+
+        buffer_size = self.buffer_steps * self.batch_size
+
+        super().__init__(buffer_size)
+
+    def sample_traces(self) -> list[Trace]:
+        """Sample traces for a batch."""
+        if self.select_strategy == "recent":
+            return self.get(self.batch_size)
+        elif self.select_strategy == "random":
+            return self.sample(self.batch_size)
+        elif self.select_strategy == "variance":
+            return self._sample_high_variance_traces()
+        else:
+            raise ValueError(f"Invalid select strategy: {self.select_strategy}")
+
+    def _sample_high_variance_traces(self) -> list[Trace]:
+        from collections import Counter, defaultdict, deque
+
+        # Expect recent window to already be grouped by task id
+
+        # Build recent window and earlier lookup (short form)
+        buf_list = list(self.buffer)
+        if len(buf_list) < self.batch_size:
+            hud_console.warning(
+                f"[group-sampler] Buffer has only {len(buf_list)} traces, need {self.batch_size}"
+            )
+            while len(buf_list) < self.batch_size:
+                take = min(len(buf_list) or 1, self.batch_size - len(buf_list))
+                buf_list.extend(buf_list[:take])
+        recent_traces = buf_list[-self.batch_size :]
+        hud_console.info(
+            f"[group-sampler] recent-window histogram: {Counter(getattr(t.task, 'id', 'NA') for t in recent_traces)}"  # noqa: E501
+        )
+
+        hud_console.info(
+            f"[group-sampler] Building earlier traces lookup, buffer size: {len(buf_list)}"
+        )
+        earlier_traces_by_task: dict[str, deque[Trace]] = defaultdict(deque)
+        for tr in buf_list[: -self.batch_size]:
+            earlier_traces_by_task[getattr(tr.task, "id", "NA")].append(tr)
+
+        # Chunk from the most-recent end
+        final_traces: list[Trace] = []
+        groups_per_batch = self.batch_size // self.group_size
+        hud_console.info(f"[group-sampler] Processing {groups_per_batch} groups")
+        for g_idx in range(groups_per_batch):
+            start = g_idx * self.group_size
+            end = start + self.group_size
+            group = recent_traces[start:end]
+
+            # Assert homogeneity: every trace in a group must share the same task id
+            cnt = Counter(getattr(t.task, "id", "NA") for t in group)
+            if len(cnt) != 1:
+                raise RuntimeError(f"Group {g_idx} is not homogeneous: {dict(cnt)}")
+            target_tid = next(iter(cnt.keys()))
+
+            # Build homogeneous group of target_tid, filling from earlier traces to increase spread
+            homogeneous: list[Trace] = [
+                t for t in group if getattr(t.task, "id", "NA") == target_tid
+            ]
+            needed = self.group_size - len(homogeneous)
+
+            # Greedy fill: choose earlier traces (same task-id) farthest from current mean reward
+            def current_mean(homogeneous: list[Trace]) -> float:
+                if not homogeneous:
+                    return 0.0
+                vals = [float(getattr(t, "reward", 0.0) or 0.0) for t in homogeneous]
+                return sum(vals) / len(vals)
+
+            while needed > 0:
+                pool = earlier_traces_by_task.get(target_tid, deque())
+                if pool:
+                    mu = current_mean(homogeneous)
+                    # pick element farthest from current mean
+                    best_i = None
+                    best_dist = -1.0
+                    for i, tr in enumerate(list(pool)):
+                        r = float(getattr(tr, "reward", 0.0) or 0.0)
+                        dist = abs(r - mu)
+                        if dist > best_dist:
+                            best_dist = dist
+                            best_i = i
+                    # pop selected
+                    chosen = list(pool)[best_i]  # type: ignore[index]
+                    # remove from deque efficiently by rotating
+                    left = list(pool)
+                    if best_i is not None:
+                        left.pop(best_i)  # O(n) but pool is small in practice
+                        earlier_traces_by_task[target_tid] = deque(left)
+                        homogeneous.append(chosen)
+                else:
+                    # duplicate extreme within current homogeneous set
+                    if not homogeneous:
+                        raise RuntimeError(f"Group {g_idx} has no traces for target {target_tid}")
+                    mu = current_mean(homogeneous)
+                    extreme = max(
+                        homogeneous, key=lambda t: abs(float(getattr(t, "reward", 0.0) or 0.0) - mu)
+                    )
+                    homogeneous.append(extreme)
+                needed -= 1
+
+            # Replacement step: swap in earlier traces to increase reward spread
+            pool = earlier_traces_by_task.get(target_tid, deque())
+            if pool:
+                # Log pool stats
+                pool_vals = [float(getattr(tr, "reward", 0.0) or 0.0) for tr in list(pool)]
+                if pool_vals:
+                    pool_mean = sum(pool_vals) / len(pool_vals)
+                    pool_var = sum((v - pool_mean) * (v - pool_mean) for v in pool_vals) / len(
+                        pool_vals
+                    )
+                    hud_console.info(
+                        f"[group-sampler] Group {g_idx}: earlier-pool size={len(pool_vals)} mean={pool_mean:.4f} std={(pool_var**0.5):.4f}"  # noqa: E501
+                    )
+
+                # Decide how many to replace (up to 1/4 of group, at least 1)
+                replace_k = max(1, self.group_size // 4)
+                replace_k = min(replace_k, len(pool), self.group_size)
+
+                if replace_k > 0:
+                    mu = current_mean(homogeneous)
+                    # Select replacement candidates from pool farthest from current mean
+                    pool_list = list(pool)
+                    pool_indices = list(range(len(pool_list)))
+                    pool_indices.sort(
+                        key=lambda i: abs(
+                            (float(getattr(pool_list[i], "reward", 0.0) or 0.0)) - mu
+                        ),
+                        reverse=True,
+                    )
+                    chosen_pool_idx = set(pool_indices[:replace_k])
+                    replacements = [pool_list[i] for i in pool_indices[:replace_k]]
+
+                    # Remove chosen from pool deque
+                    remaining = [tr for i, tr in enumerate(pool_list) if i not in chosen_pool_idx]
+                    earlier_traces_by_task[target_tid] = deque(remaining)
+
+                    # Select current group positions closest to mean to replace
+                    group_indices = list(range(len(homogeneous)))
+                    group_indices.sort(
+                        key=lambda i: abs(
+                            (float(getattr(homogeneous[i], "reward", 0.0) or 0.0)) - mu
+                        )
+                    )
+                    target_positions = group_indices[:replace_k]
+
+                    for pos, new_tr in zip(target_positions, replacements, strict=False):
+                        homogeneous[pos] = new_tr
+
+            # Validate homogeneity
+            if any(getattr(t.task, "id", "NA") != target_tid for t in homogeneous):
+                raise RuntimeError(f"Group {g_idx} is not homogeneous after sampling")
+            final_traces.extend(homogeneous)
+
+        for i in range(0, len(final_traces), self.group_size):
+            block = final_traces[i : i + self.group_size]
+            if len({getattr(t.task, "id", "NA") for t in block}) != 1:
+                raise RuntimeError(f"Homogeneity validation failed for block starting at index {i}")
+
+        hud_console.info(
+            f"[group-sampler] final histogram: {Counter(getattr(t.task, 'id', 'NA') for t in final_traces)}"  # noqa: E501
+        )
+        return final_traces
+
+        # --------------------------------------------------------------------

hud/rl/chat_template.jinja

@@ -0,0 +1,101 @@
+{% set image_count = namespace(value=0) %}
+{% set video_count = namespace(value=0) %}
+{{- '<|im_start|>system\n' }}
+{%- if messages[0]['role'] == 'system' -%}
+    {%- if messages[0]['content'] is string -%}
+        {{ messages[0]['content'] }}
+    {%- else -%}
+        {%- for content in messages[0]['content'] -%}
+            {%- if content['type'] == 'image' or 'image' in content or 'image_url' in content -%}
+                {%- set image_count.value = image_count.value + 1 -%}
+                {%- if add_vision_id -%}
+                    {{ 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif -%}
+                {{ '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif content['type'] == 'video' or 'video' in content -%}
+                {%- set video_count.value = video_count.value + 1 -%}
+                {%- if add_vision_id -%}
+                    {{ 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif -%}
+                {{ '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in content -%}
+                {{ content['text'] }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+{%- else -%}
+    {{ 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+{%- endif -%}
+{%- if tools -%}
+    {{ '\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>\n' }}
+    {{- tools | map('tojson') | join('\n') -}}
+    {{ '\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{"name": <function-name>, "arguments": <args-json-object>}\n</tool_call>' }}
+{%- endif -%}
+{{ '<|im_end|>\n' }}
+{%- for message in messages -%}
+    {# Skip the first system message as it was already rendered. #}
+    {%- if loop.first and message.role == 'system' %}{% continue %}{% endif -%}
+
+    {# Render tool messages. The logic is slightly different with other messages. #}
+    {%- if message['role'] == 'tool' -%}
+        {%- if loop.first or messages[loop.index0 - 1]['role'] != 'tool' -%}
+            {{ '<|im_start|>user' }}
+        {%- endif -%}
+        {{ '\n<tool_response>\n' }}
+    {%- else -%}
+        {{ '<|im_start|>' ~ message['role'] ~ '\n' }}
+    {%- endif -%}
+
+    {%- if message['content'] is string -%}
+        {{ message['content'] }}
+    {%- else -%}
+        {%- for content in message['content'] -%}
+            {%- if content['type'] == 'image' or 'image' in content or 'image_url' in content -%}
+                {%- set image_count.value = image_count.value + 1 -%}
+                {%- if add_vision_id -%}
+                    {{ 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif -%}
+                {{ '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif content['type'] == 'video' or 'video' in content -%}
+                {%- set video_count.value = video_count.value + 1 -%}
+                {%- if add_vision_id -%}
+                    {{ 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif -%}
+                {{ '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in content and message['role'] == 'assistant' -%}
+                {% generation %} {{ content['text'] }} {% endgeneration %}
+            {%- elif 'text' in content -%}
+                {{ content['text'] }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+    {# Render tool_calls in AI messages. #}
+    {%- if message['role'] == 'assistant' and 'tool_calls' in message -%}
+        {# It will be cleaner if I can use some map function and join them with '\n' #}
+        {%- for tool_call in message['tool_calls'] -%}
+            {%- if tool_call['function'] is defined -%}
+                {%- set tool_call = tool_call['function'] -%}
+            {%- endif -%}
+            {# Handle the case where arguments is already a JSON string (OpenAI format) #}
+            {%- if tool_call.arguments is string -%}
+                {% generation %} {{ '<tool_call>\n{"name": "' }}{{ tool_call.name }}{{ '", "arguments": ' }}{{ tool_call.arguments }}{{ '}\n</tool_call>' }} {% endgeneration %}
+            {%- else -%}
+                {% generation %} {{ '<tool_call>\n' }}{{ tool_call | tojson }}{{ '\n</tool_call>' }} {% endgeneration %}
+            {%- endif -%}
+            {%- if not loop.last -%}
+                {% generation %} {{ '\n' }} {% endgeneration %}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endif -%}
+    {%- if message['role'] == 'tool' -%}
+        {{ '\n</tool_response>' }}
+        {%- if loop.last or messages[loop.index0 + 1]['role'] != 'tool' -%}
+            {{ '<|im_end|>\n' }}
+        {%- endif -%}
+    {%- else -%}
+        {{ '<|im_end|>\n' }}
+    {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{ '<|im_start|>assistant\n' }}
+{%- endif -%}

hud/rl/config.py

@@ -0,0 +1,184 @@
+"""Configuration for RL training."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Literal
+
+# List of supported VL (Vision-Language) models
+SUPPORTED_MODELS = [
+    "Qwen/Qwen2.5-VL-3B-Instruct",
+    "Qwen/Qwen2.5-VL-7B-Instruct",
+    "Qwen/Qwen2.5-VL-14B-Instruct",
+    "Qwen/Qwen2.5-VL-32B-Instruct",
+    "Qwen/Qwen2.5-VL-72B-Instruct",
+    "Qwen/Qwen2.5-7B-Instruct",
+]
+
+
+def validate_vl_model(model_name: str) -> None:
+    """Validate that the model is a supported VL model.
+
+    Args:
+        model_name: The model name to validate
+
+    Raises:
+        ValueError: If the model is not a supported VL model
+    """
+    if not any(model_name.startswith(supported) for supported in SUPPORTED_MODELS):
+        raise ValueError(
+            f"Model '{model_name}' is not a supported VL model. "
+            f"Only VL (Vision-Language) models are supported for RL training.\n"
+            f"Supported models: {', '.join(SUPPORTED_MODELS)}\n"
+            f"Note: '{model_name}' appears to be a text-only model."
+        )
+
+
+@dataclass
+class ModelConfig:
+    """Model and LoRA configuration."""
+
+    base_model: str = "Qwen/Qwen2.5-VL-3B-Instruct"
+    lora_r: int = 8
+    lora_alpha: int = 16
+    lora_dropout: float = 0.05
+    target_modules: tuple[str, ...] = (
+        "q_proj",
+        "k_proj",
+        "v_proj",
+        "o_proj",
+        "gate_proj",
+        "up_proj",
+        "down_proj",
+    )
+    min_pixels: int = 256 * 28 * 28
+    max_pixels: int = 512 * 28 * 28
+    attn_implementation: str = "flash_attention_2"
+    use_liger: bool = True
+    gradient_checkpointing: bool = True
+
+
+@dataclass
+class TrainingConfig:
+    """Training hyperparameters."""
+
+    # Training parameters
+    training_steps: int = 100
+    shuffle_dataset: bool = False
+    save_every_batches: int = 1
+
+    # Batching parameters
+    epochs: int = 2
+    batch_size: int = 24
+    group_size: int = 4
+    mini_batch_size: int = 1
+    update_after_group: bool = True  # Whether to update the policy after each task group
+    accumulate_over_minibatches: bool = False  # Whether to accumulate over minibatches
+
+    # Advantage calculation parameters
+    batch_level: Literal["group", "batch"] = "group"
+    no_std: bool = False
+    leave_one_out: bool = True
+
+    # Replay buffer parameters
+    buffer_steps: int = 4
+    select_strategy: Literal["recent", "variance", "random"] = "variance"
+
+    # Aggregation parameters
+    ppo_mode: Literal["per_token", "per_trace"] = "per_token"
+    token_agg: Literal["mean", "sum"] = "mean"  # noqa: S105
+
+    # Regularization parameters
+    kl_beta: float = 0.0
+    entropy_beta: float = 0.0
+    top_eps: float = 0.2
+    bottom_eps: float = 0.1
+
+    # Training hyperparameters
+    lr: float = 3e-5
+    grad_clip: float = 1.0
+
+    # Adam hyperparameters
+    use_8bit_optimizer: bool = True
+    adam_betas: tuple[float, float] = (0.9, 0.999)
+    adam_eps: float = 1e-8
+
+
+@dataclass
+class ActorConfig:
+    """Actor/episode collection configuration."""
+
+    # Execution parameters
+    max_steps_per_episode: int = 5
+    max_parallel_episodes: int = 48
+    max_new_tokens: int = 1024
+    force_tool_choice: bool = True
+    allowed_tools: list[str] | None = None
+
+    # Model parameters
+    temperature: float = 0.7
+
+    # Hud agent parameters
+    system_prompt: str = "You are an expert agent. Complete the task efficiently."
+    vllm_base_url: str = "http://localhost:8000/v1"
+    vllm_api_key: str = "token-abc123"
+
+    # Episode execution timeout (seconds)
+    episode_timeout_sec: int = 600
+
+
+@dataclass
+class Config:
+    """Main configuration combining all sub-configs."""
+
+    model: ModelConfig = field(default_factory=ModelConfig)
+    training: TrainingConfig = field(default_factory=TrainingConfig)
+    actor: ActorConfig = field(default_factory=ActorConfig)
+
+    # Telemetry configuration
+    job_name: str = "RL Training"
+    job_id: str | None = None  # Use existing job ID if provided
+    stats_interval: int = 1
+    verbose: bool = False
+
+    # Paths
+    out_dir: str = "./checkpoints"
+    adapter_prefix: str = "cua-grpo-step"
+
+    # Misc
+    seed: int = 1234
+
+    @classmethod
+    def from_dict(cls, d: dict) -> Config:
+        """Create config from dictionary."""
+        model = ModelConfig(**d.get("model", {}))
+        training = TrainingConfig(**d.get("training", {}))
+        actor = ActorConfig(**d.get("actor", {}))
+
+        return cls(
+            model=model,
+            training=training,
+            actor=actor,
+            job_name=d.get("job_name", "RL Training"),
+            job_id=d.get("job_id"),
+            stats_interval=d.get("stats_interval", 1),
+            verbose=d.get("verbose", False),
+            out_dir=d.get("out_dir", "./checkpoints"),
+            adapter_prefix=d.get("adapter_prefix", "cua-grpo-step"),
+            seed=d.get("seed", 1234),
+        )
+
+    def to_dict(self) -> dict:
+        """Convert config to dictionary."""
+        return {
+            "model": self.model.__dict__,
+            "training": self.training.__dict__,
+            "actor": self.actor.__dict__,
+            "job_name": self.job_name,
+            "job_id": self.job_id,
+            "stats_interval": self.stats_interval,
+            "verbose": self.verbose,
+            "out_dir": self.out_dir,
+            "adapter_prefix": self.adapter_prefix,
+            "seed": self.seed,
+        }