synth-ai 0.2.13.dev2__py3-none-any.whl → 0.2.16__py3-none-any.whl

examples/task_apps/verilog/task_app/grpo_verilog.py

@@ -34,6 +34,7 @@ from synth_ai.task.contracts import (
 from synth_ai.task.datasets import TaskDatasetRegistry, TaskDatasetSpec
 from synth_ai.task.rubrics import load_rubric
 from synth_ai.task.server import ProxyConfig, RubricBundle, TaskAppConfig
+from synth_ai.task.validators import normalize_inference_url
 from synth_ai.task.tracing_utils import (
     build_tracer_factory,
     resolve_sft_output_dir,
@@ -45,7 +46,36 @@ from synth_ai.tracing_v3.session_tracer import SessionTracer
 logger = logging.getLogger(__name__)
 
 _HERE = Path(__file__).resolve()
-REPO_ROOT = _HERE.parents[4]
+
+
+def _resolve_repo_root() -> Path:
+    """Find synth-ai repo root, checking env var and parent traversal."""
+    candidates: list[Path] = []
+    env_root = os.getenv("SYNTH_AI_REPO_ROOT")
+    if env_root:
+        candidates.append(Path(env_root).expanduser())
+    
+    # Try Modal mount point
+    candidates.append(Path("/opt/synth_ai_repo"))
+    
+    # Traverse up from current file
+    current = _HERE
+    for _ in range(6):
+        current = current.parent
+        candidates.append(current)
+        if (current / "synth_ai").is_dir() and (current / "examples").is_dir():
+            return current
+    
+    # Return first existing candidate
+    for candidate in candidates:
+        if candidate.is_dir() and (candidate / "synth_ai").exists():
+            return candidate
+    
+    # Fallback to current parent structure (may not work in Modal)
+    return _HERE.parent.parent.parent.parent
+
+
+REPO_ROOT = _resolve_repo_root()
 
 DATASET_SPEC = TaskDatasetSpec(
     id="verilog_eval_v2",
@@ -161,23 +191,6 @@ def _base_task_info(dataset: VerilogDataset) -> TaskInfo:
     )
 
 
-def _normalize_inference_url(url: str | None) -> str:
-    candidate = (url or DEFAULT_INFERENCE_URL).strip()
-    if not candidate:
-        candidate = DEFAULT_INFERENCE_URL
-    if candidate.endswith("/v1/chat/completions"):
-        return candidate
-    if candidate.endswith("/chat/completions"):
-        return candidate
-    if candidate.endswith("/v1"):
-        return f"{candidate.rstrip('/')}/chat/completions"
-    if candidate.endswith("/v1/"):
-        return f"{candidate.rstrip('/')}/chat/completions"
-    if candidate.endswith("/chat"):
-        return f"{candidate.rstrip('/')}/completions"
-    if candidate.endswith("/chat/"):
-        return f"{candidate.rstrip('/')}/completions"
-    return f"{candidate.rstrip('/')}/v1/chat/completions"
 
 
 def _format_file_previews(files: dict[str, str]) -> str:
@@ -336,7 +349,7 @@ class VerilogLLMAgent:
         max_tokens: int,
     ) -> None:
         self.instructions = instructions.strip()
-        self.inference_url = _normalize_inference_url(inference_url)
+        self.inference_url = normalize_inference_url(inference_url, default=DEFAULT_INFERENCE_URL)
         self.model = model or DEFAULT_MODEL
         self.temperature = temperature
         self.max_tokens = max_tokens
@@ -349,7 +362,16 @@ class VerilogLLMAgent:
             if not api_key:
                 raise RuntimeError("GROQ_API_KEY is not configured for Verilog inference.")
             self.headers["Authorization"] = f"Bearer {api_key.strip()}"
-        elif "openai" in lowered:
+        # If target is Synth backend (any deployment), use SYNTH_API_KEY
+        elif any(pattern in lowered for pattern in [
+            "synth-backend", "synth.run", "agent-learning",
+            "localhost:8000", "127.0.0.1:8000"
+        ]):
+            api_key = os.getenv("SYNTH_API_KEY")
+            if not api_key:
+                raise RuntimeError("SYNTH_API_KEY is not configured for Verilog inference with Synth backend.")
+            self.headers["Authorization"] = f"Bearer {api_key.strip()}"
+        elif "openai" in lowered or "api.openai.com" in lowered:
             api_key = os.getenv("OPENAI_API_KEY")
             if not api_key:
                 raise RuntimeError("OPENAI_API_KEY is not configured for Verilog inference.")
@@ -574,6 +596,21 @@ async def rollout_executor(
     total_reward = 0.0
     final_observation: dict[str, Any] | None = None
     truncated_due_to_limit = False
+    
+    # Log episode start
+    problem_id = getattr(instance, "problem_id", "unknown")
+    logger.info("=" * 80)
+    logger.info(f"[EPISODE START] run_id={request.run_id}")
+    logger.info(f"  Problem ID:        {problem_id}")
+    logger.info(f"  Policy:            {policy_id}")
+    logger.info(f"  Model:             {policy_model}")
+    logger.info(f"  Max steps:         {max_steps}")
+    logger.info(f"  Temperature:       {temperature}")
+    logger.info(f"  Max tokens:        {max_tokens}")
+    if instructions:
+        instructions_preview = instructions[:150] + "..." if len(instructions) > 150 else instructions
+        logger.info(f"  Instructions:      {instructions_preview}")
+    logger.info("=" * 80)
     code_dirty = False
     last_compile_success = False
     simulate_since_last_compile = False
@@ -648,7 +685,7 @@ async def rollout_executor(
                             and not code_dirty
                         )
                         if skip_env_step:
-                            reward_last = -0.01
+                            reward_last = 0.0  # No reward for blocked operations
                             total_reward += reward_last
                             current_observation = dict(current_observation)
                             current_observation["reward_last"] = reward_last
@@ -669,6 +706,23 @@ async def rollout_executor(
                                 or current_observation.get("task_completed")
                             )
                             truncated_flag = bool(current_observation.get("truncated"))
+                            
+                            # Log what the environment returned
+                            print(f"\n{'='*80}")
+                            print(f"[STEP {step_index}] TOOL CALL:")
+                            print(f"  Tool: {env_call.tool}")
+                            print(f"  Args: {env_call.args}")
+                            print(f"\n[STEP {step_index}] ENVIRONMENT RESPONSE:")
+                            print(f"  Reward: {reward_last:.4f} (cumulative: {total_reward:.4f})")
+                            print(f"  Task completed: {step_observation.get('task_completed')}")
+                            print(f"  Done: {done_flag} | Truncated: {truncated_flag}")
+                            if 'compile_status' in step_observation and step_observation.get('compile_status'):
+                                print(f"  Compile status:\n{step_observation.get('compile_status')}")
+                            if 'simulate_status' in step_observation and step_observation.get('simulate_status'):
+                                print(f"  Simulate status:\n{step_observation.get('simulate_status')}")
+                            if 'files' in step_observation:
+                                print(f"  Files: {list(step_observation.get('files', {}).keys())}")
+                            print(f"{'='*80}\n")
 
                         executed_tool_name = str(primary_call["tool"])
                         normalized_executed_tool = executed_tool_name.strip().lower()
@@ -698,10 +752,40 @@ async def rollout_executor(
                             {"tool_name": call["tool"], "arguments": call["args"]}
                             for call in tool_calls
                         ]
+                        
+                        # Print tool calls for debugging
+                        logger.info(f"[STEP {step_index}] Tool calls executed:")
+                        for call in tool_calls:
+                            tool_name = call["tool"]
+                            args = call["args"]
+                            # Truncate long arguments for readability
+                            if "code" in args or "content" in args:
+                                args_preview = {k: (v[:100] + "..." if isinstance(v, str) and len(v) > 100 else v) 
+                                               for k, v in args.items()}
+                            else:
+                                args_preview = args
+                            logger.info(f"  └─ {tool_name}({args_preview})")
+                        
+                        # Log reward details for debugging
+                        logger.info(f"[STEP {step_index}] Reward details:")
+                        logger.info(f"  └─ reward_last: {reward_last:.4f}")
+                        logger.info(f"  └─ total_reward: {total_reward:.4f}")
+                        logger.info(f"  └─ skip_env_step: {skip_env_step}")
+                        if not skip_env_step:
+                            logger.info(f"  └─ obs.task_completed: {current_observation.get('task_completed', False)}")
+                            logger.info(f"  └─ obs.compile_status: {current_observation.get('compile_status', 'N/A')}")
+                            logger.info(f"  └─ obs.simulate_status: {current_observation.get('simulate_status', 'N/A')}")
+                            logger.info(f"  └─ obs.terminated: {current_observation.get('terminated', False)}")
+                        else:
+                            logger.info(f"  └─ (blocked operation - no env step)")
+                        
                         step_info = {
                             "assistant_message": assistant_text,
                             "model_response": raw_response,
                             "llm_request": request_payload,
+                            "meta": {
+                                "inference_url": policy_config.get("inference_url") or resolved_inference,  # CRITICAL: Required by RL trainer for trace extraction (must have ?cid=...)
+                            },
                         }
                         if override_info:
                             step_info["auto_override"] = override_info
@@ -756,6 +840,9 @@ async def rollout_executor(
                             "model_response": raw_response,
                             "llm_request": request_payload,
                             "error": error_text,
+                            "meta": {
+                                "inference_url": policy_config.get("inference_url") or resolved_inference,  # CRITICAL: Required by RL trainer
+                            },
                         }
                         steps.append(
                             RolloutStep(
@@ -797,6 +884,25 @@ async def rollout_executor(
         },
     )
 
+    # Extract inference_url from policy config (REQUIRED for RL trace correlation)
+    # The trainer injects this with ?cid=trace_xxxxx parameter for trace linking
+    final_inference_url = policy_config.get("inference_url")
+    if not isinstance(final_inference_url, str) or not final_inference_url.strip():
+        # Fallback to agent's inference_url if not in policy config
+        final_inference_url = agent.inference_url
+        logger.warning(
+            "VERILOG_ROLLOUT: inference_url not found in policy_config, using agent.inference_url run_id=%s url=%s",
+            request.run_id,
+            final_inference_url,
+        )
+    else:
+        logger.info(
+            "VERILOG_ROLLOUT: using inference_url from policy_config run_id=%s url=%s has_cid=%s",
+            request.run_id,
+            final_inference_url,
+            "?cid=" in final_inference_url,
+        )
+    
     trajectory = RolloutTrajectory(
         env_id=str(env_id),
         policy_id=str(policy_id),
@@ -810,11 +916,11 @@ async def rollout_executor(
                 "total_reward": final_total_reward,
                 "task_completed": bool(final_observation.get("task_completed")),
                 "policy_model": policy_model,
-                "inference_url": agent.inference_url,
+                "inference_url": final_inference_url,
             },
         },
         length=len(steps),
-        inference_url=agent.inference_url,  # NEW: Required for trace correlation
+        inference_url=final_inference_url,  # CRITICAL: Must contain ?cid=... for trace correlation
         decision_samples=None,
     )
 
@@ -836,6 +942,133 @@ async def rollout_executor(
         }
     }
     
+    # Build pipeline_metadata (required for RL training)
+    pipeline_metadata = {
+        "reward_score": final_total_reward,
+        "policy_id": policy_id,
+        "inference_url": final_inference_url,  # CRITICAL: Must be at top level for RL trainer (expects ?cid=...)
+        "inference": {
+            "provider": "groq",
+            "model": policy_model,
+            "url": final_inference_url,  # Use final_inference_url (has ?cid=...)
+        },
+        "env_name": env_id,
+        "task_id": getattr(instance, "problem_id", None),
+        "task_split": getattr(instance, "split", "val"),
+    }
+    
+    # Log episode summary with reward breakdown
+    compile_status = final_observation.get("compile_status", "N/A")
+    simulate_status = final_observation.get("simulate_status", "N/A")
+    task_completed = bool(final_observation.get("task_completed", False))
+    
+    logger.info("=" * 80)
+    logger.info(f"[EPISODE COMPLETE] run_id={request.run_id}")
+    logger.info(f"  Steps taken:       {len(steps)}")
+    logger.info(f"  Total reward:      {final_total_reward:.3f}")
+    logger.info(f"  Task completed:    {task_completed}")
+    logger.info(f"  Compile status:    {compile_status}")
+    logger.info(f"  Simulate status:   {simulate_status}")
+    logger.info(f"  Done/Truncated:    {final_done}/{final_truncated}")
+    logger.info(f"  Problem ID:        {getattr(instance, 'problem_id', 'N/A')}")
+    
+    # DEBUG: Log each step's reward for RL debugging
+    print(f"\n[REWARD DEBUG] Step-by-step breakdown:")
+    for idx, step in enumerate(steps):
+        print(f"  Step {idx}: reward={step.reward:.4f} tool_calls={[tc.get('tool_name') for tc in step.tool_calls]}")
+    print(f"[REWARD DEBUG] Final observation keys: {list(final_observation.keys())}")
+    print(f"[REWARD DEBUG] Final obs total_reward: {final_observation.get('total_reward')}")
+    print(f"[REWARD DEBUG] Metrics outcome_score: {metrics.outcome_score}")
+    print(f"[REWARD DEBUG] Metrics mean_return: {metrics.mean_return}")
+    
+    # Reward breakdown for debugging
+    logger.info("\n[REWARD BREAKDOWN]")
+    compile_count = sum(1 for s in steps if any(tc.get("tool_name") == "compile" for tc in s.tool_calls))
+    simulate_count = sum(1 for s in steps if any(tc.get("tool_name") == "simulate" for tc in s.tool_calls))
+    submit_count = sum(1 for s in steps if any(tc.get("tool_name") == "submit" for tc in s.tool_calls))
+    write_count = sum(1 for s in steps if any(tc.get("tool_name") == "write_file" for tc in s.tool_calls))
+    
+    logger.info(f"  Tool usage: write_file={write_count}, compile={compile_count}, simulate={simulate_count}, submit={submit_count}")
+    
+    # Show per-step rewards
+    step_rewards = [s.reward for s in steps]
+    nonzero_rewards = [r for r in step_rewards if r != 0.0]
+    logger.info(f"  Step rewards: {step_rewards}")
+    if nonzero_rewards:
+        logger.info(f"  Non-zero rewards: {nonzero_rewards}")
+    else:
+        logger.info(f"  ⚠️  ALL REWARDS ZERO! Possible reasons:")
+        logger.info(f"    - No successful compiles (compile reward = 0.01)")
+        logger.info(f"    - No successful simulations (simulate reward = 0.1)")
+        logger.info(f"    - No successful submits (submit reward = 1.0)")
+        logger.info(f"    - Check if task_completed={task_completed}")
+        logger.info(f"    - Check compile_status='{compile_status}'")
+        logger.info(f"    - Check simulate_status='{simulate_status}'")
+    logger.info("=" * 80)
+    
+    # Log for debugging RL training
+    logger.info(
+        "VERILOG_ROLLOUT: pipeline_metadata run_id=%s reward=%.3f inference_url=%s",
+        request.run_id,
+        final_total_reward,
+        final_inference_url,
+    )
+    
+    # DEBUG: Log what we're returning to the RL trainer
+    print(f"\n[RETURN DEBUG] Trajectory structure being returned:")
+    print(f"  trajectory.steps count: {len(steps)}")
+    print(f"  trajectory.final.reward: {trajectory.final.get('reward') if trajectory.final else 'None'}")
+    print(f"  trajectory.length: {trajectory.length}")
+    print(f"  metrics.outcome_score: {metrics.outcome_score}")
+    print(f"  metrics.mean_return: {metrics.mean_return}")
+    print(f"  metrics.episode_returns: {metrics.episode_returns}")
+    print(f"  pipeline_metadata.reward_score: {pipeline_metadata.get('reward_score')}")
+    
+    # ASSERTIONS: Validate RL-required fields before returning
+    # These catch structural issues early (before they reach the backend trainer)
+    # Only enforce for RL mode, not EVAL mode
+    is_rl_mode = hasattr(request, 'mode') and str(getattr(request, 'mode', '')).lower() == 'rl'
+    
+    assert isinstance(pipeline_metadata, dict), (
+        f"VERILOG_ROLLOUT_VALIDATION: pipeline_metadata must be dict, got {type(pipeline_metadata).__name__}"
+    )
+    assert "inference_url" in pipeline_metadata, (
+        f"VERILOG_ROLLOUT_VALIDATION: pipeline_metadata missing 'inference_url' (REQUIRED for RL training)"
+    )
+    assert isinstance(pipeline_metadata["inference_url"], str), (
+        f"VERILOG_ROLLOUT_VALIDATION: pipeline_metadata['inference_url'] must be string, got {type(pipeline_metadata['inference_url']).__name__}"
+    )
+    # Only require ?cid= for RL mode (not needed for EVAL)
+    if is_rl_mode:
+        assert "?cid=" in pipeline_metadata["inference_url"], (
+            f"VERILOG_ROLLOUT_VALIDATION: pipeline_metadata['inference_url'] must contain '?cid=' for trace correlation in RL mode. "
+            f"Got: {pipeline_metadata['inference_url'][:100]}"
+        )
+    
+    # Validate each step has meta.inference_url (backend expects this nested structure)
+    for step_idx, step in enumerate(steps):
+        step_dict = step if isinstance(step, dict) else (step.model_dump() if hasattr(step, "model_dump") else {})
+        step_info = step_dict.get("info", {})
+        assert isinstance(step_info, dict), (
+            f"VERILOG_ROLLOUT_VALIDATION: step[{step_idx}].info must be dict, got {type(step_info).__name__}"
+        )
+        step_meta = step_info.get("meta", {})
+        assert isinstance(step_meta, dict), (
+            f"VERILOG_ROLLOUT_VALIDATION: step[{step_idx}].info.meta must be dict, got {type(step_meta).__name__}"
+        )
+        assert "inference_url" in step_meta, (
+            f"VERILOG_ROLLOUT_VALIDATION: step[{step_idx}].info.meta missing 'inference_url' (REQUIRED for RL training)"
+        )
+        assert isinstance(step_meta["inference_url"], str), (
+            f"VERILOG_ROLLOUT_VALIDATION: step[{step_idx}].info.meta['inference_url'] must be string, got {type(step_meta['inference_url']).__name__}"
+        )
+    
+    logger.info(
+        "VERILOG_ROLLOUT_VALIDATION: ✓ All RL-required fields present run_id=%s steps=%d",
+        request.run_id,
+        len(steps),
+    )
+    
     return RolloutResponse(
         run_id=request.run_id,
         trajectories=[trajectory],
@@ -844,6 +1077,7 @@ async def rollout_executor(
         aborted=False,
         ops_executed=len(steps),
         trace=trace_payload,
+        pipeline_metadata=pipeline_metadata,
     )
 
 
@@ -917,6 +1151,7 @@ register_task_app(
                 "python-dotenv>=1.0.1",
                 "datasets>=2.10.0",
             ),
+            apt_packages=("iverilog",),  # Icarus Verilog compiler and simulator (provides iverilog and vvp)
             extra_local_dirs=(
                 (str(REPO_ROOT), "/opt/synth_ai_repo"),
                 (str(REPO_ROOT / "synth_ai"), "/opt/synth_ai_repo/synth_ai"),

examples/task_apps/verilog/tests/__init__.py

@@ -1,2 +1,4 @@
 # Verilog task app tests
 
+
+

examples/task_apps/verilog/tests/integration/__init__.py

@@ -1,2 +1,4 @@
 # Integration tests for Verilog task app
 
+
+

examples/task_apps/verilog/tests/integration/test_verilog_eval.py

@@ -177,3 +177,5 @@ def test_verilog_eval_with_groq(verilog_server: str) -> None:
     # Check that we got a meaningful outcome score
     assert "outcome" in result.stdout.lower() or "mean_return" in result.stdout.lower()
 
+
+

examples/task_apps/verilog/tests/unit/__init__.py

@@ -1,2 +1,4 @@
 # Unit tests for Verilog task app
 
+
+

examples/vlm/README.md

@@ -21,8 +21,8 @@ plumbing with lightweight utilities for dataset curation and training.
 3. **Export multimodal SFT rows**  
    ```
    uv run python examples/warming_up_to_rl/export_trace_sft.py \
-       --db traces/v3/synth_ai.db \
-       --output examples/vlm/output/crafter_traces_full.jsonl
+      --db traces/v3/task_app_traces_<timestamp>.db \
+       --output examples/vlm/output/crafter_sft_full.jsonl
    ```
    The exporter now emits `metadata.has_image`, `metadata.user_has_image`, and
    `metadata.assistant_has_image` flags per turn.
@@ -30,7 +30,7 @@ plumbing with lightweight utilities for dataset curation and training.
 4. **Filter to image-rich turns**  
    ```
    uv run python examples/vlm/filter_image_rows.py \
-       --input examples/vlm/output/crafter_traces_full.jsonl \
+       --input examples/vlm/output/crafter_sft_full.jsonl \
        --output examples/vlm/output/crafter_vlm_dataset.jsonl
    ```
 

examples/vlm/configs/crafter_vlm_gpt4o.toml

@@ -1,3 +1,5 @@
+type = "sft"
+
 [job]
 model = "openai/gpt-4o-mini-2024-07-18"
 modalities = ["text", "image"]

examples/vlm/crafter_openai_vlm_agent.py

@@ -24,6 +24,7 @@ import asyncio
 import base64
 import json
 import os
+from contextlib import suppress
 from pathlib import Path
 from typing import Any
 from uuid import uuid4
@@ -62,7 +63,7 @@ class EpisodeResult:
                 if unlocked:
                     self.achievements.add(str(name))
         reward = obs.get("reward_last_step")
-        if isinstance(reward, (int, float)):
+        if isinstance(reward, int | float):
             self.total_reward += float(reward)
 
 
@@ -107,11 +108,8 @@ def _decode_and_save_image(observation: dict[str, Any], path: Path) -> None:
     if not isinstance(base64_data, str) or not base64_data:
         return
     path.parent.mkdir(parents=True, exist_ok=True)
-    try:
+    with suppress(Exception):
         path.write_bytes(base64.b64decode(base64_data))
-    except Exception:
-        # Best-effort; corrupted frames should not halt rollout
-        pass
 
 
 def _normalise_openai_request(payload: dict[str, Any], model: str, temperature: float) -> dict[str, Any]:

examples/vlm/filter_image_rows.py

@@ -8,7 +8,7 @@ output now that each record's metadata includes `has_image`, `user_has_image`, a
 
 Usage:
     uv run python examples/vlm/filter_image_rows.py \
-        --input examples/sft/ft_data/crafter_traces.jsonl \
+        --input examples/sft/ft_data/crafter_sft.jsonl \
         --output examples/vlm/output/crafter_vlm_dataset.jsonl
 """
 

examples/vlm/run_crafter_vlm_benchmark.py

@@ -224,7 +224,7 @@ async def _run_episode(
                         if unlocked:
                             achievements.add(str(name))
                 reward = obs.get("reward_last_step")
-                if isinstance(reward, (int, float)):
+                if isinstance(reward, int | float):
                     total_reward += float(reward)
 
             _save_observation_frame(env_response, frames_dir / f"step_{step_idx + 1:03d}.png")
@@ -263,7 +263,7 @@ def _summarise(results: list[EpisodeResult]) -> dict[str, Any]:
             "mean_steps": round(mean_steps, 2),
             "mean_achievements": round(mean_achievements, 2),
             "total_tool_calls": sum(r.tool_calls for r in mode_results),
-            "achievements": {name: count for name, count in sorted(achievement_counts.items())},
+            "achievements": dict(sorted(achievement_counts.items())),
         }
     return summary
 

examples/warming_up_to_rl/_utils.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+
+from synth_ai.task import (
+    RolloutEnvSpec,
+    RolloutPolicySpec,
+    RolloutRecordConfig,
+    RolloutRequest,
+    RolloutSafetyConfig,
+)
+
+DEFAULT_POLICY_NAME = "crafter-react"
+DEFAULT_ENV_NAME = "crafter"
+
+
+def parse_ops(spec: str | None) -> list[str] | None:
+    """Parse a comma-separated operations string into a list."""
+
+    if spec is None:
+        return None
+    ops = [op.strip() for op in spec.split(",") if op.strip()]
+    if not ops:
+        raise ValueError("Ops must contain at least one entry")
+    return ops
+
+
+def ops_from_pairs(max_llm_calls: int, *, cap: int | None = None) -> list[str]:
+    """Return alternating agent/env ops for the requested number of LLM calls."""
+
+    pairs = max(1, int(max_llm_calls or 0))
+    if cap is not None:
+        pairs = min(pairs, cap)
+    ops: list[str] = []
+    for _ in range(pairs):
+        ops.extend(["agent", "env"])
+    return ops
+
+
+def build_rollout_request(
+    *,
+    seed: int,
+    run_id: str,
+    model: str,
+    inference_url: str,
+    ops: Sequence[str] | Iterable[str],
+    inference_api_key: str | None = None,
+    extra_headers: dict[str, str] | None = None,
+    trace_format: str = "compact",
+    return_trace: bool = False,
+    policy_name: str = DEFAULT_POLICY_NAME,
+    env_name: str = DEFAULT_ENV_NAME,
+    max_policy_tokens: int | None = None,
+    record_trajectories: bool = True,
+) -> RolloutRequest:
+    """Construct a RolloutRequest shared across local rollout utilities."""
+
+    policy_config: dict[str, object] = {
+        "model": model,
+        "inference_url": inference_url,
+    }
+    if inference_api_key is not None:
+        policy_config["api_key"] = inference_api_key
+    if extra_headers:
+        policy_config["extra_headers"] = extra_headers
+    if max_policy_tokens is not None:
+        policy_config["max_completion_tokens"] = max_policy_tokens
+        policy_config["max_tokens"] = max_policy_tokens
+
+    record_cfg = RolloutRecordConfig(
+        trajectories=record_trajectories,
+        trace_format=trace_format,
+        return_trace=return_trace,
+    )
+    return RolloutRequest(
+        run_id=run_id,
+        env=RolloutEnvSpec(env_name=env_name, seed=seed, config={}),
+        policy=RolloutPolicySpec(policy_name=policy_name, config=policy_config),
+        ops=list(ops),
+        record=record_cfg,
+        on_done="reset",
+        safety=RolloutSafetyConfig(),
+    )
+
+
+__all__ = [
+    "DEFAULT_POLICY_NAME",
+    "DEFAULT_ENV_NAME",
+    "build_rollout_request",
+    "ops_from_pairs",
+    "parse_ops",
+]

examples/warming_up_to_rl/analyze_trace_db.py

@@ -383,7 +383,7 @@ def main() -> None:
     parser.add_argument(
         "--db",
         type=Path,
-        default=Path("traces/v3/synth_ai.db"),
+        default=Path("traces/task_app_traces.db"),
         help="Path to the tracing_v3 SQLite database",
     )
     args = parser.parse_args()

examples/warming_up_to_rl/configs/crafter_fft.toml

@@ -1,6 +1,8 @@
 # Crafter Full Finetune (FFT) example on H100
 # Adjust paths and hyperparameters to your environment before running.
 
+type = "sft"
+
 [job]
 model = "Qwen/Qwen3-4B"               # base model to finetune
 # Path to your SFT JSONL dataset

examples/warming_up_to_rl/configs/crafter_fft_4b.toml

@@ -1,5 +1,7 @@
 # FFT job config for Qwen/Qwen3-4B on Crafter SFT dataset
 
+type = "sft"
+
 [algorithm]
 type = "offline"
 method = "supervised_finetune"

examples/warming_up_to_rl/configs/eval_fft_qwen4b.toml

@@ -1,5 +1,7 @@
 # Eval config for finetuned Qwen/Qwen3-4B (FFT) via task app rollout
 
+type = "sft"
+
 # Required
 task_app_url = "https://synth-laboratories--grpo-crafter-task-app-final-warming--ceb5b2.modal.run"
 # Replace with your finished job id if different

examples/warming_up_to_rl/configs/eval_groq_qwen32b.toml

@@ -1,6 +1,8 @@
 # Eval config for Groq Qwen3-32B
 # Fields mirror run_eval.py expectations
 
+type = "rl"
+
 # Required
 task_app_url = "https://synth-laboratories--grpo-crafter-task-app-final-warming--ceb5b2.modal.run"
 model = "qwen/qwen3-32b"

examples/warming_up_to_rl/configs/eval_modal_qwen4b.toml

@@ -1,5 +1,7 @@
 # Eval config for Synth Modal inference Qwen/Qwen3-4B via task app rollout
 
+type = "rl"
+
 # Required
 task_app_url = "https://synth-laboratories--grpo-crafter-task-app-final-warming--ceb5b2.modal.run"
 model = "Qwen/Qwen3-4B"
@@ -20,4 +22,3 @@ concurrency = 10
 #   fetch the vLLM base from the task app /info to use as inference_url.
 # - Ensure the task app mounts the openai-api-key secret if your vLLM gateway
 #   requires a bearer token (OPENAI_API_KEY). Otherwise it will call unauthenticated.
-

examples/warming_up_to_rl/configs/rl_from_base_qwen4b.toml

@@ -1,11 +1,12 @@
 # RL training starting from base Qwen/Qwen3-4B (TOML-only model selection)
 
+type = "rl"
+
 [algorithm]
 type = "online"
 method = "policy_gradient"
 variety = "gspo"
 
-
 [services]
 task_url = "https://synth-laboratories--grpo-crafter-task-app-final-warming--ceb5b2.modal.run"