synth-ai 0.2.9.dev5__py3-none-any.whl → 0.2.10__py3-none-any.whl

examples/rl/task_app/math_single_step.py

@@ -1,21 +1,22 @@
-from __future__ import annotations
-
 """Task app configuration for a single-step math reasoning environment."""
 
+from __future__ import annotations
+
 import contextlib
 import os
 import random
 import re
 import uuid
+from collections.abc import Iterable, Mapping, MutableMapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional, Sequence, cast
+from typing import Any, cast
 
 import httpx
 from datasets import load_dataset
 from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel, Field
-
+from synth_ai.task.apps import ModalDeploymentConfig, TaskAppEntry, register_task_app
 from synth_ai.task.contracts import (
     RolloutMetrics,
     RolloutRequest,
@@ -25,9 +26,9 @@ from synth_ai.task.contracts import (
     TaskInfo,
 )
 from synth_ai.task.datasets import TaskDatasetRegistry, TaskDatasetSpec
+from synth_ai.task.errors import http_exception
 from synth_ai.task.rubrics import Rubric, load_rubric
 from synth_ai.task.server import ProxyConfig, RubricBundle, TaskAppConfig
-from synth_ai.task.errors import http_exception
 from synth_ai.task.tracing_utils import (
     build_tracer_factory,
     resolve_sft_output_dir,
@@ -35,13 +36,14 @@ from synth_ai.task.tracing_utils import (
     tracing_env_enabled,
 )
 from synth_ai.task.vendors import normalize_vendor_keys
-from synth_ai.task.apps import ModalDeploymentConfig, TaskAppEntry, register_task_app
 from synth_ai.tracing_v3.session_tracer import SessionTracer
 
 REPO_ROOT = Path(__file__).resolve().parents[3]
 
-_modal_volume_candidate = Path(os.getenv("MATH_MODAL_DATASET_DIR", "/modal_volumes/math_dataset")).expanduser()
-_modal_volume_root: Optional[Path] = None
+_modal_volume_candidate = Path(
+    os.getenv("MATH_MODAL_DATASET_DIR", "/modal_volumes/math_dataset")
+).expanduser()
+_modal_volume_root: Path | None = None
 try:
     _modal_volume_candidate.mkdir(parents=True, exist_ok=True)
     _modal_volume_root = _modal_volume_candidate
@@ -55,7 +57,9 @@ if _modal_volume_root is not None:
     local_dataset_dir.mkdir(parents=True, exist_ok=True)
     os.environ.setdefault("MATH_DATASET_LOCAL_DIR", str(local_dataset_dir))
 else:
-    hf_cache_path = Path(os.getenv("MATH_DATASET_CACHE_DIR", str(REPO_ROOT / ".cache" / "hf-datasets")) ).expanduser()
+    hf_cache_path = Path(
+        os.getenv("MATH_DATASET_CACHE_DIR", str(REPO_ROOT / ".cache" / "hf-datasets"))
+    ).expanduser()
 
 hf_cache_path.mkdir(parents=True, exist_ok=True)
 os.environ.setdefault("MATH_DATASET_CACHE_DIR", str(hf_cache_path))
@@ -101,7 +105,7 @@ MATH_DATASET_SPEC = TaskDatasetSpec(
 _BOXED_MARKERS: tuple[str, ...] = ("\\boxed", "boxed")
 
 
-def _extract_boxed(text: str) -> Optional[str]:
+def _extract_boxed(text: str) -> str | None:
     if not text:
         return None
     for marker in _BOXED_MARKERS:
@@ -170,9 +174,9 @@ class MathDataset:
         self.name = name
         self.config = config
         self.splits = [split for split in splits if split]
-        self._cache: Dict[str, Any] = {}
+        self._cache: dict[str, Any] = {}
         self._local_dir = os.getenv("MATH_DATASET_LOCAL_DIR")
-        self._hf_token: Optional[str] = None
+        self._hf_token: str | None = None
         for key in HF_TOKEN_ENV_KEYS:
             value = os.getenv(key)
             if value:
@@ -182,7 +186,7 @@ class MathDataset:
                     break
         # No multi-candidate fallback: enforce explicit dataset id
 
-    def _local_file_for_split(self, split: str) -> Optional[Path]:
+    def _local_file_for_split(self, split: str) -> Path | None:
         specific = os.getenv(f"MATH_DATASET_LOCAL_{split.upper()}_FILE")
         if specific:
             path = Path(specific).expanduser()
@@ -203,11 +207,13 @@ class MathDataset:
         if split not in self._cache:
             local_file = self._local_file_for_split(split)
             if local_file is not None:
-                dataset = load_dataset("json", data_files=str(local_file), cache_dir=str(HF_DATASETS_CACHE))
+                dataset = load_dataset(
+                    "json", data_files=str(local_file), cache_dir=str(HF_DATASETS_CACHE)
+                )
                 self._cache[split] = dataset["train"]
             else:
                 try:
-                    load_kwargs: Dict[str, Any] = {"split": split}
+                    load_kwargs: dict[str, Any] = {"split": split}
                     if self.config:
                         load_kwargs["name"] = self.config
                     if self._hf_token:
@@ -221,7 +227,7 @@ class MathDataset:
                             tmp_path = target.with_name(target.name + ".tmp")
                             try:
                                 local_dir.mkdir(parents=True, exist_ok=True)
-                                getattr(ds, "to_json")(str(tmp_path))
+                                ds.to_json(str(tmp_path))
                                 tmp_path.replace(target)
                             except Exception:
                                 with contextlib.suppress(FileNotFoundError):
@@ -235,7 +241,7 @@ class MathDataset:
                     raise RuntimeError(" ".join(hints)) from exc
         return self._cache[split]
 
-    def sample(self, *, split: str, index: Optional[int] = None) -> Dict[str, Any]:
+    def sample(self, *, split: str, index: int | None = None) -> dict[str, Any]:
         dataset = self._load_split(split)
         if len(dataset) == 0:
             raise RuntimeError(f"Dataset split '{split}' is empty")
@@ -301,9 +307,7 @@ class MathDataset:
             except Exception as exc:
                 errors.append(f"{split}: {exc}")
         if errors:
-            raise RuntimeError(
-                "Dataset preparation failed:\n" + "\n".join(errors)
-            )
+            raise RuntimeError("Dataset preparation failed:\n" + "\n".join(errors))
 
 
 @dataclass
@@ -322,9 +326,9 @@ class MathEnvironmentManager:
 
     def __init__(self, dataset: MathDataset) -> None:
         self.dataset = dataset
-        self._states: Dict[str, MathEnvState] = {}
+        self._states: dict[str, MathEnvState] = {}
 
-    def create(self, *, split: str, index: Optional[int], seed: Optional[int]) -> MathEnvState:
+    def create(self, *, split: str, index: int | None, seed: int | None) -> MathEnvState:
         if index is None and seed is not None:
             index = seed
         sample = self.dataset.sample(split=split, index=index)
@@ -350,11 +354,11 @@ class MathEnvironmentManager:
 
 
 class InitializePayload(BaseModel):
-    seed: Optional[int] = None
-    config: Dict[str, Any] = Field(default_factory=dict)
+    seed: int | None = None
+    config: dict[str, Any] = Field(default_factory=dict)
 
 
-def _observation_from_state(state: MathEnvState) -> Dict[str, Any]:
+def _observation_from_state(state: MathEnvState) -> dict[str, Any]:
     return {
         "problem": state.problem,
         "split": state.split,
@@ -362,7 +366,9 @@ def _observation_from_state(state: MathEnvState) -> Dict[str, Any]:
     }
 
 
-def _score_submission(state: MathEnvState, tool_calls: Sequence[Mapping[str, Any]]) -> tuple[float, str, bool]:
+def _score_submission(
+    state: MathEnvState, tool_calls: Sequence[Mapping[str, Any]]
+) -> tuple[float, str, bool]:
     if not tool_calls:
         return REWARD_NEGATIVE_NO_TOOL, "missing_tool_call", False
     call = tool_calls[0]
@@ -374,14 +380,61 @@ def _score_submission(state: MathEnvState, tool_calls: Sequence[Mapping[str, Any
     if not answer:
         return REWARD_NEGATIVE_NO_ANSWER, "blank_answer", False
     is_correct = answer == state.answer
-    return (REWARD_POSITIVE if is_correct else 0.0), ("correct" if is_correct else "incorrect"), is_correct
+    return (
+        (REWARD_POSITIVE if is_correct else 0.0),
+        ("correct" if is_correct else "incorrect"),
+        is_correct,
+    )
 
 
 math_router = APIRouter()
 
 
+def _preview_tool_calls(tool_calls: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]:
+    """Return a compact, log-friendly preview of tool calls.
+
+    Truncates long fields to avoid noisy logs and leaking excessive content.
+    """
+    preview: list[dict[str, Any]] = []
+    for call in list(tool_calls or [])[:3]:
+        args = dict(call.get("args") or {})
+        answer = str(args.get("answer") or "")
+        # Hard truncate to keep logs compact
+        answer_short = answer[:120] + ("…" if len(answer) > 120 else "")
+        preview.append(
+            {
+                "tool": call.get("tool"),
+                "answer": answer_short,
+            }
+        )
+    return preview
+
+
+def _event_and_outcome_components(
+    tool_calls: Sequence[Mapping[str, Any]], *, correct: bool, reward: float
+) -> dict[str, float]:
+    """Approximate component-wise scores for RL-style logs.
+
+    - env:     task-level scalar reward (our single-step outcome)
+    - rubric_event: 1.0 if a valid tool call with non-empty answer was made else 0.0
+    - rubric_outcome: 1.0 if final answer was correct else 0.0
+    """
+    has_valid_tool = False
+    if tool_calls:
+        first = tool_calls[0] or {}
+        if str(first.get("tool") or "") == TOOL_NAME:
+            args = first.get("args") or {}
+            ans = str(args.get("answer") or "").strip()
+            has_valid_tool = bool(ans)
+    return {
+        "env": float(reward),
+        "rubric_event": 1.0 if has_valid_tool else 0.0,
+        "rubric_outcome": 1.0 if bool(correct) else 0.0,
+    }
+
+
 @math_router.post("/env/math/initialize")
-async def initialize_env(request: Request, payload: InitializePayload) -> Dict[str, Any]:
+async def initialize_env(request: Request, payload: InitializePayload) -> dict[str, Any]:
     manager: MathEnvironmentManager = request.app.state.math_env_manager
     split = str(payload.config.get("split") or DEFAULT_SPLIT)
     seed = payload.seed
@@ -397,7 +450,7 @@ async def initialize_env(request: Request, payload: InitializePayload) -> Dict[s
 
 
 @math_router.post("/env/math/step")
-async def step_env(request: Request, payload: Dict[str, Any]) -> Dict[str, Any]:
+async def step_env(request: Request, payload: dict[str, Any]) -> dict[str, Any]:
     manager: MathEnvironmentManager = request.app.state.math_env_manager
     env_id = str(payload.get("env_id") or "")
     if not env_id:
@@ -410,6 +463,26 @@ async def step_env(request: Request, payload: Dict[str, Any]) -> Dict[str, Any]:
     action = payload.get("action") or {}
     tool_calls = action.get("tool_calls") or payload.get("tool_calls") or []
     reward, status, correct = _score_submission(state, tool_calls)
+    with contextlib.suppress(Exception):
+        print(
+            "[MATH_STEP] env_id=",
+            state.env_id,
+            " split=",
+            state.split,
+            " index=",
+            state.index,
+            " calls=",
+            _preview_tool_calls(tool_calls),
+            " reward=",
+            reward,
+            " status=",
+            status,
+            " correct=",
+            correct,
+            " components=",
+            _event_and_outcome_components(tool_calls, correct=correct, reward=reward),
+            flush=True,
+        )
     state.done = True
 
     observation = _observation_from_state(state)
@@ -427,7 +500,7 @@ async def step_env(request: Request, payload: Dict[str, Any]) -> Dict[str, Any]:
 
 
 @math_router.post("/env/math/terminate")
-async def terminate_env(request: Request, payload: Dict[str, Any]) -> Dict[str, Any]:
+async def terminate_env(request: Request, payload: dict[str, Any]) -> dict[str, Any]:
     manager: MathEnvironmentManager = request.app.state.math_env_manager
     env_id = str(payload.get("env_id") or "")
     if env_id:
@@ -448,7 +521,9 @@ def _resolve_inference_url(base_url: str) -> str:
     return f"{normalized}/v1/chat/completions"
 
 
-async def _call_inference(policy_config: Mapping[str, Any], observation: Mapping[str, Any]) -> tuple[list[Dict[str, Any]], Dict[str, Any]]:
+async def _call_inference(
+    policy_config: Mapping[str, Any], observation: Mapping[str, Any]
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
     inference_url = str(policy_config.get("inference_url") or "").rstrip("/")
     if not inference_url:
         raise RuntimeError("policy.config.inference_url required for rollout")
@@ -480,7 +555,7 @@ async def _call_inference(policy_config: Mapping[str, Any], observation: Mapping
         },
     ]
 
-    payload: Dict[str, Any] = {
+    payload: dict[str, Any] = {
         "model": model,
         "messages": messages,
         "tools": [
@@ -549,7 +624,7 @@ async def _call_inference(policy_config: Mapping[str, Any], observation: Mapping
             function = call.get("function") or {}
             name = function.get("name")
             arguments = function.get("arguments")
-            parsed_args: Dict[str, Any]
+            parsed_args: dict[str, Any]
             if isinstance(arguments, str):
                 try:
                     import json
@@ -562,6 +637,15 @@ async def _call_inference(policy_config: Mapping[str, Any], observation: Mapping
             else:
                 parsed_args = {}
             tool_calls.append({"tool": name, "args": parsed_args})
+    # Lightweight provider-side logging
+    with contextlib.suppress(Exception):
+        print(
+            "[MATH_INFER] model=",
+            model,
+            " calls=",
+            _preview_tool_calls(tool_calls),
+            flush=True,
+        )
     return tool_calls, data
 
 
@@ -576,11 +660,13 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
         "index": sample["index"],
     }
 
-    tool_calls: list[Dict[str, Any]] = []
-    inference_payload: Dict[str, Any] | None = None
-    error_info: Dict[str, Any] = {}
+    tool_calls: list[dict[str, Any]] = []
+    inference_payload: dict[str, Any] | None = None
+    error_info: dict[str, Any] = {}
     try:
-        tool_calls, inference_payload = await _call_inference(request.policy.config or {}, observation)
+        tool_calls, inference_payload = await _call_inference(
+            request.policy.config or {}, observation
+        )
     except HTTPException as http_err:
         tool_calls = []
         error_info = {"error": http_err.detail, "code": http_err.status_code}
@@ -600,6 +686,28 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
         tool_calls,
     )
 
+    # Log a concise summary so we can debug reward=0 issues in production
+    with contextlib.suppress(Exception):
+        print(
+            "[MATH_ROLLOUT] run=",
+            request.run_id,
+            " split=",
+            sample["split"],
+            " index=",
+            sample["index"],
+            " calls=",
+            _preview_tool_calls(tool_calls),
+            " reward=",
+            reward,
+            " status=",
+            status,
+            " correct=",
+            correct,
+            " components=",
+            _event_and_outcome_components(tool_calls, correct=correct, reward=reward),
+            flush=True,
+        )
+
     step = RolloutStep(
         obs=observation,
         tool_calls=tool_calls,
@@ -610,6 +718,7 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
             "status": status,
             "correct": correct,
             "raw_solution": sample["raw_solution"],
+            "tool_call_preview": _preview_tool_calls(tool_calls),
             **error_info,
         },
     )
@@ -634,6 +743,34 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
         details={"status": status, "correct": correct},
     )
 
+    # Include a minimal trace when requested or tracing is enabled via env
+    include_trace = bool(
+        (request.record and getattr(request.record, "return_trace", False))
+        or os.getenv("TASKAPP_TRACING_ENABLED")
+    )
+    trace_payload = None
+    if include_trace:
+        try:
+            # Minimal structured trace for assertions
+            trace_payload = {
+                "session_id": str(uuid.uuid4()),
+                "events_count": 1,
+                "decision_rewards": [reward],
+                "lm_calls": (
+                    [{"prompt": str(observation.get("problem", "")), "response": str(tool_calls)}]
+                    if tool_calls
+                    else []
+                ),
+                "metadata": {
+                    "env": "math_single_step",
+                    "split": sample["split"],
+                    "index": sample["index"],
+                    "status": status,
+                },
+            }
+        except Exception:
+            trace_payload = None
+
     return RolloutResponse(
         run_id=request.run_id,
         trajectories=[trajectory],
@@ -641,7 +778,7 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
         metrics=metrics,
         aborted=False,
         ops_executed=2,
-        trace=None,
+        trace=trace_payload,
     )
 
 
@@ -739,7 +876,7 @@ EVENTS_RUBRIC: Rubric = cast(
 )
 
 
-def describe_taskset(dataset: MathDataset) -> Dict[str, Any]:
+def describe_taskset(dataset: MathDataset) -> dict[str, Any]:
     return {
         **MATH_DATASET_SPEC.model_dump(),
         "hf_dataset": DATASET_NAME,
@@ -775,10 +912,12 @@ def build_config() -> TaskAppConfig:
 
     tracing_enabled = tracing_env_enabled()
     tracing_db_url = resolve_tracing_db_url()
-    tracer_factory = build_tracer_factory(SessionTracer, enabled=tracing_enabled, db_url=tracing_db_url)
+    tracer_factory = build_tracer_factory(
+        SessionTracer, enabled=tracing_enabled, db_url=tracing_db_url
+    )
     sft_output_dir = resolve_sft_output_dir()
 
-    app_state: Dict[str, Any] = {
+    app_state: dict[str, Any] = {
         "math_dataset": dataset,
         "math_env_manager": MathEnvironmentManager(dataset),
         "tracing_enabled": tracing_enabled,

examples/rl/task_app/math_task_app.py

@@ -8,10 +8,10 @@ from pathlib import Path
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
 from starlette.requests import Request
-
+from synth_ai.task.auth import is_api_key_header_authorized, normalize_environment_api_key
 from synth_ai.task.server import create_task_app, run_task_app
+
 from .math_single_step import build_config
-from synth_ai.task.auth import is_api_key_header_authorized, normalize_environment_api_key
 
 
 def fastapi_app():
@@ -40,7 +40,10 @@ def fastapi_app():
     async def health(request: Request):
         env_key = normalize_environment_api_key()
         if not env_key:
-            return JSONResponse(status_code=503, content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"})
+            return JSONResponse(
+                status_code=503,
+                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
+            )
         if not is_api_key_header_authorized(request):
             prefix = _log_env_key_prefix("health", env_key)
             content = {"status": "healthy", "authorized": False}
@@ -53,7 +56,10 @@ def fastapi_app():
     async def health_rollout(request: Request):
         env_key = normalize_environment_api_key()
         if not env_key:
-            return JSONResponse(status_code=503, content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"})
+            return JSONResponse(
+                status_code=503,
+                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
+            )
         if not is_api_key_header_authorized(request):
             prefix = _log_env_key_prefix("health/rollout", env_key)
             content = {"status": "healthy", "authorized": False}
@@ -67,7 +73,7 @@ def fastapi_app():
         try:
             hdr = request.headers
             snapshot = {
-                "path": str(getattr(request, "url").path),
+                "path": str(request.url.path),
                 "have_x_api_key": bool(hdr.get("x-api-key")),
                 "have_x_api_keys": bool(hdr.get("x-api-keys")),
                 "have_authorization": bool(hdr.get("authorization")),
@@ -76,7 +82,9 @@ def fastapi_app():
             print("[422] validation", snapshot, flush=True)
         except Exception:
             pass
-        return JSONResponse(status_code=422, content={"status": "invalid", "detail": exc.errors()[:5]})
+        return JSONResponse(
+            status_code=422, content={"status": "invalid", "detail": exc.errors()[:5]}
+        )
 
     return app
 

examples/sft/README.md

@@ -0,0 +1,139 @@
+### Supervised Fine-Tuning for Crafter
+
+This folder provides a minimal, reusable SFT workflow that pulls out the SFT step from `examples/warming_up_to_rl/` and focuses it on LoRA/QLoRA. We've also added guidance for running full finetuning (FFT) so you can compare adapters against end-to-end weight updates.
+
+It supports distilling Groq (or other vendor) rollouts into JSONL using tracing and then training a small base model like `Qwen/Qwen3-0.6B`.
+
+---
+
+### 0) Load environment from .env.dev (recommended)
+
+Use your dev env file so keys/URLs are sourced consistently:
+
+```bash
+# Example path; update to your actual dev env
+set -a && source /Users/joshpurtell/Documents/GitHub/monorepo/backend/.env.dev && set +a
+```
+
+This ensures `ENVIRONMENT_API_KEY`, `GROQ_API_KEY`, and (optionally) `BACKEND_BASE_URL` are available to the steps below.
+
+---
+
+### 1) Collect traces and export SFT JSONL
+
+You can generate traces with the Crafter task app and then export them to SFT JSONL using the existing exporter:
+
+```bash
+# Serve the task app locally with tracing enabled (example)
+uvx synth-ai serve grpo-crafter \
+  --trace traces/v3 \
+  --trace-db traces/v3/synth_ai.db \
+  --port 8001
+
+# Or run traced local rollouts to accumulate data
+uv run python examples/warming_up_to_rl/run_local_rollout_traced.py \
+  --episodes 50 --max-turns 10
+
+# Export SFT dataset from the trace DB
+uv run python examples/warming_up_to_rl/export_trace_sft.py \
+  --db traces/v3/synth_ai.db \
+  --min-unique 0 \
+  --output examples/sft/ft_data/crafter_traces.jsonl
+```
+
+Notes:
+- The exporter uses achievements and event rewards to filter high-signal steps. Combine `--min-unique`, `--min-outcome-reward`, `--event-reward`, and `--require-achievement` to control data quality.
+- You can restrict to sessions from certain providers/models with `--provider`/`--model`.
+- Use `--limit` while debugging to reduce dataset size quickly.
+
+---
+
+### 2a) Train LoRA (QLoRA) on Qwen/Qwen3-0.6B
+
+Use the standard CLI. Do not use a custom Python finetuning script. Point the CLI at your `.env.dev` so it picks up keys automatically:
+
+```bash
+uvx synth-ai train \
+  --type sft \
+  --config examples/sft/configs/crafter_lora_qwen0p6b.toml \
+  --dataset examples/sft/ft_data/crafter_traces.jsonl \
+  --env-file /Users/joshpurtell/Documents/GitHub/monorepo/backend/.env.dev
+```
+
+The config sets `training.use_qlora = true` and `hyperparameters.train_kind = "peft"` to request LoRA adapters.
+
+Experiment tips:
+- The backend currently defaults to a LoRA rank of 16. If you need other ranks, generate the payload with `--dry-run`, add `"lora_rank": <value>` (and optional `"lora_alpha"`, `"lora_dropout"`) under `hyperparameters`, and submit it via the API until the CLI exposes these knobs directly.
+- Duplicate the TOML and adjust `hyperparameters.warmup_ratio`, `learning_rate`, or `gradient_accumulation_steps` to keep the global batch size comparable across datasets.
+
+---
+
+### 2b) Train Full Finetune (FFT) on Qwen/Qwen3-0.6B
+
+Full finetuning updates all weights and uses a near-identical CLI flow with the LoRA toggle disabled. The helper config lives alongside the LoRA sample:
+
+```bash
+uvx synth-ai train \
+  --type sft \
+  --config examples/sft/configs/crafter_fft_qwen0p6b.toml \
+  --dataset examples/sft/ft_data/crafter_traces.jsonl \
+  --env-file /Users/joshpurtell/Documents/GitHub/monorepo/backend/.env.dev
+```
+
+Key differences vs LoRA:
+- `training.use_qlora = false` and `hyperparameters.train_kind = "fft"` request a full-weight update.
+- `per_device_batch` defaults to 1 to keep memory use comfortable on a single H100; raise gradually as you confirm headroom.
+- FFT runs slower per step. Consider trimming the dataset with `--examples` or the exporter filters for quick baselines.
+
+If you want the 4B Crafter FFT baseline from the RL examples, reuse `examples/warming_up_to_rl/configs/crafter_fft_4b.toml` with the same CLI command.
+
+---
+
+### 3) Evaluate the fine-tuned models
+
+After the job completes, list your fine-tuned models and evaluate them in the Crafter loop:
+
+```bash
+# List models
+uv run python - <<'PY'
+import asyncio
+import os
+from synth_ai.learning.client import LearningClient
+
+backend = os.getenv("BACKEND_BASE_URL", "https://agent-learning.onrender.com/api")
+api_key = os.getenv("SYNTH_API_KEY", "")
+async def main():
+    client = LearningClient(backend, api_key)
+    models = await client.list_fine_tuned_models()
+    for m in models:
+        print(m)
+asyncio.run(main())
+PY
+
+# Evaluate in the Crafter eval loop (example via warming_up_to_rl)
+TASK_APP_URL=http://localhost:8001 \
+uv run python examples/warming_up_to_rl/run_eval.py \
+  --toml examples/warming_up_to_rl/configs/eval_local_vllm.toml \
+  --model ft:YOUR_FT_MODEL_ID \
+  --use-rollout
+```
+
+---
+
+### 4) Plan comparison runs
+
+Keep runs comparable by adjusting one axis at a time and logging the settings in your experiment tracker (spreadsheet, weights & biases, etc.).
+
+- **LoRA rank sweeps:** start from `crafter_lora_qwen0p6b.toml`, clone it per rank (e.g., `r=4,8,16,64`). For now add the desired `lora_rank` in the job payload manually (see note above) and include it in the run name.
+- **Dataset size:** duplicate the exported JSONL and slice with `head -n`, or pass `--examples N` to the CLI for quick subsamples. Track the effective token count using the exporter logs.
+- **Data quality:** increase `--min-unique`, require specific achievements, or exclude low-reward sessions with `export_trace_sft.py`. Capture the filter tuple in your run metadata so evaluations stay reproducible.
+- **FFT vs LoRA:** run both configs on the same dataset/cardinality so differences reflect the training method rather than the data.
+
+For each sweep, use consistent evaluation seeds and write down throughput (tokens/sec) so you can weigh quality vs cost.
+
+---
+
+### Files
+- `configs/crafter_lora_qwen0p6b.toml`: LoRA/QLoRA SFT config for `Qwen/Qwen3-0.6B`.
+- `configs/crafter_fft_qwen0p6b.toml`: Full-finetune SFT config for `Qwen/Qwen3-0.6B`.
+- `ft_data/`: place your exported JSONL here (ignored by VCS).

examples/sft/configs/crafter_fft_qwen0p6b.toml

@@ -0,0 +1,44 @@
+[job]
+model = "Qwen/Qwen3-0.6B"
+# Prefer passing --dataset at runtime for repeatability
+# data = "examples/sft/ft_data/crafter_traces.jsonl"
+
+[compute]
+gpu_type = "H100"
+gpu_count = 1
+nodes = 1
+
+[data]
+topology = {}
+# Optional validation set if you have one locally
+# validation_path = "examples/sft/ft_data/crafter_traces.val.jsonl"
+
+[training]
+mode = "sft_offline"
+use_qlora = false
+
+[training.validation]
+enabled = true
+evaluation_strategy = "steps"
+eval_steps = 50
+save_best_model_at_end = true
+metric_for_best_model = "val.loss"
+greater_is_better = false
+
+[hyperparameters]
+n_epochs = 1
+train_kind = "fft"
+per_device_batch = 1
+gradient_accumulation_steps = 32
+sequence_length = 4096
+learning_rate = 1e-5
+warmup_ratio = 0.03
+weight_decay = 0.01
+
+[hyperparameters.parallelism]
+use_deepspeed = true
+deepspeed_stage = 2
+fsdp = false
+bf16 = true
+fp16 = false
+activation_checkpointing = true