synth-ai 0.2.9.dev3__py3-none-any.whl → 0.2.9.dev5__py3-none-any.whl

examples/warming_up_to_rl/manage_secrets.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import os
+import shlex
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import Dict, Tuple
+
+
+def load_env_file(path: Path) -> Dict[str, str]:
+    env: Dict[str, str] = {}
+    if not path.exists():
+        raise FileNotFoundError(f".env not found at {path}")
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        k, v = line.split("=", 1)
+        env[k.strip()] = v.strip().strip("'").strip('"')
+    return env
+
+
+def write_temp_env(kv: Dict[str, str]) -> Path:
+    fd, p = tempfile.mkstemp(prefix="modal_secret_", suffix=".env")
+    path = Path(p)
+    with os.fdopen(fd, "w", encoding="utf-8") as fh:
+        for k, v in kv.items():
+            fh.write(f"{k}={v}\n")
+    return path
+
+
+def run(cmd: str) -> Tuple[int, str]:
+    proc = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+    return proc.returncode, proc.stdout
+
+
+def ensure_secret(secret_name: str, kv: Dict[str, str]) -> None:
+    if not kv:
+        print(f"[skip] {secret_name}: no values provided")
+        return
+    # Prefer passing KEY=VALUE pairs to avoid Typer --env-file bug under some shells
+    kv_args = " ".join([f"{shlex.quote(k)}={shlex.quote(v)}" for k, v in kv.items()])
+    # Try plain modal first; fallback to uv run modal
+    def _create() -> Tuple[int, str]:
+        return run(f"modal secret create {shlex.quote(secret_name)} {kv_args}")
+    def _delete() -> Tuple[int, str]:
+        return run(f"printf 'y\n' | modal secret delete {shlex.quote(secret_name)}")
+    rc, out = _create()
+    if rc != 0:
+        # Fallback: use uv run modal
+        rc_uv, out_uv = run(f"uv run modal secret create {shlex.quote(secret_name)} {kv_args}")
+        if rc_uv == 0:
+            print(f"[ok] secret ready: {secret_name}")
+            return
+        # Try delete+create with both variants
+        print(f"[info] create failed for {secret_name}, attempting delete+create…")
+        _ = _delete()
+        rc2, out2 = _create()
+        if rc2 != 0:
+            _ = run(f"printf 'y\n' | uv run modal secret delete {shlex.quote(secret_name)}")
+            rc3, out3 = run(f"uv run modal secret create {shlex.quote(secret_name)} {kv_args}")
+            if rc3 != 0:
+                print(out3 or out2 or out_uv or out)
+                raise RuntimeError(f"failed to create secret {secret_name}")
+    print(f"[ok] secret ready: {secret_name}")
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="Sync .env keys into Modal secret bundles for the task app")
+    ap.add_argument("--env-path", default=str(Path(__file__).parent / ".env"), help="Path to .env with keys")
+    args = ap.parse_args()
+
+    env = load_env_file(Path(args.env_path))
+
+    # Secrets used by the task app
+    env_secret = {
+        k: v
+        for k, v in {
+            "ENVIRONMENT_API_KEY": env.get("ENVIRONMENT_API_KEY", ""),
+            "dev_environment_api_key": env.get("ENVIRONMENT_API_KEY", ""),
+        }.items()
+        if v
+    }
+
+    groq_secret = {
+        k: v
+        for k, v in {
+            "GROQ_API_KEY": env.get("GROQ_API_KEY", ""),
+            "dev_groq_api_key": env.get("GROQ_API_KEY", ""),
+        }.items()
+        if v
+    }
+
+    openai_secret = {
+        k: v
+        for k, v in {
+            "OPENAI_API_KEY": env.get("OPENAI_API_KEY", ""),
+            "dev_openai_api_key": env.get("OPENAI_API_KEY", ""),
+        }.items()
+        if v
+    }
+
+    # Optional: backend key (not mounted by task app today, but useful to keep consistent)
+    synth_secret = {"SYNTH_API_KEY": env.get("SYNTH_API_KEY", "")} if env.get("SYNTH_API_KEY") else {}
+
+    ensure_secret("crafter-environment-sdk", env_secret)
+    ensure_secret("groq-api-key", groq_secret)
+    ensure_secret("openai-api-key", openai_secret)
+    if synth_secret:
+        ensure_secret("synth-api-key", synth_secret)
+
+    print("All requested secrets ensured. Redeploy the app if you updated any secrets:")
+    print("  uv run modal deploy examples/warming_up_to_rl/task_app/grpo_crafter_task_app.py")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        print(f"[error] {type(e).__name__}: {e}")
+        sys.exit(1)
+
+

examples/warming_up_to_rl/old/event_rewards.md

@@ -0,0 +1,234 @@
+# Crafter Event-Level Rewards (NOTES)
+
+This note outlines how to support event-level reward layering for Crafter across the warming_up_to_rl task app and the monorepo clustered_training RL pipeline.
+
+## Goals
+- Attribute reward at decision/step level (per tool call) instead of only using a single trajectory outcome reward.
+- Make this behavior controllable via TOML config flags (enable/disable and choose the source/kind of event reward).
+- Keep compatibility with existing trajectory-outcome paths; when disabled, the system behaves exactly as before.
+
+## Definitions
+- "Decision": one LM tool call (e.g., `interact_many`) and the sequence of environment steps it triggers.
+- "Absolute achievement delta" (AchΔ): count of achievements that became true during a decision.
+- "Unique achievement delta" (UniqueΔ): count of achievements first unlocked in the episode by a decision.
+- "Env sparse reward": the environment’s own per-step reward (e.g., `reward_last_step`).
+
+## What to compute per decision
+- From observation before and after the decision:
+  - `turned_true = achievements_after − achievements_before`
+  - `new_unique = episode_achievements_after − episode_achievements_before`
+- Scalars:
+  - `ach_delta = len(turned_true)`
+  - `unique_delta = len(new_unique)`
+- Optional: per-achievement markers for each `a ∈ new_unique` (reward 1.0) for fine-grained shaping.
+
+## Switches/Flags in TOML
+Prefer reusing existing RL trainer flags in clustered_training (already present in code):
+
+```
+[training]
+# Stepwise/event rewards
+step_rewards_enabled = true                # master switch
+step_rewards_mode = "decision_stepwise"      # "off" | "decision_stepwise" | "env_sparse"
+step_rewards_beta = 0.0                    # optional coefficient for time weighting
+step_rewards_indicator_lambda = 0.0        # optional coefficient for indicator-based flips
+
+# Crafter-specific selection (proposed extension, optional)
+# event_rewards_kind = "unique"              # "unique" | "absolute" (if omitted, default to "unique")
+```
+
+- `step_rewards_enabled`: enables all event-level aggregation.
+- `step_rewards_mode`:
+  - `off`: use only trajectory outcome reward (status quo).
+  - `decision_stepwise`: use per-decision computed deltas (from policy app or collector), aggregate as returns.
+  - `env_sparse`: use the environment’s `reward_last_step` per step.
+- `event_rewards_kind` (optional): if present, selects `unique_delta` (default) vs `ach_delta` for `decision_stepwise`.
+
+Warmup task TOML may place these under a `training` or `rollout` section; the launcher just forwards the full TOML blob to the backend, so the monorepo side should read the same keys.
+
+## Warming_up_to_rl task app – producing decision rewards
+- In the Crafter policy (or rollout coordinator), for each decision:
+  - Compute `ach_delta` and `unique_delta` as above.
+  - Attach a compact record to the step metadata, e.g.:
+    ```json
+    {
+      "decision_rewards": {
+        "turn": 5,
+        "ach_delta": 1,
+        "unique_delta": 1,
+        "all": ["collect_wood"],
+        "unique": ["collect_wood"]
+      }
+    }
+    ```
+  - When `step_rewards_enabled=false`, omit this block.
+  - When `step_rewards_mode="env_sparse"`, rely on `reward_last_step` (no decision block required).
+
+Notes:
+- The app already records previous tool calls and environment results; this simply adds a small, structured payload per decision (turn).
+- If per-step `reward_last_step` is unavailable, `decision_stepwise` remains effective as long as achievements maps are present.
+
+## Monorepo clustered_training – consuming event rewards
+Integration points (based on existing config structure):
+- `ClusteredTrainerConfig` already includes:
+  - `step_rewards_enabled: bool`
+  - `step_rewards_mode: str` (off | decision_stepwise)
+  - `step_rewards_beta: float`
+  - `step_rewards_indicator_lambda: float`
+
+Collector changes (conceptual):
+1. During trajectory collection, build a vector `r_t` of per-time-step rewards:
+   - If `step_rewards_mode == "decision_stepwise"`:
+     - For time step `t` corresponding to a decision, set:
+       - `r_t = unique_delta` if `event_rewards_kind=="unique"` (default), else `r_t = ach_delta`.
+     - For non-decision steps, `r_t = 0.0` (unless you prefer to spread rewards over sub-steps; keep simple attribution by default).
+   - If `step_rewards_mode == "env_sparse"`:
+     - For each environment step, set `r_t = reward_last_step`.
+   - Else (`off`):
+     - Use a single scalar outcome reward at the end (status quo).
+
+2. Compute returns/advantages as usual, summing event rewards:
+   - For GRPO/GRPO-Ludic, the typical group-based advantage calculation remains unchanged; only the reward signal changes from a single scalar to a sequence `[r_1, …, r_T]`.
+   - Optional time weighting: `r_t ← r_t + beta * (T − t) * indicator_flip_t`, where `indicator_flip_t` is 1 if any unique achievement flipped at `t`, else 0. Use `step_rewards_indicator_lambda` as a coefficient if needed.
+
+Pseudo-code (collector side):
+```python
+r = [0.0] * T
+if cfg.step_rewards_enabled:
+    if cfg.step_rewards_mode == "decision_stepwise":
+        for ev in decision_events:  # each with fields {turn, ach_delta, unique_delta}
+            idx = ev["turn"] - 1  # 0-based
+            base = ev["unique_delta"] if event_kind == "unique" else ev["ach_delta"]
+            r[idx] += float(base)
+            if cfg.step_rewards_indicator_lambda > 0 and ev["unique_delta"] > 0:
+                r[idx] += float(cfg.step_rewards_indicator_lambda)
+    elif cfg.step_rewards_mode == "env_sparse":
+        for t, step in enumerate(env_steps):
+            r[t] += float(step.get("reward_last_step", 0.0))
+else:
+    r[-1] += float(trajectory_outcome_reward)
+```
+
+## Respecting the TOML switch
+- warming_up_to_rl launcher (`run_rl_and_save.py`) forwards the entire TOML to the backend.
+- clustered_training should read `[training].step_rewards_enabled` and `[training].step_rewards_mode` (and optionally `event_rewards_kind`) inside its config loader (already present fields in `ClusteredTrainerConfig`).
+- When disabled, the collector must not attempt to parse or rely on any per-decision metadata.
+
+## Debugging & metrics
+- Log per-trajectory aggregates: `ΣAchΔ`, `ΣUniqueΔ`, and a breakdown by decision turn (already added to the Groq rollout table in research). These can be mirrored in the backend logs for quick checks.
+- Add simple counters to training logs:
+  - number of decisions with `unique_delta>0`
+  - sum of deltas per batch
+  - share of batches with nonzero event rewards
+
+## Backward compatibility
+- When flags are off, the pipeline uses trajectory outcome rewards only.
+- No schema migrations are required; event-level metadata is optional.
+
+## Recommended defaults
+- `step_rewards_enabled = true`
+- `step_rewards_mode = "decision_stepwise"`
+- Prefer `unique` deltas for better credit assignment; set `event_rewards_kind = "unique"` (if adopted) or implicitly default to unique deltas.
+
+Here’s the exact file-by-file implementation checklist, scoped so another engineer can implement from this alone.
+
+Warming_up_to_rl (task app) – record decision rewards and honor flags
+- Config examples (ensure flags present and documented)
+  - `examples/warming_up_to_rl/configs/*.toml`
+    - Add under [training]:
+      - `step_rewards_enabled = true|false`
+      - `step_rewards_mode = "off" | "decision_stepwise" | "env_sparse"`
+      - Optional: `event_rewards_kind = "unique" | "absolute"`
+      - Optional shaping: `step_rewards_beta`, `step_rewards_indicator_lambda`
+
+- Policy (compute ach/unique deltas per decision; emit into step metadata when enabled)
+  - `examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/policy.py`
+    - Before/after each tool call sequence, compute:
+      - `ach_delta = len(achievements_after − achievements_before)`
+      - `unique_delta = len((episode_achievements_after) − (episode_achievements_before))`
+    - When `[training].step_rewards_enabled` and `step_rewards_mode == "decision_stepwise"`:
+      - Attach to the step’s returned metadata:
+        - `decision_rewards = { turn, ach_delta, unique_delta, all: [...], unique: [...] }`
+    - If `step_rewards_mode == "env_sparse"`, do not emit `decision_rewards` (leave environment’s `reward_last_step` as the only per-step reward).
+    - Respect clipping for long “Previous tool calls” context (already added; keep).
+
+- Policy routes (surface flags to policy; store on policy instance or in request metadata)
+  - `examples/warming_up_to_rl/task_app/synth_envs_hosted/policy_routes.py`
+    - Accept training flags from create/init endpoints (if provided via config).
+    - Pass through/attach the flags into the policy or per-step metadata so `policy.step(...)` can read them.
+
+- Rollout coordinator (guarantee metadata flows out with each step)
+  - `examples/warming_up_to_rl/task_app/synth_envs_hosted/rollout.py`
+    - Ensure the step response returned to the caller includes `decision_rewards` when set by the policy.
+    - No compute here; just propagate metadata.
+
+- Environment adapter (ensure observation has fields needed by the deltas)
+  - `examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/environment.py`
+    - Confirm each step response includes `observation.achievements_status` and `observation.reward_last_step`.
+    - No reward computation changes here; just guarantee the fields exist.
+
+Monorepo (clustered training, GSPO/GRPO) – use decision/env-sparse rewards to build per-step returns
+- Config loader (read flags; default behavior preserved)
+  - `backend/app/routes/clustered_training/core/algorithms/gspo/training/clustered_trainer.py`
+    - In `ClusteredTrainerConfig.from_dict(...)`:
+      - Already present: `step_rewards_enabled`, `step_rewards_mode`, `step_rewards_beta`, `step_rewards_indicator_lambda`.
+      - Add (optional) read: `event_rewards_kind` with default `"unique"` if not present.
+
+- Collector/rollout trajectory builder (construct r_t per episode)
+  - The module that converts environment/policy step records into trajectories (collector). If it’s split, cover the point where step arrays are built just before advantage computation.
+    - New logic:
+      - Initialize `r = [0.0] * T`.
+      - If `step_rewards_enabled`:
+        - If `step_rewards_mode == "decision_stepwise"`:
+          - For each step metadata with `decision_rewards`:
+            - `idx = turn - 1`
+            - `base = unique_delta` if `event_rewards_kind == "unique"` else `ach_delta`
+            - `r[idx] += float(base)`
+            - If `step_rewards_indicator_lambda > 0` and `unique_delta > 0`, `r[idx] += step_rewards_indicator_lambda`
+        - Else if `step_rewards_mode == "env_sparse"`:
+          - For each step, `r[t] += float(observation.reward_last_step or 0.0)`
+      - Else (`off`): `r[-1] += float(outcome_reward)`
+      - Optional shaping: `r[t] += step_rewards_beta * (T - t) * indicator_flip_t` where `indicator_flip_t = 1` if the step had `unique_delta > 0`, else 0.
+    - Ensure this path does not run when flags are off; old outcome-only behavior remains.
+
+- Advantage/returns computation (no API change; just consume r)
+  - The function/module that currently builds returns/advantages from rewards.
+    - No interface changes; ensure it takes `r` from the collector path above instead of a single scalar outcome reward when event rewards are enabled.
+
+- Logging/metrics (help ops confirm it’s working)
+  - Add counters in the training loop logs:
+    - Sum of `r` per batch (stepwise mode).
+    - Count of decisions with `unique_delta > 0`.
+    - Mode/flags echoed on startup.
+
+- RL configs (dev example TOMLs with flags)
+  - `backend/app/routes/clustered_training/dev/configs/crafter_online.toml`
+    - Add the `[training]` keys above with comments showing choices.
+  - Any job start scripts that inline TOML (e.g. `tests/applications/crafter/rl/start_qwen_full_clustered.py` if used)
+    - Ensure they don’t strip the new keys; no code change needed if they pass through the TOML.
+
+Research (optional reference; not required for GSPO)
+- Reference rollout script demonstrating decision-delta computation
+  - `research/testing/crafter/eval_rollout_table_groq.py`
+    - Already computes/prints per-decision deltas; use as validation aid (no further changes required for GSPO).
+
+Docs/notes (keep implementers aligned)
+- Warming up to RL notes
+  - `examples/warming_up_to_rl/event_rewards.md`
+    - Already describes flags and expectations; keep this in sync if any naming changes happen.
+
+- Research spec
+  - `research/testing/crafter/event_rewards.txt`
+    - Already contains the full design and the “recording AND using stepwise rewards” plan.
+
+Sanity checklist (engineer can validate with these)
+- With `[training].step_rewards_enabled=false`: identical behavior to today (only outcome reward used).
+- With `decision_stepwise`:
+  - The task app emits `decision_rewards` per decision (check one trajectory).
+  - The collector constructs `r_t` from `unique_delta` (or `ach_delta` if configured).
+  - Training logs show nonzero stepwise batch reward sums.
+- With `env_sparse`:
+  - No decision payload; rewards come strictly from `reward_last_step`.
+- Switching `event_rewards_kind` between `"unique"` and `"absolute"` changes which scalar lands in r at a decision turn.
+
+If you want, I can generate minimal code diffs for each target file after you confirm these paths and flag names.

examples/warming_up_to_rl/old/notes.md

@@ -0,0 +1,73 @@
+# Crafter Task App Ops Cheatsheet
+
+## Discover available task apps
+- `uvx synth-ai task-app list`
+  - Lists the registered apps plus any aliases (e.g. `grpo-crafter`, `crafter`).
+
+## Run locally with uvicorn
+- Launch the FastAPI server:
+  - `uvx synth-ai serve grpo-crafter --port 8010 --force`
+    - `--force` frees the port if a previous run is still bound.
+    - Add `--reload` while iterating on code.
+- Enable tracing + SFT dumps while serving:
+  - `uvx synth-ai serve grpo-crafter --port 8010 --force --trace ./traces --trace-db ./traces/v3/synth_ai.db`
+    - `--trace` writes JSONL trajectories into the folder.
+    - `--trace-db` points the sqlite/Turso-compatible tracing DB (defaults to `traces/v3/synth_ai.db`).
+
+## Modal hot-reload (`modal serve`)
+- Run the hosted app locally inside Modal’s hot-reload loop:
+  - `uvx synth-ai task-app modal-serve grpo-crafter --env-file .env`
+    - CLI will prompt for a `.env` file if not supplied; secrets are loaded via `Secret.from_dotenv`.
+    - Keeps watching the repo for changes and streams logs in your terminal.
+
+## Modal deploy (persistent endpoint)
+- Build + deploy to the `modal deploy` target:
+  - `uvx synth-ai task-app deploy grpo-crafter --env-file .env`
+    - Use `--dry-run` first to inspect the generated `modal deploy …` command.
+    - `--modal-cli` lets you point at a non-default Modal binary if needed.
+
+## Collecting traces & rollouts
+- Local rollouts against a running server with full trace payloads:
+  - `uv run python examples/warming_up_to_rl/run_local_rollout_traced.py --api-key "$ENVIRONMENT_API_KEY" --base-url http://localhost:8010 --model gpt-4o-mini --trace-format full --trace-path ./trace_full.json`
+    - This script prints a reward summary, dumps the trace JSON, and warns if episode returns don’t line up with event rewards.
+- Remote rollouts against a deployed Modal endpoint:
+  - `uv run python examples/warming_up_to_rl/run_rollout_remote.py --base-url https://<modal-app-url> --api-key "$ENVIRONMENT_API_KEY" --model gpt-4o-mini --max-llm-calls 10`
+
+## Trace analytics
+- Summarise model usage, reward breakdowns, and achievement histograms:
+  - `uv run python examples/warming_up_to_rl/analyze_trace_db.py --db traces/v3/synth_ai.db`
+    - Output includes per-model achievement tallies and episode reward stats.
+
+## Exporting behavioural-cloning datasets
+- Filter sessions via model, achievements, rewards, etc., then export JSONL:
+  - `uv run python examples/warming_up_to_rl/export_trace_sft.py \`
+    `  --db traces/v3/synth_ai.db \`
+    `  --output traces/qwen32b_filtered.jsonl \`
+    `  --model qwen/qwen3-32b \`
+    `  --exclude-achievement collect_sapling \`
+    `  --exclude-achievement collect_drink \`
+    `  --min-unique 3 \`
+    `  --event-reward unique_achievement_delta:1.0 \`
+    `  --limit 100`
+    - `--exclude-achievement` makes it easy to ignore easier unlocks when enforcing `--min-unique`.
+    - Combine `--require-achievement`, `--min-outcome-reward`, or provider filters as needed.
+
+## Training jobs (RL + SFT)
+- `uvx synth-ai train` is the consolidated entry point for RL or SFT launches.
+  - Omit `--config` to let the CLI enumerate candidate TOMLs (RL + FFT) and pick interactively.
+  - Omit `--env-file` to browse available `.env` files; the CLI never auto-selects.
+  - Missing secrets trigger an interactive loop: enter manually, switch `.env`, or fetch from Modal (secrets/apps) before proceeding.
+- RL run (local backend + local task app):
+  - `uvx synth-ai train --type rl --config examples/warming_up_to_rl/configs/crafter_cluster.toml --backend http://localhost:8000/api --task-url http://localhost:8010`
+    - Performs task-app health checks using the resolved `ENVIRONMENT_API_KEY` before posting to `/rl/jobs`.
+    - Polls job status until terminal unless `--no-poll` is supplied.
+- SFT run (FFT fine-tune):
+  - `uvx synth-ai train --type sft --config examples/warming_up_to_rl/configs/fft_crafter.toml --dataset traces/crafter_sft.jsonl`
+    - Uploads training/validation JSONL to `/learning/files` and starts the job.
+    - Poll output mirrors the legacy `run_fft_and_save.py` script.
+- Common flags:
+  - `--dry-run` previews payloads/uploads without making requests.
+  - `--idempotency` sets the `Idempotency-Key` header for RL submissions.
+  - `--poll-timeout` / `--poll-interval` tune the backend polling cadence.
+
+> Tip: all `uvx synth-ai …` subcommands accept `--help` if you need to inspect additional options on the fly.

examples/warming_up_to_rl/readme.md

@@ -0,0 +1,172 @@
+# Warming Up to RL (Crafter)
+
+The Crafter example demonstrates the full Synth AI workflow: task app serving, Groq rollouts, tracing, SFT dataset export, FFT training, evaluation of fine-tuned models, and RL training.
+
+## Quick Reference Commands
+
+- Serve task app locally with tracing:
+  ```bash
+  uvx synth-ai serve --port 8001 --env-file examples/warming_up_to_rl/.env --trace traces/v3
+  ```
+- Deploy to Modal:
+  ```bash
+  uvx synth-ai deploy grpo-crafter --name grpo-crafter-task-app
+  ```
+- Groq rollout (server-side):
+  ```bash
+  uv run python examples/warming_up_to_rl/run_eval.py --toml examples/warming_up_to_rl/configs/eval_groq_qwen32b.toml --use-rollout
+  ```
+- Export SFT data from traced runs:
+  ```bash
+  python examples/warming_up_to_rl/export_trace_sft.py --db traces/v3/synth_ai.db --output ft_data/crafter_traces.jsonl
+  ```
+- FFT via CLI:
+  ```bash
+  uvx synth-ai train --type sft --config examples/warming_up_to_rl/configs/crafter_fft.toml --dataset /absolute/path/to/data.jsonl
+  ```
+- Evaluate FFT checkpoint:
+  ```bash
+  uv run python examples/warming_up_to_rl/run_eval.py --toml examples/warming_up_to_rl/configs/eval_fft_qwen4b.toml --use-rollout
+  ```
+- RL via CLI (FFT-first):
+  ```bash
+  uvx synth-ai train --type rl --config examples/warming_up_to_rl/configs/rl_from_ft.toml
+  ```
+
+---
+
+## 1. Prerequisites
+
+- Python 3.11+
+- `uv`/`uvx` available (or install Synth in a virtualenv)
+- Modal CLI (`modal token new`) if you plan to deploy the task app
+- `.env` in this directory with at least:
+  - `SYNTH_API_KEY`
+  - `ENVIRONMENT_API_KEY`
+  - `TASK_APP_URL` (when running against a hosted task app)
+  - Optional: `GROQ_API_KEY`, `OPENAI_API_KEY` for proxy endpoints
+
+`uvx synth-ai setup` can populate the `.env` by guiding you through the dashboard handshake.
+
+> All commands below assume you are running from the repository root unless noted.
+
+## 2. Task App Operations
+
+### Local development
+
+```bash
+uvx synth-ai serve --port 8001 --env-file examples/warming_up_to_rl/.env --trace traces/v3 --trace-db traces/v3/synth_ai.db
+```
+
+- `--trace` and `--trace-db` enable tracing v3 and SFT JSONL dumps.
+- Add `--reload` for uvicorn auto-reload while editing code.
+
+### Modal deploy / serve
+
+```bash
+uvx synth-ai deploy grpo-crafter --name grpo-crafter-task-app --env-file examples/warming_up_to_rl/.env
+uvx synth-ai modal-serve grpo-crafter --name grpo-crafter-task-app --env-file examples/warming_up_to_rl/.env
+```
+
+Both commands preflight the environment key with the backend when `SYNTH_API_KEY` is present.
+
+## 3. Baseline Evaluations (Groq and Synth vLLM)
+
+Evaluation scripts auto-load `.env` values. Update TOMLs under `configs/` with the correct `task_app_url` and provider-specific model names.
+
+- Groq Qwen3-32B:
+  ```bash
+  uv run python examples/warming_up_to_rl/run_eval.py --toml examples/warming_up_to_rl/configs/eval_groq_qwen32b.toml --use-rollout
+  ```
+- Synth vLLM Qwen3-4B (Modal-hosted inference URL specified in TOML):
+  ```bash
+  uv run python examples/warming_up_to_rl/run_eval.py --toml examples/warming_up_to_rl/configs/eval_modal_qwen4b.toml --use-rollout
+  ```
+
+`--use-rollout` drives the task app’s `/rollout` endpoint so achievements and metrics are captured. Without it the script issues per-step `initialize/step/terminate` calls.
+
+## 4. Tracing and SFT Dataset Export
+
+1. Serve the task app with tracing enabled (see Section 2) or run the traced rollout helper:
+   ```bash
+   uv run python examples/warming_up_to_rl/run_local_rollout_traced.py --episodes 10 --difficulty easy
+   ```
+2. Inspect local trace databases:
+   ```bash
+   uvx synth-ai traces --limit 10
+   ```
+3. Export JSONL suitable for SFT:
+   ```bash
+   python examples/warming_up_to_rl/export_trace_sft.py \
+     --db traces/v3/synth_ai.db \
+     --min-achievements 3 \
+     --output ft_data/crafter_traces.jsonl
+   ```
+
+The exporter enriches each example with achievements unlocked, model metadata, and reward summaries.
+
+## 5. SFT / FFT Training
+
+### Preferred: `uvx synth-ai train`
+
+```bash
+uvx synth-ai train \
+  --type sft \
+  --config examples/warming_up_to_rl/configs/crafter_fft.toml \
+  --dataset /absolute/path/to/crafter_traces.jsonl
+```
+
+The CLI will:
+- Prompt for `.env` selection (or use `--env-file`).
+- Upload training (and optional validation) data to `/learning/files`.
+- Submit the job and poll until completion unless `--no-poll` is set.
+
+### Legacy script
+
+```bash
+uv run python examples/warming_up_to_rl/run_fft_and_save.py \
+  --toml examples/warming_up_to_rl/configs/crafter_fft.toml \
+  --data /absolute/path/to/crafter_traces.jsonl \
+  --poll-seconds 1800
+```
+
+The script writes the resulting model ID to `ft_model_id.txt`. Use that ID in evaluation and RL configs (e.g., `model = "ft:abc123"`).
+
+## 6. Evaluate the Fine-tuned Model
+
+After FFT completes, update `configs/eval_fft_qwen4b.toml` so `model = "ft:<model_id>"`, then rerun the evaluation:
+
+```bash
+uv run python examples/warming_up_to_rl/run_eval.py --toml examples/warming_up_to_rl/configs/eval_fft_qwen4b.toml --use-rollout
+```
+
+This reuses the same Groq/vLLM pipeline but exercises the finetuned checkpoint.
+
+## 7. RL Training
+
+### Preferred: `uvx synth-ai train --type rl`
+
+```bash
+uvx synth-ai train \
+  --type rl \
+  --config examples/warming_up_to_rl/configs/rl_from_base_qwen4b.toml
+```
+
+During the interactive setup the CLI ensures `SYNTH_API_KEY`, `ENVIRONMENT_API_KEY`, and `TASK_APP_URL` are present, health-checks the task app, and submits the RL job to `/rl/jobs`.
+
+### Legacy script
+
+```bash
+uv run python examples/warming_up_to_rl/run_rl_and_save.py \
+  --config examples/warming_up_to_rl/configs/rl_from_ft.toml
+```
+
+To start directly from a base model, switch the config to `rl_from_base_qwen4b.toml` and ensure `[model].base` is populated.
+
+## 8. Additional Utilities
+
+- `manage_secrets.py` – convenience helpers for Modal secret management.
+- `run_local_rollout.py`, `run_local_rollout_parallel.py`, `run_rollout_remote.py` – alternative rollout launchers for benchmarking.
+- `analyze_trace_db.py` – inspect trace quality/achievements before exporting.
+
+Refer to `docs/workflows/` for end-to-end guidance that mirrors these commands.