synth-ai 0.2.9.dev0__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

examples/warming_up_to_rl/groq_test.py

@@ -0,0 +1,97 @@
+"""Quick smoke test that drives a rollout through the Groq proxy-backed Crafter Task App."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import os
+from typing import Any
+
+from synth_ai.task import (
+    INTERACT_TOOL_SCHEMA,
+    RolloutEnvSpec,
+    RolloutPolicySpec,
+    RolloutRequest,
+    TaskAppClient,
+    to_jsonable,
+)
+
+
+def _build_policy_payload(seed: int, model: str) -> dict[str, Any]:
+    return {
+        "model": model,
+        "tools": INTERACT_TOOL_SCHEMA,
+        "messages": [
+            {
+                "role": "system",
+                "content": "You control the Crafter agent. Think briefly, then call the interact tool with 3-5 actions to maximize achievements.",
+            },
+            {
+                "role": "user",
+                "content": (
+                    f"Environment seed {seed}. Plan initial survival/crafting steps and then call interact with concrete actions."
+                ),
+            },
+        ],
+    }
+
+
+async def run(args: argparse.Namespace) -> None:
+    client = TaskAppClient(args.base_url, api_key=args.api_key, timeout=args.timeout)
+
+    health = await client.health()
+    print("/health →", to_jsonable(health))
+
+    info = await client.info()
+    print("/info →", to_jsonable(info))
+
+    inference_url = args.inference_url or f"{args.base_url.rstrip('/')}/proxy/groq"
+
+    from synth_ai.task.contracts import RolloutMode
+    request = RolloutRequest(
+        run_id=args.run_id,
+        mode=RolloutMode.EVAL,
+        env=RolloutEnvSpec(env_name="crafter", seed=args.seed, config={"seed": args.seed}),
+        policy=RolloutPolicySpec(
+            policy_name="groq-smoke",
+            config={"model": args.model, "inference_url": inference_url.rstrip("/")},
+        ),
+        ops=[
+            {"type": "policy", "payload": _build_policy_payload(args.seed, args.model)},
+            {"type": "env"},
+        ],
+    )
+
+    response = await client.rollout(request)
+    print("rollout.metrics →", to_jsonable(response.metrics.model_dump()))
+    for idx, step in enumerate(response.trajectories[0].steps, start=1):
+        print(
+            f"step[{idx}] tool_calls={step.tool_calls} reward={step.reward} info={to_jsonable(step.info)}"
+        )
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--base-url", default=os.getenv("TASK_APP_BASE_URL", "http://localhost:8000")
+    )
+    parser.add_argument(
+        "--api-key",
+        default=os.getenv("TASK_APP_API_KEY"),
+        required=os.getenv("TASK_APP_API_KEY") is None,
+    )
+    parser.add_argument("--model", default=os.getenv("GROQ_MODEL", "groq/mixtral-8x7b"))
+    parser.add_argument("--inference-url", default=os.getenv("TASK_APP_INFERENCE_URL"))
+    parser.add_argument("--seed", type=int, default=int(os.getenv("CRAFTER_TEST_SEED", "42")))
+    parser.add_argument("--run-id", default=os.getenv("TASK_APP_RUN_ID", "groq-test"))
+    parser.add_argument("--timeout", type=float, default=float(os.getenv("TASK_APP_TIMEOUT", "60")))
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = _parse_args()
+    asyncio.run(run(args))
+
+
+if __name__ == "__main__":
+    main()

examples/warming_up_to_rl/manage_secrets.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import os
+import shlex
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+def load_env_file(path: Path) -> dict[str, str]:
+    env: dict[str, str] = {}
+    if not path.exists():
+        raise FileNotFoundError(f".env not found at {path}")
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        k, v = line.split("=", 1)
+        env[k.strip()] = v.strip().strip("'").strip('"')
+    return env
+
+
+def write_temp_env(kv: dict[str, str]) -> Path:
+    fd, p = tempfile.mkstemp(prefix="modal_secret_", suffix=".env")
+    path = Path(p)
+    with os.fdopen(fd, "w", encoding="utf-8") as fh:
+        for k, v in kv.items():
+            fh.write(f"{k}={v}\n")
+    return path
+
+
+def run(cmd: str) -> tuple[int, str]:
+    proc = subprocess.run(
+        cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
+    )
+    return proc.returncode, proc.stdout
+
+
+def ensure_secret(secret_name: str, kv: dict[str, str]) -> None:
+    if not kv:
+        print(f"[skip] {secret_name}: no values provided")
+        return
+    # Prefer passing KEY=VALUE pairs to avoid Typer --env-file bug under some shells
+    kv_args = " ".join([f"{shlex.quote(k)}={shlex.quote(v)}" for k, v in kv.items()])
+
+    # Try plain modal first; fallback to uv run modal
+    def _create() -> tuple[int, str]:
+        return run(f"modal secret create {shlex.quote(secret_name)} {kv_args}")
+
+    def _delete() -> tuple[int, str]:
+        return run(f"printf 'y\n' | modal secret delete {shlex.quote(secret_name)}")
+
+    rc, out = _create()
+    if rc != 0:
+        # Fallback: use uv run modal
+        rc_uv, out_uv = run(f"uv run modal secret create {shlex.quote(secret_name)} {kv_args}")
+        if rc_uv == 0:
+            print(f"[ok] secret ready: {secret_name}")
+            return
+        # Try delete+create with both variants
+        print(f"[info] create failed for {secret_name}, attempting delete+create…")
+        _ = _delete()
+        rc2, out2 = _create()
+        if rc2 != 0:
+            _ = run(f"printf 'y\n' | uv run modal secret delete {shlex.quote(secret_name)}")
+            rc3, out3 = run(f"uv run modal secret create {shlex.quote(secret_name)} {kv_args}")
+            if rc3 != 0:
+                print(out3 or out2 or out_uv or out)
+                raise RuntimeError(f"failed to create secret {secret_name}")
+    print(f"[ok] secret ready: {secret_name}")
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(
+        description="Sync .env keys into Modal secret bundles for the task app"
+    )
+    ap.add_argument(
+        "--env-path", default=str(Path(__file__).parent / ".env"), help="Path to .env with keys"
+    )
+    args = ap.parse_args()
+
+    env = load_env_file(Path(args.env_path))
+
+    # Secrets used by the task app
+    groq_secret = {
+        k: v
+        for k, v in {
+            "GROQ_API_KEY": env.get("GROQ_API_KEY", ""),
+            "dev_groq_api_key": env.get("GROQ_API_KEY", ""),
+        }.items()
+        if v
+    }
+
+    openai_secret = {
+        k: v
+        for k, v in {
+            "OPENAI_API_KEY": env.get("OPENAI_API_KEY", ""),
+            "dev_openai_api_key": env.get("OPENAI_API_KEY", ""),
+        }.items()
+        if v
+    }
+
+    # Optional: backend key (not mounted by task app today, but useful to keep consistent)
+    synth_secret = (
+        {"SYNTH_API_KEY": env.get("SYNTH_API_KEY", "")} if env.get("SYNTH_API_KEY") else {}
+    )
+
+    env_key = env.get("ENVIRONMENT_API_KEY", "")
+    if env_key:
+        print(
+            "Skipping Modal secret 'crafter-environment-sdk'; the task app now expects "
+            "ENVIRONMENT_API_KEY via --env-file so the CLI-minted value stays in sync."
+        )
+    ensure_secret("groq-api-key", groq_secret)
+    ensure_secret("openai-api-key", openai_secret)
+    if synth_secret:
+        ensure_secret("synth-api-key", synth_secret)
+
+    print("All requested secrets ensured. Redeploy the app if you updated any secrets:")
+    print("  uv run modal deploy examples/warming_up_to_rl/task_app/grpo_crafter_task_app.py")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        print(f"[error] {type(e).__name__}: {e}")
+        sys.exit(1)

examples/warming_up_to_rl/old/event_rewards.md

@@ -0,0 +1,234 @@
+# Crafter Event-Level Rewards (NOTES)
+
+This note outlines how to support event-level reward layering for Crafter across the warming_up_to_rl task app and the monorepo clustered_training RL pipeline.
+
+## Goals
+- Attribute reward at decision/step level (per tool call) instead of only using a single trajectory outcome reward.
+- Make this behavior controllable via TOML config flags (enable/disable and choose the source/kind of event reward).
+- Keep compatibility with existing trajectory-outcome paths; when disabled, the system behaves exactly as before.
+
+## Definitions
+- "Decision": one LM tool call (e.g., `interact_many`) and the sequence of environment steps it triggers.
+- "Absolute achievement delta" (AchΔ): count of achievements that became true during a decision.
+- "Unique achievement delta" (UniqueΔ): count of achievements first unlocked in the episode by a decision.
+- "Env sparse reward": the environment’s own per-step reward (e.g., `reward_last_step`).
+
+## What to compute per decision
+- From observation before and after the decision:
+  - `turned_true = achievements_after − achievements_before`
+  - `new_unique = episode_achievements_after − episode_achievements_before`
+- Scalars:
+  - `ach_delta = len(turned_true)`
+  - `unique_delta = len(new_unique)`
+- Optional: per-achievement markers for each `a ∈ new_unique` (reward 1.0) for fine-grained shaping.
+
+## Switches/Flags in TOML
+Prefer reusing existing RL trainer flags in clustered_training (already present in code):
+
+```
+[training]
+# Stepwise/event rewards
+step_rewards_enabled = true                # master switch
+step_rewards_mode = "decision_stepwise"      # "off" | "decision_stepwise" | "env_sparse"
+step_rewards_beta = 0.0                    # optional coefficient for time weighting
+step_rewards_indicator_lambda = 0.0        # optional coefficient for indicator-based flips
+
+# Crafter-specific selection (proposed extension, optional)
+# event_rewards_kind = "unique"              # "unique" | "absolute" (if omitted, default to "unique")
+```
+
+- `step_rewards_enabled`: enables all event-level aggregation.
+- `step_rewards_mode`:
+  - `off`: use only trajectory outcome reward (status quo).
+  - `decision_stepwise`: use per-decision computed deltas (from policy app or collector), aggregate as returns.
+  - `env_sparse`: use the environment’s `reward_last_step` per step.
+- `event_rewards_kind` (optional): if present, selects `unique_delta` (default) vs `ach_delta` for `decision_stepwise`.
+
+Warmup task TOML may place these under a `training` or `rollout` section; the launcher just forwards the full TOML blob to the backend, so the monorepo side should read the same keys.
+
+## Warming_up_to_rl task app – producing decision rewards
+- In the Crafter policy (or rollout coordinator), for each decision:
+  - Compute `ach_delta` and `unique_delta` as above.
+  - Attach a compact record to the step metadata, e.g.:
+    ```json
+    {
+      "decision_rewards": {
+        "turn": 5,
+        "ach_delta": 1,
+        "unique_delta": 1,
+        "all": ["collect_wood"],
+        "unique": ["collect_wood"]
+      }
+    }
+    ```
+  - When `step_rewards_enabled=false`, omit this block.
+  - When `step_rewards_mode="env_sparse"`, rely on `reward_last_step` (no decision block required).
+
+Notes:
+- The app already records previous tool calls and environment results; this simply adds a small, structured payload per decision (turn).
+- If per-step `reward_last_step` is unavailable, `decision_stepwise` remains effective as long as achievements maps are present.
+
+## Monorepo clustered_training – consuming event rewards
+Integration points (based on existing config structure):
+- `ClusteredTrainerConfig` already includes:
+  - `step_rewards_enabled: bool`
+  - `step_rewards_mode: str` (off | decision_stepwise)
+  - `step_rewards_beta: float`
+  - `step_rewards_indicator_lambda: float`
+
+Collector changes (conceptual):
+1. During trajectory collection, build a vector `r_t` of per-time-step rewards:
+   - If `step_rewards_mode == "decision_stepwise"`:
+     - For time step `t` corresponding to a decision, set:
+       - `r_t = unique_delta` if `event_rewards_kind=="unique"` (default), else `r_t = ach_delta`.
+     - For non-decision steps, `r_t = 0.0` (unless you prefer to spread rewards over sub-steps; keep simple attribution by default).
+   - If `step_rewards_mode == "env_sparse"`:
+     - For each environment step, set `r_t = reward_last_step`.
+   - Else (`off`):
+     - Use a single scalar outcome reward at the end (status quo).
+
+2. Compute returns/advantages as usual, summing event rewards:
+   - For GRPO/GRPO-Ludic, the typical group-based advantage calculation remains unchanged; only the reward signal changes from a single scalar to a sequence `[r_1, …, r_T]`.
+   - Optional time weighting: `r_t ← r_t + beta * (T − t) * indicator_flip_t`, where `indicator_flip_t` is 1 if any unique achievement flipped at `t`, else 0. Use `step_rewards_indicator_lambda` as a coefficient if needed.
+
+Pseudo-code (collector side):
+```python
+r = [0.0] * T
+if cfg.step_rewards_enabled:
+    if cfg.step_rewards_mode == "decision_stepwise":
+        for ev in decision_events:  # each with fields {turn, ach_delta, unique_delta}
+            idx = ev["turn"] - 1  # 0-based
+            base = ev["unique_delta"] if event_kind == "unique" else ev["ach_delta"]
+            r[idx] += float(base)
+            if cfg.step_rewards_indicator_lambda > 0 and ev["unique_delta"] > 0:
+                r[idx] += float(cfg.step_rewards_indicator_lambda)
+    elif cfg.step_rewards_mode == "env_sparse":
+        for t, step in enumerate(env_steps):
+            r[t] += float(step.get("reward_last_step", 0.0))
+else:
+    r[-1] += float(trajectory_outcome_reward)
+```
+
+## Respecting the TOML switch
+- warming_up_to_rl launcher (`run_rl_and_save.py`) forwards the entire TOML to the backend.
+- clustered_training should read `[training].step_rewards_enabled` and `[training].step_rewards_mode` (and optionally `event_rewards_kind`) inside its config loader (already present fields in `ClusteredTrainerConfig`).
+- When disabled, the collector must not attempt to parse or rely on any per-decision metadata.
+
+## Debugging & metrics
+- Log per-trajectory aggregates: `ΣAchΔ`, `ΣUniqueΔ`, and a breakdown by decision turn (already added to the Groq rollout table in research). These can be mirrored in the backend logs for quick checks.
+- Add simple counters to training logs:
+  - number of decisions with `unique_delta>0`
+  - sum of deltas per batch
+  - share of batches with nonzero event rewards
+
+## Backward compatibility
+- When flags are off, the pipeline uses trajectory outcome rewards only.
+- No schema migrations are required; event-level metadata is optional.
+
+## Recommended defaults
+- `step_rewards_enabled = true`
+- `step_rewards_mode = "decision_stepwise"`
+- Prefer `unique` deltas for better credit assignment; set `event_rewards_kind = "unique"` (if adopted) or implicitly default to unique deltas.
+
+Here’s the exact file-by-file implementation checklist, scoped so another engineer can implement from this alone.
+
+Warming_up_to_rl (task app) – record decision rewards and honor flags
+- Config examples (ensure flags present and documented)
+  - `examples/warming_up_to_rl/configs/*.toml`
+    - Add under [training]:
+      - `step_rewards_enabled = true|false`
+      - `step_rewards_mode = "off" | "decision_stepwise" | "env_sparse"`
+      - Optional: `event_rewards_kind = "unique" | "absolute"`
+      - Optional shaping: `step_rewards_beta`, `step_rewards_indicator_lambda`
+
+- Policy (compute ach/unique deltas per decision; emit into step metadata when enabled)
+  - `examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/policy.py`
+    - Before/after each tool call sequence, compute:
+      - `ach_delta = len(achievements_after − achievements_before)`
+      - `unique_delta = len((episode_achievements_after) − (episode_achievements_before))`
+    - When `[training].step_rewards_enabled` and `step_rewards_mode == "decision_stepwise"`:
+      - Attach to the step’s returned metadata:
+        - `decision_rewards = { turn, ach_delta, unique_delta, all: [...], unique: [...] }`
+    - If `step_rewards_mode == "env_sparse"`, do not emit `decision_rewards` (leave environment’s `reward_last_step` as the only per-step reward).
+    - Respect clipping for long “Previous tool calls” context (already added; keep).
+
+- Policy routes (surface flags to policy; store on policy instance or in request metadata)
+  - `examples/warming_up_to_rl/task_app/synth_envs_hosted/policy_routes.py`
+    - Accept training flags from create/init endpoints (if provided via config).
+    - Pass through/attach the flags into the policy or per-step metadata so `policy.step(...)` can read them.
+
+- Rollout coordinator (guarantee metadata flows out with each step)
+  - `examples/warming_up_to_rl/task_app/synth_envs_hosted/rollout.py`
+    - Ensure the step response returned to the caller includes `decision_rewards` when set by the policy.
+    - No compute here; just propagate metadata.
+
+- Environment adapter (ensure observation has fields needed by the deltas)
+  - `examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/environment.py`
+    - Confirm each step response includes `observation.achievements_status` and `observation.reward_last_step`.
+    - No reward computation changes here; just guarantee the fields exist.
+
+Monorepo (clustered training, GSPO/GRPO) – use decision/env-sparse rewards to build per-step returns
+- Config loader (read flags; default behavior preserved)
+  - `backend/app/routes/clustered_training/core/algorithms/gspo/training/clustered_trainer.py`
+    - In `ClusteredTrainerConfig.from_dict(...)`:
+      - Already present: `step_rewards_enabled`, `step_rewards_mode`, `step_rewards_beta`, `step_rewards_indicator_lambda`.
+      - Add (optional) read: `event_rewards_kind` with default `"unique"` if not present.
+
+- Collector/rollout trajectory builder (construct r_t per episode)
+  - The module that converts environment/policy step records into trajectories (collector). If it’s split, cover the point where step arrays are built just before advantage computation.
+    - New logic:
+      - Initialize `r = [0.0] * T`.
+      - If `step_rewards_enabled`:
+        - If `step_rewards_mode == "decision_stepwise"`:
+          - For each step metadata with `decision_rewards`:
+            - `idx = turn - 1`
+            - `base = unique_delta` if `event_rewards_kind == "unique"` else `ach_delta`
+            - `r[idx] += float(base)`
+            - If `step_rewards_indicator_lambda > 0` and `unique_delta > 0`, `r[idx] += step_rewards_indicator_lambda`
+        - Else if `step_rewards_mode == "env_sparse"`:
+          - For each step, `r[t] += float(observation.reward_last_step or 0.0)`
+      - Else (`off`): `r[-1] += float(outcome_reward)`
+      - Optional shaping: `r[t] += step_rewards_beta * (T - t) * indicator_flip_t` where `indicator_flip_t = 1` if the step had `unique_delta > 0`, else 0.
+    - Ensure this path does not run when flags are off; old outcome-only behavior remains.
+
+- Advantage/returns computation (no API change; just consume r)
+  - The function/module that currently builds returns/advantages from rewards.
+    - No interface changes; ensure it takes `r` from the collector path above instead of a single scalar outcome reward when event rewards are enabled.
+
+- Logging/metrics (help ops confirm it’s working)
+  - Add counters in the training loop logs:
+    - Sum of `r` per batch (stepwise mode).
+    - Count of decisions with `unique_delta > 0`.
+    - Mode/flags echoed on startup.
+
+- RL configs (dev example TOMLs with flags)
+  - `backend/app/routes/clustered_training/dev/configs/crafter_online.toml`
+    - Add the `[training]` keys above with comments showing choices.
+  - Any job start scripts that inline TOML (e.g. `tests/applications/crafter/rl/start_qwen_full_clustered.py` if used)
+    - Ensure they don’t strip the new keys; no code change needed if they pass through the TOML.
+
+Research (optional reference; not required for GSPO)
+- Reference rollout script demonstrating decision-delta computation
+  - `research/testing/crafter/eval_rollout_table_groq.py`
+    - Already computes/prints per-decision deltas; use as validation aid (no further changes required for GSPO).
+
+Docs/notes (keep implementers aligned)
+- Warming up to RL notes
+  - `examples/warming_up_to_rl/event_rewards.md`
+    - Already describes flags and expectations; keep this in sync if any naming changes happen.
+
+- Research spec
+  - `research/testing/crafter/event_rewards.txt`
+    - Already contains the full design and the “recording AND using stepwise rewards” plan.
+
+Sanity checklist (engineer can validate with these)
+- With `[training].step_rewards_enabled=false`: identical behavior to today (only outcome reward used).
+- With `decision_stepwise`:
+  - The task app emits `decision_rewards` per decision (check one trajectory).
+  - The collector constructs `r_t` from `unique_delta` (or `ach_delta` if configured).
+  - Training logs show nonzero stepwise batch reward sums.
+- With `env_sparse`:
+  - No decision payload; rewards come strictly from `reward_last_step`.
+- Switching `event_rewards_kind` between `"unique"` and `"absolute"` changes which scalar lands in r at a decision turn.
+
+If you want, I can generate minimal code diffs for each target file after you confirm these paths and flag names.

examples/warming_up_to_rl/old/notes.md

@@ -0,0 +1,73 @@
+# Crafter Task App Ops Cheatsheet
+
+## Discover available task apps
+- `uvx synth-ai task-app list`
+  - Lists the registered apps plus any aliases (e.g. `grpo-crafter`, `crafter`).
+
+## Run locally with uvicorn
+- Launch the FastAPI server:
+  - `uvx synth-ai serve grpo-crafter --port 8010 --force`
+    - `--force` frees the port if a previous run is still bound.
+    - Add `--reload` while iterating on code.
+- Enable tracing + SFT dumps while serving:
+  - `uvx synth-ai serve grpo-crafter --port 8010 --force --trace ./traces --trace-db ./traces/v3/synth_ai.db`
+    - `--trace` writes JSONL trajectories into the folder.
+    - `--trace-db` points the sqlite/Turso-compatible tracing DB (defaults to `traces/v3/synth_ai.db`).
+
+## Modal hot-reload (`modal serve`)
+- Run the hosted app locally inside Modal’s hot-reload loop:
+  - `uvx synth-ai task-app modal-serve grpo-crafter --env-file .env`
+    - CLI will prompt for a `.env` file if not supplied; secrets are loaded via `Secret.from_dotenv`.
+    - Keeps watching the repo for changes and streams logs in your terminal.
+
+## Modal deploy (persistent endpoint)
+- Build + deploy to the `modal deploy` target:
+  - `uvx synth-ai task-app deploy grpo-crafter --env-file .env`
+    - Use `--dry-run` first to inspect the generated `modal deploy …` command.
+    - `--modal-cli` lets you point at a non-default Modal binary if needed.
+
+## Collecting traces & rollouts
+- Local rollouts against a running server with full trace payloads:
+  - `uv run python examples/warming_up_to_rl/run_local_rollout_traced.py --api-key "$ENVIRONMENT_API_KEY" --base-url http://localhost:8010 --model gpt-4o-mini --trace-format full --trace-path ./trace_full.json`
+    - This script prints a reward summary, dumps the trace JSON, and warns if episode returns don’t line up with event rewards.
+- Remote rollouts against a deployed Modal endpoint:
+  - `uv run python examples/warming_up_to_rl/run_rollout_remote.py --base-url https://<modal-app-url> --api-key "$ENVIRONMENT_API_KEY" --model gpt-4o-mini --max-llm-calls 10`
+
+## Trace analytics
+- Summarise model usage, reward breakdowns, and achievement histograms:
+  - `uv run python examples/warming_up_to_rl/analyze_trace_db.py --db traces/v3/synth_ai.db`
+    - Output includes per-model achievement tallies and episode reward stats.
+
+## Exporting behavioural-cloning datasets
+- Filter sessions via model, achievements, rewards, etc., then export JSONL:
+  - `uv run python examples/warming_up_to_rl/export_trace_sft.py \`
+    `  --db traces/v3/synth_ai.db \`
+    `  --output traces/qwen32b_filtered.jsonl \`
+    `  --model qwen/qwen3-32b \`
+    `  --exclude-achievement collect_sapling \`
+    `  --exclude-achievement collect_drink \`
+    `  --min-unique 3 \`
+    `  --event-reward unique_achievement_delta:1.0 \`
+    `  --limit 100`
+    - `--exclude-achievement` makes it easy to ignore easier unlocks when enforcing `--min-unique`.
+    - Combine `--require-achievement`, `--min-outcome-reward`, or provider filters as needed.
+
+## Training jobs (RL + SFT)
+- `uvx synth-ai train` is the consolidated entry point for RL or SFT launches.
+  - Omit `--config` to let the CLI enumerate candidate TOMLs (RL + FFT) and pick interactively.
+  - Omit `--env-file` to browse available `.env` files; the CLI never auto-selects.
+  - Missing secrets trigger an interactive loop: enter manually, switch `.env`, or fetch from Modal (secrets/apps) before proceeding.
+- RL run (local backend + local task app):
+  - `uvx synth-ai train --type rl --config examples/warming_up_to_rl/configs/crafter_cluster.toml --backend http://localhost:8000/api --task-url http://localhost:8010`
+    - Performs task-app health checks using the resolved `ENVIRONMENT_API_KEY` before posting to `/rl/jobs`.
+    - Polls job status until terminal unless `--no-poll` is supplied.
+- SFT run (FFT fine-tune):
+  - `uvx synth-ai train --type sft --config examples/warming_up_to_rl/configs/fft_crafter.toml --dataset traces/crafter_sft.jsonl`
+    - Uploads training/validation JSONL to `/learning/files` and starts the job.
+    - Poll output mirrors the legacy `run_fft_and_save.py` script.
+- Common flags:
+  - `--dry-run` previews payloads/uploads without making requests.
+  - `--idempotency` sets the `Idempotency-Key` header for RL submissions.
+  - `--poll-timeout` / `--poll-interval` tune the backend polling cadence.
+
+> Tip: all `uvx synth-ai …` subcommands accept `--help` if you need to inspect additional options on the fly.

examples/warming_up_to_rl/readme.md

@@ -0,0 +1,110 @@
+# Warming Up to RL (Crafter)
+
+This folder contains an end-to-end Crafter workflow: stand up the task app, collect Groq-powered rollouts, export tracing data for supervised fine-tuning, run FFT/RL jobs, and evaluate checkpoints. Commands assume the repository root as the working directory unless stated otherwise.
+
+## 1. Prerequisites
+
+- Python 3.11+
+- [`uv`](https://docs.astral.sh/uv/) / `uvx` (or install `synth-ai` inside a virtualenv)
+- Modal CLI (`modal token new`) if you plan to deploy the task app remotely
+- API keys:
+  - `SYNTH_API_KEY` and `ENVIRONMENT_API_KEY` are required for CLI flows
+  - `GROQ_API_KEY` (used by the Groq policy) and optional `OPENAI_API_KEY`
+- Run `uvx synth-ai setup` once to pair with the Synth dashboard and populate `~/.synth-ai/user_config.json`
+
+## 2. Task App
+
+### Local serve (FastAPI)
+
+```bash
+uvx synth-ai serve \
+  --env-file examples/warming_up_to_rl/.env \
+  --host 127.0.0.1 --port 8001 \
+  --trace traces/v3
+```
+
+- `--trace` creates/uses `traces/v3/task_app_traces_<timestamp>.db` for the lifetime of the server. All rollouts append to this file.
+- Add `--trace-db` to override the SQLite path (one DB per server instance).
+- Pass `--reload` during development for auto-reload.
+
+### Modal deploy / serve
+
+```bash
+uvx synth-ai deploy grpo-crafter --name grpo-crafter-task-app
+uvx synth-ai modal-serve grpo-crafter --name grpo-crafter-task-app
+```
+
+Both commands reuse the same tracing defaults; the backend persists rollouts into the configured SQLite/Turso store.
+
+## 3. Collect rollouts
+
+Hit the running task app with the local helper to gather a traced rollout (Groq policy shown below):
+
+```bash
+python examples/warming_up_to_rl/run_local_rollout_traced.py \
+  --base-url http://localhost:8001 \
+  --api-key "$ENVIRONMENT_API_KEY" \
+  --inference-api-key "$GROQ_API_KEY" \
+  --model qwen/qwen3-32b \
+  --inference-url https://api.groq.com/openai \
+  --max-llm-calls 3 \
+  --run-id local-trace
+```
+
+Artifacts produced per rollout:
+- `traces/v3/task_app_traces_<timestamp>.db`: the task app’s append-only database (one per server lifetime; new rollouts append rows).
+- `local-trace_trace.json`: single-run JSON snapshot for inspection.
+
+## 4. Export SFT-ready data
+
+```bash
+python examples/warming_up_to_rl/export_trace_sft.py
+```
+
+- When run without `--in`, the script lists every `task_app_traces*.db` under the current directory (and subdirectories), sorted by recency, and prompts you to pick one (the newest is marked `← most recent`).
+- The exporter validates the trace data, filters sessions, and writes JSONL to `ft_data/crafter_sft.jsonl` by default (override with `--out`).
+
+## 5. FFT / SFT Training
+
+Recommended via CLI:
+
+```bash
+uvx synth-ai train \
+  --type sft \
+  --config examples/warming_up_to_rl/configs/crafter_fft.toml \
+  --dataset /absolute/path/to/crafter_sft.jsonl
+```
+
+The CLI uploads training data, submits the job to the Synth backend, and polls for completion. A legacy helper (`run_fft_and_save.py`) is still provided for ad-hoc usage.
+
+## 6. Evaluate checkpoints
+
+Update the relevant TOML with the model identifier (e.g., `model = "ft:<model_id>"`) and run:
+
+```bash
+uv run python examples/warming_up_to_rl/run_eval.py \
+  --toml examples/warming_up_to_rl/configs/eval_fft_qwen4b.toml \
+  --use-rollout
+```
+
+`--use-rollout` exercises the `/rollout` endpoint so achievements/rewards are surfaced in traces.
+
+## 7. RL Training
+
+```bash
+uvx synth-ai train \
+  --type rl \
+  --config examples/warming_up_to_rl/configs/rl_from_base_qwen4b.toml
+```
+
+Start from `rl_from_ft.toml` if you want to bootstrap from a previously fine-tuned checkpoint.
+
+---
+
+### Notes on tracing
+
+- **One SQLite DB per server:** every task app instance maintains a single `task_app_traces_<timestamp>.db` and appends each new rollout. If you want a fresh file, start another `synth-ai serve` with a different `--trace-db` path.
+- **JSON snapshots per run:** `run_local_rollout_traced.py` writes `<run_id>_trace.json` so you can inspect or hand-edit individual runs.
+- **Exporter discovery:** the SFT exporter recursively catalogs all `task_app_traces*.db` files beneath the task app directory, allowing you to select any historical snapshot when exporting training data.
+
+These conventions keep tracing predictable: continuous history per server, easy selection of historical DBs, and one-off JSON exports for quick analysis.