synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

examples/warming_up_to_rl/groq_test.py

@@ -1,97 +0,0 @@
-"""Quick smoke test that drives a rollout through the Groq proxy-backed Crafter Task App."""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import os
-from typing import Any
-
-from synth_ai.task import (
-    INTERACT_TOOL_SCHEMA,
-    RolloutEnvSpec,
-    RolloutPolicySpec,
-    RolloutRequest,
-    TaskAppClient,
-    to_jsonable,
-)
-
-
-def _build_policy_payload(seed: int, model: str) -> dict[str, Any]:
-    return {
-        "model": model,
-        "tools": INTERACT_TOOL_SCHEMA,
-        "messages": [
-            {
-                "role": "system",
-                "content": "You control the Crafter agent. Think briefly, then call the interact tool with 3-5 actions to maximize achievements.",
-            },
-            {
-                "role": "user",
-                "content": (
-                    f"Environment seed {seed}. Plan initial survival/crafting steps and then call interact with concrete actions."
-                ),
-            },
-        ],
-    }
-
-
-async def run(args: argparse.Namespace) -> None:
-    client = TaskAppClient(args.base_url, api_key=args.api_key, timeout=args.timeout)
-
-    health = await client.health()
-    print("/health →", to_jsonable(health))
-
-    info = await client.info()
-    print("/info →", to_jsonable(info))
-
-    inference_url = args.inference_url or f"{args.base_url.rstrip('/')}/proxy/groq"
-
-    from synth_ai.task.contracts import RolloutMode
-    request = RolloutRequest(
-        run_id=args.run_id,
-        mode=RolloutMode.EVAL,
-        env=RolloutEnvSpec(env_name="crafter", seed=args.seed, config={"seed": args.seed}),
-        policy=RolloutPolicySpec(
-            policy_name="groq-smoke",
-            config={"model": args.model, "inference_url": inference_url.rstrip("/")},
-        ),
-        ops=[
-            {"type": "policy", "payload": _build_policy_payload(args.seed, args.model)},
-            {"type": "env"},
-        ],
-    )
-
-    response = await client.rollout(request)
-    print("rollout.metrics →", to_jsonable(response.metrics.model_dump()))
-    for idx, step in enumerate(response.trajectories[0].steps, start=1):
-        print(
-            f"step[{idx}] tool_calls={step.tool_calls} reward={step.reward} info={to_jsonable(step.info)}"
-        )
-
-
-def _parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument(
-        "--base-url", default=os.getenv("TASK_APP_BASE_URL", "http://localhost:8000")
-    )
-    parser.add_argument(
-        "--api-key",
-        default=os.getenv("TASK_APP_API_KEY"),
-        required=os.getenv("TASK_APP_API_KEY") is None,
-    )
-    parser.add_argument("--model", default=os.getenv("GROQ_MODEL", "groq/mixtral-8x7b"))
-    parser.add_argument("--inference-url", default=os.getenv("TASK_APP_INFERENCE_URL"))
-    parser.add_argument("--seed", type=int, default=int(os.getenv("CRAFTER_TEST_SEED", "42")))
-    parser.add_argument("--run-id", default=os.getenv("TASK_APP_RUN_ID", "groq-test"))
-    parser.add_argument("--timeout", type=float, default=float(os.getenv("TASK_APP_TIMEOUT", "60")))
-    return parser.parse_args()
-
-
-def main() -> None:
-    args = _parse_args()
-    asyncio.run(run(args))
-
-
-if __name__ == "__main__":
-    main()

examples/warming_up_to_rl/manage_secrets.py

@@ -1,131 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import annotations
-
-import argparse
-import os
-import shlex
-import subprocess
-import sys
-import tempfile
-from pathlib import Path
-
-
-def load_env_file(path: Path) -> dict[str, str]:
-    env: dict[str, str] = {}
-    if not path.exists():
-        raise FileNotFoundError(f".env not found at {path}")
-    for line in path.read_text(encoding="utf-8").splitlines():
-        line = line.strip()
-        if not line or line.startswith("#") or "=" not in line:
-            continue
-        k, v = line.split("=", 1)
-        env[k.strip()] = v.strip().strip("'").strip('"')
-    return env
-
-
-def write_temp_env(kv: dict[str, str]) -> Path:
-    fd, p = tempfile.mkstemp(prefix="modal_secret_", suffix=".env")
-    path = Path(p)
-    with os.fdopen(fd, "w", encoding="utf-8") as fh:
-        for k, v in kv.items():
-            fh.write(f"{k}={v}\n")
-    return path
-
-
-def run(cmd: str) -> tuple[int, str]:
-    proc = subprocess.run(
-        cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
-    )
-    return proc.returncode, proc.stdout
-
-
-def ensure_secret(secret_name: str, kv: dict[str, str]) -> None:
-    if not kv:
-        print(f"[skip] {secret_name}: no values provided")
-        return
-    # Prefer passing KEY=VALUE pairs to avoid Typer --env-file bug under some shells
-    kv_args = " ".join([f"{shlex.quote(k)}={shlex.quote(v)}" for k, v in kv.items()])
-
-    # Try plain modal first; fallback to uv run modal
-    def _create() -> tuple[int, str]:
-        return run(f"modal secret create {shlex.quote(secret_name)} {kv_args}")
-
-    def _delete() -> tuple[int, str]:
-        return run(f"printf 'y\n' | modal secret delete {shlex.quote(secret_name)}")
-
-    rc, out = _create()
-    if rc != 0:
-        # Fallback: use uv run modal
-        rc_uv, out_uv = run(f"uv run modal secret create {shlex.quote(secret_name)} {kv_args}")
-        if rc_uv == 0:
-            print(f"[ok] secret ready: {secret_name}")
-            return
-        # Try delete+create with both variants
-        print(f"[info] create failed for {secret_name}, attempting delete+create…")
-        _ = _delete()
-        rc2, out2 = _create()
-        if rc2 != 0:
-            _ = run(f"printf 'y\n' | uv run modal secret delete {shlex.quote(secret_name)}")
-            rc3, out3 = run(f"uv run modal secret create {shlex.quote(secret_name)} {kv_args}")
-            if rc3 != 0:
-                print(out3 or out2 or out_uv or out)
-                raise RuntimeError(f"failed to create secret {secret_name}")
-    print(f"[ok] secret ready: {secret_name}")
-
-
-def main() -> None:
-    ap = argparse.ArgumentParser(
-        description="Sync .env keys into Modal secret bundles for the task app"
-    )
-    ap.add_argument(
-        "--env-path", default=str(Path(__file__).parent / ".env"), help="Path to .env with keys"
-    )
-    args = ap.parse_args()
-
-    env = load_env_file(Path(args.env_path))
-
-    # Secrets used by the task app
-    groq_secret = {
-        k: v
-        for k, v in {
-            "GROQ_API_KEY": env.get("GROQ_API_KEY", ""),
-            "dev_groq_api_key": env.get("GROQ_API_KEY", ""),
-        }.items()
-        if v
-    }
-
-    openai_secret = {
-        k: v
-        for k, v in {
-            "OPENAI_API_KEY": env.get("OPENAI_API_KEY", ""),
-            "dev_openai_api_key": env.get("OPENAI_API_KEY", ""),
-        }.items()
-        if v
-    }
-
-    # Optional: backend key (not mounted by task app today, but useful to keep consistent)
-    synth_secret = (
-        {"SYNTH_API_KEY": env.get("SYNTH_API_KEY", "")} if env.get("SYNTH_API_KEY") else {}
-    )
-
-    env_key = env.get("ENVIRONMENT_API_KEY", "")
-    if env_key:
-        print(
-            "Skipping Modal secret 'crafter-environment-sdk'; the task app now expects "
-            "ENVIRONMENT_API_KEY via --env-file so the CLI-minted value stays in sync."
-        )
-    ensure_secret("groq-api-key", groq_secret)
-    ensure_secret("openai-api-key", openai_secret)
-    if synth_secret:
-        ensure_secret("synth-api-key", synth_secret)
-
-    print("All requested secrets ensured. Redeploy the app if you updated any secrets:")
-    print("  uv run modal deploy examples/warming_up_to_rl/task_app/grpo_crafter_task_app.py")
-
-
-if __name__ == "__main__":
-    try:
-        main()
-    except Exception as e:
-        print(f"[error] {type(e).__name__}: {e}")
-        sys.exit(1)

examples/warming_up_to_rl/old/event_rewards.md

@@ -1,234 +0,0 @@
-# Crafter Event-Level Rewards (NOTES)
-
-This note outlines how to support event-level reward layering for Crafter across the warming_up_to_rl task app and the monorepo clustered_training RL pipeline.
-
-## Goals
-- Attribute reward at decision/step level (per tool call) instead of only using a single trajectory outcome reward.
-- Make this behavior controllable via TOML config flags (enable/disable and choose the source/kind of event reward).
-- Keep compatibility with existing trajectory-outcome paths; when disabled, the system behaves exactly as before.
-
-## Definitions
-- "Decision": one LM tool call (e.g., `interact_many`) and the sequence of environment steps it triggers.
-- "Absolute achievement delta" (AchΔ): count of achievements that became true during a decision.
-- "Unique achievement delta" (UniqueΔ): count of achievements first unlocked in the episode by a decision.
-- "Env sparse reward": the environment’s own per-step reward (e.g., `reward_last_step`).
-
-## What to compute per decision
-- From observation before and after the decision:
-  - `turned_true = achievements_after − achievements_before`
-  - `new_unique = episode_achievements_after − episode_achievements_before`
-- Scalars:
-  - `ach_delta = len(turned_true)`
-  - `unique_delta = len(new_unique)`
-- Optional: per-achievement markers for each `a ∈ new_unique` (reward 1.0) for fine-grained shaping.
-
-## Switches/Flags in TOML
-Prefer reusing existing RL trainer flags in clustered_training (already present in code):
-
-```
-[training]
-# Stepwise/event rewards
-step_rewards_enabled = true                # master switch
-step_rewards_mode = "decision_stepwise"      # "off" | "decision_stepwise" | "env_sparse"
-step_rewards_beta = 0.0                    # optional coefficient for time weighting
-step_rewards_indicator_lambda = 0.0        # optional coefficient for indicator-based flips
-
-# Crafter-specific selection (proposed extension, optional)
-# event_rewards_kind = "unique"              # "unique" | "absolute" (if omitted, default to "unique")
-```
-
-- `step_rewards_enabled`: enables all event-level aggregation.
-- `step_rewards_mode`:
-  - `off`: use only trajectory outcome reward (status quo).
-  - `decision_stepwise`: use per-decision computed deltas (from policy app or collector), aggregate as returns.
-  - `env_sparse`: use the environment’s `reward_last_step` per step.
-- `event_rewards_kind` (optional): if present, selects `unique_delta` (default) vs `ach_delta` for `decision_stepwise`.
-
-Warmup task TOML may place these under a `training` or `rollout` section; the launcher just forwards the full TOML blob to the backend, so the monorepo side should read the same keys.
-
-## Warming_up_to_rl task app – producing decision rewards
-- In the Crafter policy (or rollout coordinator), for each decision:
-  - Compute `ach_delta` and `unique_delta` as above.
-  - Attach a compact record to the step metadata, e.g.:
-    ```json
-    {
-      "decision_rewards": {
-        "turn": 5,
-        "ach_delta": 1,
-        "unique_delta": 1,
-        "all": ["collect_wood"],
-        "unique": ["collect_wood"]
-      }
-    }
-    ```
-  - When `step_rewards_enabled=false`, omit this block.
-  - When `step_rewards_mode="env_sparse"`, rely on `reward_last_step` (no decision block required).
-
-Notes:
-- The app already records previous tool calls and environment results; this simply adds a small, structured payload per decision (turn).
-- If per-step `reward_last_step` is unavailable, `decision_stepwise` remains effective as long as achievements maps are present.
-
-## Monorepo clustered_training – consuming event rewards
-Integration points (based on existing config structure):
-- `ClusteredTrainerConfig` already includes:
-  - `step_rewards_enabled: bool`
-  - `step_rewards_mode: str` (off | decision_stepwise)
-  - `step_rewards_beta: float`
-  - `step_rewards_indicator_lambda: float`
-
-Collector changes (conceptual):
-1. During trajectory collection, build a vector `r_t` of per-time-step rewards:
-   - If `step_rewards_mode == "decision_stepwise"`:
-     - For time step `t` corresponding to a decision, set:
-       - `r_t = unique_delta` if `event_rewards_kind=="unique"` (default), else `r_t = ach_delta`.
-     - For non-decision steps, `r_t = 0.0` (unless you prefer to spread rewards over sub-steps; keep simple attribution by default).
-   - If `step_rewards_mode == "env_sparse"`:
-     - For each environment step, set `r_t = reward_last_step`.
-   - Else (`off`):
-     - Use a single scalar outcome reward at the end (status quo).
-
-2. Compute returns/advantages as usual, summing event rewards:
-   - For GRPO/GRPO-Ludic, the typical group-based advantage calculation remains unchanged; only the reward signal changes from a single scalar to a sequence `[r_1, …, r_T]`.
-   - Optional time weighting: `r_t ← r_t + beta * (T − t) * indicator_flip_t`, where `indicator_flip_t` is 1 if any unique achievement flipped at `t`, else 0. Use `step_rewards_indicator_lambda` as a coefficient if needed.
-
-Pseudo-code (collector side):
-```python
-r = [0.0] * T
-if cfg.step_rewards_enabled:
-    if cfg.step_rewards_mode == "decision_stepwise":
-        for ev in decision_events:  # each with fields {turn, ach_delta, unique_delta}
-            idx = ev["turn"] - 1  # 0-based
-            base = ev["unique_delta"] if event_kind == "unique" else ev["ach_delta"]
-            r[idx] += float(base)
-            if cfg.step_rewards_indicator_lambda > 0 and ev["unique_delta"] > 0:
-                r[idx] += float(cfg.step_rewards_indicator_lambda)
-    elif cfg.step_rewards_mode == "env_sparse":
-        for t, step in enumerate(env_steps):
-            r[t] += float(step.get("reward_last_step", 0.0))
-else:
-    r[-1] += float(trajectory_outcome_reward)
-```
-
-## Respecting the TOML switch
-- warming_up_to_rl launcher (`run_rl_and_save.py`) forwards the entire TOML to the backend.
-- clustered_training should read `[training].step_rewards_enabled` and `[training].step_rewards_mode` (and optionally `event_rewards_kind`) inside its config loader (already present fields in `ClusteredTrainerConfig`).
-- When disabled, the collector must not attempt to parse or rely on any per-decision metadata.
-
-## Debugging & metrics
-- Log per-trajectory aggregates: `ΣAchΔ`, `ΣUniqueΔ`, and a breakdown by decision turn (already added to the Groq rollout table in research). These can be mirrored in the backend logs for quick checks.
-- Add simple counters to training logs:
-  - number of decisions with `unique_delta>0`
-  - sum of deltas per batch
-  - share of batches with nonzero event rewards
-
-## Backward compatibility
-- When flags are off, the pipeline uses trajectory outcome rewards only.
-- No schema migrations are required; event-level metadata is optional.
-
-## Recommended defaults
-- `step_rewards_enabled = true`
-- `step_rewards_mode = "decision_stepwise"`
-- Prefer `unique` deltas for better credit assignment; set `event_rewards_kind = "unique"` (if adopted) or implicitly default to unique deltas.
-
-Here’s the exact file-by-file implementation checklist, scoped so another engineer can implement from this alone.
-
-Warming_up_to_rl (task app) – record decision rewards and honor flags
-- Config examples (ensure flags present and documented)
-  - `examples/warming_up_to_rl/configs/*.toml`
-    - Add under [training]:
-      - `step_rewards_enabled = true|false`
-      - `step_rewards_mode = "off" | "decision_stepwise" | "env_sparse"`
-      - Optional: `event_rewards_kind = "unique" | "absolute"`
-      - Optional shaping: `step_rewards_beta`, `step_rewards_indicator_lambda`
-
-- Policy (compute ach/unique deltas per decision; emit into step metadata when enabled)
-  - `examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/policy.py`
-    - Before/after each tool call sequence, compute:
-      - `ach_delta = len(achievements_after − achievements_before)`
-      - `unique_delta = len((episode_achievements_after) − (episode_achievements_before))`
-    - When `[training].step_rewards_enabled` and `step_rewards_mode == "decision_stepwise"`:
-      - Attach to the step’s returned metadata:
-        - `decision_rewards = { turn, ach_delta, unique_delta, all: [...], unique: [...] }`
-    - If `step_rewards_mode == "env_sparse"`, do not emit `decision_rewards` (leave environment’s `reward_last_step` as the only per-step reward).
-    - Respect clipping for long “Previous tool calls” context (already added; keep).
-
-- Policy routes (surface flags to policy; store on policy instance or in request metadata)
-  - `examples/warming_up_to_rl/task_app/synth_envs_hosted/policy_routes.py`
-    - Accept training flags from create/init endpoints (if provided via config).
-    - Pass through/attach the flags into the policy or per-step metadata so `policy.step(...)` can read them.
-
-- Rollout coordinator (guarantee metadata flows out with each step)
-  - `examples/warming_up_to_rl/task_app/synth_envs_hosted/rollout.py`
-    - Ensure the step response returned to the caller includes `decision_rewards` when set by the policy.
-    - No compute here; just propagate metadata.
-
-- Environment adapter (ensure observation has fields needed by the deltas)
-  - `examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/environment.py`
-    - Confirm each step response includes `observation.achievements_status` and `observation.reward_last_step`.
-    - No reward computation changes here; just guarantee the fields exist.
-
-Monorepo (clustered training, GSPO/GRPO) – use decision/env-sparse rewards to build per-step returns
-- Config loader (read flags; default behavior preserved)
-  - `backend/app/routes/clustered_training/core/algorithms/gspo/training/clustered_trainer.py`
-    - In `ClusteredTrainerConfig.from_dict(...)`:
-      - Already present: `step_rewards_enabled`, `step_rewards_mode`, `step_rewards_beta`, `step_rewards_indicator_lambda`.
-      - Add (optional) read: `event_rewards_kind` with default `"unique"` if not present.
-
-- Collector/rollout trajectory builder (construct r_t per episode)
-  - The module that converts environment/policy step records into trajectories (collector). If it’s split, cover the point where step arrays are built just before advantage computation.
-    - New logic:
-      - Initialize `r = [0.0] * T`.
-      - If `step_rewards_enabled`:
-        - If `step_rewards_mode == "decision_stepwise"`:
-          - For each step metadata with `decision_rewards`:
-            - `idx = turn - 1`
-            - `base = unique_delta` if `event_rewards_kind == "unique"` else `ach_delta`
-            - `r[idx] += float(base)`
-            - If `step_rewards_indicator_lambda > 0` and `unique_delta > 0`, `r[idx] += step_rewards_indicator_lambda`
-        - Else if `step_rewards_mode == "env_sparse"`:
-          - For each step, `r[t] += float(observation.reward_last_step or 0.0)`
-      - Else (`off`): `r[-1] += float(outcome_reward)`
-      - Optional shaping: `r[t] += step_rewards_beta * (T - t) * indicator_flip_t` where `indicator_flip_t = 1` if the step had `unique_delta > 0`, else 0.
-    - Ensure this path does not run when flags are off; old outcome-only behavior remains.
-
-- Advantage/returns computation (no API change; just consume r)
-  - The function/module that currently builds returns/advantages from rewards.
-    - No interface changes; ensure it takes `r` from the collector path above instead of a single scalar outcome reward when event rewards are enabled.
-
-- Logging/metrics (help ops confirm it’s working)
-  - Add counters in the training loop logs:
-    - Sum of `r` per batch (stepwise mode).
-    - Count of decisions with `unique_delta > 0`.
-    - Mode/flags echoed on startup.
-
-- RL configs (dev example TOMLs with flags)
-  - `backend/app/routes/clustered_training/dev/configs/crafter_online.toml`
-    - Add the `[training]` keys above with comments showing choices.
-  - Any job start scripts that inline TOML (e.g. `tests/applications/crafter/rl/start_qwen_full_clustered.py` if used)
-    - Ensure they don’t strip the new keys; no code change needed if they pass through the TOML.
-
-Research (optional reference; not required for GSPO)
-- Reference rollout script demonstrating decision-delta computation
-  - `research/testing/crafter/eval_rollout_table_groq.py`
-    - Already computes/prints per-decision deltas; use as validation aid (no further changes required for GSPO).
-
-Docs/notes (keep implementers aligned)
-- Warming up to RL notes
-  - `examples/warming_up_to_rl/event_rewards.md`
-    - Already describes flags and expectations; keep this in sync if any naming changes happen.
-
-- Research spec
-  - `research/testing/crafter/event_rewards.txt`
-    - Already contains the full design and the “recording AND using stepwise rewards” plan.
-
-Sanity checklist (engineer can validate with these)
-- With `[training].step_rewards_enabled=false`: identical behavior to today (only outcome reward used).
-- With `decision_stepwise`:
-  - The task app emits `decision_rewards` per decision (check one trajectory).
-  - The collector constructs `r_t` from `unique_delta` (or `ach_delta` if configured).
-  - Training logs show nonzero stepwise batch reward sums.
-- With `env_sparse`:
-  - No decision payload; rewards come strictly from `reward_last_step`.
-- Switching `event_rewards_kind` between `"unique"` and `"absolute"` changes which scalar lands in r at a decision turn.
-
-If you want, I can generate minimal code diffs for each target file after you confirm these paths and flag names.

examples/warming_up_to_rl/old/notes.md

@@ -1,73 +0,0 @@
-# Crafter Task App Ops Cheatsheet
-
-## Discover available task apps
-- `uvx synth-ai task-app list`
-  - Lists the registered apps plus any aliases (e.g. `grpo-crafter`, `crafter`).
-
-## Run locally with uvicorn
-- Launch the FastAPI server:
-  - `uvx synth-ai serve grpo-crafter --port 8010 --force`
-    - `--force` frees the port if a previous run is still bound.
-    - Add `--reload` while iterating on code.
-- Enable tracing + SFT dumps while serving:
-  - `uvx synth-ai serve grpo-crafter --port 8010 --force --trace ./traces --trace-db ./traces/v3/synth_ai.db`
-    - `--trace` writes JSONL trajectories into the folder.
-    - `--trace-db` points the sqlite/Turso-compatible tracing DB (defaults to `traces/v3/synth_ai.db`).
-
-## Modal hot-reload (`modal serve`)
-- Run the hosted app locally inside Modal’s hot-reload loop:
-  - `uvx synth-ai task-app modal-serve grpo-crafter --env-file .env`
-    - CLI will prompt for a `.env` file if not supplied; secrets are loaded via `Secret.from_dotenv`.
-    - Keeps watching the repo for changes and streams logs in your terminal.
-
-## Modal deploy (persistent endpoint)
-- Build + deploy to the `modal deploy` target:
-  - `uvx synth-ai task-app deploy grpo-crafter --env-file .env`
-    - Use `--dry-run` first to inspect the generated `modal deploy …` command.
-    - `--modal-cli` lets you point at a non-default Modal binary if needed.
-
-## Collecting traces & rollouts
-- Local rollouts against a running server with full trace payloads:
-  - `uv run python examples/warming_up_to_rl/run_local_rollout_traced.py --api-key "$ENVIRONMENT_API_KEY" --base-url http://localhost:8010 --model gpt-4o-mini --trace-format full --trace-path ./trace_full.json`
-    - This script prints a reward summary, dumps the trace JSON, and warns if episode returns don’t line up with event rewards.
-- Remote rollouts against a deployed Modal endpoint:
-  - `uv run python examples/warming_up_to_rl/run_rollout_remote.py --base-url https://<modal-app-url> --api-key "$ENVIRONMENT_API_KEY" --model gpt-4o-mini --max-llm-calls 10`
-
-## Trace analytics
-- Summarise model usage, reward breakdowns, and achievement histograms:
-  - `uv run python examples/warming_up_to_rl/analyze_trace_db.py --db traces/v3/synth_ai.db`
-    - Output includes per-model achievement tallies and episode reward stats.
-
-## Exporting behavioural-cloning datasets
-- Filter sessions via model, achievements, rewards, etc., then export JSONL:
-  - `uv run python examples/warming_up_to_rl/export_trace_sft.py \`
-    `  --db traces/v3/synth_ai.db \`
-    `  --output traces/qwen32b_filtered.jsonl \`
-    `  --model qwen/qwen3-32b \`
-    `  --exclude-achievement collect_sapling \`
-    `  --exclude-achievement collect_drink \`
-    `  --min-unique 3 \`
-    `  --event-reward unique_achievement_delta:1.0 \`
-    `  --limit 100`
-    - `--exclude-achievement` makes it easy to ignore easier unlocks when enforcing `--min-unique`.
-    - Combine `--require-achievement`, `--min-outcome-reward`, or provider filters as needed.
-
-## Training jobs (RL + SFT)
-- `uvx synth-ai train` is the consolidated entry point for RL or SFT launches.
-  - Omit `--config` to let the CLI enumerate candidate TOMLs (RL + FFT) and pick interactively.
-  - Omit `--env-file` to browse available `.env` files; the CLI never auto-selects.
-  - Missing secrets trigger an interactive loop: enter manually, switch `.env`, or fetch from Modal (secrets/apps) before proceeding.
-- RL run (local backend + local task app):
-  - `uvx synth-ai train --type rl --config examples/warming_up_to_rl/configs/crafter_cluster.toml --backend http://localhost:8000/api --task-url http://localhost:8010`
-    - Performs task-app health checks using the resolved `ENVIRONMENT_API_KEY` before posting to `/rl/jobs`.
-    - Polls job status until terminal unless `--no-poll` is supplied.
-- SFT run (FFT fine-tune):
-  - `uvx synth-ai train --type sft --config examples/warming_up_to_rl/configs/fft_crafter.toml --dataset traces/crafter_sft.jsonl`
-    - Uploads training/validation JSONL to `/learning/files` and starts the job.
-    - Poll output mirrors the legacy `run_fft_and_save.py` script.
-- Common flags:
-  - `--dry-run` previews payloads/uploads without making requests.
-  - `--idempotency` sets the `Idempotency-Key` header for RL submissions.
-  - `--poll-timeout` / `--poll-interval` tune the backend polling cadence.
-
-> Tip: all `uvx synth-ai …` subcommands accept `--help` if you need to inspect additional options on the fly.