synth-ai 0.2.14__py3-none-any.whl → 0.2.16__py3-none-any.whl

examples/warming_up_to_rl/export_trace_sft.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 
 import argparse
 import json
+import os
 import sqlite3
 import sys
 from collections import Counter, defaultdict
@@ -12,6 +13,13 @@ from collections.abc import Iterable
 from pathlib import Path
 from typing import Any
 
+from synth_ai._utils.prompts import ensure_required_args
+from synth_ai.tracing_v3.constants import (
+    TRACE_DB_BASENAME,
+    TRACE_DB_DIR,
+    canonical_trace_db_name,
+)
+
 Row = sqlite3.Row
 
 
@@ -489,55 +497,81 @@ def _validate_dataset(records: list[dict[str, Any]]) -> None:
 
 
 def _find_trace_database() -> Path | None:
-    """Automatically discover the trace database in common locations."""
+    """Automatically discover the most recent trace database in common locations."""
 
-    # Check for demo directory from state
-    try:
-        state_path = Path.home() / ".synth-ai" / "demo.json"
-        if state_path.exists():
-            import json
-
-            with state_path.open() as f:
-                data = json.load(f)
-                demo_dir = data.get("DEMO_DIR")
-                if demo_dir:
-                    candidate = Path(demo_dir) / "traces" / "v3" / "synth_ai.db"
-                    if candidate.exists():
-                        return candidate
-    except Exception:
-        pass
+    candidates: list[Path] = []
 
-    # Search upward from current directory
+    # Walk up parent directories from CWD
     cwd = Path.cwd()
     for parent in [cwd] + list(cwd.parents):
-        candidate = parent / "traces" / "v3" / "synth_ai.db"
-        if candidate.exists():
-            return candidate
-
-    # Check standard locations
-    standard_locations = [
-        Path("traces/v3/synth_ai.db"),
-        Path("../traces/v3/synth_ai.db"),
-        Path.home() / "synth-ai" / "traces" / "v3" / "synth_ai.db",
-    ]
-
-    for location in standard_locations:
+        candidates.append(parent / "traces" / "v3")
+
+    # Standard fallback locations
+    candidates.extend(
+        [
+            TRACE_DB_DIR,
+            Path("../traces"),
+            Path.home() / "synth-ai" / "traces" / "v3",
+        ]
+    )
+
+    found: list[Path] = []
+    for directory in candidates:
         try:
-            if location.exists():
-                return location.resolve()
+            if not directory.exists():
+                continue
+            for pattern in (
+                f"{TRACE_DB_BASENAME}_*.db",
+                canonical_trace_db_name(),
+            ):
+                for candidate in directory.glob(pattern):
+                    found.append(candidate.resolve())
         except Exception:
             continue
 
-    return None
+    if not found:
+        return None
+
+    found.sort(key=lambda p: p.stat().st_mtime, reverse=True)
+    return found[0]
+
+
+def _discover_local_trace_dbs(root: Path) -> list[Path]:
+    """Return trace DBs under *root* (recursively), newest first."""
+
+    candidates: set[Path] = set()
+    ignore_dirs = {".git", ".venv", "__pycache__", "node_modules", "dist", "build"}
+    target_exact = canonical_trace_db_name()
+
+    for dirpath, dirnames, filenames in os.walk(root):
+        dirnames[:] = [d for d in dirnames if d not in ignore_dirs]
+        for filename in filenames:
+            if filename == target_exact or (
+                filename.startswith(f"{TRACE_DB_BASENAME}_") and filename.endswith(".db")
+            ):
+                path = Path(dirpath) / filename
+                try:
+                    candidates.add(path.resolve())
+                except Exception:
+                    continue
+
+    return sorted(candidates, key=lambda p: p.stat().st_mtime, reverse=True)
 
 
 def main() -> None:
     parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--db", type=Path, default=None, help="Path to tracing_v3 SQLite DB")
     parser.add_argument(
-        "--output",
+        "--in",
+        dest="input_path",
+        type=Path,
+        default=None,
+        help="Path to tracing_v3 SQLite DB",
+    )
+    parser.add_argument(
+        "--out",
+        dest="output_path",
         type=Path,
-        required=False,
+        default=None,
         help="Destination JSONL path for the exported dataset",
     )
     parser.add_argument(
@@ -593,25 +627,109 @@ def main() -> None:
     )
     args = parser.parse_args()
 
-    # Auto-discover database if not specified
-    db_path = args.db
-    if db_path is None:
-        db_path = _find_trace_database()
-        if db_path:
-            print(f"Found trace database: {db_path}")
-        else:
-            print("\nTrace database configuration:")
-            db_input = input("Trace database path [traces/v3/synth_ai.db]: ").strip()
-            db_path = Path(db_input) if db_input else Path("traces/v3/synth_ai.db")
+    default_output_path = (Path.cwd() / "ft_data" / "crafter_sft.jsonl").resolve()
+
+    initial_path: Path | None = None
+    if args.input_path is not None:
+        initial_path = Path(args.input_path).expanduser().resolve()
+    else:
+        discovered = _find_trace_database()
+        if discovered is not None:
+            initial_path = discovered.expanduser().resolve()
+            args.input_path = initial_path
+
+    if args.output_path is None:
+        args.output_path = default_output_path
+
+    local_candidates = _discover_local_trace_dbs(Path.cwd())
+    if local_candidates:
+        print("\nDiscovered trace databases:")
+        for idx, path in enumerate(local_candidates, start=1):
+            marker = " <- most recent" if idx == 1 else ""
+            print(f"  {idx}) {path}{marker}")
+        print("  m) Enter path manually")
+        print("  0) Abort")
+
+        default_index = 1
+        if initial_path:
+            for idx, candidate in enumerate(local_candidates, start=1):
+                if candidate == initial_path:
+                    default_index = idx
+                    break
 
+        while True:
+            prompt = f"Select database [{default_index}]: "
+            choice = input(prompt).strip().lower()
+            if not choice:
+                args.input_path = local_candidates[default_index - 1]
+                break
+            if choice == "0":
+                raise SystemExit("Aborted by user.")
+            if choice in {"m", "manual"}:
+                manual = input("Enter trace database path: ").strip()
+                if manual:
+                    args.input_path = Path(manual)
+                    break
+                print("Path required; try again.")
+                continue
+            try:
+                idx = int(choice)
+            except ValueError:
+                print("Invalid selection; enter a number, 'm', or 0 to abort.")
+                continue
+            if 1 <= idx <= len(local_candidates):
+                args.input_path = local_candidates[idx - 1]
+                break
+            print(f"Select between 1 and {len(local_candidates)}, 'm', or 0.")
+    elif initial_path is not None:
+        args.input_path = initial_path
+
+    # If output wasn't overridden, derive it from the chosen DB name
+    if args.output_path == default_output_path and args.input_path:
+        db_name = Path(args.input_path).name  # e.g., task_app_traces_2025-10-23_13-23-02.db
+        timestamp = db_name[:-3] if db_name.endswith(".db") else db_name
+        if timestamp.startswith("task_app_traces_"):
+            timestamp = timestamp[len("task_app_traces_") :]
+        derived_name = f"sft_dataset_{timestamp}.jsonl"
+        args.output_path = (Path.cwd() / "ft_data" / derived_name).resolve()
+
+    input_default = (
+        Path(args.input_path).expanduser().resolve()
+        if args.input_path is not None
+        else (TRACE_DB_DIR / canonical_trace_db_name()).expanduser().resolve()
+    )
+    output_default = Path(args.output_path).expanduser().resolve() if args.output_path else default_output_path
+
+    args = ensure_required_args(
+        args,
+        {
+            "input_path": "Trace database path",
+            "output_path": "Output JSONL path",
+        },
+        coerce={
+            "input_path": lambda raw: Path(raw).expanduser().resolve(),
+            "output_path": lambda raw: Path(raw).expanduser().resolve(),
+        },
+        defaults={
+            "input_path": input_default,
+            "output_path": output_default,
+        },
+    )
+
+    db_path = Path(args.input_path).expanduser().resolve()
+    print(f"Trace database: {db_path}")
     if not db_path.exists():
-        print(f"Database not found: {db_path}", file=sys.stderr)
-        raise SystemExit(1)
+        discovered = _find_trace_database()
+        if discovered and discovered.exists():
+            discovered = discovered.resolve()
+            print(f"Discovered trace database: {discovered}")
+            db_path = discovered
+        else:
+            print(f"Database not found: {db_path}", file=sys.stderr)
+            raise SystemExit(1)
 
-    output_path = args.output
-    if not output_path:
-        output_path = Path("ft_data/crafter_traces.jsonl")
-        print(f"Output will be written to: {output_path.resolve()}")
+    output_path = Path(args.output_path).expanduser().resolve()
+    print(f"Output dataset: {output_path}")
 
     min_unique = args.min_unique
     if min_unique is None:
@@ -619,15 +737,11 @@ def main() -> None:
         print(f"Minimum unique achievements filter: {min_unique} (all traces)")
 
     # Override args with prompted values
-    args.db = db_path
-    args.output = output_path
+    args.input_path = db_path
+    args.output_path = output_path
     args.min_unique = min_unique
 
-    if not args.db.exists():
-        print(f"Database not found: {args.db}", file=sys.stderr)
-        raise SystemExit(1)
-
-    conn = connect(args.db)
+    conn = connect(args.input_path)
     try:
         (
             achievements_map,
@@ -708,11 +822,11 @@ def main() -> None:
             raise SystemExit(1)
 
         _validate_dataset(dataset)
-        write_jsonl(args.output, dataset)
+        write_jsonl(args.output_path, dataset)
         session_ids = {item.get("metadata", {}).get("session_id") for item in dataset}
         session_ids.discard(None)
         print(
-            f"Wrote {len(dataset)} examples from {len(session_ids)} session(s) -> {args.output.resolve()}",
+            f"Wrote {len(dataset)} examples from {len(session_ids)} session(s) -> {args.output_path.resolve()}",
             file=sys.stderr,
         )
     finally:

examples/warming_up_to_rl/readme.md

@@ -1,179 +1,110 @@
 # Warming Up to RL (Crafter)
 
-The Crafter example demonstrates the full Synth AI workflow: task app serving, Groq rollouts, tracing, SFT dataset export, FFT training, evaluation of fine-tuned models, and RL training.
-
-## Quick Reference Commands
-
-- Serve task app locally with tracing:
-  ```bash
-  uvx synth-ai serve --port 8001 --env-file examples/warming_up_to_rl/.env --trace traces/v3
-  ```
-- Deploy to Modal:
-  ```bash
-  uvx synth-ai deploy grpo-crafter --name grpo-crafter-task-app
-  ```
-- Groq rollout (server-side):
-  ```bash
-  uv run python examples/warming_up_to_rl/run_eval.py --toml examples/warming_up_to_rl/configs/eval_groq_qwen32b.toml --use-rollout
-  ```
-- Export SFT data from traced runs:
-  ```bash
-  python examples/warming_up_to_rl/export_trace_sft.py --db traces/v3/synth_ai.db --output ft_data/crafter_traces.jsonl
-  ```
-- FFT via CLI:
-  ```bash
-  uvx synth-ai train --type sft --config examples/warming_up_to_rl/configs/crafter_fft.toml --dataset /absolute/path/to/data.jsonl
-  ```
-- Evaluate FFT checkpoint:
-  ```bash
-  uv run python examples/warming_up_to_rl/run_eval.py --toml examples/warming_up_to_rl/configs/eval_fft_qwen4b.toml --use-rollout
-  ```
-- RL via CLI (FFT-first):
-  ```bash
-  uvx synth-ai train --type rl --config examples/warming_up_to_rl/configs/rl_from_ft.toml
-  ```
-
----
+This folder contains an end-to-end Crafter workflow: stand up the task app, collect Groq-powered rollouts, export tracing data for supervised fine-tuning, run FFT/RL jobs, and evaluate checkpoints. Commands assume the repository root as the working directory unless stated otherwise.
 
 ## 1. Prerequisites
 
 - Python 3.11+
-- `uv`/`uvx` available (or install Synth in a virtualenv)
-- Modal CLI (`modal token new`) if you plan to deploy the task app
-- `.env` in this directory with at least:
-  - `SYNTH_API_KEY`
-  - `ENVIRONMENT_API_KEY`
-  - `TASK_APP_URL` (when running against a hosted task app)
-  - Optional: `GROQ_API_KEY`, `OPENAI_API_KEY` for proxy endpoints
-
-`uvx synth-ai setup` can populate the `.env` by guiding you through the dashboard handshake.
+- [`uv`](https://docs.astral.sh/uv/) / `uvx` (or install `synth-ai` inside a virtualenv)
+- Modal CLI (`modal token new`) if you plan to deploy the task app remotely
+- API keys:
+  - `SYNTH_API_KEY` and `ENVIRONMENT_API_KEY` are required for CLI flows
+  - `GROQ_API_KEY` (used by the Groq policy) and optional `OPENAI_API_KEY`
+- Run `uvx synth-ai setup` once to pair with the Synth dashboard and populate `~/.synth-ai/user_config.json`
 
-> All commands below assume you are running from the repository root unless noted.
+## 2. Task App
 
-## 2. Task App Operations
-
-### Local development
+### Local serve (FastAPI)
 
 ```bash
-uvx synth-ai serve --port 8001 --env-file examples/warming_up_to_rl/.env --trace traces/v3 --trace-db traces/v3/synth_ai.db
+uvx synth-ai serve \
+  --env-file examples/warming_up_to_rl/.env \
+  --host 127.0.0.1 --port 8001 \
+  --trace traces/v3
 ```
 
-- `--trace` and `--trace-db` enable tracing v3 and SFT JSONL dumps.
-- Add `--reload` for uvicorn auto-reload while editing code.
+- `--trace` creates/uses `traces/v3/task_app_traces_<timestamp>.db` for the lifetime of the server. All rollouts append to this file.
+- Add `--trace-db` to override the SQLite path (one DB per server instance).
+- Pass `--reload` during development for auto-reload.
 
 ### Modal deploy / serve
 
 ```bash
-uvx synth-ai deploy grpo-crafter --name grpo-crafter-task-app --env-file examples/warming_up_to_rl/.env
-uvx synth-ai modal-serve grpo-crafter --name grpo-crafter-task-app --env-file examples/warming_up_to_rl/.env
+uvx synth-ai deploy grpo-crafter --name grpo-crafter-task-app
+uvx synth-ai modal-serve grpo-crafter --name grpo-crafter-task-app
 ```
 
-Both commands preflight the environment key with the backend when `SYNTH_API_KEY` is present.
-
-## 3. Baseline Evaluations (Groq and Synth vLLM)
-
-Evaluation scripts auto-load `.env` values. Update TOMLs under `configs/` with the correct `task_app_url` and provider-specific model names.
-
-- Groq Qwen3-32B:
-  ```bash
-  uv run python examples/warming_up_to_rl/run_eval.py --toml examples/warming_up_to_rl/configs/eval_groq_qwen32b.toml --use-rollout
-  ```
-- Synth vLLM Qwen3-4B (Modal-hosted inference URL specified in TOML):
-  ```bash
-  uv run python examples/warming_up_to_rl/run_eval.py --toml examples/warming_up_to_rl/configs/eval_modal_qwen4b.toml --use-rollout
-  ```
-
-`--use-rollout` drives the task app’s `/rollout` endpoint so achievements and metrics are captured. Without it the script issues per-step `initialize/step/terminate` calls.
-
-## 4. Tracing and SFT Dataset Export
-
-1. Serve the task app with tracing enabled (see Section 2). Optionally, run the traced rollout helper against the running server:
-   ```bash
-   uv run python examples/warming_up_to_rl/run_local_rollout_traced.py \
-     --base-url http://localhost:8001 \
-     --api-key "$ENVIRONMENT_API_KEY" \
-     --inference-api-key "$GROQ_API_KEY" \
-     --model qwen/qwen3-32b \
-     --inference-url https://api.groq.com/openai \
-     --max-llm-calls 3 \
-     --run-id local-trace
-   ```
-2. Inspect local trace databases:
-   ```bash
-   uvx synth-ai traces --limit 10
-   ```
-3. Export JSONL suitable for SFT:
-   ```bash
-   python examples/warming_up_to_rl/export_trace_sft.py \
-     --db traces/v3/synth_ai.db \
-     --min-achievements 3 \
-     --output ft_data/crafter_traces.jsonl
-   ```
-
-The exporter enriches each example with achievements unlocked, model metadata, and reward summaries.
-
-## 5. SFT / FFT Training
-
-### Preferred: `uvx synth-ai train`
+Both commands reuse the same tracing defaults; the backend persists rollouts into the configured SQLite/Turso store.
+
+## 3. Collect rollouts
+
+Hit the running task app with the local helper to gather a traced rollout (Groq policy shown below):
 
 ```bash
-uvx synth-ai train \
-  --type sft \
-  --config examples/warming_up_to_rl/configs/crafter_fft.toml \
-  --dataset /absolute/path/to/crafter_traces.jsonl
+python examples/warming_up_to_rl/run_local_rollout_traced.py \
+  --base-url http://localhost:8001 \
+  --api-key "$ENVIRONMENT_API_KEY" \
+  --inference-api-key "$GROQ_API_KEY" \
+  --model qwen/qwen3-32b \
+  --inference-url https://api.groq.com/openai \
+  --max-llm-calls 3 \
+  --run-id local-trace
 ```
 
-The CLI will:
-- Prompt for `.env` selection (or use `--env-file`).
-- Upload training (and optional validation) data to `/learning/files`.
-- Submit the job and poll until completion unless `--no-poll` is set.
+Artifacts produced per rollout:
+- `traces/v3/task_app_traces_<timestamp>.db`: the task app’s append-only database (one per server lifetime; new rollouts append rows).
+- `local-trace_trace.json`: single-run JSON snapshot for inspection.
 
-### Legacy script
+## 4. Export SFT-ready data
 
 ```bash
-uv run python examples/warming_up_to_rl/run_fft_and_save.py \
-  --toml examples/warming_up_to_rl/configs/crafter_fft.toml \
-  --data /absolute/path/to/crafter_traces.jsonl \
-  --poll-seconds 1800
+python examples/warming_up_to_rl/export_trace_sft.py
 ```
 
-The script writes the resulting model ID to `ft_model_id.txt`. Use that ID in evaluation and RL configs (e.g., `model = "ft:abc123"`).
+- When run without `--in`, the script lists every `task_app_traces*.db` under the current directory (and subdirectories), sorted by recency, and prompts you to pick one (the newest is marked `← most recent`).
+- The exporter validates the trace data, filters sessions, and writes JSONL to `ft_data/crafter_sft.jsonl` by default (override with `--out`).
 
-## 6. Evaluate the Fine-tuned Model
+## 5. FFT / SFT Training
 
-After FFT completes, update `configs/eval_fft_qwen4b.toml` so `model = "ft:<model_id>"`, then rerun the evaluation:
+Recommended via CLI:
 
 ```bash
-uv run python examples/warming_up_to_rl/run_eval.py --toml examples/warming_up_to_rl/configs/eval_fft_qwen4b.toml --use-rollout
+uvx synth-ai train \
+  --type sft \
+  --config examples/warming_up_to_rl/configs/crafter_fft.toml \
+  --dataset /absolute/path/to/crafter_sft.jsonl
 ```
 
-This reuses the same Groq/vLLM pipeline but exercises the finetuned checkpoint.
+The CLI uploads training data, submits the job to the Synth backend, and polls for completion. A legacy helper (`run_fft_and_save.py`) is still provided for ad-hoc usage.
 
-## 7. RL Training
+## 6. Evaluate checkpoints
 
-### Preferred: `uvx synth-ai train --type rl`
+Update the relevant TOML with the model identifier (e.g., `model = "ft:<model_id>"`) and run:
 
 ```bash
-uvx synth-ai train \
-  --type rl \
-  --config examples/warming_up_to_rl/configs/rl_from_base_qwen4b.toml
+uv run python examples/warming_up_to_rl/run_eval.py \
+  --toml examples/warming_up_to_rl/configs/eval_fft_qwen4b.toml \
+  --use-rollout
 ```
 
-During the interactive setup the CLI ensures `SYNTH_API_KEY`, `ENVIRONMENT_API_KEY`, and `TASK_APP_URL` are present, health-checks the task app, and submits the RL job to `/rl/jobs`.
+`--use-rollout` exercises the `/rollout` endpoint so achievements/rewards are surfaced in traces.
 
-### Legacy script
+## 7. RL Training
 
 ```bash
-uv run python examples/warming_up_to_rl/run_rl_and_save.py \
-  --config examples/warming_up_to_rl/configs/rl_from_ft.toml
+uvx synth-ai train \
+  --type rl \
+  --config examples/warming_up_to_rl/configs/rl_from_base_qwen4b.toml
 ```
 
-To start directly from a base model, switch the config to `rl_from_base_qwen4b.toml` and ensure `[model].base` is populated.
+Start from `rl_from_ft.toml` if you want to bootstrap from a previously fine-tuned checkpoint.
+
+---
 
-## 8. Additional Utilities
+### Notes on tracing
 
-- `manage_secrets.py` – convenience helpers for Modal secret management.
-- `run_local_rollout.py`, `run_local_rollout_parallel.py`, `run_rollout_remote.py` – alternative rollout launchers for benchmarking.
-- `analyze_trace_db.py` – inspect trace quality/achievements before exporting.
+- **One SQLite DB per server:** every task app instance maintains a single `task_app_traces_<timestamp>.db` and appends each new rollout. If you want a fresh file, start another `synth-ai serve` with a different `--trace-db` path.
+- **JSON snapshots per run:** `run_local_rollout_traced.py` writes `<run_id>_trace.json` so you can inspect or hand-edit individual runs.
+- **Exporter discovery:** the SFT exporter recursively catalogs all `task_app_traces*.db` files beneath the task app directory, allowing you to select any historical snapshot when exporting training data.
 
-Refer to `docs/workflows/` for end-to-end guidance that mirrors these commands.
+These conventions keep tracing predictable: continuous history per server, easy selection of historical DBs, and one-off JSON exports for quick analysis.

examples/warming_up_to_rl/run_fft_and_save.py

@@ -12,7 +12,7 @@ from typing import Any
 
 import requests
 from dotenv import load_dotenv
-from synth_ai.config.base_url import PROD_BASE_URL_DEFAULT
+from synth_ai._utils.base_url import PROD_BASE_URL_DEFAULT
 
 
 def mask(val: str) -> str:

examples/warming_up_to_rl/run_rl_and_save.py

@@ -10,7 +10,7 @@ from pathlib import Path
 from typing import Any
 
 import requests
-from synth_ai.config.base_url import PROD_BASE_URL_DEFAULT
+from synth_ai._utils.base_url import PROD_BASE_URL_DEFAULT
 
 
 def _load_toml(path: Path) -> dict[str, Any]:

examples/warming_up_to_rl/task_app/README.md

@@ -0,0 +1,42 @@
+# Crafter Task App
+
+This example is now wired through the shared Synth task-app harness. Use the
+`uvx synth-ai` CLI to run it locally or deploy it to Modal without touching the
+underlying FastAPI plumbing.
+
+## Local development
+```bash
+uvx synth-ai serve grpo-crafter --port 8001
+# Optional extras:
+#   --env-file path/to/.env    # load additional environment variables
+#   --reload                   # enable uvicorn auto-reload
+```
+
+Useful endpoints while the server is running:
+- `GET http://localhost:8001/health`
+- `GET http://localhost:8001/info`
+- `GET http://localhost:8001/task_info?seed=42`
+- `POST http://localhost:8001/rollout`
+
+## Deploy to Modal
+```bash
+uvx synth-ai deploy grpo-crafter --name grpo-crafter-task-app
+```
+
+Requirements:
+- Modal CLI installed and authenticated (`modal token new`).
+- Either provide an `.env` with `ENVIRONMENT_API_KEY`, `GROQ_API_KEY`, and `OPENAI_API_KEY`
+  (recommended; pass via `--env-file`). The deploy command injects these values via an inline
+  Modal secret plus `Secret.from_dotenv`, so the minted environment key stays in sync with
+  what the CLI sends.
+- Or ensure Modal secrets `groq-api-key` and `openai-api-key` exist and continue to supply
+  model vendor credentials that way.
+
+The CLI generates a Modal entrypoint on the fly using the shared
+`TaskAppConfig`, ensuring the container matches the local FastAPI behavior.
+
+## Compatibility note
+`examples/warming_up_to_rl/task_app/grpo_crafter_task_app.py` remains as a
+legacy wrapper exposing `fastapi_app()` and a `__main__` entrypoint. Behind the
+scenes it proxies to the shared configuration; prefer the CLI workflow above
+for new automation and tests.