synth-ai 0.2.9.dev0__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

examples/warming_up_to_rl/analyze_trace_db.py

@@ -0,0 +1,422 @@
+#!/usr/bin/env python3
+"""Summarise tracing_v3 SQLite data (models, rewards, achievements)."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sqlite3
+import sys
+from collections import Counter, defaultdict
+from pathlib import Path
+from typing import Any
+
+Row = sqlite3.Row
+
+
+def connect(db_path: Path) -> sqlite3.Connection:
+    conn = sqlite3.connect(str(db_path))
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def fetch_model_usage(conn: sqlite3.Connection) -> list[dict[str, Any]]:
+    rows = conn.execute(
+        """
+        SELECT
+            model_name,
+            provider,
+            COUNT(*) AS calls,
+            COALESCE(SUM(total_tokens), 0) AS total_tokens,
+            COALESCE(SUM(input_tokens), 0) AS input_tokens,
+            COALESCE(SUM(output_tokens), 0) AS output_tokens,
+            COALESCE(AVG(latency_ms), 0) AS avg_latency_ms
+        FROM events
+        WHERE event_type = 'cais' AND model_name IS NOT NULL
+        GROUP BY model_name, provider
+        ORDER BY calls DESC
+        """
+    ).fetchall()
+    stats: list[dict[str, Any]] = []
+    for row in rows:
+        stats.append(
+            {
+                "model_name": row["model_name"],
+                "provider": row["provider"],
+                "calls": int(row["calls"] or 0),
+                "total_tokens": int(row["total_tokens"] or 0),
+                "input_tokens": int(row["input_tokens"] or 0),
+                "output_tokens": int(row["output_tokens"] or 0),
+                "avg_latency_ms": float(row["avg_latency_ms"] or 0.0),
+            }
+        )
+    return stats
+
+
+def _parse_json(value: Any) -> Any:
+    if value is None:
+        return None
+    if isinstance(value, dict | list):
+        return value
+    try:
+        return json.loads(value)
+    except Exception:
+        return None
+
+
+AchievementMap = dict[tuple[str, int], dict[str, list[str]]]
+
+
+def fetch_achievement_data(
+    conn: sqlite3.Connection,
+) -> tuple[
+    AchievementMap,
+    Counter,
+    Counter,
+    Counter,
+    dict[str, set[str]],
+    dict[str, set[str]],
+]:
+    """Return per-turn achievement map and summary counters.
+
+    Returns:
+        achievements_map: {(session_id, turn) -> {"unique": [...], "all": [...]}}
+        unique_counts_per_session: Counter mapping session -> total unique achievements
+        achievement_name_counts: Counter mapping achievement name -> occurrences (unique)
+        achievement_size_counts: Counter mapping number of unique achievements per session -> frequency
+    """
+
+    achievements_map: AchievementMap = defaultdict(lambda: {"unique": [], "all": []})
+    session_unique_sets: dict[str, set[str]] = defaultdict(set)
+    session_final_achievements: dict[str, set[str]] = defaultdict(set)
+    achievement_name_counts: Counter = Counter()
+
+    # Unique achievements (reward_type = unique_achievement_delta)
+    rows = conn.execute(
+        """
+        SELECT er.session_id, er.reward_value, er.annotation, ev.metadata
+        FROM event_rewards er
+        JOIN events ev ON er.event_id = ev.id
+        WHERE er.reward_type = 'unique_achievement_delta' AND er.reward_value > 0
+        """
+    ).fetchall()
+    for row in rows:
+        session_id = row["session_id"]
+        annotation = _parse_json(row["annotation"]) or {}
+        metadata = _parse_json(row["metadata"]) or {}
+        turn = metadata.get("turn")
+        if turn is None:
+            continue
+        new_unique = annotation.get("new_unique") or []
+        if not isinstance(new_unique, list):
+            continue
+        if new_unique:
+            achievements_map[(session_id, int(turn))]["unique"].extend(new_unique)
+            session_unique_sets[session_id].update(new_unique)
+
+    # All achievements (reward_type = achievement_delta)
+    rows = conn.execute(
+        """
+        SELECT er.session_id, er.reward_value, er.annotation, ev.metadata
+        FROM event_rewards er
+        JOIN events ev ON er.event_id = ev.id
+        WHERE er.reward_type = 'achievement_delta' AND er.reward_value > 0
+        """
+    ).fetchall()
+    for row in rows:
+        session_id = row["session_id"]
+        annotation = _parse_json(row["annotation"]) or {}
+        metadata = _parse_json(row["metadata"]) or {}
+        turn = metadata.get("turn")
+        if turn is None:
+            continue
+        turned_true = annotation.get("turned_true") or []
+        if not isinstance(turned_true, list):
+            continue
+        if turned_true:
+            achievements_map[(session_id, int(turn))]["all"].extend(turned_true)
+
+    # Fallback to outcome rewards metadata to capture final achievements
+    rows = conn.execute(
+        """
+        SELECT session_id, reward_metadata
+        FROM outcome_rewards
+        WHERE reward_metadata IS NOT NULL
+        """
+    ).fetchall()
+    for row in rows:
+        session_id = row["session_id"]
+        metadata = _parse_json(row["reward_metadata"])
+        if not isinstance(metadata, dict):
+            continue
+        final_achievements = metadata.get("achievements") or []
+        if isinstance(final_achievements, list):
+            cleaned = [a for a in final_achievements if isinstance(a, str)]
+            session_unique_sets[session_id].update(cleaned)
+            session_final_achievements[session_id].update(cleaned)
+
+    # Build counters from the unique sets
+    unique_counts_per_session: Counter = Counter()
+    for session_id, achievement_set in session_unique_sets.items():
+        unique_counts_per_session[session_id] = len(achievement_set)
+        achievement_name_counts.update(achievement_set)
+
+    achievement_size_counts: Counter = Counter()
+    for _session_id, count in unique_counts_per_session.items():
+        achievement_size_counts[count] += 1
+
+    return (
+        achievements_map,
+        unique_counts_per_session,
+        achievement_name_counts,
+        achievement_size_counts,
+        session_unique_sets,
+        session_final_achievements,
+    )
+
+
+def fetch_reward_summary(conn: sqlite3.Connection) -> tuple[dict[str, Any], list[dict[str, Any]]]:
+    """Aggregate reward information from outcome_rewards and event_rewards."""
+
+    outcome_row = conn.execute(
+        """
+        SELECT
+            COUNT(*) AS episodes,
+            COALESCE(SUM(total_reward), 0) AS total_reward,
+            COALESCE(AVG(total_reward), 0) AS avg_reward,
+            COALESCE(MIN(total_reward), 0) AS min_reward,
+            COALESCE(MAX(total_reward), 0) AS max_reward
+        FROM outcome_rewards
+        """
+    ).fetchone()
+
+    reward_breakdown_rows = conn.execute(
+        """
+        SELECT
+            reward_type,
+            COUNT(*) AS events,
+            COALESCE(SUM(reward_value), 0) AS total_value,
+            COALESCE(AVG(reward_value), 0) AS avg_value
+        FROM event_rewards
+        GROUP BY reward_type
+        ORDER BY events DESC
+        """
+    ).fetchall()
+
+    breakdown: list[dict[str, Any]] = []
+    for row in reward_breakdown_rows:
+        breakdown.append(
+            {
+                "reward_type": row["reward_type"],
+                "events": int(row["events"] or 0),
+                "total_value": float(row["total_value"] or 0.0),
+                "avg_value": float(row["avg_value"] or 0.0),
+            }
+        )
+
+    outcome = {
+        "episodes": int(outcome_row["episodes"] or 0),
+        "total_reward": float(outcome_row["total_reward"] or 0.0),
+        "avg_reward": float(outcome_row["avg_reward"] or 0.0),
+        "min_reward": float(outcome_row["min_reward"] or 0.0),
+        "max_reward": float(outcome_row["max_reward"] or 0.0),
+    }
+
+    return outcome, breakdown
+
+
+def format_model_stats(stats: list[dict[str, Any]]) -> str:
+    if not stats:
+        return "No model usage recorded."
+    lines = ["Model usage (by LLM calls):"]
+    header = (
+        f"{'Model':30} {'Provider':10} {'Calls':>7} {'Tokens (in/out)':>20} {'Avg latency ms':>15}"
+    )
+    lines.append(header)
+    lines.append("-" * len(header))
+    for item in stats:
+        lines.append(
+            f"{item['model_name'][:30]:30} "
+            f"{(item['provider'] or '')[:10]:10} "
+            f"{item['calls']:7d} "
+            f"{item['total_tokens']:10d}/{item['output_tokens']:>8d} "
+            f"{item['avg_latency_ms']:15.1f}"
+        )
+    return "\n".join(lines)
+
+
+def format_achievement_summary(name_counts: Counter, size_counts: Counter) -> str:
+    lines = ["Unique achievements unlocked:"]
+    if name_counts:
+        top = name_counts.most_common()
+        for name, count in top:
+            lines.append(f"  {name:25} -> {count}")
+    else:
+        lines.append("  (none recorded)")
+
+    lines.append("")
+    lines.append("Sessions bucketed by unique achievement count:")
+    if size_counts:
+        for size in sorted(size_counts):
+            lines.append(f"  {size:2d} unique -> {size_counts[size]} session(s)")
+    else:
+        lines.append("  (no sessions with achievements)")
+    return "\n".join(lines)
+
+
+def format_reward_summary(outcome: dict[str, Any], breakdown: list[dict[str, Any]]) -> str:
+    lines = ["Episode outcome rewards:"]
+    episodes = outcome.get("episodes", 0)
+    if episodes:
+        lines.append(
+            f"  Episodes: {episodes} | total={outcome['total_reward']:.2f} | "
+            f"avg={outcome['avg_reward']:.2f} | min/max={outcome['min_reward']:.2f}/{outcome['max_reward']:.2f}"
+        )
+    else:
+        lines.append("  (no outcome rewards recorded)")
+
+    lines.append("")
+    lines.append("Event reward breakdown (event_rewards table):")
+    if breakdown:
+        header = f"{'Reward type':20} {'Events':>8} {'Total value':>14} {'Avg value':>12}"
+        lines.append(header)
+        lines.append("-" * len(header))
+        for row in breakdown:
+            lines.append(
+                f"{row['reward_type'][:20]:20} "
+                f"{row['events']:8d} "
+                f"{row['total_value']:14.3f} "
+                f"{row['avg_value']:12.3f}"
+            )
+    else:
+        lines.append("  (no event rewards recorded)")
+
+    return "\n".join(lines)
+
+
+def compute_model_achievement_stats(
+    conn: sqlite3.Connection, session_unique_sets: dict[str, set[str]]
+) -> dict[str, dict[str, Any]]:
+    """Aggregate unique-achievement stats per model."""
+
+    rows = conn.execute(
+        """
+        SELECT session_id, model_name, provider, COUNT(*) AS calls
+        FROM events
+        WHERE event_type = 'cais' AND model_name IS NOT NULL
+        GROUP BY session_id, model_name, provider
+        """
+    ).fetchall()
+
+    session_models: dict[str, tuple[str, str, int]] = {}
+    for row in rows:
+        session_id = row["session_id"]
+        calls = int(row["calls"] or 0)
+        current = session_models.get(session_id)
+        if current is None or calls > current[2]:
+            session_models[session_id] = (row["model_name"], row["provider"], calls)
+
+    model_stats: dict[str, dict[str, Any]] = {}
+    for session_id, (model_name, provider, _calls) in session_models.items():
+        achievements = session_unique_sets.get(session_id, set())
+        unique_count = len(achievements)
+
+        stats = model_stats.setdefault(
+            model_name,
+            {
+                "providers": set(),
+                "sessions": 0,
+                "sessions_with_unique": 0,
+                "total_unique": 0,
+                "max_unique": 0,
+                "achievement_counts": Counter(),
+            },
+        )
+
+        stats["providers"].add(provider or "unknown")
+        stats["sessions"] += 1
+        stats["total_unique"] += unique_count
+        stats["max_unique"] = max(stats["max_unique"], unique_count)
+        if unique_count > 0:
+            stats["sessions_with_unique"] += 1
+            stats["achievement_counts"].update(achievements)
+
+    return model_stats
+
+
+def format_model_achievement_stats(model_stats: dict[str, dict[str, Any]]) -> str:
+    if not model_stats:
+        return "Achievement stats by model:\n  (no model sessions recorded)"
+
+    lines = ["Achievement stats by model:"]
+    for model_name in sorted(
+        model_stats.keys(), key=lambda m: model_stats[m]["sessions"], reverse=True
+    ):
+        stats = model_stats[model_name]
+        providers = ", ".join(sorted(stats["providers"])) if stats["providers"] else "-"
+        sessions = stats["sessions"]
+        total_unique = stats["total_unique"]
+        avg_unique = total_unique / sessions if sessions else 0.0
+        sessions_with_unique = stats["sessions_with_unique"]
+        max_unique = stats["max_unique"]
+        lines.append(
+            f"  {model_name} (providers: {providers})\n"
+            f"    sessions={sessions} with_unique={sessions_with_unique} "
+            f"avg_unique={avg_unique:.2f} max_unique={max_unique}"
+        )
+
+        achievement_counts = stats["achievement_counts"]
+        if achievement_counts:
+            lines.append("    achievements:")
+            for name, count in sorted(
+                achievement_counts.items(), key=lambda item: item[1], reverse=True
+            ):
+                lines.append(f"      {name}: {count}")
+        else:
+            lines.append("    achievements: none")
+
+    return "\n".join(lines)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--db",
+        type=Path,
+        default=Path("traces/task_app_traces.db"),
+        help="Path to the tracing_v3 SQLite database",
+    )
+    args = parser.parse_args()
+
+    if not args.db.exists():
+        print(f"Database not found: {args.db}", file=sys.stderr)
+        raise SystemExit(1)
+
+    conn = connect(args.db)
+    try:
+        model_stats = fetch_model_usage(conn)
+        print(format_model_stats(model_stats))
+        print("")
+
+        (
+            _achievements_map,
+            _unique_counts_per_session,
+            name_counts,
+            size_counts,
+            session_unique_sets,
+            _session_final_achievements,
+        ) = fetch_achievement_data(conn)
+        outcome_summary, reward_breakdown = fetch_reward_summary(conn)
+
+        print(format_reward_summary(outcome_summary, reward_breakdown))
+        print("")
+        print(format_achievement_summary(name_counts, size_counts))
+        print("")
+        model_achievement_stats = compute_model_achievement_stats(conn, session_unique_sets)
+        print(format_model_achievement_stats(model_achievement_stats))
+    finally:
+        conn.close()
+
+
+if __name__ == "__main__":
+    main()

examples/warming_up_to_rl/configs/crafter_fft.toml

@@ -0,0 +1,53 @@
+# Crafter Full Finetune (FFT) example on H100
+# Adjust paths and hyperparameters to your environment before running.
+
+[algorithm]
+type = "offline"
+method = "sft"
+variety = "fft"
+
+[job]
+model = "Qwen/Qwen3-4B"               # base model to finetune
+# Path to your SFT JSONL dataset
+# You can point this to an absolute path or keep relative to this TOML
+# data = "../data/crafter_sft.jsonl"
+
+# Optional: how long to poll the job (seconds)
+poll_seconds = 1800
+
+[compute]
+# Cluster shape
+gpu_type = "H100"
+gpu_count = 4
+nodes = 1
+
+[data.topology]
+# world_size / container count (optional; inferred when omitted)
+container_count = 4
+
+[training]
+mode = "full_finetune"    # for documentation; backend decides based on metadata
+use_qlora = false
+
+[hyperparameters]
+# epochs
+n_epochs = 2
+
+# global batch shape (examples; adjust to your budget)
+world_size = 4
+sequence_length = 2048
+# provide either global_batch OR (per_device_batch×grad_accum×world_size)
+# global_batch = 512
+per_device_batch = 2
+gradient_accumulation_steps = 64
+
+# optimizer/schedule
+learning_rate = 8e-6
+warmup_ratio = 0.03
+
+[hyperparameters.parallelism]
+use_deepspeed = true
+deepspeed_stage = 3
+fsdp = false
+bf16 = true
+fp16 = false

examples/warming_up_to_rl/configs/crafter_fft_4b.toml

@@ -0,0 +1,54 @@
+# FFT job config for Qwen/Qwen3-4B on Crafter SFT dataset
+
+[algorithm]
+type = "offline"
+method = "supervised_finetune"
+variety = "fft"
+
+
+[job]
+model = "Qwen/Qwen3-4B"
+# Limit training to the first 100 conversations (export a 100-row JSONL and point to it here)
+# data = "../ft_data/qwen3_32b_ach_ge3_raw_filtered.head100.jsonl"
+
+[compute]
+# Adjust as needed for your quota
+gpu_type = "H100"
+gpu_count = 1
+nodes = 1
+
+[data]
+# Optional topology metadata (left empty for now)
+topology = {}
+
+# Optional local validation dataset path (JSONL). If set, the client will upload
+# this file and wire up validation so the frontend can display val.loss.
+validation_path = "../ft_data/qwen3_32b_ach_ge3_raw_filtered.tokens_1000000_seed_123.val_2000.jsonl"
+
+[training]
+mode = "sft_offline"
+use_qlora = false
+
+# Validation settings to emit val.loss on the frontend
+[training.validation]
+enabled = true
+evaluation_strategy = "steps"
+eval_steps = 20
+save_best_model_at_end = true
+metric_for_best_model = "val.loss"
+greater_is_better = false
+
+[hyperparameters]
+# Minimal safe defaults; backend can override
+n_epochs = 1
+batch_size = 1
+gradient_accumulation_steps = 64
+sequence_length = 4096
+learning_rate = 5e-6
+warmup_ratio = 0.03
+train_kind = "fft"
+
+# Optional parallelism block example
+#[hyperparameters.parallelism]
+# tensor_parallel_size = 1
+# pipeline_parallel_size = 1

examples/warming_up_to_rl/configs/eval_fft_qwen4b.toml

@@ -0,0 +1,22 @@
+# Eval config for finetuned Qwen/Qwen3-4B (FFT) via task app rollout
+
+type = "sft"
+
+# Required
+task_app_url = "https://synth-laboratories--grpo-crafter-task-app-final-warming--ceb5b2.modal.run"
+# Replace with your finished job id if different
+model = "fft:Qwen/Qwen3-4B:job_a53914f51dc146b5"
+policy_name = "crafter"
+# Backend inference proxy base (no /v1); server will append /v1/chat/completions
+inference_url = "https://synth-backend-dev-docker.onrender.com/api/inference"
+
+# Optional
+num_episodes = 10
+max_turns = 10
+concurrency = 10
+# difficulty = "easy"
+
+# Thinking params (optional; align with your service expectations)
+thinking_mode = "think"
+thinking_budget = 1024
+max_tokens = 1024

examples/warming_up_to_rl/configs/eval_groq_qwen32b.toml

@@ -0,0 +1,15 @@
+# Eval config for Groq Qwen3-32B
+# Fields mirror run_eval.py expectations
+
+type = "rl"
+
+# Required
+task_app_url = "https://synth-laboratories--grpo-crafter-task-app-final-warming--ceb5b2.modal.run"
+model = "qwen/qwen3-32b"
+inference_url = "https://api.groq.com/openai"
+
+# Optional
+num_episodes = 20
+max_turns = 10
+concurrency = 10
+# difficulty = "easy"

examples/warming_up_to_rl/configs/eval_modal_qwen4b.toml

@@ -0,0 +1,24 @@
+# Eval config for Synth Modal inference Qwen/Qwen3-4B via task app rollout
+
+type = "rl"
+
+# Required
+task_app_url = "https://synth-laboratories--grpo-crafter-task-app-final-warming--ceb5b2.modal.run"
+model = "Qwen/Qwen3-4B"
+policy_name = "crafter"
+inference_url = "https://synth-backend-dev-docker.onrender.com/api/inference"  # Modal inference base (no /v1)
+max_tokens = 1024
+thinking_mode = "think"
+thinking_budget = 1024
+
+# Optional
+num_episodes = 20
+max_turns = 10
+concurrency = 10
+# difficulty = "easy"
+
+# Notes:
+# - run_eval.py --use-rollout will detect provider=vllm for this model and
+#   fetch the vLLM base from the task app /info to use as inference_url.
+# - Ensure the task app mounts the openai-api-key secret if your vLLM gateway
+#   requires a bearer token (OPENAI_API_KEY). Otherwise it will call unauthenticated.

examples/warming_up_to_rl/configs/eval_stepwise_complex.toml

@@ -0,0 +1,35 @@
+# Crafter eval (stepwise: complex)
+
+# Replace at runtime by exporting TASK_APP_URL, or put the full https://<modal>.run URL here
+# task_app_url = "https://YOUR-TASK-APP.modal.run"
+
+model = "qwen/qwen3-32b"
+# Route inference to local task app Groq proxy
+inference_url = "http://localhost:8001/proxy/groq"
+num_episodes = 10
+max_turns = 10
+concurrency = 10
+# difficulty = "easy"  # optional
+
+[rollout]
+env_name = "crafter"
+policy_name = "crafter-react"
+max_turns = 10
+
+  [rollout.env_config]
+
+    [rollout.env_config.step_rewards]
+    enabled = true
+    mode = "decision_stepwise"
+    strategy = "complex"
+    indicator_lambda = 0.0  # ignored by complex mode if weights are present
+    
+    # Example weights (tune per need)
+    weights = { collect_sapling = 0.1, craft_wood_pickaxe = 0.3, collect_diamond = 1.0 }
+    # Allow rewarding an achievement up to K times per episode
+    k_limits = { collect_sapling = 1, craft_wood_pickaxe = 2, collect_diamond = 3 }
+
+  [rollout.policy_config]
+  temperature = 0.2
+  top_p = 0.95
+  max_tokens = 512

examples/warming_up_to_rl/configs/eval_stepwise_consistent.toml

@@ -0,0 +1,26 @@
+# Crafter evaluation with consistent stepwise shaping.
+task_app_url = "https://synth-laboratories--grpo-crafter-task-app-final-warming--ceb5b2.modal.run"
+model = "qwen/qwen3-32b"
+inference_url = "https://api.groq.com/openai"
+num_episodes = 20
+max_turns = 10
+concurrency = 10
+
+[rollout]
+env_name = "crafter"
+policy_name = "crafter-react"
+max_turns = 10
+
+[rollout.env_config]
+difficulty = "easy"
+
+[rollout.env_config.step_rewards]
+enabled = true
+mode = "decision_stepwise"
+strategy = "consistent"
+indicator_lambda = 1.0
+
+[rollout.policy_config]
+temperature = 0.2
+top_p = 0.95
+max_tokens = 512

examples/warming_up_to_rl/configs/eval_stepwise_per_achievement.toml

@@ -0,0 +1,36 @@
+# Crafter evaluation with achievement-weighted stepwise shaping.
+task_app_url = "https://synth-laboratories--grpo-crafter-task-app-final-warming--ceb5b2.modal.run"
+model = "qwen/qwen3-32b"
+inference_url = "https://api.groq.com/openai"
+num_episodes = 20
+max_turns = 10
+concurrency = 10
+
+[rollout]
+env_name = "crafter"
+policy_name = "crafter-react"
+max_turns = 10
+
+[rollout.env_config]
+difficulty = "easy"
+
+[rollout.env_config.step_rewards]
+enabled = true
+mode = "decision_stepwise"
+strategy = "per_achievement"
+indicator_lambda = 1.0
+
+[rollout.env_config.step_rewards.weights]
+collect_sapling = 0.1
+craft_wood_pickaxe = 0.3
+collect_diamond = 1.0
+
+[rollout.env_config.step_rewards.k_limits]
+collect_sapling = 1
+craft_wood_pickaxe = 2
+collect_diamond = 3
+
+[rollout.policy_config]
+temperature = 0.2
+top_p = 0.95
+max_tokens = 512