synth-ai 0.2.9.dev0__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

examples/blog_posts/warming_up_to_rl/configs/train_rl_from_sft.toml

@@ -0,0 +1,91 @@
+# Continue training the finetuned Crafter policy with GRPO-style RL.
+# Fill in task_url with your deployed task app and set model.source to the
+# finetuned model id returned by `uvx synth-ai train --type sft`.
+
+type = "rl"
+
+# [smoke] section is OPTIONAL and only used by `synth-ai smoke` command for local testing.
+# This section is completely IGNORED by the RL trainer and will not affect training jobs.
+# It allows you to quickly test your task app without passing many CLI arguments:
+#   uvx synth-ai smoke --config this-file.toml
+# All values are optional; CLI args override TOML values.
+[smoke]
+task_url = "https://synth-laboratories--crafter-blogpost-fastapi-app-dev.modal.run"
+env_name = "crafter"
+policy_name = "crafter-react"
+max_steps = 10
+policy = "mock"  # mock, gpt-5-nano, openai, groq
+model = "gpt-5-nano"
+mock_backend = "openai"  # synthetic or openai
+mock_port = 0  # 0 = auto-assign
+return_trace = true
+use_mock = true
+
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[services]
+task_url = "https://synth-laboratories--crafter-blogpost-fastapi-app-dev.modal.run"
+judge_url = "https://synth-backend-dev-docker.onrender.com/api"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2
+[compute.topology]
+reference_placement = "none"
+
+[topology]
+type = "single_node_split"
+reference_placement = "none"
+gpus_for_vllm = 1
+gpus_for_training = 1
+gpus_for_ref = 0
+tensor_parallel = 1
+
+[vllm]
+tensor_parallel_size = 1
+max_model_len = 8192
+
+[reference]
+placement = "none"
+
+[model]
+base = "Qwen/Qwen3-4B"
+trainer_mode = "lora"
+label = "crafter-rl-baseline"
+
+[rollout]
+env_name = "crafter"
+policy_name = "crafter-react"
+max_turns = 10
+episodes_per_batch = 20
+max_concurrent_rollouts = 8
+rubric_rewards_only = false
+task_app_origin_rewards_only = true
+
+[evaluation]
+instances = 100
+every_n_iters = 20
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]
+
+[training]
+num_epochs = 1
+iterations_per_epoch = 1
+max_turns = 10
+batch_size = 2
+group_size = 2
+learning_rate = 5e-6
+weight_sync_interval = 1
+log_interval = 1
+max_completion_tokens = 256
+async_semaphore_max = 4
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+weight_sync_interval = 1
+
+[rubric]
+enabled = false

examples/blog_posts/warming_up_to_rl/configs/train_sft_qwen4b.toml

@@ -0,0 +1,40 @@
+# Full-finetune Qwen3-4B on filtered Crafter traces.
+# Update the `data` path once `uvx synth-ai filter` produces your JSONL.
+
+[algorithm]
+type = "offline"
+method = "sft"
+variety = "fft"
+
+[job]
+model = "Qwen/Qwen3-4B"
+data = "examples/blog_posts/warming_up_to_rl/ft_data/crafter_blog_high_reward.jsonl"
+poll_seconds = 1800
+
+[compute]
+gpu_type = "H100"
+gpu_count = 4
+nodes = 1
+
+[data.topology]
+container_count = 4
+
+[training]
+mode = "full_finetune"
+use_qlora = false
+
+[hyperparameters]
+n_epochs = 2
+world_size = 4
+sequence_length = 2048
+per_device_batch = 2
+gradient_accumulation_steps = 64
+learning_rate = 8e-6
+warmup_ratio = 0.03
+
+[hyperparameters.parallelism]
+use_deepspeed = true
+deepspeed_stage = 3
+fsdp = false
+bf16 = true
+fp16 = false

examples/blog_posts/warming_up_to_rl/warming_up_to_rl_baseline.py

@@ -0,0 +1,187 @@
+"""Warming Up to RL baseline for Crafter.
+
+This baseline demonstrates how to evaluate an LLM agent on the Crafter survival game
+without requiring a deployed task app. This is the recommended starting point for coding
+agents to get a baseline score before making changes.
+
+Quick Start:
+    # Run a quick 3-task baseline
+    uvx synth-ai baseline warming_up_to_rl --split train --seeds 0,1,2
+
+    # Full train evaluation
+    uvx synth-ai baseline warming_up_to_rl --split train
+
+    # Compare models
+    uvx synth-ai baseline warming_up_to_rl --model groq:openai/gpt-oss-20b
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+try:
+    import crafter
+    CRAFTER_AVAILABLE = True
+except ImportError:
+    CRAFTER_AVAILABLE = False
+
+from synth_ai.baseline import BaselineConfig, BaselineTaskRunner, DataSplit, TaskResult
+from synth_ai.types import EventReward, OutcomeReward
+
+
+class CrafterRunner(BaselineTaskRunner):
+    """Task runner for Crafter environment."""
+
+    def __init__(self, policy_config: dict[str, Any], env_config: dict[str, Any]):
+        super().__init__(policy_config, env_config)
+        self.max_steps = env_config.get("max_steps", 1000)
+
+    async def run_task(self, seed: int) -> TaskResult:
+        """Run a single Crafter episode."""
+        if not CRAFTER_AVAILABLE:
+            raise ImportError(
+                "Crafter not installed. Install with: pip install crafter"
+            )
+
+        # Create environment
+        env = crafter.Env()
+        env.reset()
+
+        # Initialize tracking
+        event_rewards: list[EventReward] = []
+        achievements = {}
+        step_count = 0
+
+        # Get model configuration
+        from synth_ai.inference.client import InferenceClient
+
+        client = InferenceClient()
+        model = self.policy_config.get("model", "gpt-4o-mini")
+        temperature = self.policy_config.get("temperature", 0.7)
+
+        # Define action tool
+        actions = [
+            "noop", "move_left", "move_right", "move_up", "move_down",
+            "do", "sleep", "place_stone", "place_table", "place_furnace",
+            "place_plant", "make_wood_pickaxe", "make_stone_pickaxe",
+            "make_iron_pickaxe", "make_wood_sword", "make_stone_sword",
+            "make_iron_sword"
+        ]
+
+        action_tool = {
+            "type": "function",
+            "function": {
+                "name": "take_action",
+                "description": "Take an action in the Crafter world",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "action": {
+                            "type": "string",
+                            "enum": actions,
+                            "description": f"Action to take. Available: {', '.join(actions)}",
+                        }
+                    },
+                    "required": ["action"],
+                },
+            },
+        }
+
+        # Run episode
+        done = False
+        while not done and step_count < self.max_steps:
+            # Get observation (would include visual state in full implementation)
+            obs_str = f"Crafter Step {step_count}\n"
+            obs_str += f"Current achievements: {achievements}\n"
+            obs_str += "What action should you take to survive and progress?"
+
+            # Get action from model
+            try:
+                response = await client.generate(
+                    model=model,
+                    messages=[
+                        {
+                            "role": "system",
+                            "content": "You are an expert at survival games. Use the take_action tool to survive and achieve goals in Crafter.",
+                        },
+                        {"role": "user", "content": obs_str},
+                    ],
+                    tools=[action_tool],
+                    temperature=temperature,
+                    max_tokens=100,
+                )
+
+                # Extract action
+                action_name = "noop"
+                if response.get("tool_calls"):
+                    tool_call = response["tool_calls"][0]
+                    args = json.loads(tool_call["function"]["arguments"])
+                    action_name = args.get("action", "noop")
+
+                action_idx = actions.index(action_name) if action_name in actions else 0
+
+                # Take step
+                obs, reward, done, info = env.step(action_idx)
+
+                # Update achievements
+                if "achievements" in info:
+                    achievements.update(info["achievements"])
+
+                # Track rewards
+                if reward > 0:
+                    event_rewards.append(
+                        EventReward(
+                            event_id=f"step_{step_count}",
+                            reward=reward,
+                            metadata={"action": action_name, "achievements": achievements.copy()},
+                        )
+                    )
+
+                step_count += 1
+
+            except Exception as e:
+                done = True
+                break
+
+        # Calculate outcome reward based on achievements
+        total_achievements = sum(achievements.values())
+        success = total_achievements >= 3  # At least 3 achievements
+
+        return TaskResult(
+            success=success,
+            outcome_reward=OutcomeReward(
+                reward=float(total_achievements),
+                metadata={
+                    "steps": step_count,
+                    "achievements": achievements,
+                    "seed": seed,
+                },
+            ),
+            event_rewards=event_rewards,
+            total_steps=step_count,
+            metadata={"achievements": achievements},
+        )
+
+
+# Define baseline configuration (only if Crafter is available)
+if CRAFTER_AVAILABLE:
+    warming_up_to_rl_baseline = BaselineConfig(
+        baseline_id="warming_up_to_rl",
+        name="Warming Up to RL - Crafter",
+        description="Crafter survival game baseline for comparing agent performance on RL tasks",
+        task_runner=CrafterRunner,
+        splits={
+            "train": DataSplit(name="train", seeds=list(range(20))),
+            "val": DataSplit(name="val", seeds=list(range(20, 25))),
+            "test": DataSplit(name="test", seeds=list(range(25, 30))),
+        },
+        default_policy_config={
+            "model": "gpt-4o-mini",
+            "temperature": 0.7,
+        },
+        default_env_config={
+            "max_steps": 1000,
+        },
+        tags=["rl", "survival", "achievements", "blog-post"],
+    )

examples/crafter_debug_render.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python3
+"""
+Quick local Crafter observation inspector.
+
+- Initializes a CrafterClassic env via local service (default http://localhost:8901)
+- Fetches one observation
+- Renders a 7x7 semantic view around the player with best-effort item names
+- Prints status (health/food/energy), inventory, and achievements
+
+Run:
+  uv run python examples/crafter_debug_render.py --base-url http://localhost:8901 --seed 1
+"""
+
+import argparse
+import contextlib
+import math
+import os
+from typing import Any
+
+import httpx
+
+
+def try_import_crafter_mapping():
+    try:
+        import crafter  # type: ignore
+
+        env = crafter.Env()
+        try:
+            max_id = (
+                max(max(env._world._mat_ids.values()), max(env._sem_view._obj_ids.values())) + 1
+            )
+            id_to_item = ["void"] * max_id
+            for name, ind in env._world._mat_ids.items():
+                label = name.__name__ if hasattr(name, "__name__") else str(name)
+                id_to_item[ind] = label.lower()
+            for name, ind in env._sem_view._obj_ids.items():
+                label = name.__name__ if hasattr(name, "__name__") else str(name)
+                id_to_item[ind] = label.lower()
+            return id_to_item
+        finally:
+            with contextlib.suppress(Exception):
+                env.close()
+    except Exception:
+        return None
+
+
+def format_semantic_map_view(obs: dict[str, Any], view_size: int = 7) -> str:
+    sem = obs.get("semantic_map") or obs.get("sem_map") or obs.get("map")
+    if sem is None:
+        return "No semantic map available"
+
+    # Normalize to 2D grid
+    grid: list[list[int]]
+    if isinstance(sem, list) and sem and isinstance(sem[0], list):
+        grid = sem
+    elif isinstance(sem, list):
+        try:
+            n = int(math.sqrt(len(sem)))
+            if n * n != len(sem) or n == 0:
+                return "Semantic map format not recognized"
+            grid = [sem[i * n : (i + 1) * n] for i in range(n)]
+        except Exception:
+            return "Semantic map format not recognized"
+    else:
+        return "Semantic map format not recognized"
+
+    rows = len(grid)
+    cols = len(grid[0]) if rows > 0 else 0
+    if rows == 0 or cols == 0:
+        return "Empty semantic map"
+
+    # Resolve item mapping if available
+    id_to_item = try_import_crafter_mapping()
+
+    # Player position if provided; otherwise center
+    ppos = obs.get("player_position") or [rows // 2, cols // 2]
+    try:
+        px = int(ppos[0])
+        py = int(ppos[1])
+    except Exception:
+        px, py = rows // 2, cols // 2
+
+    half = max(1, view_size // 2)
+    lines: list[str] = []
+    visible: set[str] = set()
+    for dy in range(-half, half + 1):
+        row_cells: list[str] = []
+        for dx in range(-half, half + 1):
+            x = px + dx
+            y = py + dy
+            if dx == 0 and dy == 0:
+                row_cells.append("you")
+            elif 0 <= x < rows and 0 <= y < cols:
+                try:
+                    val = int(grid[x][y])
+                except Exception:
+                    val = -1
+                if id_to_item and 0 <= val < len(id_to_item):
+                    name = id_to_item[val]
+                else:
+                    # Fallback: simple mapping for common ids (best-effort)
+                    name = {
+                        0: "grass",
+                        1: "stone",
+                        2: "stone",
+                        3: "tree",
+                        4: "coal",
+                        5: "iron",
+                        6: "water",
+                        7: "zombie",
+                        14: "wood",
+                    }.get(val, str(val))
+                row_cells.append(name)
+                if name not in {"grass", "you", "0"}:
+                    visible.add(name)
+            else:
+                row_cells.append("void")
+        lines.append(" ".join(row_cells))
+
+    legend = (
+        f"Visible items: {', '.join(sorted(visible))}" if visible else "No notable items visible"
+    )
+    return "\n".join(lines) + "\n" + legend
+
+
+async def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--base-url", default=os.getenv("CRAFTER_BASE_URL", "http://localhost:8901")
+    )
+    parser.add_argument("--seed", type=int, default=1)
+    args = parser.parse_args()
+
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        init = await client.post(
+            f"{args.base_url}/env/CrafterClassic/initialize",
+            json={"config": {"difficulty": "easy", "seed": args.seed}},
+        )
+        init.raise_for_status()
+        data = init.json()
+        env_id = data["env_id"]
+        obs = data["observation"]
+
+        print("=== INITIAL OBSERVATION ===")
+        print(format_semantic_map_view(obs, view_size=7))
+        inv = obs.get("inventory", {})
+        ach = obs.get("achievements_status", {})
+        print("\n=== STATUS ===")
+        print(f"Health: {obs.get('health', 10)}/10")
+        print(f"Hunger: {obs.get('food', 10)}/10")
+        print(f"Energy: {obs.get('energy', 10)}/10")
+        inv_items = (
+            ", ".join([f"{k}: {v}" for k, v in inv.items() if v])
+            if isinstance(inv, dict)
+            else str(inv)
+        )
+        print(f"Inventory: {inv_items if inv_items else 'empty'}")
+        if isinstance(ach, dict):
+            unlocked = sum(1 for v in ach.values() if v)
+            print(f"Achievements: {unlocked}/{len(ach)} unlocked")
+
+        # Take one step right to get a new obs
+        step = await client.post(
+            f"{args.base_url}/env/CrafterClassic/step",
+            json={
+                "env_id": env_id,
+                "action": {"tool_calls": [{"tool": "interact", "args": {"action": 2}}]},
+            },
+        )
+        step.raise_for_status()
+        sdata = step.json()
+        sobs = sdata["observation"]
+        print("\n=== NEXT OBSERVATION (after move_right) ===")
+        print(format_semantic_map_view(sobs, view_size=7))
+
+        # Cleanup
+        with contextlib.suppress(Exception):
+            await client.post(
+                f"{args.base_url}/env/CrafterClassic/terminate", json={"env_id": env_id}
+            )
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

examples/dev/qwen3_32b_qlora_4xh100.toml

@@ -0,0 +1,45 @@
+[algorithm]
+type = "offline"
+method = "sft"
+variety = "qlora"
+
+[job]
+model = "Qwen/Qwen3-32B"
+# Optionally set here; you can also pass --dataset
+# data = "/abs/path/to/train.jsonl"
+
+[compute]
+gpu_type = "H100"
+gpu_count = 4
+nodes = 1
+
+[data]
+# Optional; forwarded into metadata.effective_config.data.topology
+topology = { container_count = 4 }
+
+[training]
+mode = "sft_offline"
+use_qlora = true
+
+[training.validation]
+enabled = true
+evaluation_strategy = "steps"
+eval_steps = 20
+save_best_model_at_end = true
+metric_for_best_model = "val.loss"
+greater_is_better = false
+
+[hyperparameters]
+n_epochs = 1
+per_device_batch = 1
+gradient_accumulation_steps = 64
+sequence_length = 4096
+learning_rate = 5e-6
+warmup_ratio = 0.03
+
+[hyperparameters.parallelism]
+use_deepspeed = true
+deepspeed_stage = 2
+bf16 = true
+fp16 = false
+fsdp = false

examples/gepa/banking77_pipeline_gepa.toml

@@ -0,0 +1,96 @@
+# Multi-stage GEPA configuration for Banking77 Pipeline
+# Two-stage pipeline: classifier → calibrator
+
+[prompt_learning]
+algorithm = "gepa"
+task_app_url = "https://synth-laboratories-dev--synth-banking77-pipeline-web-web.modal.run"
+task_app_id = "banking77-pipeline"
+
+# Pipeline-level initial prompt (pattern format - now supported for multi-stage!)
+[prompt_learning.initial_prompt]
+id = "banking77_pipeline_gepa"
+name = "Banking77 Pipeline GEPA"
+
+[[prompt_learning.initial_prompt.messages]]
+role = "system"
+pattern = "Pipeline placeholder. Actual instructions provided via metadata.pipeline_modules."
+order = 0
+
+[prompt_learning.initial_prompt.metadata]
+pipeline_modules = [
+  { name = "classifier", instruction_text = "You are an expert banking assistant. Classify the customer query into one of the Banking77 intents using the banking77_classify tool.", few_shots = [] },
+  { name = "calibrator", instruction_text = "Review the classifier's suggestion. Confirm if appropriate, or choose the best Banking77 intent and return it via banking77_classify.", few_shots = [] }
+]
+
+# Policy configuration (applies to all stages)
+[prompt_learning.policy]
+inference_mode = "synth_hosted"
+model = "llama-3.3-70b-versatile"
+provider = "groq"
+temperature = 0.0
+max_completion_tokens = 128
+policy_name = "banking77-pipeline-gepa"
+
+[prompt_learning.env_config]
+pool = "train"
+
+# GEPA-specific configuration
+[prompt_learning.gepa]
+env_name = "banking77_pipeline"
+rng_seed = 42
+proposer_type = "dspy"
+
+# Multi-stage pipeline configuration
+# Note: These are called "modules" in the config but represent stages in the pipeline
+# One pipeline, multiple stages/calls
+[[prompt_learning.gepa.modules]]
+module_id = "classifier"
+max_instruction_slots = 3
+max_tokens = 1024
+
+[[prompt_learning.gepa.modules]]
+module_id = "calibrator"
+max_instruction_slots = 3
+max_tokens = 1024
+
+# Rollout configuration
+[prompt_learning.gepa.rollout]
+budget = 500
+max_concurrent = 10
+minibatch_size = 5
+
+# Evaluation configuration
+[prompt_learning.gepa.evaluation]
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+validation_seeds = [10, 11, 12, 13, 14]
+validation_pool = "test"
+
+# Mutation configuration (LLM-guided)
+[prompt_learning.gepa.mutation]
+rate = 0.3
+llm_model = "llama-3.3-70b-versatile"
+llm_provider = "groq"
+
+# Population/evolution configuration
+[prompt_learning.gepa.population]
+initial_size = 15
+num_generations = 8
+children_per_generation = 5
+crossover_rate = 0.5
+selection_pressure = 1.0
+patience_generations = 3
+
+# Archive/Pareto configuration
+[prompt_learning.gepa.archive]
+size = 32
+pareto_set_size = 32
+pareto_eps = 1e-6
+feedback_fraction = 0.5
+
+# Token/budget configuration
+[prompt_learning.gepa.token]
+max_limit = 30000
+counting_model = "gpt-4"
+enforce_pattern_limit = true
+max_spend_usd = 50.0
+