synth-ai 0.2.16__py3-none-any.whl → 0.2.17__py3-none-any.whl

examples/analyze_semantic_words.sh

@@ -5,7 +5,7 @@
 # Output: Markdown tables and JSON data (no plotting dependencies)
 
 echo "🔍 Analyzing semantic map words from Crafter agent..."
-echo "Make sure the synth-ai service is running: uvx synth-ai serve"
+echo "Make sure the synth-ai service is running: uvx synth-ai deploy --runtime uvicorn"
 echo ""
 
 cd synth_ai/environments/examples/crafter_classic/agent_demos/
@@ -14,4 +14,4 @@ cd synth_ai/environments/examples/crafter_classic/agent_demos/
 python analyze_semantic_words_markdown.py --model gemini-1.5-flash --episodes 3 --max-turns 30
 
 echo ""
-echo "✅ Analysis complete! Check the generated markdown report and JSON files."
+echo "✅ Analysis complete! Check the generated markdown report and JSON files."

examples/blog_posts/pokemon_vl/README.md

@@ -0,0 +1,98 @@
+# Pokémon VL: Vision-Language RL Pipeline
+
+This playbook demonstrates end-to-end vision-language reinforcement learning on Pokémon Red using Synth AI's CLI tools. We follow the eval → collect data → SFT → RL → eval pipeline, but with vision models throughout.
+
+## Overview
+
+**Model**: Qwen3-VL-4B-Instruct (4B parameter vision-language model via Synth API)
+**Environment**: Pokémon Red (Game Boy emulation with vision support)
+**Benchmark**: Pallet Town progression task (leave bedroom → get starter → win first battle)
+
+## Pipeline Steps
+
+1. **Deploy Task App** - Host the Pokémon Red environment
+2. **Collect Vision Rollouts** - Generate high-quality demonstrations using Qwen3-VL
+3. **Filter Dataset** - Extract successful trajectories for supervised fine-tuning
+4. **Fine-Tune Qwen3-4B VL** - Train vision-language model on filtered data
+5. **Vision-Language RL** - Bootstrap RL training from SFT checkpoint
+6. **Final Evaluation** - Compare SFT and RL performance
+
+## Prerequisites
+
+```bash
+# Install dependencies
+uv pip install -e .
+
+# Setup authentication
+uvx synth-ai setup
+
+# Copy environment template
+cp examples/blog_posts/pokemon_vl/.env.example .env
+```
+
+## Quick Start
+
+```bash
+# Export trace database path
+export POKEMON_VL_TRACE_DB=traces/v3/pokemon_vl_blog.db
+
+# 1. Deploy task app
+uvx synth-ai deploy pokemon_red --runtime modal --name pokemon-vl-blog --env-file .env
+
+# 2. Collect vision rollouts with Qwen3-VL
+uvx synth-ai eval pokemon_red --config examples/blog_posts/pokemon_vl/configs/eval_qwen3_vl.toml --trace-db "${POKEMON_VL_TRACE_DB}"
+
+# 3. Filter high-reward trajectories
+uvx synth-ai filter --config examples/blog_posts/pokemon_vl/configs/filter_high_reward.toml
+
+# 4. Fine-tune Qwen3-4B VL
+uvx synth-ai train --type sft --config examples/blog_posts/pokemon_vl/configs/train_sft_qwen4b_vl.toml --env-file .env --poll
+
+# 5. RL from SFT checkpoint (replace JOB_ID)
+uvx synth-ai train --type rl --config examples/blog_posts/pokemon_vl/configs/train_rl_from_sft.toml --env-file .env --poll
+
+# 6. Evaluate final RL model
+uvx synth-ai eval pokemon_red --config examples/blog_posts/pokemon_vl/configs/eval_rl_final.toml --trace-db "${POKEMON_VL_TRACE_DB}"
+```
+
+## Vision Features
+
+- **Full Game Boy Frames**: Base64-encoded PNG screenshots (160x144 resolution)
+- **Vision-Only Mode**: Pure image understanding without text state
+- **Vision + Text Mode**: Combined visual and structured state information
+- **Efficient Action Batching**: `execute_sequence` tool for 5-10 actions per inference call
+
+## Expected Results
+
+| Stage | Model | Mean Reward | Success Rate | Best Achievement |
+|-------|-------|-------------|--------------|------------------|
+| Initial | Qwen3-VL (vision) | ~150 | 60% | Win first battle |
+| SFT | Qwen3-4B VL | ~200 | 75% | Win first battle + explore |
+| RL | Qwen3-4B VL + RL | ~350 | 85% | Complete Pallet Town |
+
+## Files
+
+- `configs/` - All TOML configuration files
+- `ft_data/` - Filtered datasets for fine-tuning
+- `.env.example` - Environment variables template
+
+## Vision Model Configuration
+
+The vision models receive:
+- **Input**: Game Boy screenshot + optional structured state (position, HP, party, etc.)
+- **Output**: Sequence of button presses via `execute_sequence` tool
+- **Action Space**: UP, DOWN, LEFT, RIGHT, A, B, START, SELECT with frame counts
+
+## Reward Function
+
+Dense rewards for Pallet Town progression:
+- Leave bedroom (+20)
+- Exit house (+30)
+- Find Oak's lab (+40)
+- Talk to Oak (+50)
+- Get starter Pokémon (+100)
+- Enter battle (+75)
+- Deal damage (+50 per 10HP)
+- Win battle (+150)
+
+Total possible: ~700 points

examples/blog_posts/pokemon_vl/configs/eval_qwen3_vl.toml

@@ -0,0 +1,25 @@
+[eval]
+app_id = "pokemon_red"
+task_app_url = "https://synth-laboratories--pokemon-vl-qwen-xml-fastapi-app.modal.run"
+model = "Qwen/Qwen3-VL-8B-Instruct"  # Vision-capable Qwen3-VL model
+seeds = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+max_turns = 10
+concurrency = 2
+env_name = "pokemon_red"
+policy_name = "pokemon_vl_qwen3_vl"
+trace_format = "full"
+return_trace = true
+
+[eval.policy_config]
+provider = "synth"  # Use Synth internal API for vision models
+model = "Qwen/Qwen3-VL-8B-Instruct"  # Vision-capable Qwen3-VL model
+inference_url = "http://localhost:8000/api/inference/v1/chat/completions"
+temperature = 1.0  # Higher temperature to encourage tool calling
+top_p = 0.95
+max_tokens = 4096
+use_vision = true
+image_only_mode = false
+max_llm_calls = 10
+
+[eval.env_config.env_params]
+max_steps_per_episode = 10

examples/blog_posts/pokemon_vl/configs/eval_rl_final.toml

@@ -0,0 +1,24 @@
+[eval]
+app_id = "pokemon_red"
+task_app_url = "http://127.0.0.1:8914"
+model = "fft:REPLACE-WITH-RL-JOB-ID"  # Update with final RL job ID
+seeds = [100, 101, 102, 103, 104, 105, 106, 107, 108, 109]
+max_turns = 15  # Allow more steps for trained model
+concurrency = 3
+env_name = "pokemon_red"
+policy_name = "pokemon_vl_rl_final"
+trace_format = "full"
+return_trace = true
+
+[eval.policy_config]
+provider = "synth"
+model = "fft:REPLACE-WITH-RL-JOB-ID"  # Update with final RL job ID
+temperature = 0.1  # Lower temperature for evaluation
+top_p = 0.9
+max_tokens = 4096
+use_vision = true
+image_only_mode = false
+max_llm_calls = 15
+
+[eval.env_config.env_params]
+max_steps_per_episode = 15

examples/blog_posts/pokemon_vl/configs/filter_high_reward.toml

@@ -0,0 +1,10 @@
+# Filter high-quality vision-language rollouts for SFT training
+# Assumes traces stored in pokemon_vl_blog.db via eval commands
+
+[filter]
+db = "traces/v3/pokemon_vl_blog.db"
+output = "examples/blog_posts/pokemon_vl/ft_data/pokemon_vl_high_reward.jsonl"
+min_official_score = 0.3  # Require at least 30% completion (Pallet Town progression)
+models = ["Qwen/Qwen3-VL-4B-Instruct"]  # Vision models used for rollouts
+shuffle = true
+shuffle_seed = 42

examples/blog_posts/pokemon_vl/configs/train_rl_from_sft.toml

@@ -0,0 +1,42 @@
+# Vision-Language RL: Continue training Qwen3-4B VL from SFT checkpoint
+# Update task_url with deployed Modal task app URL
+# Set model.source to the SFT job id from `uvx synth-ai train --type sft`
+
+type = "rl"
+
+[services]
+task_url = "http://127.0.0.1:8914"
+
+[compute]
+gpu_type = "H100"
+gpu_count = 8
+
+[topology]
+gpus_for_vllm = 4
+gpus_for_training = 3
+gpus_for_ref = 1
+
+[vllm]
+tensor_parallel_size = 4
+
+[model]
+source = "fft:REPLACE-WITH-SFT-JOB-ID"  # Update with actual SFT job ID
+label = "pokemon_vl_rl_blog"
+supports_vision = true
+
+[rollout]
+max_turns = 10
+episodes_per_batch = 64
+
+[evaluation]
+instances = 100
+every_n_iters = 20
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]
+
+[training]
+log_interval = 1
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+weight_sync_interval = 1

examples/blog_posts/pokemon_vl/configs/train_sft_qwen4b_vl.toml

@@ -0,0 +1,40 @@
+# Vision-Language Supervised Fine-Tuning: Qwen3-4B VL on filtered Pokémon rollouts
+# Update the `data` path once `uvx synth-ai filter` produces your JSONL
+
+[algorithm]
+type = "offline"
+method = "sft"
+variety = "fft"
+
+[job]
+model = "Qwen/Qwen3-VL-4B-Instruct"  # Vision-enabled Qwen3-VL model
+data = "../ft_data/pokemon_vl_high_reward.jsonl"
+poll_seconds = 1800
+
+[compute]
+gpu_type = "H100"
+gpu_count = 4
+nodes = 1
+
+[data.topology]
+container_count = 4
+
+[training]
+mode = "full_finetune"
+use_qlora = false
+
+[hyperparameters]
+n_epochs = 2
+world_size = 4
+sequence_length = 4096  # Longer for vision tokens + text
+per_device_batch = 2
+gradient_accumulation_steps = 64
+learning_rate = 8e-6
+warmup_ratio = 0.03
+
+[hyperparameters.parallelism]
+use_deepspeed = true
+deepspeed_stage = 3
+fsdp = false
+bf16 = true
+fp16 = false

examples/blog_posts/warming_up_to_rl/README.md

@@ -0,0 +1,158 @@
+# Crafter: From Rollouts to RL with the Synth AI CLI
+
+This playbook mirrors the original “Warming Up to RL” walkthrough, but swaps the bespoke scripts for the first–class `uvx synth-ai` helpers. Every step—from deploying the task app to filtering rollouts, fine-tuning, and bootstrapping RL— now uses the same CLI you’d reach for in production.
+
+All commands assume you are inside the repository root and have `uv`/`uvx` available.
+
+---
+
+## 0. Prerequisites
+
+1. Install dependencies and authenticate once:
+   ```bash
+   uv pip install -e .
+   uvx synth-ai setup
+   ```
+   The setup wizard writes the required `SYNTH_API_KEY`, `ENVIRONMENT_API_KEY`, and local `.env` helpers.
+
+2. Copy the example secrets if you need a starter file:
+   ```bash
+   cp examples/warming_up_to_rl/.env.example .env
+   ```
+
+3. Export the path we use for trace capture (optional but keeps things tidy):
+   ```bash
+   export CRAFTER_TRACE_DB=traces/v3/crafter_blog.db
+   ```
+
+---
+
+## 1. Ship the Crafter Task App
+
+Deploy the hosted Crafter environment once. The Modal URL that prints at the end is reused by eval, SFT, and RL.
+
+```bash
+uvx synth-ai deploy grpo-crafter \
+  --runtime modal \
+  --modal-mode serve \
+  --name crafter-blogpost \
+  --env-file .env
+```
+
+For local testing you can run:
+
+```bash
+uvx synth-ai deploy grpo-crafter \
+  --runtime uvicorn \
+  --port 8001 \
+  --trace traces/v3 \
+  --env-file .env
+```
+
+Copy the Modal URL (e.g. `https://your-app.modal.run`) and replace the `task_app_url` placeholders inside every config under `examples/blog_posts/warming_up_to_rl/configs/`.
+
+---
+
+## 2. Collect High-Quality Rollouts
+
+We lean on large teacher models to produce demonstrations. The configs in `configs/` already request full traces so we retain chain-of-thought.
+
+ Groq Qwen3-32B (text-only prompt):
+```bash
+uvx synth-ai eval grpo-crafter \
+  --config examples/blog_posts/warming_up_to_rl/configs/eval_groq_qwen32b.toml \
+  --trace-db "${CRAFTER_TRACE_DB}"
+```
+
+ GPT-OSS-120B via Groq’s OpenAI-compatible endpoint (also text-only):
+```bash
+uvx synth-ai eval grpo-crafter \
+  --config examples/blog_posts/warming_up_to_rl/configs/eval_openai_gpt_oss_120b.toml \
+  --trace-db "${CRAFTER_TRACE_DB}"
+```
+
+ Both configs disable image attachments and rely on the textual observation renderer (`format_observation`) so Groq stays within its supported modalities. If you want to try other models, keep `use_vision = false` unless the provider explicitly supports image inputs.
+
+---
+
+## 3. Filter Into an SFT Dataset
+
+Once traces are stored in `CRAFT_TRACE_DB`, trim to the crisp trajectories:
+
+```bash
+uvx synth-ai filter \
+  --config examples/blog_posts/warming_up_to_rl/configs/filter_high_reward_dataset.toml
+```
+
+The output JSONL lands in `ft_data/crafter_blog_high_reward.jsonl`, ready for supervised fine-tuning.
+
+---
+
+## 4. Fine-Tune Qwen3-4B with `uvx synth-ai train`
+
+Update the dataset path (and optionally hyperparameters) in `train_sft_qwen4b.toml`, then launch:
+
+```bash
+uvx synth-ai train \
+  --type sft \
+  --config examples/blog_posts/warming_up_to_rl/configs/train_sft_qwen4b.toml \
+  --env-file .env \
+  --poll
+```
+
+Capture the returned job id (it looks like `fft:Qwen/Qwen3-4B:job_xxxxx`). We reuse that identifier in the evaluation and RL configs.
+At any time you can list recently minted checkpoints with:
+
+```bash
+uvx synth-ai status models
+```
+
+The output table shows the canonical model name/ID alongside the source job.
+
+---
+
+## 5. Evaluate the Fine-Tuned Checkpoint
+
+Replace both `REPLACE-WITH-SFT-JOB-ID` strings inside `eval_ft_qwen4b.toml`, then run:
+
+```bash
+uvx synth-ai eval grpo-crafter \
+  --config examples/blog_posts/warming_up_to_rl/configs/eval_ft_qwen4b.toml \
+  --trace-db "${CRAFTER_TRACE_DB}"
+```
+
+This provides a clean, CLI-native comparison between the teacher rollouts and the fine-tuned model.
+
+---
+
+## 6. Kick Off RL from the Fine-Tuned Model
+
+Point `train_rl_from_sft.toml` at the same Modal task app and set `model.source` to your SFT job id:
+
+```bash
+uvx synth-ai train \
+  --type rl \
+  --config examples/blog_posts/warming_up_to_rl/configs/train_rl_from_sft.toml \
+  --env-file .env \
+  --poll
+```
+
+The CLI streams rollout and judge metrics in real time. When the run finishes, you can re-use the Stage 5 config (substituting the RL job id) to quantify the uplift.
+If you lose track of the produced RL label or want to confirm the latest status, run:
+
+```bash
+uvx synth-ai status jobs
+uvx synth-ai status models
+```
+
+The first command shows job completion state; the second surfaces model IDs you can plug into new eval configs.
+
+---
+
+## 7. Where to Go Next
+
+- The original `examples/warming_up_to_rl` folder still contains deeper experiments (auto-curricula, modal renderers, etc.).
+- Add more `eval_*.toml` configs to compare alternative judges or reward shaping strategies.
+- Plug the filtered dataset into `uvx synth-ai files upload` if you want to share it with a teammate without copying JSONL around.
+
+This directory now holds everything a blog post needs: configs, output locations, and the CLI entrypoints to reproduce the Crafter SFT → RL pipeline end-to-end.

examples/blog_posts/warming_up_to_rl/configs/eval_ft_qwen4b.toml

@@ -0,0 +1,25 @@
+# Evaluate the finetuned Qwen3-4B checkpoint on Crafter.
+# Replace model with the fft: job id returned by the SFT run.
+
+[eval]
+app_id = "grpo-crafter"
+task_app_url = "https://synth-laboratories--grpo-crafter-task-app-fastapi-app-dev.modal.run"
+model = "fft:REPLACE-WITH-SFT-JOB-ID"
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+max_turns = 10
+concurrency = 4
+env_name = "crafter"
+policy_name = "crafter-react"
+trace_format = "compact"
+return_trace = false
+
+[eval.policy_config]
+provider = "synth"
+model = "fft:REPLACE-WITH-SFT-JOB-ID"
+temperature = 0.2
+top_p = 0.8
+max_tokens = 512
+use_vision = true
+image_only_mode = false
+max_llm_calls = 10
+tool_choice = "auto"

examples/blog_posts/warming_up_to_rl/configs/eval_groq_qwen32b.toml

@@ -0,0 +1,25 @@
+[eval]
+app_id = "grpo-crafter"
+task_app_url = "https://synth-laboratories--grpo-crafter-task-app-fastapi-app-dev.modal.run"
+model = "qwen/qwen3-32b"
+seeds = [ 0, 1, 2,]
+max_turns = 10
+concurrency = 1
+env_name = "crafter"
+policy_name = "crafter-react"
+trace_format = "full"
+return_trace = true
+
+[eval.policy_config]
+provider = "groq"
+model = "qwen/qwen3-32b"
+inference_url = "https://api.groq.com/openai"
+temperature = 0.6
+top_p = 0.95
+max_tokens = 8192
+use_vision = false
+image_only_mode = false
+max_llm_calls = 10
+
+[eval.env_config.env_params]
+max_steps_per_episode = 10

examples/blog_posts/warming_up_to_rl/configs/eval_openai_gpt_oss_120b.toml

@@ -0,0 +1,29 @@
+# Crafter rollout config for GPT-OSS-120B served from OpenAI-compatible APIs.
+# Replace the task_app_url with your deployed Crafter task app URL.
+# The run stores full traces so we can keep the LLM reasoning for fine-tuning.
+
+[eval]
+app_id = "grpo-crafter"
+task_app_url = "https://synth-laboratories--grpo-crafter-task-app-fastapi-app-dev.modal.run"
+model = "openai/gpt-oss-120b"
+seeds = [0, 1, 2]
+max_turns = 10
+concurrency = 1
+env_name = "crafter"
+policy_name = "crafter-react"
+trace_format = "full"
+return_trace = true
+
+[eval.env_config]
+env_params = { max_steps_per_episode = 10 }
+
+[eval.policy_config]
+provider = "groq"
+model = "openai/gpt-oss-120b"
+inference_url = "https://api.groq.com/openai"
+temperature = 0.6
+top_p = 0.9
+max_tokens = 768
+use_vision = false
+image_only_mode = false
+max_llm_calls = 10

examples/blog_posts/warming_up_to_rl/configs/filter_high_reward_dataset.toml

@@ -0,0 +1,10 @@
+# Filters Crafter traces into an instruction-tuning dataset.
+# Assumes you stored rollouts in traces/v3/crafter_blog.db via `uvx synth-ai eval`.
+
+[filter]
+db = "traces/v3/crafter_blog.db"
+output = "examples/blog_posts/warming_up_to_rl/ft_data/crafter_blog_high_reward.jsonl"
+min_official_score = 0.1
+models = ["qwen/qwen3-32b", "openai/gpt-oss-120b"]
+shuffle = true
+shuffle_seed = 42

examples/blog_posts/warming_up_to_rl/configs/train_rl_from_sft.toml

@@ -0,0 +1,41 @@
+# Continue training the finetuned Crafter policy with GRPO-style RL.
+# Fill in task_url with your deployed task app and set model.source to the
+# finetuned model id returned by `uvx synth-ai train --type sft`.
+
+type = "rl"
+
+[services]
+task_url = "https://synth-laboratories--grpo-crafter-task-app-fastapi-app-dev.modal.run"
+
+[compute]
+gpu_type = "H100"
+gpu_count = 8
+
+[topology]
+gpus_for_vllm = 4
+gpus_for_training = 3
+gpus_for_ref = 1
+
+[vllm]
+tensor_parallel_size = 4
+
+[model]
+source = "fft:REPLACE-WITH-SFT-JOB-ID"
+label = "crafter-rl-blogpost"
+
+[rollout]
+max_turns = 10
+episodes_per_batch = 64
+
+[evaluation]
+instances = 100
+every_n_iters = 20
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99]
+
+[training]
+log_interval = 1
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+weight_sync_interval = 1

examples/blog_posts/warming_up_to_rl/configs/train_sft_qwen4b.toml

@@ -0,0 +1,40 @@
+# Full-finetune Qwen3-4B on filtered Crafter traces.
+# Update the `data` path once `uvx synth-ai filter` produces your JSONL.
+
+[algorithm]
+type = "offline"
+method = "sft"
+variety = "fft"
+
+[job]
+model = "Qwen/Qwen3-4B"
+data = "../ft_data/crafter_blog_high_reward.jsonl"
+poll_seconds = 1800
+
+[compute]
+gpu_type = "H100"
+gpu_count = 4
+nodes = 1
+
+[data.topology]
+container_count = 4
+
+[training]
+mode = "full_finetune"
+use_qlora = false
+
+[hyperparameters]
+n_epochs = 2
+world_size = 4
+sequence_length = 2048
+per_device_batch = 2
+gradient_accumulation_steps = 64
+learning_rate = 8e-6
+warmup_ratio = 0.03
+
+[hyperparameters.parallelism]
+use_deepspeed = true
+deepspeed_stage = 3
+fsdp = false
+bf16 = true
+fp16 = false

examples/dev/qwen3_32b_qlora_4xh100.toml

@@ -1,3 +1,8 @@
+[algorithm]
+type = "offline"
+method = "sft"
+variety = "qlora"
+
 [job]
 model = "Qwen/Qwen3-32B"
 # Optionally set here; you can also pass --dataset

examples/multi_step/configs/crafter_rl_outcome.toml

@@ -6,7 +6,7 @@ method = "policy_gradient"
 variety = "gspo"
 
 [services]
-# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
+# Replace with the Modal URL printed by `uvx synth-ai deploy --runtime modal --modal-mode serve grpo-crafter`
 task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
 
 [compute]