synth-ai 0.2.14__py3-none-any.whl → 0.2.17__py3-none-any.whl

examples/multi_step/SFT_README.md

@@ -0,0 +1,147 @@
+# SFT Training for Qwen3-Coder-30B with LoRA
+
+Supervised Fine-Tuning configuration for the same 30B MoE model used in RL training.
+
+## Configuration Overview
+
+**Model:** `Qwen/Qwen3-Coder-30B-A3B-Instruct` (Mixture of Experts)
+
+**Hardware:** 4x H200 GPUs (561GB total VRAM)
+
+**Parallelism Strategy:**
+- **Tensor Parallel (TP)**: 2 GPUs - Splits the model across 2 GPUs for inference/forward pass
+- **Data Parallel (DP)**: 2 GPUs - Splits batches across 2 GPUs for training throughput
+
+**LoRA Configuration:**
+- Rank (r): 16
+- Alpha: 32
+- Dropout: 0.05
+- Target modules: `["all-linear"]` - Applies LoRA to all linear layers
+
+## Memory Breakdown per GPU
+
+With 4x H200 (141GB each):
+
+**Model Split (TP=2):**
+- 2 GPUs hold the base model (70GB each)
+- ~70GB free per GPU for activations and gradients
+
+**Training (DP=2):**
+- 2 GPUs process different batches
+- LoRA adapters: ~5-10GB per GPU
+- Gradients/optimizer states: ~20-30GB per GPU
+- **Total per training GPU: ~50-60GB** ✅
+
+## Quick Start
+
+### 1. Prepare Your Dataset
+
+Your dataset should be in JSONL format with conversation turns:
+
+```jsonl
+{"messages": [{"role": "system", "content": "..."}, {"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]}
+{"messages": [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]}
+```
+
+### 2. Run Training
+
+```bash
+# Using the helper script
+./examples/multi_step/run_sft_qwen30b.sh path/to/your/dataset.jsonl
+
+# Or directly with synth-ai CLI
+uvx synth-ai train \
+  --type sft \
+  --config examples/multi_step/configs/crafter_sft_qwen30b_lora.toml \
+  --dataset path/to/your/dataset.jsonl \
+  --env-file backend/.env.dev
+```
+
+### 3. Monitor Training
+
+Check the Synth dashboard for:
+- Training loss curve
+- Validation metrics (if validation set provided)
+- GPU utilization
+- Training throughput (tokens/sec)
+
+## Hyperparameters
+
+**Batch Configuration:**
+- Per-device batch size: 1
+- Gradient accumulation: 64 steps
+- **Effective global batch size: 128** (1 × 64 × 2 GPUs)
+
+**Learning Rate:**
+- Initial LR: 5e-6
+- Warmup ratio: 3%
+- Schedule: Linear decay
+
+**Sequence Length:** 4096 tokens
+
+**Training:**
+- Epochs: 1
+- Mixed precision: BF16
+- DeepSpeed: Stage 2 (optimizer state sharding)
+- Activation checkpointing: Enabled
+
+## Configuration File Structure
+
+```toml
+[algorithm]
+type = "offline"        # Supervised (not RL)
+method = "sft"          # Supervised fine-tuning
+variety = "lora"        # Using LoRA adapters
+
+[compute]
+gpu_type = "H200"
+gpu_count = 4
+
+[data.topology]
+tensor_parallel = 2     # Split model across 2 GPUs
+data_parallel = 2       # Split batches across 2 GPUs
+
+[training]
+mode = "lora"
+use_qlora = true        # Quantized LoRA (4-bit base model)
+
+[lora]
+r = 16                  # LoRA rank
+alpha = 32              # LoRA scaling
+dropout = 0.05
+target_modules = ["all-linear"]  # Apply to all linear layers
+```
+
+## Comparison with RL Config
+
+| Aspect | SFT | RL |
+|--------|-----|-----|
+| Purpose | Supervised learning | Reinforcement learning |
+| Data | Labeled examples | Environment interactions |
+| Topology | TP=2, DP=2 | Split: 2 inference + 2 training |
+| Batch size | 128 (effective) | Variable (episode-based) |
+| Training | Standard backprop | Policy gradient (GSPO) |
+
+## Tips
+
+1. **Start Small:** Test with a small dataset first to verify the pipeline
+2. **Validation:** Add a validation set to monitor overfitting
+3. **Checkpointing:** Training saves checkpoints every 100 steps
+4. **Resume:** Can resume from checkpoint if training is interrupted
+5. **Inference:** After training, use the LoRA adapter with the base model
+
+## Output
+
+After training completes, you'll get:
+- LoRA adapter weights (saved to volume)
+- Training metrics and logs
+- Best checkpoint (based on validation loss)
+- Model ready for inference or RL initialization
+
+## Next Steps
+
+1. **Evaluate:** Test your fine-tuned model on held-out data
+2. **RL Training:** Use this as initialization for RL (`init_from_sft = true`)
+3. **Deploy:** Load LoRA adapter for inference
+4. **Iterate:** Adjust hyperparameters based on performance
+

examples/multi_step/configs/crafter_rl_outcome.toml

@@ -6,7 +6,7 @@ method = "policy_gradient"
 variety = "gspo"
 
 [services]
-# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
+# Replace with the Modal URL printed by `uvx synth-ai deploy --runtime modal --modal-mode serve grpo-crafter`
 task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
 
 [compute]

examples/multi_step/configs/crafter_rl_stepwise_hosted_judge.toml

@@ -1,39 +1,32 @@
-# Crafter RL experiment – stepwise shaping with hosted judge rubrics
-#
-# This configuration extends the stepwise LoRA baseline by wiring the Synth judge
-# service so evaluation rolls combine dense step rewards with hosted rubric scoring.
-
 [algorithm]
 type = "online"
 method = "policy_gradient"
 variety = "gspo"
 
 [services]
-# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
 task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
-# Point at the Synth backend (or compatible service) that exposes /api/judge/v1/*
 judge_url = "https://synth-backend-dev-docker.onrender.com/api"
 
 [compute]
 gpu_type = "H200"
-gpu_count = 2
+gpu_count = 4
 
 [topology]
 type = "single_node_split"
-gpus_for_vllm = 1
-gpus_for_training = 1
+gpus_for_vllm = 2
+gpus_for_training = 2
 gpus_for_ref = 0
-tensor_parallel = 1
+tensor_parallel = 2
 
 [vllm]
-tensor_parallel_size = 1
-max_model_len = 8192
+tensor_parallel_size = 2
+max_model_len = 4096
 
 [reference]
 placement = "none"
 
 [model]
-base = "Qwen/Qwen3-4B"
+base = "Qwen/Qwen3-Coder-30B-A3B-Instruct"
 trainer_mode = "lora"
 label = "crafter-rl-stepwise-hosted-judge"
 
@@ -41,7 +34,7 @@ label = "crafter-rl-stepwise-hosted-judge"
 r = 16
 alpha = 32
 dropout = 0.05
-target_modules = ["all-linear"]
+target_modules = [ "all-linear",]
 
 [rollout]
 env_name = "crafter"
@@ -50,31 +43,16 @@ episodes_per_batch = 2
 policy_name = "crafter-react"
 max_concurrent_rollouts = 8
 batches_per_step = 2
-ops = ["agent", "env"]
-
-  [rollout.env_config]
-  difficulty = "easy"
-
-    [rollout.env_config.step_rewards]
-    enabled = true
-    mode = "decision_stepwise"
-    strategy = "consistent"       # +1 for each decision that unlocks a new achievement
-    indicator_lambda = 1.0
-    step_beta = 0.0
-
-  [rollout.policy_config]
-  temperature = 0.2
-  top_p = 0.95
-  max_tokens = 512
+ops = [ "agent", "env",]
 
 [evaluation]
 instances = 16
 every_n_iters = 10
-seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+seeds = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,]
 
 [training]
 num_epochs = 1
-iterations_per_epoch = 20
+iterations_per_epoch = 5
 gradient_accumulation_steps = 1
 max_accumulated_minibatch = 1
 max_turns = 10
@@ -84,104 +62,84 @@ learning_rate = 5e-5
 log_interval = 1
 weight_sync_interval = 1
 event_rewards_kind = "unique"
-async_semaphore_max = 40  # Max concurrent rollouts in streaming pipeline
-
-# Enable dense decision rewards in the trainer to mirror env_config step rewards.
+async_semaphore_max = 4
 step_rewards_enabled = true
 step_rewards_mode = "decision_stepwise"
 step_rewards_indicator_lambda = 1.0
 step_rewards_beta = 0.0
 step_rewards_strategy = "consistent"
 
+[rubric]
+enabled = true
+
+[rollout.env_config]
+difficulty = "easy"
+
+[rollout.policy_config]
+temperature = 0.2
+top_p = 0.95
+max_tokens = 512
+
 [training.weight_sync]
 enable = true
-targets = ["policy"]
+targets = [ "policy",]
 mode = "direct"
 direct = true
 verify_every_k = 0
 
-[rubric]
-enabled = true
-model = "openai/gpt-oss-120b"
-api_base = "https://synth-backend-dev-docker.onrender.com/api/judge"
-api_key_env = "OPENAI_API_KEY"
-# Blend the hosted judge scores with environment returns inside the trainer.
 [rubric.weights]
 env = 0.2
 event = 0.4
 outcome = 0.4
 
-[rubric.event]
-# Hosted judge rubric for per-decision progress scoring.
-rubric_id = "crafter/event@v1"
-criteria = [
-  { key = "progress.unique_achievements", weight = 0.9, description = "Return 1 when this decision explicitly unlocks a brand-new Crafter achievement (inventory or status text confirms it this turn). Otherwise return 0.", aggregation = "weighted_sum" },
-  { key = "process.intent_alignment", weight = 0.1, description = "Use at most 0.3 to acknowledge tightly coupled setup that finishes the last prerequisite; keep ≤0.1 when the agent only repositions or gathers without an imminent unlock.", aggregation = "weighted_sum" },
-]
-
-[rubric.outcome]
-# Hosted judge rubric for final trajectory scoring.
-rubric_id = "crafter/outcome@v1"
-criteria = [
-  { key = "outcome.goal_completion", weight = 0.6, description = "Full credit when the agent ends with strong survival metrics and a clear crafted milestone (e.g., iron tools, furnace).", aggregation = "weighted_sum" },
-  { key = "outcome.achievement_depth", weight = 0.4, description = "Partial credit for intermediate achievements (saplings, wood/stone tools) that set up future success.", aggregation = "weighted_sum" },
-]
-
-[judge]
-type = "groq"                            # or "groq" when routing to Groq-hosted judges
+[judge.options]
+event = true
+outcome = true
+provider = "openai"
+model = "openai/gpt-oss-120b"
+rubric_id = "crafter/bundle@v1"
 timeout_s = 45
 
-  [judge.options]
-  event = true
-  outcome = true
-  provider = "openai"
-  model = "openai/gpt-oss-120b"
-  rubric_id = "crafter/bundle@v1"
-  max_concurrency = 6
-  tracks = ["process", "reasoning", "progress", "outcome"]
-
-    [judge.options.rubric_overrides]
-
-      [judge.options.rubric_overrides.event]
-      goal_text = """
-Treat each decision as a check for new Crafter achievements.
-Award the top score only when the log shows a fresh achievement unlock or an immediately verifiable deterministic completion.
-Keep otherwise useful setup actions in a narrow low band so non-achievement turns stay near zero."""
-      aggregation = "weighted_sum"
-
-        [[judge.options.rubric_overrides.event.criteria]]
-        id = "progress.unique_achievements"
-        weight = 0.9
-        scale = "binary"
-        description = "Return 1 when this decision explicitly unlocks a brand-new Crafter achievement (inventory or status text confirms it this turn). Otherwise return 0."
-
-        [[judge.options.rubric_overrides.event.criteria]]
-        id = "process.intent_alignment"
-        weight = 0.1
-        scale = "bounded"
-        description = "Use at most 0.3 to acknowledge tightly coupled setup that finishes the last prerequisite; keep ≤0.1 when the agent only repositions or gathers without an imminent unlock."
-
-      [judge.options.rubric_overrides.outcome]
-      goal_text = """
-Summarise the episode outcome in relation to Crafter’s win condition:
-survive, accumulate resources, and craft advanced tools or structures.
-Highlight notable achievements, safety failures, and preparedness for future exploration."""
-      aggregation = "weighted_sum"
-
-        [[judge.options.rubric_overrides.outcome.criteria]]
-        id = "outcome.goal_completion"
-        weight = 0.6
-        scale = "binary"
-        description = "Full credit when the agent ends with strong survival metrics and a clear crafted milestone (e.g., iron tools, furnace)."
-
-        [[judge.options.rubric_overrides.outcome.criteria]]
-        id = "outcome.achievement_depth"
-        weight = 0.4
-        scale = "bounded"
-        description = "Partial credit for intermediate achievements (saplings, wood/stone tools) that set up future success."
-
-    [judge.options.weights]
-    process = 0.05
-    reasoning = 0.15
-    progress = 0.30
-    outcome = 0.50
+[rollout.env_config.step_rewards]
+enabled = true
+mode = "decision_stepwise"
+strategy = "consistent"
+indicator_lambda = 1.0
+step_beta = 0.0
+
+[judge.options.weights]
+process = 0.05
+reasoning = 0.15
+progress = 0.3
+outcome = 0.5
+
+[judge.options.rubric_overrides.event]
+goal_text = "Treat each decision as a check for new Crafter achievements.\nAward the top score only when the log shows a fresh achievement unlock or an immediately verifiable deterministic completion.\nKeep otherwise useful setup actions in a narrow low band so non-achievement turns stay near zero."
+aggregation = "weighted_sum"
+[[judge.options.rubric_overrides.event.criteria]]
+id = "progress.unique_achievements"
+weight = 0.9
+scale = "binary"
+description = "Return 1 when this decision explicitly unlocks a brand-new Crafter achievement (inventory or status text confirms it this turn). Otherwise return 0."
+
+[[judge.options.rubric_overrides.event.criteria]]
+id = "process.intent_alignment"
+weight = 0.1
+scale = "bounded"
+description = "Use at most 0.3 to acknowledge tightly coupled setup that finishes the last prerequisite; keep ≤0.1 when the agent only repositions or gathers without an imminent unlock."
+
+[judge.options.rubric_overrides.outcome]
+goal_text = "Summarise the episode outcome in relation to Crafter’s win condition:\nsurvive, accumulate resources, and craft advanced tools or structures.\nHighlight notable achievements, safety failures, and preparedness for future exploration."
+aggregation = "weighted_sum"
+[[judge.options.rubric_overrides.outcome.criteria]]
+id = "outcome.goal_completion"
+weight = 0.6
+scale = "binary"
+description = "Full credit when the agent ends with strong survival metrics and a clear crafted milestone (e.g., iron tools, furnace)."
+
+[[judge.options.rubric_overrides.outcome.criteria]]
+id = "outcome.achievement_depth"
+weight = 0.4
+scale = "bounded"
+description = "Partial credit for intermediate achievements (saplings, wood/stone tools) that set up future success."
+

examples/multi_step/configs/crafter_rl_stepwise_shaped.toml

@@ -6,7 +6,7 @@ method = "policy_gradient"
 variety = "gspo"
 
 [services]
-# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
+# Replace with the Modal URL printed by `uvx synth-ai deploy --runtime modal --modal-mode serve grpo-crafter`
 task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
 
 [compute]

examples/multi_step/configs/crafter_rl_stepwise_simple.toml

@@ -6,7 +6,7 @@ method = "policy_gradient"
 variety = "gspo"
 
 [services]
-# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
+# Replace with the Modal URL printed by `uvx synth-ai deploy --runtime modal --modal-mode serve grpo-crafter`
 task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
 
 [compute]

examples/multi_step/configs/crafter_rl_stepwise_simple_NEW_FORMAT.toml

@@ -0,0 +1,105 @@
+# Crafter RL experiment – simple stepwise rewards (1 point per new achievement)
+# This config uses the NEW unified [policy] section format
+
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[services]
+# Replace with the Modal URL printed by `uvx synth-ai deploy --runtime modal --modal-mode serve grpo-crafter`
+task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2
+
+  [compute.topology]  # Nested: topology is part of compute
+  type = "single_node_split"
+  gpus_for_vllm = 1
+  gpus_for_training = 1
+  gpus_for_ref = 0
+  tensor_parallel = 1
+  reference_placement = "none"  # Reference model placement
+
+[vllm]
+tensor_parallel_size = 1
+max_model_len = 8192
+
+[judge]
+enabled = false  # Set to true to enable judge/rubric scoring
+
+# Uncomment to enable judge-based reward blending:
+# enabled = true
+# timeout_s = 45
+#
+#  [judge.reward_blend]  # How to blend env/event/outcome reward sources
+#  env = 0.2
+#  event = 0.4
+#  outcome = 0.4
+#
+#  [judge.options]
+#  provider = "openai"
+#  model = "openai/gpt-oss-120b"
+#  event = true
+#  outcome = true
+#  max_concurrency = 6
+
+# NEW: Unified [policy] section - single source of truth for model and sampling
+[policy]
+model_name = "Qwen/Qwen3-4B"
+trainer_mode = "lora"
+label = "crafter-rl-stepwise-simple"
+
+# Sampling parameters for rollouts
+max_tokens = 512
+temperature = 0.6
+top_p = 0.95
+
+[rollout]
+env_name = "crafter"
+max_turns = 10
+episodes_per_batch = 4
+policy_name = "crafter-react"
+max_concurrent_rollouts = 8
+batches_per_step = 2
+ops = ["agent", "env"]
+
+[evaluation]
+instances = 10
+every_n_iters = 10
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+[training]
+num_epochs = 1
+iterations_per_epoch = 10
+gradient_accumulation_steps = 1
+max_accumulated_minibatch = 1
+max_turns = 10
+batch_size = 4
+group_size = 4
+learning_rate = 5e-5
+log_interval = 1
+weight_sync_interval = 1
+
+  [training.rewards]  # Nested: Reward config under training
+  step_rewards_enabled = true
+  step_rewards_mode = "decision_stepwise"
+  step_rewards_indicator_lambda = 1.0
+  step_rewards_beta = 0.0
+  step_rewards_strategy = "consistent"
+  event_rewards_kind = "unique"
+
+  [training.lora]  # Nested: LoRA config under training
+  r = 16
+  alpha = 32
+  dropout = 0.05
+  target_modules = ["all-linear"]
+
+  [training.weight_sync]
+  enable = true
+  targets = ["policy"]
+  mode = "direct"
+  direct = true
+  verify_every_k = 0
+

examples/multi_step/configs/crafter_sft_qwen30b_lora.toml

@@ -0,0 +1,62 @@
+# Crafter SFT LoRA configuration
+# Train Qwen3-Coder-30B on Crafter agent traces
+
+[algorithm]
+type = "offline"
+method = "sft"
+variety = "lora"
+
+[job]
+model = "Qwen/Qwen3-Coder-30B-A3B-Instruct"
+# Default dataset - can override with --dataset flag
+data = "traces/crafter_sft_converted.jsonl"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2
+nodes = 1
+
+[data]
+# Forwarded into metadata.effective_config
+topology = {}
+# Optional validation set if you have one locally
+# validation_path = "examples/multi_step/ft_data/crafter_sft.val.jsonl"
+
+[training]
+mode = "lora"
+use_qlora = true
+
+[training.validation]
+enabled = true
+evaluation_strategy = "steps"
+eval_steps = 100
+save_best_model_at_end = true
+metric_for_best_model = "val.loss"
+greater_is_better = false
+
+[hyperparameters]
+n_epochs = 1
+train_kind = "peft"
+per_device_batch = 1
+gradient_accumulation_steps = 64
+sequence_length = 4096
+learning_rate = 5e-6
+warmup_ratio = 0.03
+lora_rank = 16
+lora_alpha = 32
+lora_dropout = 0.05
+lora_target_modules = ["all-linear"]
+
+[hyperparameters.parallelism]
+use_deepspeed = true
+deepspeed_stage = 2
+fsdp = false
+bf16 = true
+fp16 = false
+activation_checkpointing = true
+
+[tags]
+experiment = "crafter_sft_lora_qwen_coder_30b"
+task = "crafter_agent"
+model_size = "30b"
+