synth-ai 0.2.12__py3-none-any.whl → 0.2.13.dev2__py3-none-any.whl

examples/multi_step/configs/crafter_rl_outcome.toml

@@ -0,0 +1,74 @@
+# Crafter RL experiment – outcome rewards only (step rewards disabled)
+
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[services]
+# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
+task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2
+
+[topology]
+type = "single_node_split"
+gpus_for_vllm = 1
+gpus_for_training = 1
+gpus_for_ref = 0
+tensor_parallel = 1
+
+[vllm]
+tensor_parallel_size = 1
+max_model_len = 8192
+
+[reference]
+placement = "none"
+
+[model]
+base = "Qwen/Qwen3-4B"
+trainer_mode = "lora"
+label = "crafter-rl-outcome"
+
+[lora]
+r = 16
+alpha = 32
+dropout = 0.05
+target_modules = ["all-linear"]
+
+[rollout]
+env_name = "crafter"
+max_turns = 10
+episodes_per_batch = 4
+policy_name = "crafter-react"
+max_concurrent_rollouts = 12
+batches_per_step = 2
+ops = ["agent", "env"]
+
+[evaluation]
+instances = 10
+every_n_iters = 5
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+[training]
+num_epochs = 1
+iterations_per_epoch = 20
+gradient_accumulation_steps = 1
+max_accumulated_minibatch = 1
+max_turns = 8
+batch_size = 3
+group_size = 4
+learning_rate = 5e-5
+log_interval = 1
+weight_sync_interval = 1
+step_rewards_enabled = false
+event_rewards_kind = "unique"
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+mode = "direct"
+direct = true
+verify_every_k = 0

examples/multi_step/configs/crafter_rl_stepwise_hosted_judge.toml

@@ -0,0 +1,186 @@
+# Crafter RL experiment – stepwise shaping with hosted judge rubrics
+#
+# This configuration extends the stepwise LoRA baseline by wiring the Synth judge
+# service so evaluation rolls combine dense step rewards with hosted rubric scoring.
+
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[services]
+# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
+task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
+# Point at the Synth backend (or compatible service) that exposes /api/judge/v1/*
+judge_url = "https://synth-backend-dev-docker.onrender.com/api"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2
+
+[topology]
+type = "single_node_split"
+gpus_for_vllm = 1
+gpus_for_training = 1
+gpus_for_ref = 0
+tensor_parallel = 1
+
+[vllm]
+tensor_parallel_size = 1
+max_model_len = 8192
+
+[reference]
+placement = "none"
+
+[model]
+base = "Qwen/Qwen3-4B"
+trainer_mode = "lora"
+label = "crafter-rl-stepwise-hosted-judge"
+
+[lora]
+r = 16
+alpha = 32
+dropout = 0.05
+target_modules = ["all-linear"]
+
+[rollout]
+env_name = "crafter"
+max_turns = 10
+episodes_per_batch = 4
+policy_name = "crafter-react"
+max_concurrent_rollouts = 8
+batches_per_step = 2
+ops = ["agent", "env"]
+
+  [rollout.env_config]
+  difficulty = "easy"
+
+    [rollout.env_config.step_rewards]
+    enabled = true
+    mode = "decision_stepwise"
+    strategy = "consistent"       # +1 for each decision that unlocks a new achievement
+    indicator_lambda = 1.0
+    step_beta = 0.0
+
+  [rollout.policy_config]
+  temperature = 0.2
+  top_p = 0.95
+  max_tokens = 512
+
+[evaluation]
+instances = 16
+every_n_iters = 8
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+
+[training]
+num_epochs = 1
+iterations_per_epoch = 16
+gradient_accumulation_steps = 1
+max_accumulated_minibatch = 1
+max_turns = 10
+batch_size = 4
+group_size = 4
+learning_rate = 5e-5
+log_interval = 1
+weight_sync_interval = 1
+event_rewards_kind = "unique"
+
+# Enable dense decision rewards in the trainer to mirror env_config step rewards.
+step_rewards_enabled = true
+step_rewards_mode = "decision_stepwise"
+step_rewards_indicator_lambda = 1.0
+step_rewards_beta = 0.0
+step_rewards_strategy = "consistent"
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+mode = "direct"
+direct = true
+verify_every_k = 0
+
+[rubric]
+enabled = true
+model = "openai/gpt-oss-120b"
+api_base = "https://synth-backend-dev-docker.onrender.com/api/judge"
+api_key_env = "OPENAI_API_KEY"
+# Blend the hosted judge scores with environment returns inside the trainer.
+[rubric.weights]
+env = 0.2
+event = 0.4
+outcome = 0.4
+
+[rubric.event]
+# Hosted judge rubric for per-decision progress scoring.
+rubric_id = "crafter/event@v1"
+criteria = [
+  { key = "progress.unique_achievements", weight = 0.9, description = "Return 1 when this decision explicitly unlocks a brand-new Crafter achievement (inventory or status text confirms it this turn). Otherwise return 0.", aggregation = "weighted_sum" },
+  { key = "process.intent_alignment", weight = 0.1, description = "Use at most 0.3 to acknowledge tightly coupled setup that finishes the last prerequisite; keep ≤0.1 when the agent only repositions or gathers without an imminent unlock.", aggregation = "weighted_sum" },
+]
+
+[rubric.outcome]
+# Hosted judge rubric for final trajectory scoring.
+rubric_id = "crafter/outcome@v1"
+criteria = [
+  { key = "outcome.goal_completion", weight = 0.6, description = "Full credit when the agent ends with strong survival metrics and a clear crafted milestone (e.g., iron tools, furnace).", aggregation = "weighted_sum" },
+  { key = "outcome.achievement_depth", weight = 0.4, description = "Partial credit for intermediate achievements (saplings, wood/stone tools) that set up future success.", aggregation = "weighted_sum" },
+]
+
+[judge]
+type = "gemini"                            # or "groq" when routing to Groq-hosted judges
+timeout_s = 45
+
+  [judge.options]
+  event = true
+  outcome = true
+  provider = "openai"
+  model = "openai/gpt-oss-120b"
+  rubric_id = "crafter/bundle@v1"
+  max_concurrency = 6
+  tracks = ["process", "reasoning", "progress", "outcome"]
+
+    [judge.options.rubric_overrides]
+
+      [judge.options.rubric_overrides.event]
+      goal_text = """
+Treat each decision as a check for new Crafter achievements.
+Award the top score only when the log shows a fresh achievement unlock or an immediately verifiable deterministic completion.
+Keep otherwise useful setup actions in a narrow low band so non-achievement turns stay near zero."""
+      aggregation = "weighted_sum"
+
+        [[judge.options.rubric_overrides.event.criteria]]
+        id = "progress.unique_achievements"
+        weight = 0.9
+        scale = "binary"
+        description = "Return 1 when this decision explicitly unlocks a brand-new Crafter achievement (inventory or status text confirms it this turn). Otherwise return 0."
+
+        [[judge.options.rubric_overrides.event.criteria]]
+        id = "process.intent_alignment"
+        weight = 0.1
+        scale = "bounded"
+        description = "Use at most 0.3 to acknowledge tightly coupled setup that finishes the last prerequisite; keep ≤0.1 when the agent only repositions or gathers without an imminent unlock."
+
+      [judge.options.rubric_overrides.outcome]
+      goal_text = """
+Summarise the episode outcome in relation to Crafter’s win condition:
+survive, accumulate resources, and craft advanced tools or structures.
+Highlight notable achievements, safety failures, and preparedness for future exploration."""
+      aggregation = "weighted_sum"
+
+        [[judge.options.rubric_overrides.outcome.criteria]]
+        id = "outcome.goal_completion"
+        weight = 0.6
+        scale = "binary"
+        description = "Full credit when the agent ends with strong survival metrics and a clear crafted milestone (e.g., iron tools, furnace)."
+
+        [[judge.options.rubric_overrides.outcome.criteria]]
+        id = "outcome.achievement_depth"
+        weight = 0.4
+        scale = "bounded"
+        description = "Partial credit for intermediate achievements (saplings, wood/stone tools) that set up future success."
+
+    [judge.options.weights]
+    process = 0.05
+    reasoning = 0.15
+    progress = 0.30
+    outcome = 0.50

examples/multi_step/configs/crafter_rl_stepwise_shaped.toml

@@ -0,0 +1,83 @@
+# Crafter RL experiment – shaped stepwise rewards (achievement + resource shaping)
+
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[services]
+# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
+task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2
+
+[topology]
+type = "single_node_split"
+gpus_for_vllm = 1
+gpus_for_training = 1
+gpus_for_ref = 0
+tensor_parallel = 1
+
+[vllm]
+tensor_parallel_size = 1
+max_model_len = 8192
+
+[reference]
+placement = "none"
+
+[model]
+base = "Qwen/Qwen3-4B"
+trainer_mode = "lora"
+label = "crafter-rl-stepwise-shaped"
+
+[lora]
+r = 16
+alpha = 32
+dropout = 0.05
+target_modules = ["all-linear"]
+
+[rollout]
+env_name = "crafter"
+max_turns = 10
+episodes_per_batch = 4
+policy_name = "crafter-react"
+max_concurrent_rollouts = 8
+batches_per_step = 2
+ops = ["agent", "env"]
+
+[evaluation]
+instances = 10
+every_n_iters = 10
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+
+[training]
+num_epochs = 1
+iterations_per_epoch = 10
+gradient_accumulation_steps = 1
+max_accumulated_minibatch = 1
+max_turns = 10
+batch_size = 4
+group_size = 4
+learning_rate = 5e-5
+log_interval = 1
+weight_sync_interval = 1
+step_rewards_enabled = true
+step_rewards_mode = "decision_stepwise"
+step_rewards_indicator_lambda = 0.5
+step_rewards_beta = 0.0
+event_rewards_kind = "unique"
+step_rewards_strategy = "per_achievement"
+
+# Reward each achievement up to a cap inside `compute_stepwise_reward`
+step_rewards_weights = { collect_sapling = 0.6, collect_wood = 0.8, collect_stone = 1.0, collect_iron = 1.2, collect_drink = 0.4, collect_food = 0.4 }
+step_rewards_k_limits = { collect_sapling = 2, collect_wood = 4, collect_stone = 3, collect_iron = 3, collect_drink = 3, collect_food = 3 }
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+mode = "direct"
+direct = true
+verify_every_k = 0

examples/multi_step/configs/crafter_rl_stepwise_simple.toml

@@ -0,0 +1,78 @@
+# Crafter RL experiment – simple stepwise rewards (1 point per new achievement)
+
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[services]
+# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-crafter`
+task_url = "https://YOUR-MODAL-TASK-APP.modal.run"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2
+
+[topology]
+type = "single_node_split"
+gpus_for_vllm = 1
+gpus_for_training = 1
+gpus_for_ref = 0
+tensor_parallel = 1
+
+[vllm]
+tensor_parallel_size = 1
+max_model_len = 8192
+
+[reference]
+placement = "none"
+
+[model]
+base = "Qwen/Qwen3-4B"
+trainer_mode = "lora"
+label = "crafter-rl-stepwise-simple"
+
+[lora]
+r = 16
+alpha = 32
+dropout = 0.05
+target_modules = ["all-linear"]
+
+[rollout]
+env_name = "crafter"
+max_turns = 10
+episodes_per_batch = 4
+policy_name = "crafter-react"
+max_concurrent_rollouts = 8
+batches_per_step = 2
+ops = ["agent", "env"]
+
+[evaluation]
+instances = 10
+every_n_iters = 10
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+
+[training]
+num_epochs = 1
+iterations_per_epoch = 10
+gradient_accumulation_steps = 1
+max_accumulated_minibatch = 1
+max_turns = 10
+batch_size = 4
+group_size = 4
+learning_rate = 5e-5
+log_interval = 1
+weight_sync_interval = 1
+step_rewards_enabled = true
+step_rewards_mode = "decision_stepwise"
+step_rewards_indicator_lambda = 1.0
+step_rewards_beta = 0.0
+event_rewards_kind = "unique"
+step_rewards_strategy = "consistent"
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+mode = "direct"
+direct = true
+verify_every_k = 0

examples/multi_step/crafter_rl_lora.md

@@ -2,28 +2,69 @@
 
 This walkthrough shows how to fine-tune the Crafter task app with our 10-step RL LoRA config.
 
-1. **Start the Crafter task app on Modal (with tracing + text-only prompts)**
+1. **Deploy the Crafter task app on Modal**
 
    ```bash
-   BACKEND_BASE_URL=https://agent-learning.onrender.com/api \
+   # assumes .env contains SYNTH_API_KEY, ENVIRONMENT_API_KEY, GROQ_API_KEY, etc.
    uvx synth-ai modal-serve grpo-crafter \
      --env-file examples/warming_up_to_rl/.env \
      --name grpo-crafter-task-app
    ```
 
-   * Deploys the Modal task app with the tracing/text-only fixes baked in.*
+   * The command prints the public `https://…modal.run` URL; copy it for the RL configs below.*
 
-2. **Launch the RL job using the updated LoRA config**
+2. **Wire up the three RL experiment configs**
+
+   Update the `task_url` placeholder in each config with the Modal URL from step 1:
+
+   - `examples/multi_step/configs/crafter_rl_outcome.toml`
+   - `examples/multi_step/configs/crafter_rl_stepwise_simple.toml`
+   - `examples/multi_step/configs/crafter_rl_stepwise_shaped.toml`
+
+   The difference between them (all run with LoRA on 2×H100 split 1/1 for vLLM vs. trainer):
+
+   | Config | Reward signal |
+   | ------ | ------------- |
+   | `crafter_rl_outcome.toml` | Outcome-only — step rewards disabled. |
+   | `crafter_rl_stepwise_simple.toml` | Stepwise (“consistent”) — +1 for every newly unlocked achievement. |
+   | `crafter_rl_stepwise_shaped.toml` | Stepwise (“per_achievement”) — combines achievement credit with inventory/achievement-count shaping from the rollout hook. |
+
+3. **Launch the three RL runs in parallel**
 
    ```bash
+   export SYNTH_API_KEY=...       # already sourced if examples/.env was loaded
+   export TASK_APP_URL=https://your-modal-task-app.modal.run
+
+   uvx synth-ai train --type rl \
+     --config examples/multi_step/configs/crafter_rl_outcome.toml \
+     --run-name crafter-rl-outcome \
+     --no-poll &
+
    uvx synth-ai train --type rl \
-     --config tests/artifacts/configs/rl.lora.small.toml \
-     --backend https://agent-learning.onrender.com/api \
-     --env-file .env \
-     --no-poll
+     --config examples/multi_step/configs/crafter_rl_stepwise_simple.toml \
+     --run-name crafter-rl-stepwise-simple \
+     --no-poll &
+
+   uvx synth-ai train --type rl \
+     --config examples/multi_step/configs/crafter_rl_stepwise_shaped.toml \
+     --run-name crafter-rl-stepwise-shaped \
+     --no-poll &
+
+   wait
    ```
 
-   * This config forces 10 agent turns per rollout, reduces batch size to avoid OOMs, and enforces Crafter-specific defaults.*
+   *`--no-poll` returns immediately so each run can stream logs in its own terminal; `wait` blocks until all jobs finish.*
+
+4. **Track results**
+
+   Tail each job’s logs with `uvx synth-ai train logs --run-name <name>` or open the Modal dashboard. Compare:
+
+   - Avg outcome reward (modal dashboard)
+   - Stepwise reward components (`resource_reward`, `unique_achievements_total`) in the task app logs
+   - Trace JSONL dumps under `traces/v3` if tracing is enabled
+
+
+  * This config forces 10 agent turns per rollout, reduces batch size to avoid OOMs, and enforces Crafter-specific defaults.*
 
   INFO - 🎉 Training completed successfully!
-  INFO - All batch rewards: [0.0625, 0.0625, 0.125, 0.0625, 0.0625, 0.3125, 0.375, 0.4375, 0.5, 0.9375]
+  INFO - All batch rewards: [0.0625, 0.0625, 0.125, 0.0625, 0.0625, 0.3125, 0.375, 0.4375, 0.5, 0.9375]