synth-ai 0.2.14__py3-none-any.whl → 0.2.17__py3-none-any.whl

examples/multi_step/configs/verilog_rl_lora.toml

@@ -1,40 +1,33 @@
-# Verilog RL experiment – LoRA training on Qwen3-0.6B
-#
-# This configuration adapts the Crafter RL setup for Verilog spec-to-RTL tasks.
-# Uses the same proven pipeline but optimized for 0.6B model and Verilog domain.
-
 [algorithm]
 type = "online"
 method = "policy_gradient"
 variety = "gspo"
 
 [services]
-# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-verilog`
 task_url = "https://synth-laboratories--grpo-verilog-task-app-fastapi-app-dev.modal.run"
-# Point at the Synth backend (or compatible service) that exposes /api/judge/v1/*
 judge_url = "https://synth-backend-dev-docker.onrender.com/api"
 
 [compute]
-gpu_type = "H200"  # ✅ 8B model needs H200 for larger context window
-gpu_count = 2      # ✅ Minimum 2x GPUs (1 for vLLM inference + 1 for training)
+gpu_type = "H200"
+gpu_count = 2
 nodes = 1
 
 [topology]
 type = "single_node_split"
-gpus_for_vllm = 1      # ✅ vLLM for inference
-gpus_for_training = 1  # ✅ Training GPU (8B LoRA fits well)
+gpus_for_vllm = 1
+gpus_for_training = 1
 gpus_for_ref = 0
 tensor_parallel = 1
 
 [vllm]
 tensor_parallel_size = 1
-max_model_len = 24576  # ✅ Increased to 24K to accommodate long Verilog prompts (16K + 8K buffer for testbenches + history)
+max_model_len = 24576
 
 [reference]
 placement = "none"
 
 [model]
-base = "Qwen/Qwen3-8B"  # ✅ 8B model for RL training with good balance of speed and capability
+base = "Qwen/Qwen3-8B"
 trainer_mode = "lora"
 label = "verilog-rl-lora-qwen8b"
 
@@ -42,38 +35,21 @@ label = "verilog-rl-lora-qwen8b"
 r = 16
 alpha = 32
 dropout = 0.05
-target_modules = ["all-linear"]
+target_modules = [ "all-linear",]
 
 [rollout]
-env_name = "verilog"  # ✅ Changed from "crafter" to "verilog"
-max_turns = 6        # ✅ More steps for compilation chains vs Crafter's 10
-episodes_per_batch = 4  # ✅ Good batch size for 8B model
+env_name = "verilog"
+max_turns = 6
+episodes_per_batch = 4
 policy_name = "verilog-designer"
 max_concurrent_rollouts = 8
 batches_per_step = 2
-ops = ["agent", "env"]
-
-  [rollout.env_config]
-  # Verilog-specific environment settings
-  difficulty = "medium"  # Can be "easy", "medium", or "hard"
-
-    [rollout.env_config.step_rewards]
-    enabled = true
-    mode = "decision_stepwise"
-    strategy = "consistent"
-    indicator_lambda = 0.5  # ✅ Reduced from Crafter (sparser rewards)
-    step_beta = 0.0
-
-  [rollout.policy_config]
-  provider = "openai"
-  model = "Qwen/Qwen3-8B"  # ✅ Use the model being trained (8B) for rollouts
-  temperature = 0.2
-  max_tokens = 4096  # ✅ Balanced for Verilog generation while leaving room for long input prompts (testbenches + history)
+ops = [ "agent", "env",]
 
 [evaluation]
 instances = 16
 every_n_iters = 10
-seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+seeds = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,]
 
 [training]
 num_epochs = 1
@@ -81,110 +57,91 @@ iterations_per_epoch = 5
 gradient_accumulation_steps = 1
 max_accumulated_minibatch = 1
 max_turns = 15
-batch_size = 4          # ✅ Same as Crafter (works well for 8B LoRA)
+batch_size = 4
 group_size = 4
-learning_rate = 5e-5    # ✅ Same as Crafter
+learning_rate = 5e-5
 log_interval = 1
 weight_sync_interval = 1
 event_rewards_kind = "unique"
-async_semaphore_max = 20  # Max concurrent rollouts in streaming pipeline
-
-# Enable dense decision rewards in the trainer
+async_semaphore_max = 20
 step_rewards_enabled = true
 step_rewards_mode = "decision_stepwise"
-step_rewards_indicator_lambda = 0.5  # ✅ Reduced for Verilog's sparser rewards
+step_rewards_indicator_lambda = 0.5
 step_rewards_beta = 0.0
 step_rewards_strategy = "consistent"
 
+[judge]
+enabled = true
+
+[rollout.env_config]
+difficulty = "medium"
+
+[rollout.policy_config]
+provider = "openai"
+model = "Qwen/Qwen3-8B"
+temperature = 0.2
+max_tokens = 4096
+
 [training.weight_sync]
 enable = true
-targets = ["policy"]
+targets = [ "policy",]
 mode = "direct"
 direct = true
 verify_every_k = 0
 
-[rubric]
-enabled = true
+[judge.reward_blend]
+env = 0.3
+event = 0.3
+outcome = 0.4
+
+[judge.options]
+event = true
+outcome = true
+provider = "openai"
 model = "openai/gpt-oss-120b"
-api_base = "https://synth-backend-dev-docker.onrender.com/api/judge"
-api_key_env = "OPENAI_API_KEY"
+rubric_id = "verilog/bundle@v1"
+timeout_s = 45
 
-# Blend the hosted judge scores with environment returns
-[rubric.weights]
-env = 0.3     # ✅ Higher weight on env rewards for Verilog (vs Crafter's 0.2)
-event = 0.3   # ✅ Adjusted for Verilog's different reward structure
+[rollout.env_config.step_rewards]
+enabled = true
+mode = "decision_stepwise"
+strategy = "consistent"
+indicator_lambda = 0.5
+step_beta = 0.0
+
+[judge.options.weights]
+process = 0.1
+reasoning = 0.2
+progress = 0.3
 outcome = 0.4
 
-[rubric.event]
-# Verilog-specific event rubric for process efficiency
-rubric_id = "verilog/event@v1"
-criteria = [
-  { key = "process.compilation_success", weight = 0.7, description = "Return 1.0 when compilation succeeds, 0.5 for partial success, 0.0 for failure", aggregation = "weighted_sum" },
-  { key = "process.design_iterations", weight = 0.3, description = "Reward efficient design iterations without unnecessary recompilation", aggregation = "weighted_sum" },
-]
-
-[rubric.outcome]
-# Verilog-specific outcome rubric for final results
-rubric_id = "verilog/outcome@v1"
-criteria = [
-  { key = "outcome.tests_passed", weight = 0.8, description = "Full credit when all tests pass, partial for some tests", aggregation = "weighted_sum" },
-  { key = "outcome.design_quality", weight = 0.2, description = "Code quality, documentation, and design efficiency", aggregation = "weighted_sum" },
-]
-
-[judge]
-type = "groq"
-timeout_s = 45
+[judge.options.rubric_overrides.event]
+goal_text = "      Evaluate each Verilog design decision for compilation success and process efficiency.\n      High scores for successful compilation and strategic tool usage.\n      Penalize unnecessary operations and compilation failures."
+aggregation = "weighted_sum"
+[[judge.options.rubric_overrides.event.criteria]]
+id = "process.compilation_success"
+weight = 0.7
+scale = "bounded"
+description = "Return 1.0 when compilation succeeds cleanly, 0.5 for warnings, 0.0 for errors"
+
+[[judge.options.rubric_overrides.event.criteria]]
+id = "process.design_iterations"
+weight = 0.3
+scale = "bounded"
+description = "Reward efficient write→compile→simulate workflow, penalize redundant operations"
+
+[judge.options.rubric_overrides.outcome]
+goal_text = "      Evaluate the final Verilog implementation for correctness and quality.\n      High scores for working designs that pass all tests with good code quality."
+aggregation = "weighted_sum"
+[[judge.options.rubric_overrides.outcome.criteria]]
+id = "outcome.tests_passed"
+weight = 0.8
+scale = "binary"
+description = "Full credit when all tests pass, partial credit for some tests passing"
+
+[[judge.options.rubric_overrides.outcome.criteria]]
+id = "outcome.design_quality"
+weight = 0.2
+scale = "bounded"
+description = "Code clarity, proper documentation, and efficient design patterns"
 
-  [judge.options]
-  event = true
-  outcome = true
-  provider = "openai"
-  model = "openai/gpt-oss-120b"
-  rubric_id = "verilog/bundle@v1"
-  max_concurrency = 6
-  tracks = ["process", "reasoning", "progress", "outcome"]
-
-    [judge.options.rubric_overrides]
-
-      [judge.options.rubric_overrides.event]
-      goal_text = """
-      Evaluate each Verilog design decision for compilation success and process efficiency.
-      High scores for successful compilation and strategic tool usage.
-      Penalize unnecessary operations and compilation failures."""
-      aggregation = "weighted_sum"
-
-        [[judge.options.rubric_overrides.event.criteria]]
-        id = "process.compilation_success"
-        weight = 0.7
-        scale = "bounded"
-        description = "Return 1.0 when compilation succeeds cleanly, 0.5 for warnings, 0.0 for errors"
-
-        [[judge.options.rubric_overrides.event.criteria]]
-        id = "process.design_iterations"
-        weight = 0.3
-        scale = "bounded"
-        description = "Reward efficient write→compile→simulate workflow, penalize redundant operations"
-
-      [judge.options.rubric_overrides.outcome]
-      goal_text = """
-      Evaluate the final Verilog implementation for correctness and quality.
-      High scores for working designs that pass all tests with good code quality."""
-      aggregation = "weighted_sum"
-
-        [[judge.options.rubric_overrides.outcome.criteria]]
-        id = "outcome.tests_passed"
-        weight = 0.8
-        scale = "binary"
-        description = "Full credit when all tests pass, partial credit for some tests passing"
-
-        [[judge.options.rubric_overrides.outcome.criteria]]
-        id = "outcome.design_quality"
-        weight = 0.2
-        scale = "bounded"
-        description = "Code clarity, proper documentation, and efficient design patterns"
-
-    [judge.options.weights]
-    process = 0.1
-    reasoning = 0.2
-    progress = 0.3
-    outcome = 0.4

examples/multi_step/convert_traces_to_sft.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Convert Crafter trace format to SFT format with messages[] structure."""
+
+import json
+import sys
+from pathlib import Path
+
+def convert_trace_to_sft(trace: dict) -> dict:
+    """Convert a single trace to SFT format."""
+    # Extract dialogue from trace
+    dialogue = trace.get("dialogue", [])
+    assistant = trace.get("assistant", {})
+    
+    # Build messages list
+    messages = []
+    
+    # Add dialogue history
+    for msg in dialogue:
+        messages.append({
+            "role": msg["role"],
+            "content": msg["content"]
+        })
+    
+    # Add assistant response if present
+    if assistant:
+        content = assistant.get("content", "")
+        tool_calls = assistant.get("tool_calls", [])
+        
+        # If there are tool calls, format them
+        if tool_calls:
+            # Convert tool calls to a simple text format for SFT
+            tool_text = "\n".join([
+                f"Tool: {tc['name']}\nArguments: {json.dumps(tc.get('arguments', {}))}"
+                for tc in tool_calls
+            ])
+            content = f"{content}\n\n{tool_text}".strip()
+        
+        messages.append({
+            "role": "assistant",
+            "content": content
+        })
+    
+    return {"messages": messages}
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python convert_traces_to_sft.py <input.jsonl> [output.jsonl]")
+        sys.exit(1)
+    
+    input_path = Path(sys.argv[1])
+    output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else input_path.with_name(f"{input_path.stem}_sft_format.jsonl")
+    
+    if not input_path.exists():
+        print(f"Error: Input file not found: {input_path}")
+        sys.exit(1)
+    
+    print(f"Converting {input_path} → {output_path}")
+    
+    converted = 0
+    skipped = 0
+    
+    with open(input_path) as f_in, open(output_path, "w") as f_out:
+        for line_no, line in enumerate(f_in, 1):
+            try:
+                trace = json.loads(line.strip())
+                sft_entry = convert_trace_to_sft(trace)
+                
+                # Only write if we have messages
+                if sft_entry["messages"]:
+                    f_out.write(json.dumps(sft_entry) + "\n")
+                    converted += 1
+                else:
+                    skipped += 1
+                    
+            except Exception as e:
+                print(f"Warning: Skipping line {line_no}: {e}")
+                skipped += 1
+    
+    print(f"✅ Converted {converted} entries, skipped {skipped}")
+    print(f"Output: {output_path}")
+
+if __name__ == "__main__":
+    main()
+

examples/multi_step/run_sft_qwen30b.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Run SFT for Qwen3-Coder-30B with LoRA on Crafter data
+
+# Usage:
+#   ./run_sft_qwen30b.sh <dataset_path> [env_file]
+#
+# Example:
+#   ./run_sft_qwen30b.sh examples/multi_step/ft_data/crafter_traces.jsonl
+#   ./run_sft_qwen30b.sh examples/multi_step/ft_data/crafter_traces.jsonl backend/.env.dev
+
+set -e
+
+DATASET_PATH="${1:-examples/sft/ft_data/crafter_traces.jsonl}"
+ENV_FILE="${2:-backend/.env.dev}"
+
+if [ ! -f "$DATASET_PATH" ]; then
+    echo "Error: Dataset not found at $DATASET_PATH"
+    echo "Usage: $0 <dataset_path> [env_file]"
+    exit 1
+fi
+
+if [ ! -f "$ENV_FILE" ]; then
+    echo "Error: Env file not found at $ENV_FILE"
+    echo "Usage: $0 <dataset_path> [env_file]"
+    exit 1
+fi
+
+echo "🚀 Starting SFT training for Qwen3-Coder-30B with LoRA"
+echo "   Model: Qwen/Qwen3-Coder-30B-A3B-Instruct"
+echo "   Dataset: $DATASET_PATH"
+echo "   Config: examples/multi_step/configs/crafter_sft_qwen30b_lora.toml"
+echo "   GPUs: 4x H200"
+echo "   LoRA: r=16, alpha=32, all-linear"
+echo ""
+
+uvx synth-ai train \
+  --type sft \
+  --config examples/multi_step/configs/crafter_sft_qwen30b_lora.toml \
+  --dataset "$DATASET_PATH" \
+  --env-file "$ENV_FILE"
+
+echo ""
+echo "✅ SFT training job submitted!"
+echo "   Monitor progress in your Synth dashboard"
+

examples/qwen_coder/configs/coder_lora_30b.toml

@@ -3,7 +3,7 @@
 [algorithm]
 type = "offline"
 method = "sft"
-variety = "lora"
+variety = "qlora"
 
 [job]
 model = "Qwen/Qwen3-Coder-30B-A3B-Instruct"
@@ -58,4 +58,3 @@ alpha = 32
 dropout = 0.05
 target_modules = ["all-linear"]
 
-

examples/qwen_coder/configs/coder_lora_4b.toml

@@ -1,5 +1,10 @@
 # Qwen3 Coder 4B LoRA SFT – all-linear adapters
 
+[algorithm]
+type = "offline"
+method = "sft"
+variety = "qlora"
+
 [job]
 model = "Qwen/Qwen3-4B"
 
@@ -54,4 +59,3 @@ dropout = 0.05
 target_modules = ["all-linear"]
 
 
-

examples/qwen_coder/configs/coder_lora_small.toml

@@ -3,7 +3,7 @@
 [algorithm]
 type = "offline"
 method = "sft"
-variety = "fft"
+variety = "qlora"
 
 [job]
 # Smallest supported Qwen3 base; replace with the smallest Coder variant when available
@@ -55,4 +55,3 @@ alpha = 32
 dropout = 0.05
 target_modules = ["all-linear"]
 
-

examples/qwen_vl/BUGS_AND_FIXES.md

@@ -0,0 +1,232 @@
+# Vision SFT Pipeline - Bugs and Fixes
+
+Complete log of issues encountered and resolved during vision data collection setup.
+
+## ✅ Issue #1: Import Error - CrafterEnvironment
+
+**Problem:**
+```python
+ImportError: cannot import name 'CrafterEnvironment' from 'examples.task_apps.crafter.task_app.synth_envs_hosted.envs.crafter.environment'
+```
+
+**Root Cause:**  
+Class is named `CrafterEnvironmentWrapper`, not `CrafterEnvironment`
+
+**Fix:**  
+Updated imports and usages in:
+- `crafter_gpt5nano_agent.py`
+- `crafter_qwen_vl_agent.py`
+- `collect_vision_traces.py`
+
+```python
+# Before
+from ...environment import CrafterEnvironment
+wrapper = CrafterEnvironment(env, seed=seed)
+
+# After
+from ...environment import CrafterEnvironmentWrapper
+wrapper = CrafterEnvironmentWrapper(env, seed=seed)
+```
+
+**Status:** FIXED ✓
+
+---
+
+## ✅ Issue #2: OpenAI API Parameter - max_tokens
+
+**Problem:**
+```
+openai.BadRequestError: Error code: 400 - {'error': {'message': "Unsupported parameter: 'max_tokens' is not supported with this model. Use 'max_completion_tokens' instead."}}
+```
+
+**Root Cause:**  
+gpt-5 models require `max_completion_tokens` parameter instead of `max_tokens`
+
+**Fix:**  
+Updated `_normalise_openai_request()` function to detect gpt-5 models:
+
+```python
+def _normalise_openai_request(payload, model, temperature):
+    request = dict(payload)
+    request["model"] = model
+    
+    # gpt-5 models use max_completion_tokens, not max_tokens
+    if "gpt-5" in model.lower():
+        request.setdefault("max_completion_tokens", 512)
+        request.pop("max_tokens", None)  # Remove if present
+    else:
+        # Older models use max_tokens
+        request.setdefault("max_tokens", 512)
+    
+    return request
+```
+
+**Files Updated:**
+- `crafter_gpt5nano_agent.py`
+- `collect_vision_traces.py`
+
+**Status:** FIXED ✓
+
+---
+
+## ✅ Issue #3: OpenAI API Parameter - temperature
+
+**Problem:**
+```
+openai.BadRequestError: Error code: 400 - {'error': {'message': "Unsupported value: 'temperature' does not support 0.6 with this model. Only the default (1) value is supported."}}
+```
+
+**Root Cause:**  
+gpt-5-nano only supports `temperature=1` (default), custom temperature values are not allowed
+
+**Fix:**  
+Remove temperature parameter for gpt-5 models:
+
+```python
+def _normalise_openai_request(payload, model, temperature):
+    # ...
+    
+    if "gpt-5" in model.lower():
+        # gpt-5-nano only supports temperature=1 (default)
+        request.pop("temperature", None)  # Remove custom temperature
+        request.setdefault("max_completion_tokens", 512)
+        request.pop("max_tokens", None)
+    else:
+        # Older models support custom temperature
+        request.setdefault("temperature", temperature)
+        request.setdefault("max_tokens", 512)
+    
+    return request
+```
+
+**Files Updated:**
+- `crafter_gpt5nano_agent.py`
+- `collect_vision_traces.py`
+
+**Status:** FIXED ✓
+
+---
+
+## ⚠️  Issue #4: gpt-5-nano Tool Calling Support
+
+**Problem:**
+```
+Seed 0: no tool calls returned by model; ending episode early at step 0.
+```
+
+**Root Cause:**  
+gpt-5-nano does not appear to support function/tool calling yet, or requires a different prompt format for tool use.
+
+**Testing Results:**
+- API returned 200 OK (auth and network fine)
+- Model processed vision inputs successfully
+- Model did not return tool calls even with tools schema provided
+- Both episodes stopped immediately (step 0)
+
+**Workaround:**  
+Switch to `gpt-4o-mini-2024-07-18` for data collection:
+- Confirmed to support both vision AND tool calling
+- Successfully completed 10 episodes with good quality
+- Mean 2.6 achievements per episode
+- 685 total tool calls across 10 episodes
+
+**Status:** WORKAROUND APPLIED (use gpt-4o-mini) ✓
+
+**Note:**  
+This is a model capability limitation, not a code bug. gpt-5-nano can be revisited when tool calling support is confirmed by OpenAI.
+
+---
+
+## 📊 Final Validation Results
+
+### Test Run #5: 10-Episode Collection with gpt-4o-mini
+
+**Command:**
+```bash
+uv run python examples/qwen_vl/crafter_gpt5nano_agent.py \
+  --model gpt-4o-mini-2024-07-18 \
+  --seeds 10 \
+  --steps 50
+```
+
+**Results:**
+```
+✓ All 10 episodes completed (50 steps each)
+✓ Mean achievements: 2.6 per episode
+✓ Total tool calls: 685
+✓ Vision processing: Working (64x64 PNG frames)
+✓ Tool calling: Working (proper tool call format)
+✓ Frame saving: Working (saved to output directory)
+✓ Performance: ~5-6 minutes for 10 episodes
+```
+
+**Quality Metrics:**
+- Episode 1: 4 achievements, 72 tool calls, reward: 97.3
+- Episode 5: 3 achievements, 62 tool calls, reward: 120.0
+- Episode 8: 1 achievement, 71 tool calls, reward: 12.9
+- Good variety in performance (1-4 achievements)
+
+---
+
+## 🔧 Code Changes Summary
+
+### Files Modified:
+1. **crafter_gpt5nano_agent.py**
+   - Import: `CrafterEnvironment` → `CrafterEnvironmentWrapper`
+   - Function: `_normalise_openai_request()` - handle gpt-5 parameters
+
+2. **crafter_qwen_vl_agent.py**
+   - Import: `CrafterEnvironment` → `CrafterEnvironmentWrapper`
+
+3. **collect_vision_traces.py**
+   - Import: `CrafterEnvironment` → `CrafterEnvironmentWrapper`
+   - Function: `_normalise_openai_request()` - handle gpt-5 parameters
+
+### Key Learnings:
+1. ✅ Always check actual class names in source code
+2. ✅ OpenAI's API evolves - newer models have different parameter requirements
+3. ✅ Test with known-working models first (gpt-4o-mini) before trying cutting-edge ones
+4. ✅ Vision + tool calling combo requires mature model support
+
+---
+
+## 🎯 Recommendations
+
+### For Production:
+- **Teacher model:** Use `gpt-4o-mini-2024-07-18` for data collection
+  - Proven to work with vision + tools
+  - Good quality (2-4 achievements per episode)
+  - Reasonable cost
+
+- **Monitor gpt-5-nano:** Revisit when tool calling support is confirmed
+
+### For Configs:
+- Update eval configs to use `gpt-4o-mini` by default:
+  ```toml
+  [eval]
+  model = "gpt-4o-mini-2024-07-18"  # Not gpt-5-nano
+  ```
+
+---
+
+## ✅ All Issues Resolved
+
+**Infrastructure Status:** READY FOR PRODUCTION ✓
+
+- Vision processing: Working
+- Tool calling: Working  
+- Frame saving: Working
+- OpenAI API integration: Working
+- 10-episode test: Successful
+
+**Next Steps:**
+1. Scale to 100 episodes for full dataset
+2. Apply filters and export to SFT format
+3. Train VLM with LoRA
+4. Fine-tune with RL
+
+---
+
+**Last Updated:** 2025-10-26  
+**Test Environment:** synth-ai dev, macOS, Python 3.11
+