synth-ai 0.2.13.dev2__py3-none-any.whl → 0.2.16__py3-none-any.whl

examples/multi_step/judges/verilog_backend_judge.py

@@ -0,0 +1,234 @@
+"""Verilog backend judge that calls the Synth judge API with inline rubric."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, List, TypedDict
+
+import httpx
+
+
+class TraceMetadata(TypedDict, total=False):
+    """Metadata for the trace."""
+    env_id: str
+    policy_id: str
+    length: int
+
+
+class JudgeTracePayload(TypedDict):
+    """Trace payload sent to backend judge."""
+    event_history: List[Dict[str, Any]]
+    markov_blanket_message_history: List[Dict[str, Any]]
+    metadata: TraceMetadata
+
+
+class JudgeOptions(TypedDict, total=False):
+    """Options for judge scoring."""
+    model: str
+    timeout_s: int
+    event: bool
+    outcome: bool
+
+
+class TaskApp(TypedDict):
+    """Task application metadata."""
+    id: str
+
+
+class JudgeScoreRequest(TypedDict):
+    """Request to backend judge API."""
+    policy_name: str
+    task_app: TaskApp
+    trace: JudgeTracePayload
+    rubric: Dict[str, Any]
+    options: JudgeOptions
+
+
+# Load rubric from file (cached at module level)
+_RUBRIC_PATH = Path(__file__).parent.parent / "rubrics" / "verilog_backend_judge.json"
+_RUBRIC: Dict[str, Any] | None = None
+
+
+def _load_rubric() -> Dict[str, Any]:
+    """Load rubric from file with fallback to inline default."""
+    global _RUBRIC
+    if _RUBRIC is None:
+        try:
+            with open(_RUBRIC_PATH, 'r') as f:
+                _RUBRIC = json.load(f)
+            assert isinstance(_RUBRIC, dict), "Rubric must be a dict"
+            assert "outcome" in _RUBRIC, "Rubric must have 'outcome' key"
+            assert isinstance(_RUBRIC["outcome"], list), "Rubric 'outcome' must be a list"
+        except Exception as e:
+            print(f"[verilog_backend_judge] Warning: Failed to load rubric from {_RUBRIC_PATH}: {e}")
+            # Fallback inline rubric (matching RubricCriteriaBlock format)
+            _RUBRIC = {
+                "event": [],
+                "outcome": [
+                    {"id": "correctness.tests_pass", "description": "Tests pass", "weight": 0.5, "scale": "bounded"},
+                    {"id": "efficiency.code_quality", "description": "Code quality", "weight": 0.3, "scale": "bounded"},
+                    {"id": "efficiency.solution_steps", "description": "Solution efficiency", "weight": 0.2, "scale": "bounded"}
+                ]
+            }
+    return _RUBRIC
+
+
+def judge(payload: Dict[str, Any], **kwargs: Any) -> float:
+    """
+    Call the Synth backend judge API to score a Verilog rollout.
+
+    Args:
+        payload: Dict with keys: seed, prompt, completion, metrics, response, trace
+        **kwargs: Additional config (backend_url, model, timeout_s, etc.)
+
+    Returns:
+        float: Aggregate score from 0.0 to 1.0
+    """
+    try:
+        # Extract configuration
+        backend_url = kwargs.get("backend_url", "http://localhost:8000/api")
+        model = kwargs.get("model", "openai/gpt-oss-120b")
+        timeout = kwargs.get("timeout_s", 45)
+        
+        assert isinstance(backend_url, str), "backend_url must be a string"
+        assert isinstance(model, str), "model must be a string"
+        assert isinstance(timeout, (int, float)), "timeout_s must be numeric"
+        
+        # Extract trajectory from response
+        response_data = payload.get("response", {})
+        assert isinstance(response_data, dict), "response must be a dict"
+        
+        trajectories = response_data.get("trajectories", [])
+        assert isinstance(trajectories, list), "trajectories must be a list"
+        
+        if not trajectories:
+            print("[verilog_backend_judge] No trajectories in response")
+            return 0.0
+        
+        trajectory = trajectories[0]  # First trajectory
+        assert isinstance(trajectory, dict), "trajectory must be a dict"
+        
+        # Load rubric
+        rubric = _load_rubric()
+        
+        # Transform trajectory into JudgeTracePayload format
+        # The backend expects: event_history, markov_blanket_message_history, metadata
+        steps = trajectory.get("steps", [])
+        assert isinstance(steps, list), "trajectory steps must be a list"
+        
+        event_history: List[Dict[str, Any]] = []
+        for idx, step in enumerate(steps):
+            assert isinstance(step, dict), f"step {idx} must be a dict"
+            # Each step becomes an event with obs, tool_calls, reward, done, info
+            event_history.append({
+                "observation": step.get("obs", {}),
+                "tool_calls": step.get("tool_calls", []),
+                "reward": step.get("reward", 0.0),
+                "done": step.get("done", False),
+                "truncated": step.get("truncated", False),
+                "info": step.get("info", {}),
+            })
+        
+        # Add final observation - backend will extract this as outcome context
+        final_data = trajectory.get("final", {})
+        if final_data:
+            assert isinstance(final_data, dict), "final data must be a dict"
+            final_obs = final_data.get("observation", {})
+            assert isinstance(final_obs, dict), "final observation must be a dict"
+            
+            event_history.append({
+                "observation": final_obs,
+                "reward": final_data.get("reward", 0.0),
+                "done": final_data.get("done", True),
+                "truncated": final_data.get("truncated", False),
+                "info": final_data.get("info", {}),
+            })
+        
+        # Build trace metadata
+        metadata: TraceMetadata = {
+            "env_id": trajectory.get("env_id", "verilog"),
+            "policy_id": trajectory.get("policy_id", "verilog-designer"),
+            "length": trajectory.get("length", len(steps)),
+        }
+        
+        # Build judge request with rubric included
+        judge_request: JudgeScoreRequest = {
+            "policy_name": "verilog-designer",
+            "task_app": {"id": "grpo-verilog"},
+            "trace": {
+                "event_history": event_history,
+                "markov_blanket_message_history": [],
+                "metadata": metadata,
+            },
+            "rubric": rubric,
+            "options": {
+                "model": model,
+                "timeout_s": timeout,
+                "event": False,  # Not scoring per-event
+                "outcome": True,  # Score the final outcome
+            }
+        }
+        
+        # Call backend judge API
+        with httpx.Client(timeout=timeout) as client:
+            # Get API key from env
+            api_key = os.environ.get("SYNTH_API_KEY") or os.environ.get("OPENAI_API_KEY")
+            headers = {}
+            if api_key:
+                headers["Authorization"] = f"Bearer {api_key}"
+            
+            url = f"{backend_url.rstrip('/')}/judge/v1/score"
+            
+            # Debug: print request details
+            print(f"\n[verilog_backend_judge] REQUEST DEBUG:")
+            print(f"  URL: {url}")
+            print(f"  Request body keys: {list(judge_request.keys())}")
+            rubric_data = judge_request.get('rubric', {})
+            print(f"  Rubric event criteria: {len(rubric_data.get('event', []))}")
+            print(f"  Rubric outcome criteria: {len(rubric_data.get('outcome', []))}")
+            trace_data = judge_request.get('trace', {})
+            event_hist = trace_data.get('event_history', [])
+            print(f"  Trace event_history count: {len(event_hist)}")
+            if event_hist:
+                last_event = event_hist[-1]
+                last_obs = last_event.get('observation', {})
+                print(f"  Last event done: {last_event.get('done', False)}")
+                print(f"  Last obs keys: {list(last_obs.keys())}")
+                print(f"  Task completed: {last_obs.get('task_completed', 'N/A')}")
+            
+            response = client.post(url, json=judge_request, headers=headers)
+            
+            # Debug: print response details
+            print(f"\n[verilog_backend_judge] RESPONSE DEBUG:")
+            print(f"  Status: {response.status_code}")
+            print(f"  Response body: {response.text[:500]}")  # First 500 chars
+            
+            response.raise_for_status()
+            result = response.json()
+            assert isinstance(result, dict), "Response must be a dict"
+            
+            # Extract aggregate score
+            aggregate_score = result.get("aggregate_score", 0.0)
+            
+            # Try outcome_review.total if aggregate_score not found
+            if aggregate_score == 0.0 and "outcome_review" in result:
+                outcome_review = result["outcome_review"]
+                if isinstance(outcome_review, dict):
+                    aggregate_score = outcome_review.get("total", 0.0)
+            
+            print(f"  Aggregate score: {aggregate_score}\n")
+            return float(aggregate_score)
+            
+    except httpx.HTTPStatusError as e:
+        print(f"\n[verilog_backend_judge] HTTP ERROR:")
+        print(f"  Status: {e.response.status_code}")
+        print(f"  Response body: {e.response.text}\n")
+        return 0.0
+    except AssertionError as e:
+        print(f"[verilog_backend_judge] Assertion error: {e}")
+        return 0.0
+    except Exception as e:
+        print(f"[verilog_backend_judge] Unexpected error: {e}")
+        return 0.0

examples/multi_step/readme.md

@@ -0,0 +1,48 @@
+
+
+Crafter
+
+cd /Users/joshpurtell/Documents/GitHub/synth-ai && uvx synth-ai modal-serve grpo-crafter-task-app --name grpo-crafter-task-app --env-file /Users/joshpurtell/Documents/GitHub/monorepo/environments/crafter/.env
+
+cd /Users/joshpurtell/Documents/GitHub/monorepo && uv run modal deploy backend/app/routes/clustered_training/core/algorithms/gspo/app.py --env dev
+
+uvx synth-ai eval --config /Users/joshpurtell/Documents/GitHub/synth-ai/examples/multi_step/configs/crafter_eval_text_only_groq_qwen32b.toml
+
+
+uvx synth-ai train \
+  --type rl \
+  --config /Users/joshpurtell/Documents/GitHub/synth-ai/examples/multi_step/configs/crafter_rl_stepwise_hosted_judge.toml \
+  --task-url https://synth-laboratories--grpo-crafter-task-app-fastapi-app-dev.modal.run \
+  --backend https://synth-backend-dev-docker.onrender.com/api \
+  --env-file /Users/joshpurtell/Documents/GitHub/monorepo/environments/crafter/.env
+
+
+
+---
+
+Verilog
+
+# 1. Deploy Verilog task app
+cd /Users/joshpurtell/Documents/GitHub/synth-ai && uvx synth-ai modal-serve grpo-verilog --name grpo-verilog-task-app --env-file /Users/joshpurtell/Documents/GitHub/monorepo/environments/verilog/.env
+
+# 2. Baseline eval using Synth backend (pre-training)
+uvx synth-ai eval --config /Users/joshpurtell/Documents/GitHub/synth-ai/examples/multi_step/configs/verilog_eval_synth_qwen4b.toml
+
+# 3. (Optional) External reference eval using Groq Qwen 32B
+uvx synth-ai eval --config /Users/joshpurtell/Documents/GitHub/synth-ai/examples/multi_step/configs/verilog_eval_groq_qwen32b.toml
+
+# 4. Deploy training backend
+cd /Users/joshpurtell/Documents/GitHub/monorepo && uv run modal deploy backend/app/routes/clustered_training/core/algorithms/gspo/app.py --env dev
+
+# 5. Run RL training
+uvx synth-ai train \
+  --type rl \
+  --config /Users/joshpurtell/Documents/GitHub/synth-ai/examples/multi_step/configs/verilog_rl_lora.toml \
+  --task-url https://synth-laboratories--grpo-verilog-task-app-fastapi-app-dev.modal.run \
+  --backend https://synth-backend-dev-docker.onrender.com/api \
+  --env-file /Users/joshpurtell/Documents/GitHub/monorepo/environments/verilog/.env
+
+# 6. Post-training eval (update job_id in config first!)
+# After training, note the job_id from logs (e.g., job_19a1823e56303de604f)
+# Update verilog_eval_synth_trained_qwen8b.toml with your job_id
+uvx synth-ai eval --config /Users/joshpurtell/Documents/GitHub/synth-ai/examples/multi_step/configs/verilog_eval_synth_trained_qwen8b.toml

examples/multi_step/run_sft_qwen30b.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Run SFT for Qwen3-Coder-30B with LoRA on Crafter data
+
+# Usage:
+#   ./run_sft_qwen30b.sh <dataset_path> [env_file]
+#
+# Example:
+#   ./run_sft_qwen30b.sh examples/multi_step/ft_data/crafter_traces.jsonl
+#   ./run_sft_qwen30b.sh examples/multi_step/ft_data/crafter_traces.jsonl backend/.env.dev
+
+set -e
+
+DATASET_PATH="${1:-examples/sft/ft_data/crafter_traces.jsonl}"
+ENV_FILE="${2:-backend/.env.dev}"
+
+if [ ! -f "$DATASET_PATH" ]; then
+    echo "Error: Dataset not found at $DATASET_PATH"
+    echo "Usage: $0 <dataset_path> [env_file]"
+    exit 1
+fi
+
+if [ ! -f "$ENV_FILE" ]; then
+    echo "Error: Env file not found at $ENV_FILE"
+    echo "Usage: $0 <dataset_path> [env_file]"
+    exit 1
+fi
+
+echo "🚀 Starting SFT training for Qwen3-Coder-30B with LoRA"
+echo "   Model: Qwen/Qwen3-Coder-30B-A3B-Instruct"
+echo "   Dataset: $DATASET_PATH"
+echo "   Config: examples/multi_step/configs/crafter_sft_qwen30b_lora.toml"
+echo "   GPUs: 4x H200"
+echo "   LoRA: r=16, alpha=32, all-linear"
+echo ""
+
+uvx synth-ai train \
+  --type sft \
+  --config examples/multi_step/configs/crafter_sft_qwen30b_lora.toml \
+  --dataset "$DATASET_PATH" \
+  --env-file "$ENV_FILE"
+
+echo ""
+echo "✅ SFT training job submitted!"
+echo "   Monitor progress in your Synth dashboard"
+

examples/multi_step/verilog_rl_lora.md

@@ -0,0 +1,218 @@
+# Verilog RL with LoRA Analysis
+
+## Executive Summary
+
+**✅ YES, Verilog can absolutely do RL with LoRA just like Crafter!** The architecture is nearly identical, but there are important considerations around model size and task complexity.
+
+## Architecture Compatibility ✅
+
+### **Same Foundation** (No changes needed)
+- ✅ **Contracts**: Uses identical `RolloutRequest`/`RolloutResponse` as Crafter
+- ✅ **Task App Framework**: Same `synth_ai.task.apps` framework
+- ✅ **Environment Pattern**: Same `StatefulEnvironment` + tool-based architecture
+- ✅ **Rubrics System**: Same evaluation and reward system
+- ✅ **Trace Correlation**: Already implemented in `rollout_executor` (line 817 in `grpo_verilog.py`)
+- ✅ **Modal Deployment**: Same deployment pattern as Crafter
+
+### **Key Differences** (Considerations for LoRA)
+
+#### 1. **Model Size: 8x Larger** ⚠️
+```toml
+# Verilog (current)
+model = "qwen/qwen3-32b"  # 32B parameters
+
+# Crafter (working)
+model = "Qwen/Qwen3-4B"   # 4B parameters
+```
+**Impact**: Memory requirements 8x higher for LoRA training
+**Solution**: Use gradient checkpointing, smaller batch sizes, or distributed training
+
+#### 2. **Tool Set: Simpler but More Structured**
+```python
+# Verilog Tools (4 tools)
+TOOLS = ["write_file", "compile", "simulate", "submit"]
+
+# Crafter Tools (20+ tools)
+# craft, move, attack, gather, etc.
+```
+
+**Verilog Advantages**:
+- ✅ **Deterministic**: Write → Compile → Simulate → Submit workflow
+- ✅ **Clear Success Criteria**: Tests pass = high reward
+- ✅ **Sparse but Meaningful Rewards**: +10 for submit success, +1 for simulation pass
+
+**Verilog Challenges**:
+- ❌ **Sparser Rewards**: Fewer intermediate signals for learning
+- ❌ **Longer Sequences**: Multi-step compilation chains
+- ❌ **Error Recovery**: Must debug compilation failures
+
+#### 3. **State Representation**
+```python
+# Verilog State (file-based)
+{
+    "files": {"TopModule.v": "module TopModule(..."},
+    "compile_status": "Last compile: Success",
+    "simulate_status": "Last simulation: Passed",
+    "task_completed": false
+}
+
+# Crafter State (world-based)
+{
+    "inventory": {"wood": 5, "stone": 3},
+    "position": [x, y],
+    "nearby_entities": [...],
+    "achievement_unlocked": true
+}
+```
+
+## Configuration for LoRA RL
+
+### **Option 1: Qwen3-0.6B (Recommended for testing)** ⭐
+```toml
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[model]
+base = "Qwen/Qwen3-0.6B"  # ✅ Same as existing SFT configs
+trainer_mode = "lora"
+
+[lora]
+r = 16
+alpha = 32
+dropout = 0.05
+target_modules = ["all-linear"]
+
+[rollout]
+env_name = "verilog"
+max_turns = 15
+policy_name = "verilog-designer"
+
+[training]
+batch_size = 4  # ✅ Same as Crafter
+gradient_accumulation_steps = 1
+```
+
+### **Option 2: Qwen3-32B (Production)** ⚠️
+```toml
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[model]
+base = "qwen/qwen3-32b"  # ⚠️ 8x memory vs Crafter's 4B
+trainer_mode = "lora"
+
+[lora]
+r = 16
+alpha = 32
+dropout = 0.05
+target_modules = ["all-linear"]
+
+[rollout]
+env_name = "verilog"
+max_turns = 15
+policy_name = "verilog-designer"
+```
+
+### **Memory Optimization** (for 32B model)
+```toml
+[vllm]
+max_model_len = 4096  # Shorter than Crafter's 8192
+tensor_parallel_size = 2  # Distribute across GPUs
+
+[training]
+batch_size = 2  # Smaller than Crafter's 4
+gradient_accumulation_steps = 4
+```
+
+## Task App Changes Needed
+
+### **1. Mode Parameter Support** ✅ (Already implemented)
+The Verilog task app already handles `mode="rl"` correctly:
+```python
+# In grpo_verilog.py rollout_executor
+policy_config = dict(policy_config_raw)
+# ... mode parameter flows through naturally
+```
+
+### **2. Trace Correlation** ✅ (Already implemented)
+```python
+# Line 817 in grpo_verilog.py
+trajectory = RolloutTrajectory(
+    # ...
+    inference_url=agent.inference_url,  # ✅ Required for trace correlation
+    decision_samples=None,
+)
+```
+
+### **3. Rubric Integration** ✅ (Already configured)
+```python
+# In grpo_verilog.py
+rubrics=RubricBundle(
+    outcome=OUTCOME_RUBRIC,  # Tests pass reward
+    events=EVENTS_RUBRIC,    # Process efficiency reward
+)
+```
+
+## RL Training Feasibility
+
+### **✅ Works Great**
+1. **Clear Success Signal**: Submit passing tests = +10 reward
+2. **Guided Process**: Natural write→compile→simulate→submit progression
+3. **Error Learning**: Agent must learn to debug compilation failures
+4. **Hardware Design**: Real-world applicable skills
+
+### **⚠️ Challenges**
+1. **Model Size**: 32B vs 4B = 8x memory, slower training
+2. **Sparse Rewards**: Fewer learning signals than Crafter's dense rewards
+3. **Longer Episodes**: 15+ steps vs Crafter's 10 steps
+4. **Compilation Errors**: Must learn to interpret and fix syntax errors
+
+## Recommended Approach
+
+### **Phase 1: Start with Qwen3-0.6B** ⭐ (as you requested)
+```toml
+# Perfect for testing - same model used in existing SFT configs
+model = "Qwen/Qwen3-0.6B"
+batch_size = 4  # Same as Crafter
+```
+- ✅ **Zero setup**: Already configured in `synth-ai/examples/sft/configs/crafter_lora_qwen0p6b.toml`
+- ✅ **Fast iteration**: 0.6B parameters = quick training cycles
+- ✅ **Memory efficient**: Fits on single GPU easily
+- ✅ **Proven baseline**: Same model used in RL demos and SFT examples
+
+### **Phase 2: Scale to Qwen3-8B** (if 0.6B works well)
+```toml
+model = "qwen/qwen3-8b"
+batch_size = 2
+gradient_accumulation_steps = 2
+```
+
+### **Phase 3: Production with Qwen3-32B**
+```toml
+model = "qwen/qwen3-32b"
+tensor_parallel_size = 2
+batch_size = 1
+gradient_accumulation_steps = 4
+```
+
+### **Phase 3: Optimize for Verilog Domain**
+Consider fine-tuning the base model on:
+- Verilog syntax and semantics
+- Hardware design patterns
+- Compilation error messages
+- Testbench writing
+
+## Conclusion
+
+**✅ Verilog RL with LoRA is absolutely feasible** and should work with the same pipeline as Crafter. The main differences are:
+
+1. **Larger model** (32B vs 4B) requires memory optimization
+2. **Sparser rewards** may need different reward shaping
+3. **More structured tasks** could actually make learning easier
+4. **Real hardware skills** make it more valuable than game tasks
+
+**Recommended next step**: Create a `verilog_rl_lora.toml` config starting with Qwen3-8B and adapt the reward rubrics for the compilation workflow.

examples/qwen_coder/configs/coder_lora_30b.toml

@@ -1,9 +1,11 @@
 # Qwen3 Coder 30B LoRA SFT – all-linear adapters
 
+type = "sft"
+
 [algorithm]
 type = "offline"
 method = "sft"
-variety = "fft"
+variety = "lora"
 
 [job]
 model = "Qwen/Qwen3-Coder-30B-A3B-Instruct"
@@ -58,4 +60,3 @@ alpha = 32
 dropout = 0.05
 target_modules = ["all-linear"]
 
-

examples/qwen_coder/configs/coder_lora_4b.toml

@@ -1,5 +1,7 @@
 # Qwen3 Coder 4B LoRA SFT – all-linear adapters
 
+type = "sft"
+
 [job]
 model = "Qwen/Qwen3-4B"
 
@@ -54,4 +56,3 @@ dropout = 0.05
 target_modules = ["all-linear"]
 
 
-

examples/qwen_coder/configs/coder_lora_small.toml

@@ -1,5 +1,7 @@
 # Qwen3 Coder LoRA SFT – all-linear adapters
 
+type = "sft"
+
 [algorithm]
 type = "offline"
 method = "sft"
@@ -55,4 +57,3 @@ alpha = 32
 dropout = 0.05
 target_modules = ["all-linear"]
 
-