synth-ai 0.2.13.dev2__py3-none-any.whl → 0.2.16__py3-none-any.whl

examples/multi_step/configs/crafter_sft_qwen30b_lora.toml

@@ -0,0 +1,62 @@
+# Crafter SFT LoRA configuration
+# Train Qwen3-Coder-30B on Crafter agent traces
+
+[algorithm]
+type = "offline"
+method = "sft"
+variety = "lora"
+
+[job]
+model = "Qwen/Qwen3-Coder-30B-A3B-Instruct"
+# Default dataset - can override with --dataset flag
+data = "traces/crafter_sft_converted.jsonl"
+
+[compute]
+gpu_type = "H200"
+gpu_count = 2
+nodes = 1
+
+[data]
+# Forwarded into metadata.effective_config
+topology = {}
+# Optional validation set if you have one locally
+# validation_path = "examples/multi_step/ft_data/crafter_sft.val.jsonl"
+
+[training]
+mode = "lora"
+use_qlora = true
+
+[training.validation]
+enabled = true
+evaluation_strategy = "steps"
+eval_steps = 100
+save_best_model_at_end = true
+metric_for_best_model = "val.loss"
+greater_is_better = false
+
+[hyperparameters]
+n_epochs = 1
+train_kind = "peft"
+per_device_batch = 1
+gradient_accumulation_steps = 64
+sequence_length = 4096
+learning_rate = 5e-6
+warmup_ratio = 0.03
+lora_rank = 16
+lora_alpha = 32
+lora_dropout = 0.05
+lora_target_modules = ["all-linear"]
+
+[hyperparameters.parallelism]
+use_deepspeed = true
+deepspeed_stage = 2
+fsdp = false
+bf16 = true
+fp16 = false
+activation_checkpointing = true
+
+[tags]
+experiment = "crafter_sft_lora_qwen_coder_30b"
+task = "crafter_agent"
+model_size = "30b"
+

examples/multi_step/configs/crafter_synth_backend.md

@@ -0,0 +1,40 @@
+# Crafter Eval Using Synth Backend with Qwen 4B
+
+## What Changed
+
+Created `crafter_eval_synth_qwen4b.toml` to evaluate Crafter using Qwen3-4B via the Synth backend inference proxy.
+
+## Key Difference from Groq Config
+
+**Before (Groq):**
+```toml
+[eval.policy_config]
+provider = "groq"
+model = "qwen/qwen3-32b"
+inference_url = "https://api.groq.com/openai/v1/chat/completions"
+```
+
+**After (Synth Backend):**
+```toml
+[eval.policy_config]
+provider = "openai"
+model = "Qwen/Qwen3-4B"
+inference_url = "https://synth-backend-dev-docker.onrender.com/api/v1/chat/completions"
+```
+
+## Usage
+
+```bash
+uvx synth-ai eval --config examples/multi_step/configs/crafter_eval_synth_qwen4b.toml
+```
+
+## Why This Works
+
+The Synth backend's `/api/v1/chat/completions` endpoint:
+1. Accepts OpenAI-compatible requests
+2. Routes to Modal vLLM service
+3. Loads the base model (Qwen/Qwen3-4B from HuggingFace)
+4. Returns OpenAI-compatible responses
+
+No code changes needed - the infrastructure already exists.
+

examples/multi_step/configs/verilog_eval_groq_qwen32b.toml

@@ -0,0 +1,31 @@
+# Verilog Eval Config for Groq Qwen3-32B
+# Quick eval to test Verilog task app before RL training
+
+[eval]
+app_id = "grpo-verilog"
+task_app_url = "https://synth-laboratories--grpo-verilog-task-app-fastapi-app-dev.modal.run"
+model = "qwen/qwen3-32b"
+seeds = [0, 1, 2]
+max_turns = 15
+concurrency = 1
+env_name = "verilog"
+policy_name = "verilog-designer"
+trace_format = "full"
+return_trace = true
+
+[eval.env_config]
+difficulty = "medium"  # Can be "easy", "medium", or "hard"
+
+[eval.policy_config]
+provider = "groq"
+model = "qwen/qwen3-32b"
+inference_url = "https://api.groq.com/openai/v1/chat/completions"
+temperature = 0.2
+max_tokens = 8192  # Large buffer for Verilog (long testbenches + module implementation)
+
+[eval.judge]
+path = "examples/multi_step/judges/verilog_backend_judge.py"
+name = "Backend"
+backend_url = "http://localhost:8000/api"
+model = "openai/gpt-oss-120b"
+timeout_s = 45

examples/multi_step/configs/verilog_eval_synth_qwen8b.toml

@@ -0,0 +1,33 @@
+# Verilog eval using Synth backend with Qwen3-8B
+
+[eval]
+app_id = "grpo-verilog"
+task_app_url = "https://synth-laboratories--grpo-verilog-task-app-fastapi-app-dev.modal.run"
+model = "Qwen/Qwen3-8B"
+seeds = [0, 1, 2]
+max_turns = 6
+concurrency = 1
+env_name = "verilog"
+policy_name = "verilog-designer"
+trace_format = "full"
+return_trace = true
+
+[eval.env_config]
+difficulty = "medium"
+
+[eval.policy_config]
+provider = "openai"
+model = "Qwen/Qwen3-8B"
+inference_url = "https://synth-backend-dev-docker.onrender.com/api/v1/chat/completions"
+temperature = 0.2
+top_p = 0.95
+max_tokens = 4096
+max_llm_calls = 6
+
+[eval.judge]
+path = "examples/multi_step/judges/verilog_backend_judge.py"
+name = "Backend"
+backend_url = "http://localhost:8000/api"
+model = "openai/gpt-oss-120b"
+timeout_s = 45
+

examples/multi_step/configs/verilog_rl_lora.toml

@@ -0,0 +1,190 @@
+# Verilog RL experiment – LoRA training on Qwen3-0.6B
+#
+# This configuration adapts the Crafter RL setup for Verilog spec-to-RTL tasks.
+# Uses the same proven pipeline but optimized for 0.6B model and Verilog domain.
+
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[services]
+# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-verilog`
+task_url = "https://synth-laboratories--grpo-verilog-task-app-fastapi-app-dev.modal.run"
+# Point at the Synth backend (or compatible service) that exposes /api/judge/v1/*
+judge_url = "https://synth-backend-dev-docker.onrender.com/api"
+
+[compute]
+gpu_type = "H200"  # ✅ 8B model needs H200 for larger context window
+gpu_count = 2      # ✅ Minimum 2x GPUs (1 for vLLM inference + 1 for training)
+nodes = 1
+
+[topology]
+type = "single_node_split"
+gpus_for_vllm = 1      # ✅ vLLM for inference
+gpus_for_training = 1  # ✅ Training GPU (8B LoRA fits well)
+gpus_for_ref = 0
+tensor_parallel = 1
+
+[vllm]
+tensor_parallel_size = 1
+max_model_len = 24576  # ✅ Increased to 24K to accommodate long Verilog prompts (16K + 8K buffer for testbenches + history)
+
+[reference]
+placement = "none"
+
+[model]
+base = "Qwen/Qwen3-8B"  # ✅ 8B model for RL training with good balance of speed and capability
+trainer_mode = "lora"
+label = "verilog-rl-lora-qwen8b"
+
+[lora]
+r = 16
+alpha = 32
+dropout = 0.05
+target_modules = ["all-linear"]
+
+[rollout]
+env_name = "verilog"  # ✅ Changed from "crafter" to "verilog"
+max_turns = 6        # ✅ More steps for compilation chains vs Crafter's 10
+episodes_per_batch = 4  # ✅ Good batch size for 8B model
+policy_name = "verilog-designer"
+max_concurrent_rollouts = 8
+batches_per_step = 2
+ops = ["agent", "env"]
+
+  [rollout.env_config]
+  # Verilog-specific environment settings
+  difficulty = "medium"  # Can be "easy", "medium", or "hard"
+
+    [rollout.env_config.step_rewards]
+    enabled = true
+    mode = "decision_stepwise"
+    strategy = "consistent"
+    indicator_lambda = 0.5  # ✅ Reduced from Crafter (sparser rewards)
+    step_beta = 0.0
+
+  [rollout.policy_config]
+  provider = "openai"
+  model = "Qwen/Qwen3-8B"  # ✅ Use the model being trained (8B) for rollouts
+  temperature = 0.2
+  max_tokens = 4096  # ✅ Balanced for Verilog generation while leaving room for long input prompts (testbenches + history)
+
+[evaluation]
+instances = 16
+every_n_iters = 10
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+
+[training]
+num_epochs = 1
+iterations_per_epoch = 5
+gradient_accumulation_steps = 1
+max_accumulated_minibatch = 1
+max_turns = 15
+batch_size = 4          # ✅ Same as Crafter (works well for 8B LoRA)
+group_size = 4
+learning_rate = 5e-5    # ✅ Same as Crafter
+log_interval = 1
+weight_sync_interval = 1
+event_rewards_kind = "unique"
+async_semaphore_max = 20  # Max concurrent rollouts in streaming pipeline
+
+# Enable dense decision rewards in the trainer
+step_rewards_enabled = true
+step_rewards_mode = "decision_stepwise"
+step_rewards_indicator_lambda = 0.5  # ✅ Reduced for Verilog's sparser rewards
+step_rewards_beta = 0.0
+step_rewards_strategy = "consistent"
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+mode = "direct"
+direct = true
+verify_every_k = 0
+
+[rubric]
+enabled = true
+model = "openai/gpt-oss-120b"
+api_base = "https://synth-backend-dev-docker.onrender.com/api/judge"
+api_key_env = "OPENAI_API_KEY"
+
+# Blend the hosted judge scores with environment returns
+[rubric.weights]
+env = 0.3     # ✅ Higher weight on env rewards for Verilog (vs Crafter's 0.2)
+event = 0.3   # ✅ Adjusted for Verilog's different reward structure
+outcome = 0.4
+
+[rubric.event]
+# Verilog-specific event rubric for process efficiency
+rubric_id = "verilog/event@v1"
+criteria = [
+  { key = "process.compilation_success", weight = 0.7, description = "Return 1.0 when compilation succeeds, 0.5 for partial success, 0.0 for failure", aggregation = "weighted_sum" },
+  { key = "process.design_iterations", weight = 0.3, description = "Reward efficient design iterations without unnecessary recompilation", aggregation = "weighted_sum" },
+]
+
+[rubric.outcome]
+# Verilog-specific outcome rubric for final results
+rubric_id = "verilog/outcome@v1"
+criteria = [
+  { key = "outcome.tests_passed", weight = 0.8, description = "Full credit when all tests pass, partial for some tests", aggregation = "weighted_sum" },
+  { key = "outcome.design_quality", weight = 0.2, description = "Code quality, documentation, and design efficiency", aggregation = "weighted_sum" },
+]
+
+[judge]
+type = "groq"
+timeout_s = 45
+
+  [judge.options]
+  event = true
+  outcome = true
+  provider = "openai"
+  model = "openai/gpt-oss-120b"
+  rubric_id = "verilog/bundle@v1"
+  max_concurrency = 6
+  tracks = ["process", "reasoning", "progress", "outcome"]
+
+    [judge.options.rubric_overrides]
+
+      [judge.options.rubric_overrides.event]
+      goal_text = """
+      Evaluate each Verilog design decision for compilation success and process efficiency.
+      High scores for successful compilation and strategic tool usage.
+      Penalize unnecessary operations and compilation failures."""
+      aggregation = "weighted_sum"
+
+        [[judge.options.rubric_overrides.event.criteria]]
+        id = "process.compilation_success"
+        weight = 0.7
+        scale = "bounded"
+        description = "Return 1.0 when compilation succeeds cleanly, 0.5 for warnings, 0.0 for errors"
+
+        [[judge.options.rubric_overrides.event.criteria]]
+        id = "process.design_iterations"
+        weight = 0.3
+        scale = "bounded"
+        description = "Reward efficient write→compile→simulate workflow, penalize redundant operations"
+
+      [judge.options.rubric_overrides.outcome]
+      goal_text = """
+      Evaluate the final Verilog implementation for correctness and quality.
+      High scores for working designs that pass all tests with good code quality."""
+      aggregation = "weighted_sum"
+
+        [[judge.options.rubric_overrides.outcome.criteria]]
+        id = "outcome.tests_passed"
+        weight = 0.8
+        scale = "binary"
+        description = "Full credit when all tests pass, partial credit for some tests passing"
+
+        [[judge.options.rubric_overrides.outcome.criteria]]
+        id = "outcome.design_quality"
+        weight = 0.2
+        scale = "bounded"
+        description = "Code clarity, proper documentation, and efficient design patterns"
+
+    [judge.options.weights]
+    process = 0.1
+    reasoning = 0.2
+    progress = 0.3
+    outcome = 0.4

examples/multi_step/convert_traces_to_sft.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Convert Crafter trace format to SFT format with messages[] structure."""
+
+import json
+import sys
+from pathlib import Path
+
+def convert_trace_to_sft(trace: dict) -> dict:
+    """Convert a single trace to SFT format."""
+    # Extract dialogue from trace
+    dialogue = trace.get("dialogue", [])
+    assistant = trace.get("assistant", {})
+    
+    # Build messages list
+    messages = []
+    
+    # Add dialogue history
+    for msg in dialogue:
+        messages.append({
+            "role": msg["role"],
+            "content": msg["content"]
+        })
+    
+    # Add assistant response if present
+    if assistant:
+        content = assistant.get("content", "")
+        tool_calls = assistant.get("tool_calls", [])
+        
+        # If there are tool calls, format them
+        if tool_calls:
+            # Convert tool calls to a simple text format for SFT
+            tool_text = "\n".join([
+                f"Tool: {tc['name']}\nArguments: {json.dumps(tc.get('arguments', {}))}"
+                for tc in tool_calls
+            ])
+            content = f"{content}\n\n{tool_text}".strip()
+        
+        messages.append({
+            "role": "assistant",
+            "content": content
+        })
+    
+    return {"messages": messages}
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python convert_traces_to_sft.py <input.jsonl> [output.jsonl]")
+        sys.exit(1)
+    
+    input_path = Path(sys.argv[1])
+    output_path = Path(sys.argv[2]) if len(sys.argv) > 2 else input_path.with_name(f"{input_path.stem}_sft_format.jsonl")
+    
+    if not input_path.exists():
+        print(f"Error: Input file not found: {input_path}")
+        sys.exit(1)
+    
+    print(f"Converting {input_path} → {output_path}")
+    
+    converted = 0
+    skipped = 0
+    
+    with open(input_path) as f_in, open(output_path, "w") as f_out:
+        for line_no, line in enumerate(f_in, 1):
+            try:
+                trace = json.loads(line.strip())
+                sft_entry = convert_trace_to_sft(trace)
+                
+                # Only write if we have messages
+                if sft_entry["messages"]:
+                    f_out.write(json.dumps(sft_entry) + "\n")
+                    converted += 1
+                else:
+                    skipped += 1
+                    
+            except Exception as e:
+                print(f"Warning: Skipping line {line_no}: {e}")
+                skipped += 1
+    
+    print(f"✅ Converted {converted} entries, skipped {skipped}")
+    print(f"Output: {output_path}")
+
+if __name__ == "__main__":
+    main()
+

examples/multi_step/judges/crafter_backend_judge.py

@@ -0,0 +1,220 @@
+"""Crafter backend judge that calls the Synth judge API with inline rubric."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, List, TypedDict
+
+import httpx
+
+
+class TraceMetadata(TypedDict, total=False):
+    """Metadata for the trace."""
+    env_id: str
+    policy_id: str
+    length: int
+
+
+class JudgeTracePayload(TypedDict):
+    """Trace payload sent to backend judge."""
+    event_history: List[Dict[str, Any]]
+    markov_blanket_message_history: List[Dict[str, Any]]
+    metadata: TraceMetadata
+
+
+class JudgeOptions(TypedDict, total=False):
+    """Options for judge scoring."""
+    model: str
+    timeout_s: int
+    event: bool
+    outcome: bool
+
+
+class TaskApp(TypedDict):
+    """Task application metadata."""
+    id: str
+
+
+class JudgeScoreRequest(TypedDict):
+    """Request to backend judge API."""
+    policy_name: str
+    task_app: TaskApp
+    trace: JudgeTracePayload
+    rubric: Dict[str, Any]
+    options: JudgeOptions
+
+
+# Load rubric from file (cached at module level)
+_RUBRIC_PATH = Path(__file__).parent.parent / "rubrics" / "crafter_backend_judge.json"
+_RUBRIC: Dict[str, Any] | None = None
+
+
+def _load_rubric() -> Dict[str, Any]:
+    """Load rubric from file with fallback to inline default."""
+    global _RUBRIC
+    if _RUBRIC is None:
+        try:
+            with open(_RUBRIC_PATH, 'r') as f:
+                _RUBRIC = json.load(f)
+            assert isinstance(_RUBRIC, dict), "Rubric must be a dict"
+            assert "outcome" in _RUBRIC, "Rubric must have 'outcome' key"
+            assert isinstance(_RUBRIC["outcome"], list), "Rubric 'outcome' must be a list"
+        except Exception as e:
+            print(f"[crafter_backend_judge] Warning: Failed to load rubric from {_RUBRIC_PATH}: {e}")
+            # Fallback inline rubric (matching RubricCriteriaBlock format)
+            _RUBRIC = {
+                "event": [],
+                "outcome": [
+                    {"id": "achievement_progression", "description": "Achievement progression", "weight": 0.35, "scale": "bounded"},
+                    {"id": "resource_stockpile", "description": "Resource stockpile", "weight": 0.2, "scale": "bounded"},
+                    {"id": "survival_state", "description": "Survival state", "weight": 0.2, "scale": "bounded"},
+                    {"id": "failure_analysis", "description": "Failure analysis", "weight": 0.15, "scale": "bounded"},
+                    {"id": "future_readiness", "description": "Future readiness", "weight": 0.1, "scale": "bounded"}
+                ]
+            }
+    return _RUBRIC
+
+
+def judge(payload: Dict[str, Any], **kwargs: Any) -> float:
+    """
+    Call the Synth backend judge API to score a Crafter rollout.
+
+    Args:
+        payload: Dict with keys: seed, prompt, completion, metrics, response, trace
+        **kwargs: Additional config (backend_url, model, timeout_s, etc.)
+
+    Returns:
+        float: Aggregate score from 0.0 to 1.0
+    """
+    try:
+        # Extract configuration
+        backend_url = kwargs.get("backend_url", "http://localhost:8000/api")
+        model = kwargs.get("model", "openai/gpt-oss-120b")
+        timeout = kwargs.get("timeout_s", 45)
+        
+        assert isinstance(backend_url, str), "backend_url must be a string"
+        assert isinstance(model, str), "model must be a string"
+        assert isinstance(timeout, (int, float)), "timeout_s must be numeric"
+        
+        # Extract trajectory from response
+        response_data = payload.get("response", {})
+        assert isinstance(response_data, dict), "response must be a dict"
+        
+        trajectories = response_data.get("trajectories", [])
+        assert isinstance(trajectories, list), "trajectories must be a list"
+        
+        if not trajectories:
+            print("[crafter_backend_judge] No trajectories in response")
+            return 0.0
+        
+        trajectory = trajectories[0]  # First trajectory
+        assert isinstance(trajectory, dict), "trajectory must be a dict"
+        
+        # Load rubric
+        rubric = _load_rubric()
+        
+        # Transform trajectory into JudgeTracePayload format
+        steps = trajectory.get("steps", [])
+        assert isinstance(steps, list), "trajectory steps must be a list"
+        
+        event_history: List[Dict[str, Any]] = []
+        for idx, step in enumerate(steps):
+            assert isinstance(step, dict), f"step {idx} must be a dict"
+            # Each step becomes an event
+            event_history.append({
+                "observation": step.get("obs", {}),
+                "tool_calls": step.get("tool_calls", []),
+                "reward": step.get("reward", 0.0),
+                "done": step.get("done", False),
+                "truncated": step.get("truncated", False),
+                "info": step.get("info", {}),
+            })
+        
+        # Add final observation - backend will extract this as outcome context
+        final_data = trajectory.get("final", {})
+        if final_data:
+            assert isinstance(final_data, dict), "final data must be a dict"
+            final_obs = final_data.get("observation", {})
+            assert isinstance(final_obs, dict), "final observation must be a dict"
+            
+            event_history.append({
+                "observation": final_obs,
+                "reward": final_data.get("reward", 0.0),
+                "done": final_data.get("done", True),
+                "truncated": final_data.get("truncated", False),
+                "info": final_data.get("info", {}),
+            })
+        
+        # Build trace metadata
+        metadata: TraceMetadata = {
+            "env_id": trajectory.get("env_id", "crafter"),
+            "policy_id": trajectory.get("policy_id", "crafter-react"),
+            "length": trajectory.get("length", len(steps)),
+        }
+        
+        # Build judge request with rubric included
+        judge_request: JudgeScoreRequest = {
+            "policy_name": "crafter-react",
+            "task_app": {"id": "grpo-crafter-task-app"},
+            "trace": {
+                "event_history": event_history,
+                "markov_blanket_message_history": [],
+                "metadata": metadata,
+            },
+            "rubric": rubric,
+            "options": {
+                "model": model,
+                "timeout_s": timeout,
+                "event": False,  # Not scoring per-event
+                "outcome": True,  # Score the final outcome
+            }
+        }
+        
+        # Call backend judge API
+        with httpx.Client(timeout=timeout) as client:
+            # Get API key from env
+            api_key = os.environ.get("SYNTH_API_KEY") or os.environ.get("OPENAI_API_KEY")
+            headers = {}
+            if api_key:
+                headers["Authorization"] = f"Bearer {api_key}"
+            
+            url = f"{backend_url.rstrip('/')}/judge/v1/score"
+            
+            # Debug: print request summary
+            print(f"\n[crafter_backend_judge] Scoring trajectory with {len(event_history)} events")
+            if event_history:
+                last_obs = event_history[-1].get('observation', {})
+                print(f"  Final observation keys: {list(last_obs.keys())[:5]}...")
+            
+            response = client.post(url, json=judge_request, headers=headers)
+            
+            response.raise_for_status()
+            result = response.json()
+            assert isinstance(result, dict), "Response must be a dict"
+            
+            # Extract aggregate score
+            aggregate_score = result.get("aggregate_score", 0.0)
+            
+            # Try outcome_review.total if aggregate_score not found
+            if aggregate_score == 0.0 and "outcome_review" in result:
+                outcome_review = result["outcome_review"]
+                if isinstance(outcome_review, dict):
+                    aggregate_score = outcome_review.get("total", 0.0)
+            
+            print(f"  Backend judge score: {aggregate_score:.3f}\n")
+            return float(aggregate_score)
+            
+    except httpx.HTTPStatusError as e:
+        print(f"\n[crafter_backend_judge] HTTP ERROR:")
+        print(f"  Status: {e.response.status_code}")
+        print(f"  Response: {e.response.text[:300]}\n")
+        return 0.0
+    except AssertionError as e:
+        print(f"[crafter_backend_judge] Assertion error: {e}")
+        return 0.0
+    except Exception as e:
+        print(f"[crafter_backend_judge] Unexpected error: {e}")
+        return 0.0
+