synth-ai 0.2.13.dev2__py3-none-any.whl → 0.2.14__py3-none-any.whl

examples/multi_step/configs/verilog_rl_lora.toml

@@ -0,0 +1,190 @@
+# Verilog RL experiment – LoRA training on Qwen3-0.6B
+#
+# This configuration adapts the Crafter RL setup for Verilog spec-to-RTL tasks.
+# Uses the same proven pipeline but optimized for 0.6B model and Verilog domain.
+
+[algorithm]
+type = "online"
+method = "policy_gradient"
+variety = "gspo"
+
+[services]
+# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-verilog`
+task_url = "https://synth-laboratories--grpo-verilog-task-app-fastapi-app-dev.modal.run"
+# Point at the Synth backend (or compatible service) that exposes /api/judge/v1/*
+judge_url = "https://synth-backend-dev-docker.onrender.com/api"
+
+[compute]
+gpu_type = "H200"  # ✅ 8B model needs H200 for larger context window
+gpu_count = 2      # ✅ Minimum 2x GPUs (1 for vLLM inference + 1 for training)
+nodes = 1
+
+[topology]
+type = "single_node_split"
+gpus_for_vllm = 1      # ✅ vLLM for inference
+gpus_for_training = 1  # ✅ Training GPU (8B LoRA fits well)
+gpus_for_ref = 0
+tensor_parallel = 1
+
+[vllm]
+tensor_parallel_size = 1
+max_model_len = 24576  # ✅ Increased to 24K to accommodate long Verilog prompts (16K + 8K buffer for testbenches + history)
+
+[reference]
+placement = "none"
+
+[model]
+base = "Qwen/Qwen3-8B"  # ✅ 8B model for RL training with good balance of speed and capability
+trainer_mode = "lora"
+label = "verilog-rl-lora-qwen8b"
+
+[lora]
+r = 16
+alpha = 32
+dropout = 0.05
+target_modules = ["all-linear"]
+
+[rollout]
+env_name = "verilog"  # ✅ Changed from "crafter" to "verilog"
+max_turns = 6        # ✅ More steps for compilation chains vs Crafter's 10
+episodes_per_batch = 4  # ✅ Good batch size for 8B model
+policy_name = "verilog-designer"
+max_concurrent_rollouts = 8
+batches_per_step = 2
+ops = ["agent", "env"]
+
+  [rollout.env_config]
+  # Verilog-specific environment settings
+  difficulty = "medium"  # Can be "easy", "medium", or "hard"
+
+    [rollout.env_config.step_rewards]
+    enabled = true
+    mode = "decision_stepwise"
+    strategy = "consistent"
+    indicator_lambda = 0.5  # ✅ Reduced from Crafter (sparser rewards)
+    step_beta = 0.0
+
+  [rollout.policy_config]
+  provider = "openai"
+  model = "Qwen/Qwen3-8B"  # ✅ Use the model being trained (8B) for rollouts
+  temperature = 0.2
+  max_tokens = 4096  # ✅ Balanced for Verilog generation while leaving room for long input prompts (testbenches + history)
+
+[evaluation]
+instances = 16
+every_n_iters = 10
+seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+
+[training]
+num_epochs = 1
+iterations_per_epoch = 5
+gradient_accumulation_steps = 1
+max_accumulated_minibatch = 1
+max_turns = 15
+batch_size = 4          # ✅ Same as Crafter (works well for 8B LoRA)
+group_size = 4
+learning_rate = 5e-5    # ✅ Same as Crafter
+log_interval = 1
+weight_sync_interval = 1
+event_rewards_kind = "unique"
+async_semaphore_max = 20  # Max concurrent rollouts in streaming pipeline
+
+# Enable dense decision rewards in the trainer
+step_rewards_enabled = true
+step_rewards_mode = "decision_stepwise"
+step_rewards_indicator_lambda = 0.5  # ✅ Reduced for Verilog's sparser rewards
+step_rewards_beta = 0.0
+step_rewards_strategy = "consistent"
+
+[training.weight_sync]
+enable = true
+targets = ["policy"]
+mode = "direct"
+direct = true
+verify_every_k = 0
+
+[rubric]
+enabled = true
+model = "openai/gpt-oss-120b"
+api_base = "https://synth-backend-dev-docker.onrender.com/api/judge"
+api_key_env = "OPENAI_API_KEY"
+
+# Blend the hosted judge scores with environment returns
+[rubric.weights]
+env = 0.3     # ✅ Higher weight on env rewards for Verilog (vs Crafter's 0.2)
+event = 0.3   # ✅ Adjusted for Verilog's different reward structure
+outcome = 0.4
+
+[rubric.event]
+# Verilog-specific event rubric for process efficiency
+rubric_id = "verilog/event@v1"
+criteria = [
+  { key = "process.compilation_success", weight = 0.7, description = "Return 1.0 when compilation succeeds, 0.5 for partial success, 0.0 for failure", aggregation = "weighted_sum" },
+  { key = "process.design_iterations", weight = 0.3, description = "Reward efficient design iterations without unnecessary recompilation", aggregation = "weighted_sum" },
+]
+
+[rubric.outcome]
+# Verilog-specific outcome rubric for final results
+rubric_id = "verilog/outcome@v1"
+criteria = [
+  { key = "outcome.tests_passed", weight = 0.8, description = "Full credit when all tests pass, partial for some tests", aggregation = "weighted_sum" },
+  { key = "outcome.design_quality", weight = 0.2, description = "Code quality, documentation, and design efficiency", aggregation = "weighted_sum" },
+]
+
+[judge]
+type = "groq"
+timeout_s = 45
+
+  [judge.options]
+  event = true
+  outcome = true
+  provider = "openai"
+  model = "openai/gpt-oss-120b"
+  rubric_id = "verilog/bundle@v1"
+  max_concurrency = 6
+  tracks = ["process", "reasoning", "progress", "outcome"]
+
+    [judge.options.rubric_overrides]
+
+      [judge.options.rubric_overrides.event]
+      goal_text = """
+      Evaluate each Verilog design decision for compilation success and process efficiency.
+      High scores for successful compilation and strategic tool usage.
+      Penalize unnecessary operations and compilation failures."""
+      aggregation = "weighted_sum"
+
+        [[judge.options.rubric_overrides.event.criteria]]
+        id = "process.compilation_success"
+        weight = 0.7
+        scale = "bounded"
+        description = "Return 1.0 when compilation succeeds cleanly, 0.5 for warnings, 0.0 for errors"
+
+        [[judge.options.rubric_overrides.event.criteria]]
+        id = "process.design_iterations"
+        weight = 0.3
+        scale = "bounded"
+        description = "Reward efficient write→compile→simulate workflow, penalize redundant operations"
+
+      [judge.options.rubric_overrides.outcome]
+      goal_text = """
+      Evaluate the final Verilog implementation for correctness and quality.
+      High scores for working designs that pass all tests with good code quality."""
+      aggregation = "weighted_sum"
+
+        [[judge.options.rubric_overrides.outcome.criteria]]
+        id = "outcome.tests_passed"
+        weight = 0.8
+        scale = "binary"
+        description = "Full credit when all tests pass, partial credit for some tests passing"
+
+        [[judge.options.rubric_overrides.outcome.criteria]]
+        id = "outcome.design_quality"
+        weight = 0.2
+        scale = "bounded"
+        description = "Code clarity, proper documentation, and efficient design patterns"
+
+    [judge.options.weights]
+    process = 0.1
+    reasoning = 0.2
+    progress = 0.3
+    outcome = 0.4

examples/multi_step/judges/crafter_backend_judge.py

@@ -0,0 +1,220 @@
+"""Crafter backend judge that calls the Synth judge API with inline rubric."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, List, TypedDict
+
+import httpx
+
+
+class TraceMetadata(TypedDict, total=False):
+    """Metadata for the trace."""
+    env_id: str
+    policy_id: str
+    length: int
+
+
+class JudgeTracePayload(TypedDict):
+    """Trace payload sent to backend judge."""
+    event_history: List[Dict[str, Any]]
+    markov_blanket_message_history: List[Dict[str, Any]]
+    metadata: TraceMetadata
+
+
+class JudgeOptions(TypedDict, total=False):
+    """Options for judge scoring."""
+    model: str
+    timeout_s: int
+    event: bool
+    outcome: bool
+
+
+class TaskApp(TypedDict):
+    """Task application metadata."""
+    id: str
+
+
+class JudgeScoreRequest(TypedDict):
+    """Request to backend judge API."""
+    policy_name: str
+    task_app: TaskApp
+    trace: JudgeTracePayload
+    rubric: Dict[str, Any]
+    options: JudgeOptions
+
+
+# Load rubric from file (cached at module level)
+_RUBRIC_PATH = Path(__file__).parent.parent / "rubrics" / "crafter_backend_judge.json"
+_RUBRIC: Dict[str, Any] | None = None
+
+
+def _load_rubric() -> Dict[str, Any]:
+    """Load rubric from file with fallback to inline default."""
+    global _RUBRIC
+    if _RUBRIC is None:
+        try:
+            with open(_RUBRIC_PATH, 'r') as f:
+                _RUBRIC = json.load(f)
+            assert isinstance(_RUBRIC, dict), "Rubric must be a dict"
+            assert "outcome" in _RUBRIC, "Rubric must have 'outcome' key"
+            assert isinstance(_RUBRIC["outcome"], list), "Rubric 'outcome' must be a list"
+        except Exception as e:
+            print(f"[crafter_backend_judge] Warning: Failed to load rubric from {_RUBRIC_PATH}: {e}")
+            # Fallback inline rubric (matching RubricCriteriaBlock format)
+            _RUBRIC = {
+                "event": [],
+                "outcome": [
+                    {"id": "achievement_progression", "description": "Achievement progression", "weight": 0.35, "scale": "bounded"},
+                    {"id": "resource_stockpile", "description": "Resource stockpile", "weight": 0.2, "scale": "bounded"},
+                    {"id": "survival_state", "description": "Survival state", "weight": 0.2, "scale": "bounded"},
+                    {"id": "failure_analysis", "description": "Failure analysis", "weight": 0.15, "scale": "bounded"},
+                    {"id": "future_readiness", "description": "Future readiness", "weight": 0.1, "scale": "bounded"}
+                ]
+            }
+    return _RUBRIC
+
+
+def judge(payload: Dict[str, Any], **kwargs: Any) -> float:
+    """
+    Call the Synth backend judge API to score a Crafter rollout.
+
+    Args:
+        payload: Dict with keys: seed, prompt, completion, metrics, response, trace
+        **kwargs: Additional config (backend_url, model, timeout_s, etc.)
+
+    Returns:
+        float: Aggregate score from 0.0 to 1.0
+    """
+    try:
+        # Extract configuration
+        backend_url = kwargs.get("backend_url", "http://localhost:8000/api")
+        model = kwargs.get("model", "openai/gpt-oss-120b")
+        timeout = kwargs.get("timeout_s", 45)
+        
+        assert isinstance(backend_url, str), "backend_url must be a string"
+        assert isinstance(model, str), "model must be a string"
+        assert isinstance(timeout, (int, float)), "timeout_s must be numeric"
+        
+        # Extract trajectory from response
+        response_data = payload.get("response", {})
+        assert isinstance(response_data, dict), "response must be a dict"
+        
+        trajectories = response_data.get("trajectories", [])
+        assert isinstance(trajectories, list), "trajectories must be a list"
+        
+        if not trajectories:
+            print("[crafter_backend_judge] No trajectories in response")
+            return 0.0
+        
+        trajectory = trajectories[0]  # First trajectory
+        assert isinstance(trajectory, dict), "trajectory must be a dict"
+        
+        # Load rubric
+        rubric = _load_rubric()
+        
+        # Transform trajectory into JudgeTracePayload format
+        steps = trajectory.get("steps", [])
+        assert isinstance(steps, list), "trajectory steps must be a list"
+        
+        event_history: List[Dict[str, Any]] = []
+        for idx, step in enumerate(steps):
+            assert isinstance(step, dict), f"step {idx} must be a dict"
+            # Each step becomes an event
+            event_history.append({
+                "observation": step.get("obs", {}),
+                "tool_calls": step.get("tool_calls", []),
+                "reward": step.get("reward", 0.0),
+                "done": step.get("done", False),
+                "truncated": step.get("truncated", False),
+                "info": step.get("info", {}),
+            })
+        
+        # Add final observation - backend will extract this as outcome context
+        final_data = trajectory.get("final", {})
+        if final_data:
+            assert isinstance(final_data, dict), "final data must be a dict"
+            final_obs = final_data.get("observation", {})
+            assert isinstance(final_obs, dict), "final observation must be a dict"
+            
+            event_history.append({
+                "observation": final_obs,
+                "reward": final_data.get("reward", 0.0),
+                "done": final_data.get("done", True),
+                "truncated": final_data.get("truncated", False),
+                "info": final_data.get("info", {}),
+            })
+        
+        # Build trace metadata
+        metadata: TraceMetadata = {
+            "env_id": trajectory.get("env_id", "crafter"),
+            "policy_id": trajectory.get("policy_id", "crafter-react"),
+            "length": trajectory.get("length", len(steps)),
+        }
+        
+        # Build judge request with rubric included
+        judge_request: JudgeScoreRequest = {
+            "policy_name": "crafter-react",
+            "task_app": {"id": "grpo-crafter-task-app"},
+            "trace": {
+                "event_history": event_history,
+                "markov_blanket_message_history": [],
+                "metadata": metadata,
+            },
+            "rubric": rubric,
+            "options": {
+                "model": model,
+                "timeout_s": timeout,
+                "event": False,  # Not scoring per-event
+                "outcome": True,  # Score the final outcome
+            }
+        }
+        
+        # Call backend judge API
+        with httpx.Client(timeout=timeout) as client:
+            # Get API key from env
+            api_key = os.environ.get("SYNTH_API_KEY") or os.environ.get("OPENAI_API_KEY")
+            headers = {}
+            if api_key:
+                headers["Authorization"] = f"Bearer {api_key}"
+            
+            url = f"{backend_url.rstrip('/')}/judge/v1/score"
+            
+            # Debug: print request summary
+            print(f"\n[crafter_backend_judge] Scoring trajectory with {len(event_history)} events")
+            if event_history:
+                last_obs = event_history[-1].get('observation', {})
+                print(f"  Final observation keys: {list(last_obs.keys())[:5]}...")
+            
+            response = client.post(url, json=judge_request, headers=headers)
+            
+            response.raise_for_status()
+            result = response.json()
+            assert isinstance(result, dict), "Response must be a dict"
+            
+            # Extract aggregate score
+            aggregate_score = result.get("aggregate_score", 0.0)
+            
+            # Try outcome_review.total if aggregate_score not found
+            if aggregate_score == 0.0 and "outcome_review" in result:
+                outcome_review = result["outcome_review"]
+                if isinstance(outcome_review, dict):
+                    aggregate_score = outcome_review.get("total", 0.0)
+            
+            print(f"  Backend judge score: {aggregate_score:.3f}\n")
+            return float(aggregate_score)
+            
+    except httpx.HTTPStatusError as e:
+        print(f"\n[crafter_backend_judge] HTTP ERROR:")
+        print(f"  Status: {e.response.status_code}")
+        print(f"  Response: {e.response.text[:300]}\n")
+        return 0.0
+    except AssertionError as e:
+        print(f"[crafter_backend_judge] Assertion error: {e}")
+        return 0.0
+    except Exception as e:
+        print(f"[crafter_backend_judge] Unexpected error: {e}")
+        return 0.0
+

examples/multi_step/judges/verilog_backend_judge.py

@@ -0,0 +1,234 @@
+"""Verilog backend judge that calls the Synth judge API with inline rubric."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Any, Dict, List, TypedDict
+
+import httpx
+
+
+class TraceMetadata(TypedDict, total=False):
+    """Metadata for the trace."""
+    env_id: str
+    policy_id: str
+    length: int
+
+
+class JudgeTracePayload(TypedDict):
+    """Trace payload sent to backend judge."""
+    event_history: List[Dict[str, Any]]
+    markov_blanket_message_history: List[Dict[str, Any]]
+    metadata: TraceMetadata
+
+
+class JudgeOptions(TypedDict, total=False):
+    """Options for judge scoring."""
+    model: str
+    timeout_s: int
+    event: bool
+    outcome: bool
+
+
+class TaskApp(TypedDict):
+    """Task application metadata."""
+    id: str
+
+
+class JudgeScoreRequest(TypedDict):
+    """Request to backend judge API."""
+    policy_name: str
+    task_app: TaskApp
+    trace: JudgeTracePayload
+    rubric: Dict[str, Any]
+    options: JudgeOptions
+
+
+# Load rubric from file (cached at module level)
+_RUBRIC_PATH = Path(__file__).parent.parent / "rubrics" / "verilog_backend_judge.json"
+_RUBRIC: Dict[str, Any] | None = None
+
+
+def _load_rubric() -> Dict[str, Any]:
+    """Load rubric from file with fallback to inline default."""
+    global _RUBRIC
+    if _RUBRIC is None:
+        try:
+            with open(_RUBRIC_PATH, 'r') as f:
+                _RUBRIC = json.load(f)
+            assert isinstance(_RUBRIC, dict), "Rubric must be a dict"
+            assert "outcome" in _RUBRIC, "Rubric must have 'outcome' key"
+            assert isinstance(_RUBRIC["outcome"], list), "Rubric 'outcome' must be a list"
+        except Exception as e:
+            print(f"[verilog_backend_judge] Warning: Failed to load rubric from {_RUBRIC_PATH}: {e}")
+            # Fallback inline rubric (matching RubricCriteriaBlock format)
+            _RUBRIC = {
+                "event": [],
+                "outcome": [
+                    {"id": "correctness.tests_pass", "description": "Tests pass", "weight": 0.5, "scale": "bounded"},
+                    {"id": "efficiency.code_quality", "description": "Code quality", "weight": 0.3, "scale": "bounded"},
+                    {"id": "efficiency.solution_steps", "description": "Solution efficiency", "weight": 0.2, "scale": "bounded"}
+                ]
+            }
+    return _RUBRIC
+
+
+def judge(payload: Dict[str, Any], **kwargs: Any) -> float:
+    """
+    Call the Synth backend judge API to score a Verilog rollout.
+
+    Args:
+        payload: Dict with keys: seed, prompt, completion, metrics, response, trace
+        **kwargs: Additional config (backend_url, model, timeout_s, etc.)
+
+    Returns:
+        float: Aggregate score from 0.0 to 1.0
+    """
+    try:
+        # Extract configuration
+        backend_url = kwargs.get("backend_url", "http://localhost:8000/api")
+        model = kwargs.get("model", "openai/gpt-oss-120b")
+        timeout = kwargs.get("timeout_s", 45)
+        
+        assert isinstance(backend_url, str), "backend_url must be a string"
+        assert isinstance(model, str), "model must be a string"
+        assert isinstance(timeout, (int, float)), "timeout_s must be numeric"
+        
+        # Extract trajectory from response
+        response_data = payload.get("response", {})
+        assert isinstance(response_data, dict), "response must be a dict"
+        
+        trajectories = response_data.get("trajectories", [])
+        assert isinstance(trajectories, list), "trajectories must be a list"
+        
+        if not trajectories:
+            print("[verilog_backend_judge] No trajectories in response")
+            return 0.0
+        
+        trajectory = trajectories[0]  # First trajectory
+        assert isinstance(trajectory, dict), "trajectory must be a dict"
+        
+        # Load rubric
+        rubric = _load_rubric()
+        
+        # Transform trajectory into JudgeTracePayload format
+        # The backend expects: event_history, markov_blanket_message_history, metadata
+        steps = trajectory.get("steps", [])
+        assert isinstance(steps, list), "trajectory steps must be a list"
+        
+        event_history: List[Dict[str, Any]] = []
+        for idx, step in enumerate(steps):
+            assert isinstance(step, dict), f"step {idx} must be a dict"
+            # Each step becomes an event with obs, tool_calls, reward, done, info
+            event_history.append({
+                "observation": step.get("obs", {}),
+                "tool_calls": step.get("tool_calls", []),
+                "reward": step.get("reward", 0.0),
+                "done": step.get("done", False),
+                "truncated": step.get("truncated", False),
+                "info": step.get("info", {}),
+            })
+        
+        # Add final observation - backend will extract this as outcome context
+        final_data = trajectory.get("final", {})
+        if final_data:
+            assert isinstance(final_data, dict), "final data must be a dict"
+            final_obs = final_data.get("observation", {})
+            assert isinstance(final_obs, dict), "final observation must be a dict"
+            
+            event_history.append({
+                "observation": final_obs,
+                "reward": final_data.get("reward", 0.0),
+                "done": final_data.get("done", True),
+                "truncated": final_data.get("truncated", False),
+                "info": final_data.get("info", {}),
+            })
+        
+        # Build trace metadata
+        metadata: TraceMetadata = {
+            "env_id": trajectory.get("env_id", "verilog"),
+            "policy_id": trajectory.get("policy_id", "verilog-designer"),
+            "length": trajectory.get("length", len(steps)),
+        }
+        
+        # Build judge request with rubric included
+        judge_request: JudgeScoreRequest = {
+            "policy_name": "verilog-designer",
+            "task_app": {"id": "grpo-verilog"},
+            "trace": {
+                "event_history": event_history,
+                "markov_blanket_message_history": [],
+                "metadata": metadata,
+            },
+            "rubric": rubric,
+            "options": {
+                "model": model,
+                "timeout_s": timeout,
+                "event": False,  # Not scoring per-event
+                "outcome": True,  # Score the final outcome
+            }
+        }
+        
+        # Call backend judge API
+        with httpx.Client(timeout=timeout) as client:
+            # Get API key from env
+            api_key = os.environ.get("SYNTH_API_KEY") or os.environ.get("OPENAI_API_KEY")
+            headers = {}
+            if api_key:
+                headers["Authorization"] = f"Bearer {api_key}"
+            
+            url = f"{backend_url.rstrip('/')}/judge/v1/score"
+            
+            # Debug: print request details
+            print(f"\n[verilog_backend_judge] REQUEST DEBUG:")
+            print(f"  URL: {url}")
+            print(f"  Request body keys: {list(judge_request.keys())}")
+            rubric_data = judge_request.get('rubric', {})
+            print(f"  Rubric event criteria: {len(rubric_data.get('event', []))}")
+            print(f"  Rubric outcome criteria: {len(rubric_data.get('outcome', []))}")
+            trace_data = judge_request.get('trace', {})
+            event_hist = trace_data.get('event_history', [])
+            print(f"  Trace event_history count: {len(event_hist)}")
+            if event_hist:
+                last_event = event_hist[-1]
+                last_obs = last_event.get('observation', {})
+                print(f"  Last event done: {last_event.get('done', False)}")
+                print(f"  Last obs keys: {list(last_obs.keys())}")
+                print(f"  Task completed: {last_obs.get('task_completed', 'N/A')}")
+            
+            response = client.post(url, json=judge_request, headers=headers)
+            
+            # Debug: print response details
+            print(f"\n[verilog_backend_judge] RESPONSE DEBUG:")
+            print(f"  Status: {response.status_code}")
+            print(f"  Response body: {response.text[:500]}")  # First 500 chars
+            
+            response.raise_for_status()
+            result = response.json()
+            assert isinstance(result, dict), "Response must be a dict"
+            
+            # Extract aggregate score
+            aggregate_score = result.get("aggregate_score", 0.0)
+            
+            # Try outcome_review.total if aggregate_score not found
+            if aggregate_score == 0.0 and "outcome_review" in result:
+                outcome_review = result["outcome_review"]
+                if isinstance(outcome_review, dict):
+                    aggregate_score = outcome_review.get("total", 0.0)
+            
+            print(f"  Aggregate score: {aggregate_score}\n")
+            return float(aggregate_score)
+            
+    except httpx.HTTPStatusError as e:
+        print(f"\n[verilog_backend_judge] HTTP ERROR:")
+        print(f"  Status: {e.response.status_code}")
+        print(f"  Response body: {e.response.text}\n")
+        return 0.0
+    except AssertionError as e:
+        print(f"[verilog_backend_judge] Assertion error: {e}")
+        return 0.0
+    except Exception as e:
+        print(f"[verilog_backend_judge] Unexpected error: {e}")
+        return 0.0