synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

examples/multi_step/configs/verilog_rl_lora.toml

@@ -1,190 +0,0 @@
-# Verilog RL experiment – LoRA training on Qwen3-0.6B
-#
-# This configuration adapts the Crafter RL setup for Verilog spec-to-RTL tasks.
-# Uses the same proven pipeline but optimized for 0.6B model and Verilog domain.
-
-[algorithm]
-type = "online"
-method = "policy_gradient"
-variety = "gspo"
-
-[services]
-# Replace with the Modal URL printed by `uvx synth-ai modal-serve grpo-verilog`
-task_url = "https://synth-laboratories--grpo-verilog-task-app-fastapi-app-dev.modal.run"
-# Point at the Synth backend (or compatible service) that exposes /api/judge/v1/*
-judge_url = "https://synth-backend-dev-docker.onrender.com/api"
-
-[compute]
-gpu_type = "H200"  # ✅ 8B model needs H200 for larger context window
-gpu_count = 2      # ✅ Minimum 2x GPUs (1 for vLLM inference + 1 for training)
-nodes = 1
-
-[topology]
-type = "single_node_split"
-gpus_for_vllm = 1      # ✅ vLLM for inference
-gpus_for_training = 1  # ✅ Training GPU (8B LoRA fits well)
-gpus_for_ref = 0
-tensor_parallel = 1
-
-[vllm]
-tensor_parallel_size = 1
-max_model_len = 24576  # ✅ Increased to 24K to accommodate long Verilog prompts (16K + 8K buffer for testbenches + history)
-
-[reference]
-placement = "none"
-
-[model]
-base = "Qwen/Qwen3-8B"  # ✅ 8B model for RL training with good balance of speed and capability
-trainer_mode = "lora"
-label = "verilog-rl-lora-qwen8b"
-
-[lora]
-r = 16
-alpha = 32
-dropout = 0.05
-target_modules = ["all-linear"]
-
-[rollout]
-env_name = "verilog"  # ✅ Changed from "crafter" to "verilog"
-max_turns = 6        # ✅ More steps for compilation chains vs Crafter's 10
-episodes_per_batch = 4  # ✅ Good batch size for 8B model
-policy_name = "verilog-designer"
-max_concurrent_rollouts = 8
-batches_per_step = 2
-ops = ["agent", "env"]
-
-  [rollout.env_config]
-  # Verilog-specific environment settings
-  difficulty = "medium"  # Can be "easy", "medium", or "hard"
-
-    [rollout.env_config.step_rewards]
-    enabled = true
-    mode = "decision_stepwise"
-    strategy = "consistent"
-    indicator_lambda = 0.5  # ✅ Reduced from Crafter (sparser rewards)
-    step_beta = 0.0
-
-  [rollout.policy_config]
-  provider = "openai"
-  model = "Qwen/Qwen3-8B"  # ✅ Use the model being trained (8B) for rollouts
-  temperature = 0.2
-  max_tokens = 4096  # ✅ Balanced for Verilog generation while leaving room for long input prompts (testbenches + history)
-
-[evaluation]
-instances = 16
-every_n_iters = 10
-seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
-
-[training]
-num_epochs = 1
-iterations_per_epoch = 5
-gradient_accumulation_steps = 1
-max_accumulated_minibatch = 1
-max_turns = 15
-batch_size = 4          # ✅ Same as Crafter (works well for 8B LoRA)
-group_size = 4
-learning_rate = 5e-5    # ✅ Same as Crafter
-log_interval = 1
-weight_sync_interval = 1
-event_rewards_kind = "unique"
-async_semaphore_max = 20  # Max concurrent rollouts in streaming pipeline
-
-# Enable dense decision rewards in the trainer
-step_rewards_enabled = true
-step_rewards_mode = "decision_stepwise"
-step_rewards_indicator_lambda = 0.5  # ✅ Reduced for Verilog's sparser rewards
-step_rewards_beta = 0.0
-step_rewards_strategy = "consistent"
-
-[training.weight_sync]
-enable = true
-targets = ["policy"]
-mode = "direct"
-direct = true
-verify_every_k = 0
-
-[rubric]
-enabled = true
-model = "openai/gpt-oss-120b"
-api_base = "https://synth-backend-dev-docker.onrender.com/api/judge"
-api_key_env = "OPENAI_API_KEY"
-
-# Blend the hosted judge scores with environment returns
-[rubric.weights]
-env = 0.3     # ✅ Higher weight on env rewards for Verilog (vs Crafter's 0.2)
-event = 0.3   # ✅ Adjusted for Verilog's different reward structure
-outcome = 0.4
-
-[rubric.event]
-# Verilog-specific event rubric for process efficiency
-rubric_id = "verilog/event@v1"
-criteria = [
-  { key = "process.compilation_success", weight = 0.7, description = "Return 1.0 when compilation succeeds, 0.5 for partial success, 0.0 for failure", aggregation = "weighted_sum" },
-  { key = "process.design_iterations", weight = 0.3, description = "Reward efficient design iterations without unnecessary recompilation", aggregation = "weighted_sum" },
-]
-
-[rubric.outcome]
-# Verilog-specific outcome rubric for final results
-rubric_id = "verilog/outcome@v1"
-criteria = [
-  { key = "outcome.tests_passed", weight = 0.8, description = "Full credit when all tests pass, partial for some tests", aggregation = "weighted_sum" },
-  { key = "outcome.design_quality", weight = 0.2, description = "Code quality, documentation, and design efficiency", aggregation = "weighted_sum" },
-]
-
-[judge]
-type = "groq"
-timeout_s = 45
-
-  [judge.options]
-  event = true
-  outcome = true
-  provider = "openai"
-  model = "openai/gpt-oss-120b"
-  rubric_id = "verilog/bundle@v1"
-  max_concurrency = 6
-  tracks = ["process", "reasoning", "progress", "outcome"]
-
-    [judge.options.rubric_overrides]
-
-      [judge.options.rubric_overrides.event]
-      goal_text = """
-      Evaluate each Verilog design decision for compilation success and process efficiency.
-      High scores for successful compilation and strategic tool usage.
-      Penalize unnecessary operations and compilation failures."""
-      aggregation = "weighted_sum"
-
-        [[judge.options.rubric_overrides.event.criteria]]
-        id = "process.compilation_success"
-        weight = 0.7
-        scale = "bounded"
-        description = "Return 1.0 when compilation succeeds cleanly, 0.5 for warnings, 0.0 for errors"
-
-        [[judge.options.rubric_overrides.event.criteria]]
-        id = "process.design_iterations"
-        weight = 0.3
-        scale = "bounded"
-        description = "Reward efficient write→compile→simulate workflow, penalize redundant operations"
-
-      [judge.options.rubric_overrides.outcome]
-      goal_text = """
-      Evaluate the final Verilog implementation for correctness and quality.
-      High scores for working designs that pass all tests with good code quality."""
-      aggregation = "weighted_sum"
-
-        [[judge.options.rubric_overrides.outcome.criteria]]
-        id = "outcome.tests_passed"
-        weight = 0.8
-        scale = "binary"
-        description = "Full credit when all tests pass, partial credit for some tests passing"
-
-        [[judge.options.rubric_overrides.outcome.criteria]]
-        id = "outcome.design_quality"
-        weight = 0.2
-        scale = "bounded"
-        description = "Code clarity, proper documentation, and efficient design patterns"
-
-    [judge.options.weights]
-    process = 0.1
-    reasoning = 0.2
-    progress = 0.3
-    outcome = 0.4

examples/multi_step/crafter_rl_lora.md

@@ -1,70 +0,0 @@
-# Crafter RL LoRA (10-step runs)
-
-This walkthrough shows how to fine-tune the Crafter task app with our 10-step RL LoRA config.
-
-1. **Deploy the Crafter task app on Modal**
-
-   ```bash
-   # assumes .env contains SYNTH_API_KEY, ENVIRONMENT_API_KEY, GROQ_API_KEY, etc.
-   uvx synth-ai modal-serve grpo-crafter \
-     --env-file examples/warming_up_to_rl/.env \
-     --name grpo-crafter-task-app
-   ```
-
-   * The command prints the public `https://…modal.run` URL; copy it for the RL configs below.*
-
-2. **Wire up the three RL experiment configs**
-
-   Update the `task_url` placeholder in each config with the Modal URL from step 1:
-
-   - `examples/multi_step/configs/crafter_rl_outcome.toml`
-   - `examples/multi_step/configs/crafter_rl_stepwise_simple.toml`
-   - `examples/multi_step/configs/crafter_rl_stepwise_shaped.toml`
-
-   The difference between them (all run with LoRA on 2×H100 split 1/1 for vLLM vs. trainer):
-
-   | Config | Reward signal |
-   | ------ | ------------- |
-   | `crafter_rl_outcome.toml` | Outcome-only — step rewards disabled. |
-   | `crafter_rl_stepwise_simple.toml` | Stepwise (“consistent”) — +1 for every newly unlocked achievement. |
-   | `crafter_rl_stepwise_shaped.toml` | Stepwise (“per_achievement”) — combines achievement credit with inventory/achievement-count shaping from the rollout hook. |
-
-3. **Launch the three RL runs in parallel**
-
-   ```bash
-   export SYNTH_API_KEY=...       # already sourced if examples/.env was loaded
-   export TASK_APP_URL=https://your-modal-task-app.modal.run
-
-   uvx synth-ai train --type rl \
-     --config examples/multi_step/configs/crafter_rl_outcome.toml \
-     --run-name crafter-rl-outcome \
-     --no-poll &
-
-   uvx synth-ai train --type rl \
-     --config examples/multi_step/configs/crafter_rl_stepwise_simple.toml \
-     --run-name crafter-rl-stepwise-simple \
-     --no-poll &
-
-   uvx synth-ai train --type rl \
-     --config examples/multi_step/configs/crafter_rl_stepwise_shaped.toml \
-     --run-name crafter-rl-stepwise-shaped \
-     --no-poll &
-
-   wait
-   ```
-
-   *`--no-poll` returns immediately so each run can stream logs in its own terminal; `wait` blocks until all jobs finish.*
-
-4. **Track results**
-
-   Tail each job’s logs with `uvx synth-ai train logs --run-name <name>` or open the Modal dashboard. Compare:
-
-   - Avg outcome reward (modal dashboard)
-   - Stepwise reward components (`resource_reward`, `unique_achievements_total`) in the task app logs
-   - Trace JSONL dumps under `traces/v3` if tracing is enabled
-
-
-  * This config forces 10 agent turns per rollout, reduces batch size to avoid OOMs, and enforces Crafter-specific defaults.*
-
-  INFO - 🎉 Training completed successfully!
-  INFO - All batch rewards: [0.0625, 0.0625, 0.125, 0.0625, 0.0625, 0.3125, 0.375, 0.4375, 0.5, 0.9375]

examples/multi_step/judges/crafter_backend_judge.py

@@ -1,220 +0,0 @@
-"""Crafter backend judge that calls the Synth judge API with inline rubric."""
-
-from __future__ import annotations
-
-import json
-import os
-from pathlib import Path
-from typing import Any, Dict, List, TypedDict
-
-import httpx
-
-
-class TraceMetadata(TypedDict, total=False):
-    """Metadata for the trace."""
-    env_id: str
-    policy_id: str
-    length: int
-
-
-class JudgeTracePayload(TypedDict):
-    """Trace payload sent to backend judge."""
-    event_history: List[Dict[str, Any]]
-    markov_blanket_message_history: List[Dict[str, Any]]
-    metadata: TraceMetadata
-
-
-class JudgeOptions(TypedDict, total=False):
-    """Options for judge scoring."""
-    model: str
-    timeout_s: int
-    event: bool
-    outcome: bool
-
-
-class TaskApp(TypedDict):
-    """Task application metadata."""
-    id: str
-
-
-class JudgeScoreRequest(TypedDict):
-    """Request to backend judge API."""
-    policy_name: str
-    task_app: TaskApp
-    trace: JudgeTracePayload
-    rubric: Dict[str, Any]
-    options: JudgeOptions
-
-
-# Load rubric from file (cached at module level)
-_RUBRIC_PATH = Path(__file__).parent.parent / "rubrics" / "crafter_backend_judge.json"
-_RUBRIC: Dict[str, Any] | None = None
-
-
-def _load_rubric() -> Dict[str, Any]:
-    """Load rubric from file with fallback to inline default."""
-    global _RUBRIC
-    if _RUBRIC is None:
-        try:
-            with open(_RUBRIC_PATH, 'r') as f:
-                _RUBRIC = json.load(f)
-            assert isinstance(_RUBRIC, dict), "Rubric must be a dict"
-            assert "outcome" in _RUBRIC, "Rubric must have 'outcome' key"
-            assert isinstance(_RUBRIC["outcome"], list), "Rubric 'outcome' must be a list"
-        except Exception as e:
-            print(f"[crafter_backend_judge] Warning: Failed to load rubric from {_RUBRIC_PATH}: {e}")
-            # Fallback inline rubric (matching RubricCriteriaBlock format)
-            _RUBRIC = {
-                "event": [],
-                "outcome": [
-                    {"id": "achievement_progression", "description": "Achievement progression", "weight": 0.35, "scale": "bounded"},
-                    {"id": "resource_stockpile", "description": "Resource stockpile", "weight": 0.2, "scale": "bounded"},
-                    {"id": "survival_state", "description": "Survival state", "weight": 0.2, "scale": "bounded"},
-                    {"id": "failure_analysis", "description": "Failure analysis", "weight": 0.15, "scale": "bounded"},
-                    {"id": "future_readiness", "description": "Future readiness", "weight": 0.1, "scale": "bounded"}
-                ]
-            }
-    return _RUBRIC
-
-
-def judge(payload: Dict[str, Any], **kwargs: Any) -> float:
-    """
-    Call the Synth backend judge API to score a Crafter rollout.
-
-    Args:
-        payload: Dict with keys: seed, prompt, completion, metrics, response, trace
-        **kwargs: Additional config (backend_url, model, timeout_s, etc.)
-
-    Returns:
-        float: Aggregate score from 0.0 to 1.0
-    """
-    try:
-        # Extract configuration
-        backend_url = kwargs.get("backend_url", "http://localhost:8000/api")
-        model = kwargs.get("model", "openai/gpt-oss-120b")
-        timeout = kwargs.get("timeout_s", 45)
-        
-        assert isinstance(backend_url, str), "backend_url must be a string"
-        assert isinstance(model, str), "model must be a string"
-        assert isinstance(timeout, (int, float)), "timeout_s must be numeric"
-        
-        # Extract trajectory from response
-        response_data = payload.get("response", {})
-        assert isinstance(response_data, dict), "response must be a dict"
-        
-        trajectories = response_data.get("trajectories", [])
-        assert isinstance(trajectories, list), "trajectories must be a list"
-        
-        if not trajectories:
-            print("[crafter_backend_judge] No trajectories in response")
-            return 0.0
-        
-        trajectory = trajectories[0]  # First trajectory
-        assert isinstance(trajectory, dict), "trajectory must be a dict"
-        
-        # Load rubric
-        rubric = _load_rubric()
-        
-        # Transform trajectory into JudgeTracePayload format
-        steps = trajectory.get("steps", [])
-        assert isinstance(steps, list), "trajectory steps must be a list"
-        
-        event_history: List[Dict[str, Any]] = []
-        for idx, step in enumerate(steps):
-            assert isinstance(step, dict), f"step {idx} must be a dict"
-            # Each step becomes an event
-            event_history.append({
-                "observation": step.get("obs", {}),
-                "tool_calls": step.get("tool_calls", []),
-                "reward": step.get("reward", 0.0),
-                "done": step.get("done", False),
-                "truncated": step.get("truncated", False),
-                "info": step.get("info", {}),
-            })
-        
-        # Add final observation - backend will extract this as outcome context
-        final_data = trajectory.get("final", {})
-        if final_data:
-            assert isinstance(final_data, dict), "final data must be a dict"
-            final_obs = final_data.get("observation", {})
-            assert isinstance(final_obs, dict), "final observation must be a dict"
-            
-            event_history.append({
-                "observation": final_obs,
-                "reward": final_data.get("reward", 0.0),
-                "done": final_data.get("done", True),
-                "truncated": final_data.get("truncated", False),
-                "info": final_data.get("info", {}),
-            })
-        
-        # Build trace metadata
-        metadata: TraceMetadata = {
-            "env_id": trajectory.get("env_id", "crafter"),
-            "policy_id": trajectory.get("policy_id", "crafter-react"),
-            "length": trajectory.get("length", len(steps)),
-        }
-        
-        # Build judge request with rubric included
-        judge_request: JudgeScoreRequest = {
-            "policy_name": "crafter-react",
-            "task_app": {"id": "grpo-crafter-task-app"},
-            "trace": {
-                "event_history": event_history,
-                "markov_blanket_message_history": [],
-                "metadata": metadata,
-            },
-            "rubric": rubric,
-            "options": {
-                "model": model,
-                "timeout_s": timeout,
-                "event": False,  # Not scoring per-event
-                "outcome": True,  # Score the final outcome
-            }
-        }
-        
-        # Call backend judge API
-        with httpx.Client(timeout=timeout) as client:
-            # Get API key from env
-            api_key = os.environ.get("SYNTH_API_KEY") or os.environ.get("OPENAI_API_KEY")
-            headers = {}
-            if api_key:
-                headers["Authorization"] = f"Bearer {api_key}"
-            
-            url = f"{backend_url.rstrip('/')}/judge/v1/score"
-            
-            # Debug: print request summary
-            print(f"\n[crafter_backend_judge] Scoring trajectory with {len(event_history)} events")
-            if event_history:
-                last_obs = event_history[-1].get('observation', {})
-                print(f"  Final observation keys: {list(last_obs.keys())[:5]}...")
-            
-            response = client.post(url, json=judge_request, headers=headers)
-            
-            response.raise_for_status()
-            result = response.json()
-            assert isinstance(result, dict), "Response must be a dict"
-            
-            # Extract aggregate score
-            aggregate_score = result.get("aggregate_score", 0.0)
-            
-            # Try outcome_review.total if aggregate_score not found
-            if aggregate_score == 0.0 and "outcome_review" in result:
-                outcome_review = result["outcome_review"]
-                if isinstance(outcome_review, dict):
-                    aggregate_score = outcome_review.get("total", 0.0)
-            
-            print(f"  Backend judge score: {aggregate_score:.3f}\n")
-            return float(aggregate_score)
-            
-    except httpx.HTTPStatusError as e:
-        print(f"\n[crafter_backend_judge] HTTP ERROR:")
-        print(f"  Status: {e.response.status_code}")
-        print(f"  Response: {e.response.text[:300]}\n")
-        return 0.0
-    except AssertionError as e:
-        print(f"[crafter_backend_judge] Assertion error: {e}")
-        return 0.0
-    except Exception as e:
-        print(f"[crafter_backend_judge] Unexpected error: {e}")
-        return 0.0
-