synth-ai 0.2.12__py3-none-any.whl → 0.2.13.dev2__py3-none-any.whl

examples/task_apps/verilog/task_app/grpo_verilog_task_app.py

@@ -0,0 +1,145 @@
+"""Compatibility wrapper for the GRPO Verilog task app.
+
+This mirrors the Crafter task app wrapper while delegating configuration to
+`grpo_verilog.py`. Normal usage should prefer `uvx synth-ai serve grpo-verilog`,
+but the module remains for direct execution or importing the FastAPI app.
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from starlette.requests import Request
+from synth_ai.task.apps import ModalDeploymentConfig, registry
+from synth_ai.task.auth import is_api_key_header_authorized, normalize_environment_api_key
+from synth_ai.task.server import TaskAppConfig, create_task_app, run_task_app
+
+from .grpo_verilog import build_config
+
+APP_ID = "grpo-verilog"
+
+
+def _build_base_config() -> TaskAppConfig:
+    # Lazily construct the base config to avoid heavy work at import time.
+    return build_config()
+
+
+try:
+    _REGISTERED_ENTRY = registry.get(APP_ID)
+except Exception:  # pragma: no cover - registry unavailable in some contexts
+    MODAL_DEPLOYMENT: ModalDeploymentConfig | None = None
+    ENV_FILES: tuple[str, ...] = ()
+else:
+    MODAL_DEPLOYMENT = _REGISTERED_ENTRY.modal
+    ENV_FILES = tuple(_REGISTERED_ENTRY.env_files)
+
+
+def build_task_app_config() -> TaskAppConfig:
+    """Return a fresh TaskAppConfig for this wrapper."""
+    base = _build_base_config()
+    return base.clone()
+
+
+def fastapi_app():
+    """Return the FastAPI application for Modal or other ASGI hosts."""
+
+    app = create_task_app(build_task_app_config())
+
+    # Replace default health endpoints so we can permit soft auth failures and log 422s.
+    filtered_routes = []
+    for route in app.router.routes:
+        path = getattr(route, "path", None)
+        methods = getattr(route, "methods", set()) or set()
+        if path in {"/health", "/health/rollout"} and "GET" in methods:
+            continue
+        filtered_routes.append(route)
+    app.router.routes = filtered_routes
+
+    def _log_env_key_prefix(source: str, env_key: str | None) -> str | None:
+        if not env_key:
+            return None
+        prefix = env_key[: max(1, len(env_key) // 2)]
+        print(f"[{source}] expected ENVIRONMENT_API_KEY prefix: {prefix}")
+        return prefix
+
+    @app.get("/health")
+    async def health(request: Request):
+        env_key = normalize_environment_api_key()
+        if not env_key:
+            return JSONResponse(
+                status_code=503,
+                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
+            )
+        if not is_api_key_header_authorized(request):
+            prefix = _log_env_key_prefix("health", env_key)
+            content = {"status": "healthy", "authorized": False}
+            if prefix:
+                content["expected_api_key_prefix"] = prefix
+            return JSONResponse(status_code=200, content=content)
+        return {"status": "healthy", "authorized": True}
+
+    @app.get("/health/rollout")
+    async def health_rollout(request: Request):
+        env_key = normalize_environment_api_key()
+        if not env_key:
+            return JSONResponse(
+                status_code=503,
+                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
+            )
+        if not is_api_key_header_authorized(request):
+            prefix = _log_env_key_prefix("health/rollout", env_key)
+            content = {"status": "healthy", "authorized": False}
+            if prefix:
+                content["expected_api_key_prefix"] = prefix
+            return JSONResponse(status_code=200, content=content)
+        return {"ok": True, "authorized": True}
+
+    @app.exception_handler(RequestValidationError)
+    async def _on_validation_error(request: Request, exc: RequestValidationError):
+        try:
+            hdr = request.headers
+            snapshot = {
+                "path": str(request.url.path),
+                "have_x_api_key": bool(hdr.get("x-api-key")),
+                "have_x_api_keys": bool(hdr.get("x-api-keys")),
+                "have_authorization": bool(hdr.get("authorization")),
+                "errors": exc.errors()[:5],
+            }
+            print("[422] validation", snapshot, flush=True)
+        except Exception:
+            pass
+        return JSONResponse(
+            status_code=422,
+            content={"status": "invalid", "detail": exc.errors()[:5]},
+        )
+
+    return app
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the Verilog task app locally")
+    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=8103)
+    parser.add_argument("--reload", action="store_true", help="Enable uvicorn autoreload")
+    parser.add_argument(
+        "--env-file",
+        action="append",
+        default=[],
+        help="Additional .env files to load before startup",
+    )
+    args = parser.parse_args()
+
+    default_env = Path(__file__).resolve().parents[4] / "backend" / ".env.dev"
+    env_files = [str(default_env)] if default_env.exists() else []
+    env_files.extend(args.env_file or [])
+
+    run_task_app(
+        build_task_app_config,
+        host=args.host,
+        port=args.port,
+        reload=args.reload,
+        env_files=env_files,
+    )

examples/task_apps/verilog/tests/__init__.py

@@ -0,0 +1,2 @@
+# Verilog task app tests
+

examples/task_apps/verilog/tests/conftest.py

@@ -0,0 +1,115 @@
+"""Shared fixtures for Verilog tests."""
+import os
+import socket
+import subprocess
+from subprocess import TimeoutExpired
+import time
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+
+requests = pytest.importorskip("requests")
+
+
+def _which(executable: str) -> bool:
+    return any(
+        (Path(path) / executable).exists()
+        for path in os.getenv("PATH", "").split(os.pathsep)
+    )
+
+
+def _find_free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
+def _wait_for_server(base_url: str, timeout: float = 60.0) -> None:
+    """Wait for the Verilog server to become ready."""
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        try:
+            # Try /info first (no auth required if --insecure)
+            resp = requests.get(f"{base_url}/info", timeout=2.0)
+            if resp.status_code == 200:
+                return
+            # If 400/401, server is up but needs auth - that's OK
+            if resp.status_code in (400, 401):
+                return
+        except Exception:
+            time.sleep(0.5)
+    raise RuntimeError(f"Task app at {base_url} did not become ready")
+
+
+@pytest.fixture(scope="module")
+def verilog_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]:
+    """Start the Verilog task app server for testing."""
+    if not _which("uv"):
+        pytest.skip("uv executable not found on PATH")
+    if "GROQ_API_KEY" not in os.environ:
+        pytest.skip("GROQ_API_KEY must be set for Groq-backed tests")
+
+    port = _find_free_port()
+    base_url = f"http://127.0.0.1:{port}"
+    tmp_path = tmp_path_factory.mktemp("verilog")
+    trace_dir = tmp_path / "traces"
+    trace_dir.mkdir(parents=True, exist_ok=True)
+
+    env = os.environ.copy()
+    cmd = [
+        "uv",
+        "run",
+        "-m",
+        "synth_ai",
+        "task-app",
+        "serve",
+        "grpo-verilog",
+        "--port",
+        str(port),
+        "--no-reload",
+    ]
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        env=env,
+        stdin=subprocess.PIPE,
+    )
+    
+    # Send "n" to decline tracing
+    try:
+        if proc.stdin:
+            proc.stdin.write("n\n")
+            proc.stdin.flush()
+    except Exception:
+        pass
+    
+    stdout_capture = ""
+    try:
+        time.sleep(2)
+        if proc.poll() is not None:
+            stdout_capture, _ = proc.communicate(timeout=2)
+            tail = "\n".join(stdout_capture.strip().splitlines()[-20:]) if stdout_capture else ""
+            pytest.skip(f"Task app terminated immediately:\n{tail}")
+        
+        _wait_for_server(base_url)
+        yield base_url
+    except RuntimeError as e:
+        proc.terminate()
+        try:
+            stdout_capture, _ = proc.communicate(timeout=10)
+        except TimeoutExpired:
+            proc.kill()
+            stdout_capture, _ = proc.communicate()
+        tail = "\n".join((stdout_capture or "").strip().splitlines()[-20:])
+        pytest.skip(f"Task app failed to start: {e}\n{tail}")
+    finally:
+        if proc.poll() is None:
+            proc.terminate()
+            try:
+                proc.wait(timeout=5)
+            except TimeoutExpired:
+                proc.kill()
+

examples/task_apps/verilog/tests/integration/__init__.py

@@ -0,0 +1,2 @@
+# Integration tests for Verilog task app
+

examples/task_apps/verilog/tests/integration/test_verilog_eval.py

@@ -0,0 +1,179 @@
+"""Integration tests for Verilog task app with Groq evaluation."""
+from __future__ import annotations
+
+import os
+import socket
+import subprocess
+from subprocess import TimeoutExpired
+import time
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+
+requests = pytest.importorskip("requests")
+
+
+HERE = Path(__file__).resolve().parent
+TASK_APP_ROOT = HERE.parents[1]
+CONFIG_PATH = TASK_APP_ROOT / "eval_groq_qwen32b.toml"
+
+
+def _which(executable: str) -> bool:
+    return any(
+        (Path(path) / executable).exists()
+        for path in os.getenv("PATH", "").split(os.pathsep)
+    )
+
+
+def _find_free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
+def _wait_for_server(base_url: str, timeout: float = 60.0) -> None:
+    """Wait for the Verilog server to become ready."""
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        try:
+            resp = requests.get(f"{base_url}/info", timeout=2.0)
+            if resp.status_code == 200:
+                return
+        except Exception:
+            time.sleep(0.5)
+    raise RuntimeError(f"Task app at {base_url} did not become ready")
+
+
+@pytest.fixture
+def verilog_server(tmp_path: Path) -> Iterator[str]:
+    """Start the Verilog task app server for testing."""
+    if not _which("uv"):
+        pytest.skip("uv executable not found on PATH")
+    if "GROQ_API_KEY" not in os.environ:
+        pytest.skip("GROQ_API_KEY must be set for Groq-backed evals")
+
+    port = _find_free_port()
+    base_url = f"http://127.0.0.1:{port}"
+    trace_dir = tmp_path / "traces"
+    trace_dir.mkdir(parents=True, exist_ok=True)
+
+    env = os.environ.copy()
+    cmd = [
+        "uv",
+        "run",
+        "-m",
+        "synth_ai",
+        "task-app",
+        "serve",
+        "grpo-verilog",
+        "--port",
+        str(port),
+        "--no-reload",
+    ]
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        env=env,
+        stdin=subprocess.PIPE,  # Auto-answer tracing prompt
+    )
+    
+    # Send "n" to decline tracing
+    try:
+        if proc.stdin:
+            proc.stdin.write("n\n")
+            proc.stdin.flush()
+    except Exception:
+        pass
+    
+    stdout_capture = ""
+    try:
+        # Check if process died immediately
+        time.sleep(2)
+        if proc.poll() is not None:
+            stdout_capture, _ = proc.communicate(timeout=2)
+            tail = "\n".join(stdout_capture.strip().splitlines()[-20:]) if stdout_capture else ""
+            pytest.skip(f"Task app terminated immediately:\n{tail}")
+        
+        _wait_for_server(base_url)
+        yield base_url
+    except RuntimeError as e:
+        proc.terminate()
+        try:
+            stdout_capture, _ = proc.communicate(timeout=10)
+        except TimeoutExpired:
+            proc.kill()
+            stdout_capture, _ = proc.communicate()
+        tail = "\n".join((stdout_capture or "").strip().splitlines()[-20:])
+        pytest.skip(f"Task app failed to start: {e}\n{tail}")
+    finally:
+        if proc.poll() is None:
+            proc.terminate()
+            try:
+                proc.wait(timeout=5)
+            except TimeoutExpired:
+                proc.kill()
+
+
+@pytest.mark.slow
+def test_verilog_server_health(verilog_server: str) -> None:
+    """Test that the Verilog server health endpoint works."""
+    # Health endpoint requires auth, so we expect 400 (auth failed) or 200
+    resp = requests.get(f"{verilog_server}/health", timeout=5.0)
+    assert resp.status_code in (200, 400), f"Unexpected status: {resp.status_code}"
+
+
+@pytest.mark.slow
+def test_verilog_task_info(verilog_server: str) -> None:
+    """Test that the Verilog server returns valid task_info."""
+    resp = requests.get(f"{verilog_server}/task_info", timeout=5.0)
+    assert resp.status_code == 200
+    data = resp.json()
+    assert "task" in data
+    assert data["task"]["id"] == "verilog"
+
+
+@pytest.mark.slow
+def test_verilog_eval_with_groq(verilog_server: str) -> None:
+    """Spin up the Verilog task app and run a Groq-backed eval."""
+    if not CONFIG_PATH.exists():
+        pytest.skip(f"Config file not found: {CONFIG_PATH}")
+    
+    cmd = [
+        "uv",
+        "run",
+        "-m",
+        "synth_ai",
+        "eval",
+        "grpo-verilog",
+        "--config",
+        str(CONFIG_PATH),
+        "--url",
+        verilog_server,
+        "--model",
+        "qwen/qwen3-32b",
+        "--seeds",
+        "0",  # Just test one seed
+    ]
+    result = subprocess.run(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        env=os.environ.copy(),
+        check=False,
+        timeout=300,  # 5 minutes max
+    )
+    
+    if result.returncode != 0:
+        pytest.fail(f"Eval failed with return code {result.returncode}:\n{result.stdout}")
+    
+    # Check for success indicators
+    assert "Eval complete" in result.stdout
+    assert "1 ok, 0 failed" in result.stdout or "status=200" in result.stdout
+    
+    # Check that we got a meaningful outcome score
+    assert "outcome" in result.stdout.lower() or "mean_return" in result.stdout.lower()
+

examples/task_apps/verilog/tests/integration/test_verilog_rollout.py

@@ -0,0 +1,55 @@
+"""Integration test for Verilog rollouts via /rollout endpoint."""
+import os
+import pytest
+
+requests = pytest.importorskip("requests")
+
+# Use the actual ENVIRONMENT_API_KEY from .env
+AUTH_HEADER = {"Authorization": "Bearer sk_env_30c78a787bac223c716918181209f263"}
+
+
+@pytest.mark.slow
+def test_verilog_policy_rollout(verilog_server: str) -> None:
+    """Test a Verilog rollout using Groq policy."""
+    if "GROQ_API_KEY" not in os.environ:
+        pytest.skip("GROQ_API_KEY required for this test")
+    
+    rollout_payload = {
+        "run_id": "test_policy_verilog",
+        "env": {"seed": 0},
+        "ops": [],  # Empty ops means use policy for all steps
+        "policy": {
+            "policy_name": "qwen-groq",
+            "config": {
+                "provider": "groq",
+                "model": "qwen/qwen3-32b",
+                "max_steps": 5,  # Limit steps for test
+            },
+        },
+    }
+    
+    resp = requests.post(
+        f"{verilog_server}/rollout",
+        json=rollout_payload,
+        headers=AUTH_HEADER,
+        timeout=120.0,
+    )
+    
+    assert resp.status_code == 200, f"Rollout failed: {resp.status_code} {resp.text}"
+    data = resp.json()
+    
+    # Verify response structure
+    assert "trajectories" in data
+    assert "metrics" in data
+    assert "trace" in data
+    
+    trajectory = data["trajectories"][0]
+    assert "steps" in trajectory
+    
+    # Check that at least one step was taken
+    assert len(trajectory["steps"]) > 0
+    
+    # Verify metrics
+    metrics = data["metrics"]
+    assert "episode_returns" in metrics or "mean_return" in metrics
+

examples/task_apps/verilog/tests/unit/__init__.py

@@ -0,0 +1,2 @@
+# Unit tests for Verilog task app
+

examples/task_apps/verilog/tests/unit/test_verilog_scoring.py

@@ -0,0 +1,118 @@
+"""Unit tests for Verilog scoring and rewards."""
+import pytest
+
+from synth_ai.environments.examples.verilog.engine import (
+    VerilogCompileSuccessComponent,
+    VerilogSimulationPassComponent,
+    VerilogSubmitSuccessComponent,
+    VerilogPublicState,
+)
+
+
+@pytest.mark.asyncio
+async def test_compile_success_reward():
+    """Test that successful compilation awards 0.1 reward."""
+    component = VerilogCompileSuccessComponent()
+    state = VerilogPublicState(files={}, build_dir="/tmp", task_completed=False)
+    
+    # Successful compile (returncode 0)
+    action = {"type": "compile", "returncode": 0}
+    reward = await component.score(state, action)
+    assert reward == 0.1
+    
+    # Failed compile (returncode != 0)
+    action_fail = {"type": "compile", "returncode": 1}
+    reward_fail = await component.score(state, action_fail)
+    assert reward_fail == 0.0
+    
+    # Non-compile action
+    action_other = {"type": "write_file"}
+    reward_other = await component.score(state, action_other)
+    assert reward_other == 0.0
+
+
+@pytest.mark.asyncio
+async def test_simulation_pass_reward():
+    """Test that passing simulation awards 1.0 reward."""
+    component = VerilogSimulationPassComponent()
+    state = VerilogPublicState(files={}, build_dir="/tmp", task_completed=False)
+    
+    # Passing simulation
+    action = {"type": "simulate", "passed": True}
+    reward = await component.score(state, action)
+    assert reward == 1.0
+    
+    # Failing simulation
+    action_fail = {"type": "simulate", "passed": False}
+    reward_fail = await component.score(state, action_fail)
+    assert reward_fail == 0.0
+    
+    # Non-simulate action
+    action_other = {"type": "compile"}
+    reward_other = await component.score(state, action_other)
+    assert reward_other == 0.0
+
+
+@pytest.mark.asyncio
+async def test_submit_success_reward():
+    """Test that successful submission awards 10.0 reward."""
+    component = VerilogSubmitSuccessComponent()
+    state = VerilogPublicState(files={}, build_dir="/tmp", task_completed=False)
+    
+    # Successful submission (tests passed)
+    action = {"type": "submit", "passed": True}
+    reward = await component.score(state, action)
+    assert reward == 10.0
+    
+    # Failed submission (tests didn't pass)
+    action_fail = {"type": "submit", "passed": False}
+    reward_fail = await component.score(state, action_fail)
+    assert reward_fail == 0.0
+    
+    # Non-submit action
+    action_other = {"type": "compile"}
+    reward_other = await component.score(state, action_other)
+    assert reward_other == 0.0
+
+
+@pytest.mark.asyncio
+async def test_submit_checks_simulation_output():
+    """Test that submit() correctly checks the last simulation output."""
+    from synth_ai.environments.examples.verilog.engine import VerilogEngine
+    from synth_ai.environments.tasks.core import TaskInstance, Impetus, Intent
+    
+    # Create a minimal task instance
+    task = TaskInstance(
+        id="test",
+        impetus=Impetus(instructions="Test"),
+        intent=Intent(
+            rubric={"goal": "test"},
+            gold_trajectories=None,
+            gold_state_diff={},
+            deterministic_eval_functions=[],
+        ),
+        metadata=None,
+        is_reproducible=False,
+        initial_engine_snapshot=None,
+    )
+    task.snapshot_dir = None  # Will be set by engine
+    
+    engine = VerilogEngine(task)
+    
+    # Test 1: No simulation run yet
+    result = await engine.submit()
+    assert result["passed"] is False
+    assert "No simulation run yet" in result["detail"]
+    
+    # Test 2: Simulate with passing output
+    engine._last_simulate_output = "Mismatches: 0 in 100 samples\nALL_TESTS_PASSED"
+    result_pass = await engine.submit()
+    assert result_pass["passed"] is True
+    assert "All tests passed" in result_pass["detail"]
+    
+    # Test 3: Simulate with failing output
+    engine._last_simulate_output = "Mismatches: 5 in 100 samples\nErrors detected"
+    result_fail = await engine.submit()
+    assert result_fail["passed"] is False
+    assert "Tests failed" in result_fail["detail"]
+

examples/vlm/crafter_openai_vlm_agent.py

@@ -28,10 +28,10 @@ from pathlib import Path
 from typing import Any
 from uuid import uuid4
 
-from examples.warming_up_to_rl.task_app.synth_envs_hosted.envs.crafter.environment import (
-    CrafterEnvironmentWrapper,
+from examples.task_apps.crafter.task_app.synth_envs_hosted.envs.crafter.environment import (
+    CrafterEnvironment,
 )
-from examples.warming_up_to_rl.task_app.synth_envs_hosted.envs.crafter.policy import CrafterPolicy
+from examples.task_apps.crafter.task_app.synth_envs_hosted.envs.crafter.policy import CrafterPolicy
 from openai import OpenAI
 from synth_ai.environments.examples.crafter_classic.environment import CrafterClassicEnvironment
 from synth_ai.environments.examples.crafter_classic.taskset import (
@@ -140,7 +140,7 @@ async def _run_episode(
 ) -> EpisodeResult:
     task_instance = _build_task_instance(seed)
     env = CrafterClassicEnvironment(task_instance)
-    wrapper = CrafterEnvironmentWrapper(env, seed=seed)
+    wrapper = CrafterEnvironment(env, seed=seed)
     policy = CrafterPolicy(inference_url="openai://chat-completions", model=model)
     await policy.initialize({"use_tools": True, "model": model})
 

examples/vlm/run_crafter_vlm_benchmark.py

@@ -24,10 +24,10 @@ from pathlib import Path
 from typing import Any
 from uuid import uuid4
 
-from examples.warming_up_to_rl.task_app.synth_envs_hosted.envs.crafter.environment import (
-    CrafterEnvironmentWrapper,
+from examples.task_apps.crafter.task_app.synth_envs_hosted.envs.crafter.environment import (
+    CrafterEnvironment,
 )
-from examples.warming_up_to_rl.task_app.synth_envs_hosted.envs.crafter.policy import CrafterPolicy
+from examples.task_apps.crafter.task_app.synth_envs_hosted.envs.crafter.policy import CrafterPolicy
 from openai import AsyncOpenAI
 from synth_ai.environments.examples.crafter_classic.environment import CrafterClassicEnvironment
 from synth_ai.environments.examples.crafter_classic.taskset import (
@@ -142,7 +142,7 @@ async def _run_episode(
     async with semaphore:
         task_instance = _build_task_instance(seed)
         env = CrafterClassicEnvironment(task_instance)
-        wrapper = CrafterEnvironmentWrapper(env, seed=seed)
+        wrapper = CrafterEnvironment(env, seed=seed)
 
         policy = CrafterPolicy(inference_url="openai://chat-completions", model=model)
         await policy.initialize({"use_tools": True, "model": model})

examples/warming_up_to_rl/configs/eval_stepwise_complex.toml

@@ -4,9 +4,11 @@
 # task_app_url = "https://YOUR-TASK-APP.modal.run"
 
 model = "qwen/qwen3-32b"
-num_episodes = 50
+# Route inference to local task app Groq proxy
+inference_url = "http://localhost:8001/proxy/groq"
+num_episodes = 10
 max_turns = 10
-concurrency = 5
+concurrency = 10
 # difficulty = "easy"  # optional
 
 [rollout]