synth-ai 0.2.13.dev1__py3-none-any.whl → 0.2.14__py3-none-any.whl

examples/task_apps/sokoban/tests/__init__.py

@@ -0,0 +1,4 @@
+# Sokoban task app tests
+
+
+

examples/task_apps/sokoban/tests/conftest.py

@@ -0,0 +1,113 @@
+"""Shared fixtures for Sokoban tests."""
+import os
+import socket
+import subprocess
+from subprocess import TimeoutExpired
+import time
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+
+requests = pytest.importorskip("requests")
+
+
+def _which(executable: str) -> bool:
+    return any(
+        (Path(path) / executable).exists()
+        for path in os.getenv("PATH", "").split(os.pathsep)
+    )
+
+
+def _find_free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
+def _wait_for_server(base_url: str, timeout: float = 60.0) -> None:
+    """Wait for the Sokoban server to become ready."""
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        try:
+            # Try /info first (no auth required if --insecure)
+            resp = requests.get(f"{base_url}/info", timeout=2.0)
+            if resp.status_code == 200:
+                return
+            # If 400/401, server is up but needs auth - that's OK
+            if resp.status_code in (400, 401):
+                return
+        except Exception:
+            time.sleep(0.5)
+    raise RuntimeError(f"Task app at {base_url} did not become ready")
+
+
+@pytest.fixture(scope="module")
+def sokoban_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]:
+    """Start the Sokoban task app server for testing."""
+    if not _which("uv"):
+        pytest.skip("uv executable not found on PATH")
+
+    port = _find_free_port()
+    base_url = f"http://127.0.0.1:{port}"
+    tmp_path = tmp_path_factory.mktemp("sokoban")
+
+    env = os.environ.copy()
+    # Set the test API key
+    env["ENVIRONMENT_API_KEY"] = "sk_env_30c78a787bac223c716918181209f263"
+    cmd = [
+        "uv",
+        "run",
+        "-m",
+        "synth_ai",
+        "task-app",
+        "serve",
+        "sokoban",
+        "--port",
+        str(port),
+        "--no-reload",
+    ]
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        env=env,
+        stdin=subprocess.PIPE,
+    )
+    
+    # Send "n" to decline tracing
+    try:
+        if proc.stdin:
+            proc.stdin.write("n\n")
+            proc.stdin.flush()
+    except Exception:
+        pass
+    
+    stdout_capture = ""
+    try:
+        time.sleep(2)
+        if proc.poll() is not None:
+            stdout_capture, _ = proc.communicate(timeout=2)
+            tail = "\n".join(stdout_capture.strip().splitlines()[-20:]) if stdout_capture else ""
+            pytest.skip(f"Task app terminated immediately:\n{tail}")
+        
+        _wait_for_server(base_url)
+        yield base_url
+    except RuntimeError as e:
+        proc.terminate()
+        try:
+            stdout_capture, _ = proc.communicate(timeout=10)
+        except TimeoutExpired:
+            proc.kill()
+            stdout_capture, _ = proc.communicate()
+        tail = "\n".join((stdout_capture or "").strip().splitlines()[-20:])
+        pytest.skip(f"Task app failed to start: {e}\n{tail}")
+    finally:
+        if proc.poll() is None:
+            proc.terminate()
+            try:
+                proc.wait(timeout=5)
+            except TimeoutExpired:
+                proc.kill()
+

examples/task_apps/sokoban/tests/integration/__init__.py

@@ -0,0 +1,4 @@
+# Integration tests for Sokoban task app
+
+
+

examples/task_apps/sokoban/tests/integration/test_sokoban_eval.py

@@ -0,0 +1,57 @@
+"""Integration tests for Sokoban task app with evaluation."""
+import pytest
+
+requests = pytest.importorskip("requests")
+
+# sokoban_server fixture is in conftest.py
+# Use the actual ENVIRONMENT_API_KEY from .env
+AUTH_HEADER = {"Authorization": "Bearer sk_env_30c78a787bac223c716918181209f263"}
+
+
+@pytest.mark.slow
+def test_sokoban_server_health(sokoban_server: str) -> None:
+    """Test that the Sokoban server health endpoint works."""
+    resp = requests.get(f"{sokoban_server}/health", timeout=5.0)
+    assert resp.status_code in (200, 400), f"Unexpected status: {resp.status_code}"
+
+
+def test_sokoban_task_info(sokoban_server: str) -> None:
+    """Test that the Sokoban server returns valid task_info."""
+    resp = requests.get(f"{sokoban_server}/task_info", timeout=5.0)
+    assert resp.status_code == 200
+    data = resp.json()
+    assert "task" in data
+    assert data["task"]["id"] == "sokoban"
+
+
+@pytest.mark.fast
+def test_sokoban_manual_rollout(sokoban_server: str) -> None:
+    """Test a manual Sokoban rollout with explicit actions."""
+    # Try explicit action rollout (no LLM required)
+    # Actions: 0=left, 1=up, 2=right, 3=down
+    rollout_payload = {
+        "run_id": "test_manual",
+        "env": {"seed": 0, "config": {"difficulty": "easy", "max_steps": 50}},
+        "ops": [],
+        "policy": {
+            "policy_name": "manual",
+            "config": {
+                "provider": "noop",
+                "actions": [0, 2, 2, 3],  # left, right, right, down
+            },
+        },
+    }
+    
+    resp = requests.post(
+        f"{sokoban_server}/rollout",
+        json=rollout_payload,
+        headers=AUTH_HEADER,
+        timeout=30.0,
+    )
+    
+    assert resp.status_code == 200
+    data = resp.json()
+    assert "trajectories" in data
+    assert len(data["trajectories"]) > 0
+    assert "metrics" in data
+

examples/task_apps/sokoban/tests/integration/test_sokoban_rollout.py

@@ -0,0 +1,198 @@
+"""Integration test for Sokoban rollouts via /rollout endpoint."""
+import os
+import pytest
+
+requests = pytest.importorskip("requests")
+
+# Use the actual ENVIRONMENT_API_KEY from .env
+AUTH_HEADER = {"Authorization": "Bearer sk_env_30c78a787bac223c716918181209f263"}
+
+
+@pytest.mark.slow
+def test_sokoban_manual_rollout(sokoban_server: str) -> None:
+    """Test a manual Sokoban rollout with explicit movement actions."""
+    # Actions: 0=left, 1=up, 2=right, 3=down
+    rollout_payload = {
+        "run_id": "test_manual_sokoban",
+        "env": {"seed": 0, "config": {"difficulty": "easy", "max_steps": 20}},
+        "ops": [],  # Not used for manual actions in Sokoban
+        "policy": {
+            "policy_name": "manual",
+            "config": {
+                "provider": "noop",
+                "actions": [0, 2, 2, 3, 3, 0],  # Pass actions via policy.config
+            },
+        },
+    }
+    
+    resp = requests.post(
+        f"{sokoban_server}/rollout",
+        json=rollout_payload,
+        headers=AUTH_HEADER,
+        timeout=30.0,
+    )
+    
+    assert resp.status_code == 200, f"Rollout failed: {resp.status_code} {resp.text}"
+    data = resp.json()
+    
+    # Verify response structure
+    assert "trajectories" in data
+    assert len(data["trajectories"]) > 0
+    assert "metrics" in data
+    
+    trajectory = data["trajectories"][0]
+    assert "steps" in trajectory
+    
+    # Should have taken the requested actions
+    assert len(trajectory["steps"]) >= 6  # Initial obs + 6 actions
+    
+    # Verify each step has required fields
+    for step in trajectory["steps"]:
+        assert "obs" in step
+        assert "reward" in step or "reward_last" in step.get("obs", {})
+
+
+@pytest.mark.slow
+def test_sokoban_policy_rollout_with_openai(sokoban_server: str) -> None:
+    """Test a Sokoban rollout using OpenAI GPT-5-mini policy."""
+    if "OPENAI_API_KEY" not in os.environ:
+        pytest.skip("OPENAI_API_KEY required for this test")
+    
+    rollout_payload = {
+        "run_id": "test_policy_sokoban",
+        "env": {"seed": 0, "config": {"difficulty": "easy", "max_steps": 10}},
+        "ops": ["policy", "policy"],  # 2 policy calls
+        "policy": {
+            "policy_name": "gpt-5-mini",
+            "config": {
+                "provider": "openai",
+                "model": "gpt-5-mini",
+                "max_tokens": 512,
+            },
+        },
+    }
+    
+    resp = requests.post(
+        f"{sokoban_server}/rollout",
+        json=rollout_payload,
+        headers=AUTH_HEADER,
+        timeout=180.0,  # GPT-5-mini can be slow
+    )
+    
+    # GPT-5-mini may or may not work for Sokoban, so just check it doesn't crash
+    assert resp.status_code in (200, 500), f"Unexpected status: {resp.status_code}"
+    
+    if resp.status_code == 200:
+        data = resp.json()
+        assert "trajectories" in data
+        assert "metrics" in data
+
+
+@pytest.mark.fast
+def test_sokoban_difficulty_levels(sokoban_server: str) -> None:
+    """Test Sokoban rollouts with different difficulty levels."""
+    for difficulty in ["easy", "medium", "hard"]:
+        rollout_payload = {
+            "run_id": f"test_difficulty_{difficulty}",
+            "env": {"seed": 0, "config": {"difficulty": difficulty, "max_steps": 10}},
+            "ops": [],
+            "policy": {
+                "config": {
+                    "provider": "noop",
+                    "actions": [2, 3, 0],  # right, down, left
+                },
+            },
+        }
+        
+        resp = requests.post(
+            f"{sokoban_server}/rollout",
+            json=rollout_payload,
+            headers=AUTH_HEADER,
+            timeout=30.0,
+        )
+        
+        assert resp.status_code == 200, f"Rollout failed for {difficulty}: {resp.text}"
+        data = resp.json()
+        
+        # Verify basic structure
+        assert "trajectories" in data
+        assert len(data["trajectories"]) > 0
+
+
+@pytest.mark.fast
+def test_sokoban_max_steps_limit(sokoban_server: str) -> None:
+    """Test that Sokoban respects max_steps configuration."""
+    max_steps = 5
+    rollout_payload = {
+        "run_id": "test_max_steps",
+        "env": {"seed": 0, "config": {"difficulty": "easy", "max_steps": max_steps}},
+        "ops": [],
+        "policy": {
+            "config": {
+                "provider": "noop",
+                "actions": [0] * 20,  # Try to take 20 actions, but should be limited
+            },
+        },
+    }
+    
+    resp = requests.post(
+        f"{sokoban_server}/rollout",
+        json=rollout_payload,
+        headers=AUTH_HEADER,
+        timeout=30.0,
+    )
+    
+    assert resp.status_code == 200
+    data = resp.json()
+    
+    trajectory = data["trajectories"][0]
+    steps = trajectory["steps"]
+    
+    # Should have stopped at max_steps (plus initial observation)
+    assert len(steps) <= max_steps + 1, f"Expected <= {max_steps + 1} steps, got {len(steps)}"
+    
+    # Check if truncated
+    final_obs = steps[-1].get("obs", {})
+    if len(steps) > max_steps:
+        assert final_obs.get("truncated") is True
+
+
+@pytest.mark.fast
+def test_sokoban_completion_detection(sokoban_server: str) -> None:
+    """Test that Sokoban detects puzzle completion (terminated=True)."""
+    # This test verifies the structure, not necessarily that we solve it
+    rollout_payload = {
+        "run_id": "test_completion",
+        "env": {"seed": 0, "config": {"difficulty": "easy", "max_steps": 50}},
+        "ops": [],
+        "policy": {
+            "config": {
+                "provider": "noop",
+                "actions": [2, 3, 0, 1, 2],  # Random moves
+            },
+        },
+    }
+    
+    resp = requests.post(
+        f"{sokoban_server}/rollout",
+        json=rollout_payload,
+        headers=AUTH_HEADER,
+        timeout=30.0,
+    )
+    
+    assert resp.status_code == 200
+    data = resp.json()
+    
+    trajectory = data["trajectories"][0]
+    final_step = trajectory["steps"][-1]
+    final_obs = final_step.get("obs", {})
+    
+    # Verify that termination fields exist
+    assert "terminated" in final_obs or "done" in final_step
+    assert "boxes_on_target" in final_obs
+    assert "num_boxes" in final_obs
+    
+    # If all boxes on target, should be terminated
+    if final_obs.get("boxes_on_target") == final_obs.get("num_boxes"):
+        assert final_obs.get("terminated") is True or final_step.get("done") is True
+

examples/task_apps/sokoban/tests/unit/__init__.py

@@ -0,0 +1,4 @@
+# Unit tests for Sokoban task app
+
+
+

examples/task_apps/sokoban/tests/unit/test_sokoban_environment.py

@@ -0,0 +1,114 @@
+"""Unit tests for Sokoban environment and rewards."""
+import pytest
+
+
+@pytest.mark.fast
+def test_sokoban_module_imports():
+    """Test that Sokoban modules can be imported."""
+    from synth_ai.environments.examples.sokoban import environment, engine
+    
+    assert hasattr(environment, "SokobanEnvironment")
+    assert hasattr(engine, "SokobanEngine")
+
+
+@pytest.mark.asyncio
+async def test_sokoban_reward_components():
+    """Test that Sokoban reward components exist and work."""
+    from synth_ai.environments.examples.sokoban.engine import (
+        SokobanEngine,
+        SokobanGoalAchievedComponent,
+        SokobanStepPenaltyComponent,
+        SokobanPublicState,
+    )
+    from synth_ai.environments.tasks.core import TaskInstance, Impetus, Intent
+    
+    # Create a minimal task instance
+    task = TaskInstance(
+        id="test",
+        impetus=Impetus(instructions="Test"),
+        intent=Intent(
+            rubric={"goal": "test"},
+            gold_trajectories=None,
+            gold_state_diff={},
+            deterministic_eval_functions=[],
+        ),
+        metadata={"difficulty": "easy", "max_steps": 50, "seed": 0},
+        is_reproducible=False,
+        initial_engine_snapshot=None,
+    )
+    
+    engine = SokobanEngine(task)
+    
+    # Test that reward components exist
+    assert hasattr(engine, "reward_stack")
+    assert engine.reward_stack is not None
+    
+    # Test reward components directly
+    goal_reward = SokobanGoalAchievedComponent()
+    penalty = SokobanStepPenaltyComponent()
+    
+    # Mock state for reward calculation
+    import numpy as np
+    
+    state = SokobanPublicState(
+        dim_room=(3, 3),
+        room_fixed=np.array([[0]]),
+        room_state=np.array([[0]]),
+        player_position=(0, 0),
+        boxes_on_target=0,
+        num_steps=0,
+        max_steps=50,
+        last_action_name="NONE",
+        num_boxes=1,
+        error_info=None,
+    )
+    
+    # Test goal reward (should be 0 for incomplete puzzle)
+    reward1 = await goal_reward.score(state, {"action": 0})
+    assert reward1 == 0.0
+    
+    # Test completed state
+    state_complete = SokobanPublicState(
+        dim_room=(3, 3),
+        room_fixed=np.array([[0]]),
+        room_state=np.array([[0]]),
+        player_position=(0, 0),
+        boxes_on_target=1,
+        num_steps=10,
+        max_steps=50,
+        last_action_name="RIGHT",
+        num_boxes=1,
+        error_info=None,
+    )
+    reward_complete = await goal_reward.score(state_complete, {"action": 0})
+    assert reward_complete > 0
+    
+    # Test penalty (should be negative small value)
+    penalty_reward = await penalty.score(state, {"action": 0})
+    assert penalty_reward < 0
+    assert penalty_reward > -1  # Should be a small penalty
+
+
+def test_sokoban_difficulty_settings():
+    """Test that Sokoban engine can be created with task metadata."""
+    from synth_ai.environments.examples.sokoban.engine import SokobanEngine
+    from synth_ai.environments.tasks.core import TaskInstance, Impetus, Intent
+    
+    # Test with different difficulty metadata
+    for difficulty in ["easy", "medium", "hard"]:
+        task = TaskInstance(
+            id="test",
+            impetus=Impetus(instructions="Test"),
+            intent=Intent(
+                rubric={"goal": "test"},
+                gold_trajectories=None,
+                gold_state_diff={},
+                deterministic_eval_functions=[],
+            ),
+            metadata={"difficulty": difficulty, "max_steps": 50, "seed": 0},
+            is_reproducible=False,
+            initial_engine_snapshot=None,
+        )
+        
+        engine = SokobanEngine(task)
+        assert engine is not None

examples/task_apps/verilog/__init__.py

@@ -0,0 +1 @@
+

examples/task_apps/verilog/eval_groq_qwen32b.toml

@@ -0,0 +1,24 @@
+# Verilog Eval Config for Groq Qwen3-32B
+# Quick eval to test Verilog task app before RL training
+
+[task_app]
+# Update this with your Modal URL after deployment
+url = "https://synth-laboratories--grpo-verilog-task-app-fastapi-app-dev.modal.run"
+
+[eval]
+num_episodes = 3  # Quick test with 3 seeds
+seeds = [0, 1, 2]
+max_steps = 15    # More steps for Verilog compilation chains
+
+[policy]
+provider = "groq"
+model = "qwen/qwen3-32b"
+temperature = 0.2
+max_tokens = 768
+inference_url = "https://api.groq.com/openai/v1/chat/completions"
+
+[env]
+difficulty = "medium"  # Can be "easy", "medium", or "hard"
+
+
+

examples/task_apps/verilog/filter_sft.toml

@@ -0,0 +1,5 @@
+[filter]
+db = "traces/v3/synth_ai.db"
+output = "ft_data/verilog_sft.jsonl"
+min_official_score = 0.01
+

examples/task_apps/verilog/task_app/README.md

@@ -0,0 +1,12 @@
+# GRPO Verilog Task App
+
+This example mirrors the Crafter task app layout while targeting the Verilog
+hardware synthesis environment under `synth_ai.environments.examples.verilog`.
+The `grpo_verilog.py` module builds a lightweight dataset from the VerilogEval
+spec-to-RTL benchmark and wires a minimalist task-app configuration. The
+companion `grpo_verilog_task_app.py` acts as a compatibility wrapper for direct
+FastAPI execution or Modal deployment.
+
+The rollout bridge currently surfaces the initial observation for the selected
+task instance, providing a scaffold for future extensions that integrate the
+full hosted environment workflow and policy orchestration similar to Crafter.

examples/task_apps/verilog/task_app/__init__.py

@@ -0,0 +1 @@
+