synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

examples/task_apps/verilog/task_app/grpo_verilog_task_app.py

@@ -1,145 +0,0 @@
-"""Compatibility wrapper for the GRPO Verilog task app.
-
-This mirrors the Crafter task app wrapper while delegating configuration to
-`grpo_verilog.py`. Normal usage should prefer `uvx synth-ai serve grpo-verilog`,
-but the module remains for direct execution or importing the FastAPI app.
-"""
-
-from __future__ import annotations
-
-import argparse
-from pathlib import Path
-
-from fastapi.exceptions import RequestValidationError
-from fastapi.responses import JSONResponse
-from starlette.requests import Request
-from synth_ai.task.apps import ModalDeploymentConfig, registry
-from synth_ai.task.auth import is_api_key_header_authorized, normalize_environment_api_key
-from synth_ai.task.server import TaskAppConfig, create_task_app, run_task_app
-
-from .grpo_verilog import build_config
-
-APP_ID = "grpo-verilog"
-
-
-def _build_base_config() -> TaskAppConfig:
-    # Lazily construct the base config to avoid heavy work at import time.
-    return build_config()
-
-
-try:
-    _REGISTERED_ENTRY = registry.get(APP_ID)
-except Exception:  # pragma: no cover - registry unavailable in some contexts
-    MODAL_DEPLOYMENT: ModalDeploymentConfig | None = None
-    ENV_FILES: tuple[str, ...] = ()
-else:
-    MODAL_DEPLOYMENT = _REGISTERED_ENTRY.modal
-    ENV_FILES = tuple(_REGISTERED_ENTRY.env_files)
-
-
-def build_task_app_config() -> TaskAppConfig:
-    """Return a fresh TaskAppConfig for this wrapper."""
-    base = _build_base_config()
-    return base.clone()
-
-
-def fastapi_app():
-    """Return the FastAPI application for Modal or other ASGI hosts."""
-
-    app = create_task_app(build_task_app_config())
-
-    # Replace default health endpoints so we can permit soft auth failures and log 422s.
-    filtered_routes = []
-    for route in app.router.routes:
-        path = getattr(route, "path", None)
-        methods = getattr(route, "methods", set()) or set()
-        if path in {"/health", "/health/rollout"} and "GET" in methods:
-            continue
-        filtered_routes.append(route)
-    app.router.routes = filtered_routes
-
-    def _log_env_key_prefix(source: str, env_key: str | None) -> str | None:
-        if not env_key:
-            return None
-        prefix = env_key[: max(1, len(env_key) // 2)]
-        print(f"[{source}] expected ENVIRONMENT_API_KEY prefix: {prefix}")
-        return prefix
-
-    @app.get("/health")
-    async def health(request: Request):
-        env_key = normalize_environment_api_key()
-        if not env_key:
-            return JSONResponse(
-                status_code=503,
-                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
-            )
-        if not is_api_key_header_authorized(request):
-            prefix = _log_env_key_prefix("health", env_key)
-            content = {"status": "healthy", "authorized": False}
-            if prefix:
-                content["expected_api_key_prefix"] = prefix
-            return JSONResponse(status_code=200, content=content)
-        return {"status": "healthy", "authorized": True}
-
-    @app.get("/health/rollout")
-    async def health_rollout(request: Request):
-        env_key = normalize_environment_api_key()
-        if not env_key:
-            return JSONResponse(
-                status_code=503,
-                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
-            )
-        if not is_api_key_header_authorized(request):
-            prefix = _log_env_key_prefix("health/rollout", env_key)
-            content = {"status": "healthy", "authorized": False}
-            if prefix:
-                content["expected_api_key_prefix"] = prefix
-            return JSONResponse(status_code=200, content=content)
-        return {"ok": True, "authorized": True}
-
-    @app.exception_handler(RequestValidationError)
-    async def _on_validation_error(request: Request, exc: RequestValidationError):
-        try:
-            hdr = request.headers
-            snapshot = {
-                "path": str(request.url.path),
-                "have_x_api_key": bool(hdr.get("x-api-key")),
-                "have_x_api_keys": bool(hdr.get("x-api-keys")),
-                "have_authorization": bool(hdr.get("authorization")),
-                "errors": exc.errors()[:5],
-            }
-            print("[422] validation", snapshot, flush=True)
-        except Exception:
-            pass
-        return JSONResponse(
-            status_code=422,
-            content={"status": "invalid", "detail": exc.errors()[:5]},
-        )
-
-    return app
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Run the Verilog task app locally")
-    parser.add_argument("--host", default="0.0.0.0")
-    parser.add_argument("--port", type=int, default=8103)
-    parser.add_argument("--reload", action="store_true", help="Enable uvicorn autoreload")
-    parser.add_argument(
-        "--env-file",
-        action="append",
-        default=[],
-        help="Additional .env files to load before startup",
-    )
-    args = parser.parse_args()
-
-    default_env = Path(__file__).resolve().parents[4] / "backend" / ".env.dev"
-    env_files = [str(default_env)] if default_env.exists() else []
-    env_files.extend(args.env_file or [])
-
-    run_task_app(
-        build_task_app_config,
-        host=args.host,
-        port=args.port,
-        reload=args.reload,
-        env_files=env_files,
-    )

examples/task_apps/verilog/tests/__init__.py

@@ -1,4 +0,0 @@
-# Verilog task app tests
-
-
-

examples/task_apps/verilog/tests/conftest.py

@@ -1,115 +0,0 @@
-"""Shared fixtures for Verilog tests."""
-import os
-import socket
-import subprocess
-from subprocess import TimeoutExpired
-import time
-from pathlib import Path
-from typing import Iterator
-
-import pytest
-
-requests = pytest.importorskip("requests")
-
-
-def _which(executable: str) -> bool:
-    return any(
-        (Path(path) / executable).exists()
-        for path in os.getenv("PATH", "").split(os.pathsep)
-    )
-
-
-def _find_free_port() -> int:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        sock.bind(("127.0.0.1", 0))
-        return sock.getsockname()[1]
-
-
-def _wait_for_server(base_url: str, timeout: float = 60.0) -> None:
-    """Wait for the Verilog server to become ready."""
-    deadline = time.time() + timeout
-    while time.time() < deadline:
-        try:
-            # Try /info first (no auth required if --insecure)
-            resp = requests.get(f"{base_url}/info", timeout=2.0)
-            if resp.status_code == 200:
-                return
-            # If 400/401, server is up but needs auth - that's OK
-            if resp.status_code in (400, 401):
-                return
-        except Exception:
-            time.sleep(0.5)
-    raise RuntimeError(f"Task app at {base_url} did not become ready")
-
-
-@pytest.fixture(scope="module")
-def verilog_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]:
-    """Start the Verilog task app server for testing."""
-    if not _which("uv"):
-        pytest.skip("uv executable not found on PATH")
-    if "GROQ_API_KEY" not in os.environ:
-        pytest.skip("GROQ_API_KEY must be set for Groq-backed tests")
-
-    port = _find_free_port()
-    base_url = f"http://127.0.0.1:{port}"
-    tmp_path = tmp_path_factory.mktemp("verilog")
-    trace_dir = tmp_path / "traces"
-    trace_dir.mkdir(parents=True, exist_ok=True)
-
-    env = os.environ.copy()
-    cmd = [
-        "uv",
-        "run",
-        "-m",
-        "synth_ai",
-        "task-app",
-        "serve",
-        "grpo-verilog",
-        "--port",
-        str(port),
-        "--no-reload",
-    ]
-    proc = subprocess.Popen(
-        cmd,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        text=True,
-        env=env,
-        stdin=subprocess.PIPE,
-    )
-    
-    # Send "n" to decline tracing
-    try:
-        if proc.stdin:
-            proc.stdin.write("n\n")
-            proc.stdin.flush()
-    except Exception:
-        pass
-    
-    stdout_capture = ""
-    try:
-        time.sleep(2)
-        if proc.poll() is not None:
-            stdout_capture, _ = proc.communicate(timeout=2)
-            tail = "\n".join(stdout_capture.strip().splitlines()[-20:]) if stdout_capture else ""
-            pytest.skip(f"Task app terminated immediately:\n{tail}")
-        
-        _wait_for_server(base_url)
-        yield base_url
-    except RuntimeError as e:
-        proc.terminate()
-        try:
-            stdout_capture, _ = proc.communicate(timeout=10)
-        except TimeoutExpired:
-            proc.kill()
-            stdout_capture, _ = proc.communicate()
-        tail = "\n".join((stdout_capture or "").strip().splitlines()[-20:])
-        pytest.skip(f"Task app failed to start: {e}\n{tail}")
-    finally:
-        if proc.poll() is None:
-            proc.terminate()
-            try:
-                proc.wait(timeout=5)
-            except TimeoutExpired:
-                proc.kill()
-

examples/task_apps/verilog/tests/integration/__init__.py

@@ -1,4 +0,0 @@
-# Integration tests for Verilog task app
-
-
-

examples/task_apps/verilog/tests/integration/test_verilog_eval.py

@@ -1,181 +0,0 @@
-"""Integration tests for Verilog task app with Groq evaluation."""
-from __future__ import annotations
-
-import os
-import socket
-import subprocess
-from subprocess import TimeoutExpired
-import time
-from pathlib import Path
-from typing import Iterator
-
-import pytest
-
-requests = pytest.importorskip("requests")
-
-
-HERE = Path(__file__).resolve().parent
-TASK_APP_ROOT = HERE.parents[1]
-CONFIG_PATH = TASK_APP_ROOT / "eval_groq_qwen32b.toml"
-
-
-def _which(executable: str) -> bool:
-    return any(
-        (Path(path) / executable).exists()
-        for path in os.getenv("PATH", "").split(os.pathsep)
-    )
-
-
-def _find_free_port() -> int:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        sock.bind(("127.0.0.1", 0))
-        return sock.getsockname()[1]
-
-
-def _wait_for_server(base_url: str, timeout: float = 60.0) -> None:
-    """Wait for the Verilog server to become ready."""
-    deadline = time.time() + timeout
-    while time.time() < deadline:
-        try:
-            resp = requests.get(f"{base_url}/info", timeout=2.0)
-            if resp.status_code == 200:
-                return
-        except Exception:
-            time.sleep(0.5)
-    raise RuntimeError(f"Task app at {base_url} did not become ready")
-
-
-@pytest.fixture
-def verilog_server(tmp_path: Path) -> Iterator[str]:
-    """Start the Verilog task app server for testing."""
-    if not _which("uv"):
-        pytest.skip("uv executable not found on PATH")
-    if "GROQ_API_KEY" not in os.environ:
-        pytest.skip("GROQ_API_KEY must be set for Groq-backed evals")
-
-    port = _find_free_port()
-    base_url = f"http://127.0.0.1:{port}"
-    trace_dir = tmp_path / "traces"
-    trace_dir.mkdir(parents=True, exist_ok=True)
-
-    env = os.environ.copy()
-    cmd = [
-        "uv",
-        "run",
-        "-m",
-        "synth_ai",
-        "task-app",
-        "serve",
-        "grpo-verilog",
-        "--port",
-        str(port),
-        "--no-reload",
-    ]
-    proc = subprocess.Popen(
-        cmd,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        text=True,
-        env=env,
-        stdin=subprocess.PIPE,  # Auto-answer tracing prompt
-    )
-    
-    # Send "n" to decline tracing
-    try:
-        if proc.stdin:
-            proc.stdin.write("n\n")
-            proc.stdin.flush()
-    except Exception:
-        pass
-    
-    stdout_capture = ""
-    try:
-        # Check if process died immediately
-        time.sleep(2)
-        if proc.poll() is not None:
-            stdout_capture, _ = proc.communicate(timeout=2)
-            tail = "\n".join(stdout_capture.strip().splitlines()[-20:]) if stdout_capture else ""
-            pytest.skip(f"Task app terminated immediately:\n{tail}")
-        
-        _wait_for_server(base_url)
-        yield base_url
-    except RuntimeError as e:
-        proc.terminate()
-        try:
-            stdout_capture, _ = proc.communicate(timeout=10)
-        except TimeoutExpired:
-            proc.kill()
-            stdout_capture, _ = proc.communicate()
-        tail = "\n".join((stdout_capture or "").strip().splitlines()[-20:])
-        pytest.skip(f"Task app failed to start: {e}\n{tail}")
-    finally:
-        if proc.poll() is None:
-            proc.terminate()
-            try:
-                proc.wait(timeout=5)
-            except TimeoutExpired:
-                proc.kill()
-
-
-@pytest.mark.slow
-def test_verilog_server_health(verilog_server: str) -> None:
-    """Test that the Verilog server health endpoint works."""
-    # Health endpoint requires auth, so we expect 400 (auth failed) or 200
-    resp = requests.get(f"{verilog_server}/health", timeout=5.0)
-    assert resp.status_code in (200, 400), f"Unexpected status: {resp.status_code}"
-
-
-@pytest.mark.slow
-def test_verilog_task_info(verilog_server: str) -> None:
-    """Test that the Verilog server returns valid task_info."""
-    resp = requests.get(f"{verilog_server}/task_info", timeout=5.0)
-    assert resp.status_code == 200
-    data = resp.json()
-    assert "task" in data
-    assert data["task"]["id"] == "verilog"
-
-
-@pytest.mark.slow
-def test_verilog_eval_with_groq(verilog_server: str) -> None:
-    """Spin up the Verilog task app and run a Groq-backed eval."""
-    if not CONFIG_PATH.exists():
-        pytest.skip(f"Config file not found: {CONFIG_PATH}")
-    
-    cmd = [
-        "uv",
-        "run",
-        "-m",
-        "synth_ai",
-        "eval",
-        "grpo-verilog",
-        "--config",
-        str(CONFIG_PATH),
-        "--url",
-        verilog_server,
-        "--model",
-        "qwen/qwen3-32b",
-        "--seeds",
-        "0",  # Just test one seed
-    ]
-    result = subprocess.run(
-        cmd,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        text=True,
-        env=os.environ.copy(),
-        check=False,
-        timeout=300,  # 5 minutes max
-    )
-    
-    if result.returncode != 0:
-        pytest.fail(f"Eval failed with return code {result.returncode}:\n{result.stdout}")
-    
-    # Check for success indicators
-    assert "Eval complete" in result.stdout
-    assert "1 ok, 0 failed" in result.stdout or "status=200" in result.stdout
-    
-    # Check that we got a meaningful outcome score
-    assert "outcome" in result.stdout.lower() or "mean_return" in result.stdout.lower()
-
-
-

examples/task_apps/verilog/tests/integration/test_verilog_rollout.py

@@ -1,55 +0,0 @@
-"""Integration test for Verilog rollouts via /rollout endpoint."""
-import os
-import pytest
-
-requests = pytest.importorskip("requests")
-
-# Use the actual ENVIRONMENT_API_KEY from .env
-AUTH_HEADER = {"Authorization": "Bearer sk_env_30c78a787bac223c716918181209f263"}
-
-
-@pytest.mark.slow
-def test_verilog_policy_rollout(verilog_server: str) -> None:
-    """Test a Verilog rollout using Groq policy."""
-    if "GROQ_API_KEY" not in os.environ:
-        pytest.skip("GROQ_API_KEY required for this test")
-    
-    rollout_payload = {
-        "run_id": "test_policy_verilog",
-        "env": {"seed": 0},
-        "ops": [],  # Empty ops means use policy for all steps
-        "policy": {
-            "policy_name": "qwen-groq",
-            "config": {
-                "provider": "groq",
-                "model": "qwen/qwen3-32b",
-                "max_steps": 5,  # Limit steps for test
-            },
-        },
-    }
-    
-    resp = requests.post(
-        f"{verilog_server}/rollout",
-        json=rollout_payload,
-        headers=AUTH_HEADER,
-        timeout=120.0,
-    )
-    
-    assert resp.status_code == 200, f"Rollout failed: {resp.status_code} {resp.text}"
-    data = resp.json()
-    
-    # Verify response structure
-    assert "trajectories" in data
-    assert "metrics" in data
-    assert "trace" in data
-    
-    trajectory = data["trajectories"][0]
-    assert "steps" in trajectory
-    
-    # Check that at least one step was taken
-    assert len(trajectory["steps"]) > 0
-    
-    # Verify metrics
-    metrics = data["metrics"]
-    assert "episode_returns" in metrics or "mean_return" in metrics
-

examples/task_apps/verilog/tests/unit/__init__.py

@@ -1,4 +0,0 @@
-# Unit tests for Verilog task app
-
-
-

examples/task_apps/verilog/tests/unit/test_verilog_scoring.py

@@ -1,118 +0,0 @@
-"""Unit tests for Verilog scoring and rewards."""
-import pytest
-
-from synth_ai.environments.examples.verilog.engine import (
-    VerilogCompileSuccessComponent,
-    VerilogSimulationPassComponent,
-    VerilogSubmitSuccessComponent,
-    VerilogPublicState,
-)
-
-
-@pytest.mark.asyncio
-async def test_compile_success_reward():
-    """Test that successful compilation awards 0.1 reward."""
-    component = VerilogCompileSuccessComponent()
-    state = VerilogPublicState(files={}, build_dir="/tmp", task_completed=False)
-    
-    # Successful compile (returncode 0)
-    action = {"type": "compile", "returncode": 0}
-    reward = await component.score(state, action)
-    assert reward == 0.1
-    
-    # Failed compile (returncode != 0)
-    action_fail = {"type": "compile", "returncode": 1}
-    reward_fail = await component.score(state, action_fail)
-    assert reward_fail == 0.0
-    
-    # Non-compile action
-    action_other = {"type": "write_file"}
-    reward_other = await component.score(state, action_other)
-    assert reward_other == 0.0
-
-
-@pytest.mark.asyncio
-async def test_simulation_pass_reward():
-    """Test that passing simulation awards 1.0 reward."""
-    component = VerilogSimulationPassComponent()
-    state = VerilogPublicState(files={}, build_dir="/tmp", task_completed=False)
-    
-    # Passing simulation
-    action = {"type": "simulate", "passed": True}
-    reward = await component.score(state, action)
-    assert reward == 1.0
-    
-    # Failing simulation
-    action_fail = {"type": "simulate", "passed": False}
-    reward_fail = await component.score(state, action_fail)
-    assert reward_fail == 0.0
-    
-    # Non-simulate action
-    action_other = {"type": "compile"}
-    reward_other = await component.score(state, action_other)
-    assert reward_other == 0.0
-
-
-@pytest.mark.asyncio
-async def test_submit_success_reward():
-    """Test that successful submission awards 10.0 reward."""
-    component = VerilogSubmitSuccessComponent()
-    state = VerilogPublicState(files={}, build_dir="/tmp", task_completed=False)
-    
-    # Successful submission (tests passed)
-    action = {"type": "submit", "passed": True}
-    reward = await component.score(state, action)
-    assert reward == 10.0
-    
-    # Failed submission (tests didn't pass)
-    action_fail = {"type": "submit", "passed": False}
-    reward_fail = await component.score(state, action_fail)
-    assert reward_fail == 0.0
-    
-    # Non-submit action
-    action_other = {"type": "compile"}
-    reward_other = await component.score(state, action_other)
-    assert reward_other == 0.0
-
-
-@pytest.mark.asyncio
-async def test_submit_checks_simulation_output():
-    """Test that submit() correctly checks the last simulation output."""
-    from synth_ai.environments.examples.verilog.engine import VerilogEngine
-    from synth_ai.environments.tasks.core import TaskInstance, Impetus, Intent
-    
-    # Create a minimal task instance
-    task = TaskInstance(
-        id="test",
-        impetus=Impetus(instructions="Test"),
-        intent=Intent(
-            rubric={"goal": "test"},
-            gold_trajectories=None,
-            gold_state_diff={},
-            deterministic_eval_functions=[],
-        ),
-        metadata=None,
-        is_reproducible=False,
-        initial_engine_snapshot=None,
-    )
-    task.snapshot_dir = None  # Will be set by engine
-    
-    engine = VerilogEngine(task)
-    
-    # Test 1: No simulation run yet
-    result = await engine.submit()
-    assert result["passed"] is False
-    assert "No simulation run yet" in result["detail"]
-    
-    # Test 2: Simulate with passing output
-    engine._last_simulate_output = "Mismatches: 0 in 100 samples\nALL_TESTS_PASSED"
-    result_pass = await engine.submit()
-    assert result_pass["passed"] is True
-    assert "All tests passed" in result_pass["detail"]
-    
-    # Test 3: Simulate with failing output
-    engine._last_simulate_output = "Mismatches: 5 in 100 samples\nErrors detected"
-    result_fail = await engine.submit()
-    assert result_fail["passed"] is False
-    assert "Tests failed" in result_fail["detail"]
-

examples/vlm/PROPOSAL.md

@@ -1,53 +0,0 @@
-# Crafter VLM Fine-Tuning Plan
-
-## Why
-
-Crafter observations already expose 64×64 RGB frames, but until now the stack only
-surfaced text summaries. The new multimodal trace format allows us to:
-
-- Retrieve and persist `observation_image_base64` on every step.
-- Thread data URLs through policy prompts so LLMs see the raw frame.
-- Emit multimodal content parts inside LLM call records (and therefore SFT datasets).
-- Capture the same information inside the session trace (`policy_user_prompt` entries).
-
-This unlocks a supervised dataset for vision‑language models that treat the frame as a
-first-class token alongside the textual state summary.
-
-## What Changed
-
-1. **Environment wrappers** now attach `observation_image_base64`, `..._data_url`,
-   and image metadata to every observation (initialise + step).
-2. **Crafter policy** augments the user prompt with `{"type":"image_url"}` segments
-   while still emitting the textual summary for language-only models.
-3. **Tracing** stores structured prompt content and serialises it to JSON so prompts
-   with images survive the round-trip to SQLite/Turso.
-4. **Dataset exporter** preserves multimodal content, flags rows with images, and
-   carries this metadata through SFT JSONL.
-5. **Utility scripts** under `examples/vlm/` make it easy to sanity check frames,
-   filter datasets, and spin up an image-aware training job.
-
-## Proposed Workflow
-
-1. **Collect rollouts** with tracing enabled (either via the task app or scripted
-   runs) to populate `traces/v3/synth_ai.db`.
-2. **Export** using `export_trace_sft.py` — images are embedded automatically.
-3. **Filter** to rows with user images (see `filter_image_rows.py`) and optionally
-   build validation splits.
-4. **Upload** the JSONL via `run_fft_and_save.py` or the Synth CLI.
-5. **Train** with a VLM-capable base model (e.g. `openai/gpt-4o-mini-2024-07-18`)
-   using `configs/crafter_vlm_gpt4o.toml`.
-6. **Evaluate** the resulting checkpoint on Crafter tasks (reuse the evaluation
-   harness from `examples/warming_up_to_rl` but now with multimodal prompts).
-
-## Open Questions / Future Work
-
-- **Longer context**: investigate packing multiple sequential frames per turn
-  (e.g. last N frames) as either separate image parts or a stitched sprite sheet.
-- **Reward shaping**: extend metadata so SFT rows carry frame-level reward deltas,
-  enabling hybrid BC + value regression objectives.
-- **Assistant images**: currently most assistant messages are textual; we could
-  experiment with returning thumbnails explaining the plan.
-- **Automated filtering**: add CLI helpers to keep only turns whose tool calls led
-  to high-reward outcomes (e.g. achievements).
-- **Evaluation**: define a reference set of human-labelled “vision checkpoints”
-  (e.g. recognise nearby resource, detect threats) to quantify multimodal progress.