synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

examples/task_apps/enron/task_app/grpo_enron_task_app.py

@@ -1,146 +0,0 @@
-"""Compatibility wrapper for the GRPO Enron task app.
-
-This mirrors the structure of the Crafter task app wrapper while delegating
-all configuration to the colocated `grpo_enron.py` module. Normal usage should
-prefer invoking `uvx synth-ai serve grpo-enron`, but this module remains for
-direct execution or importing the FastAPI app object.
-"""
-
-from __future__ import annotations
-
-import argparse
-from pathlib import Path
-
-from fastapi.exceptions import RequestValidationError
-from fastapi.responses import JSONResponse
-from starlette.requests import Request
-from synth_ai.task.apps import ModalDeploymentConfig, registry
-from synth_ai.task.auth import is_api_key_header_authorized, normalize_environment_api_key
-from synth_ai.task.server import TaskAppConfig, create_task_app, run_task_app
-
-from .grpo_enron import build_config
-
-APP_ID = "grpo-enron"
-
-
-def _build_base_config() -> TaskAppConfig:
-    # Lazily construct the base config to avoid heavy work at import time.
-    return build_config()
-
-
-try:
-    _REGISTERED_ENTRY = registry.get(APP_ID)
-except Exception:  # pragma: no cover - registry unavailable in some contexts
-    MODAL_DEPLOYMENT: ModalDeploymentConfig | None = None
-    ENV_FILES: tuple[str, ...] = ()
-else:
-    MODAL_DEPLOYMENT = _REGISTERED_ENTRY.modal
-    ENV_FILES = tuple(_REGISTERED_ENTRY.env_files)
-
-
-def build_task_app_config() -> TaskAppConfig:
-    """Return a fresh TaskAppConfig for this wrapper."""
-    base = _build_base_config()
-    return base.clone()
-
-
-def fastapi_app():
-    """Return the FastAPI application for Modal or other ASGI hosts."""
-
-    app = create_task_app(build_task_app_config())
-
-    # Replace default health endpoints so we can permit soft auth failures and log 422s.
-    filtered_routes = []
-    for route in app.router.routes:
-        path = getattr(route, "path", None)
-        methods = getattr(route, "methods", set()) or set()
-        if path in {"/health", "/health/rollout"} and "GET" in methods:
-            continue
-        filtered_routes.append(route)
-    app.router.routes = filtered_routes
-
-    def _log_env_key_prefix(source: str, env_key: str | None) -> str | None:
-        if not env_key:
-            return None
-        prefix = env_key[: max(1, len(env_key) // 2)]
-        print(f"[{source}] expected ENVIRONMENT_API_KEY prefix: {prefix}")
-        return prefix
-
-    @app.get("/health")
-    async def health(request: Request):
-        env_key = normalize_environment_api_key()
-        if not env_key:
-            return JSONResponse(
-                status_code=503,
-                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
-            )
-        if not is_api_key_header_authorized(request):
-            prefix = _log_env_key_prefix("health", env_key)
-            content = {"status": "healthy", "authorized": False}
-            if prefix:
-                content["expected_api_key_prefix"] = prefix
-            return JSONResponse(status_code=200, content=content)
-        return {"status": "healthy", "authorized": True}
-
-    @app.get("/health/rollout")
-    async def health_rollout(request: Request):
-        env_key = normalize_environment_api_key()
-        if not env_key:
-            return JSONResponse(
-                status_code=503,
-                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
-            )
-        if not is_api_key_header_authorized(request):
-            prefix = _log_env_key_prefix("health/rollout", env_key)
-            content = {"status": "healthy", "authorized": False}
-            if prefix:
-                content["expected_api_key_prefix"] = prefix
-            return JSONResponse(status_code=200, content=content)
-        return {"ok": True, "authorized": True}
-
-    @app.exception_handler(RequestValidationError)
-    async def _on_validation_error(request: Request, exc: RequestValidationError):
-        try:
-            hdr = request.headers
-            snapshot = {
-                "path": str(request.url.path),
-                "have_x_api_key": bool(hdr.get("x-api-key")),
-                "have_x_api_keys": bool(hdr.get("x-api-keys")),
-                "have_authorization": bool(hdr.get("authorization")),
-                "errors": exc.errors()[:5],
-            }
-            print("[422] validation", snapshot, flush=True)
-        except Exception:
-            pass
-        return JSONResponse(
-            status_code=422,
-            content={"status": "invalid", "detail": exc.errors()[:5]},
-        )
-
-    return app
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Run the Enron task app locally")
-    parser.add_argument("--host", default="0.0.0.0")
-    parser.add_argument("--port", type=int, default=8102)
-    parser.add_argument("--reload", action="store_true", help="Enable uvicorn autoreload")
-    parser.add_argument(
-        "--env-file",
-        action="append",
-        default=[],
-        help="Additional .env files to load before startup",
-    )
-    args = parser.parse_args()
-
-    default_env = Path(__file__).resolve().parents[4] / "backend" / ".env.dev"
-    env_files = [str(default_env)] if default_env.exists() else []
-    env_files.extend(args.env_file or [])
-
-    run_task_app(
-        build_task_app_config,
-        host=args.host,
-        port=args.port,
-        reload=args.reload,
-        env_files=env_files,
-    )

examples/task_apps/enron/tests/__init__.py

@@ -1,4 +0,0 @@
-# Enron task app tests
-
-
-

examples/task_apps/enron/tests/conftest.py

@@ -1,115 +0,0 @@
-"""Shared fixtures for Enron tests."""
-import os
-import socket
-import subprocess
-from subprocess import TimeoutExpired
-import time
-from pathlib import Path
-from typing import Iterator
-
-import pytest
-
-requests = pytest.importorskip("requests")
-
-
-def _which(executable: str) -> bool:
-    return any(
-        (Path(path) / executable).exists()
-        for path in os.getenv("PATH", "").split(os.pathsep)
-    )
-
-
-def _find_free_port() -> int:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        sock.bind(("127.0.0.1", 0))
-        return sock.getsockname()[1]
-
-
-def _wait_for_server(base_url: str, timeout: float = 60.0) -> None:
-    """Wait for the Enron server to become ready."""
-    deadline = time.time() + timeout
-    while time.time() < deadline:
-        try:
-            # Try /info first (no auth required if --insecure)
-            resp = requests.get(f"{base_url}/info", timeout=2.0)
-            if resp.status_code == 200:
-                return
-            # If 400/401, server is up but needs auth - that's OK
-            if resp.status_code in (400, 401):
-                return
-        except Exception:
-            time.sleep(0.5)
-    raise RuntimeError(f"Task app at {base_url} did not become ready")
-
-
-@pytest.fixture(scope="module")
-def enron_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]:
-    """Start the Enron task app server for testing."""
-    if not _which("uv"):
-        pytest.skip("uv executable not found on PATH")
-    if "GROQ_API_KEY" not in os.environ:
-        pytest.skip("GROQ_API_KEY must be set for Groq-backed tests")
-
-    port = _find_free_port()
-    base_url = f"http://127.0.0.1:{port}"
-    tmp_path = tmp_path_factory.mktemp("enron")
-    trace_dir = tmp_path / "traces"
-    trace_dir.mkdir(parents=True, exist_ok=True)
-
-    env = os.environ.copy()
-    cmd = [
-        "uv",
-        "run",
-        "-m",
-        "synth_ai",
-        "task-app",
-        "serve",
-        "grpo-enron",
-        "--port",
-        str(port),
-        "--no-reload",
-    ]
-    proc = subprocess.Popen(
-        cmd,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        text=True,
-        env=env,
-        stdin=subprocess.PIPE,
-    )
-    
-    # Send "n" to decline tracing
-    try:
-        if proc.stdin:
-            proc.stdin.write("n\n")
-            proc.stdin.flush()
-    except Exception:
-        pass
-    
-    stdout_capture = ""
-    try:
-        time.sleep(2)
-        if proc.poll() is not None:
-            stdout_capture, _ = proc.communicate(timeout=2)
-            tail = "\n".join(stdout_capture.strip().splitlines()[-20:]) if stdout_capture else ""
-            pytest.skip(f"Task app terminated immediately:\n{tail}")
-        
-        _wait_for_server(base_url)
-        yield base_url
-    except RuntimeError as e:
-        proc.terminate()
-        try:
-            stdout_capture, _ = proc.communicate(timeout=10)
-        except TimeoutExpired:
-            proc.kill()
-            stdout_capture, _ = proc.communicate()
-        tail = "\n".join((stdout_capture or "").strip().splitlines()[-20:])
-        pytest.skip(f"Task app failed to start: {e}\n{tail}")
-    finally:
-        if proc.poll() is None:
-            proc.terminate()
-            try:
-                proc.wait(timeout=5)
-            except TimeoutExpired:
-                proc.kill()
-

examples/task_apps/enron/tests/integration/__init__.py

@@ -1,4 +0,0 @@
-# Integration tests for Enron task app
-
-
-

examples/task_apps/enron/tests/integration/test_enron_eval.py

@@ -1,179 +0,0 @@
-"""Integration tests for Enron task app with Groq evaluation."""
-from __future__ import annotations
-
-import os
-import socket
-import subprocess
-from subprocess import TimeoutExpired
-import time
-from pathlib import Path
-from typing import Iterator
-
-import pytest
-
-requests = pytest.importorskip("requests")
-
-
-HERE = Path(__file__).resolve().parent
-TASK_APP_ROOT = HERE.parents[1]
-CONFIG_PATH = TASK_APP_ROOT / "eval_groq_qwen32.toml"
-
-
-def _which(executable: str) -> bool:
-    return any(
-        (Path(path) / executable).exists()
-        for path in os.getenv("PATH", "").split(os.pathsep)
-    )
-
-
-def _find_free_port() -> int:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
-        sock.bind(("127.0.0.1", 0))
-        return sock.getsockname()[1]
-
-
-def _wait_for_server(base_url: str, timeout: float = 60.0) -> None:
-    """Wait for the Enron server to become ready."""
-    deadline = time.time() + timeout
-    while time.time() < deadline:
-        try:
-            resp = requests.get(f"{base_url}/info", timeout=2.0)
-            if resp.status_code == 200:
-                return
-        except Exception:
-            time.sleep(0.5)
-    raise RuntimeError(f"Task app at {base_url} did not become ready")
-
-
-@pytest.fixture
-def enron_server(tmp_path: Path) -> Iterator[str]:
-    """Start the Enron task app server for testing."""
-    if not _which("uv"):
-        pytest.skip("uv executable not found on PATH")
-    if "GROQ_API_KEY" not in os.environ:
-        pytest.skip("GROQ_API_KEY must be set for Groq-backed evals")
-
-    port = _find_free_port()
-    base_url = f"http://127.0.0.1:{port}"
-    trace_dir = tmp_path / "traces"
-    trace_dir.mkdir(parents=True, exist_ok=True)
-
-    env = os.environ.copy()
-    cmd = [
-        "uv",
-        "run",
-        "-m",
-        "synth_ai",
-        "task-app",
-        "serve",
-        "grpo-enron",
-        "--port",
-        str(port),
-        "--no-reload",
-    ]
-    proc = subprocess.Popen(
-        cmd,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        text=True,
-        env=env,
-        stdin=subprocess.PIPE,
-    )
-    
-    # Send "n" to decline tracing
-    try:
-        if proc.stdin:
-            proc.stdin.write("n\n")
-            proc.stdin.flush()
-    except Exception:
-        pass
-    
-    stdout_capture = ""
-    try:
-        time.sleep(2)
-        if proc.poll() is not None:
-            stdout_capture, _ = proc.communicate(timeout=2)
-            tail = "\n".join(stdout_capture.strip().splitlines()[-20:]) if stdout_capture else ""
-            pytest.skip(f"Task app terminated immediately:\n{tail}")
-        
-        _wait_for_server(base_url)
-        yield base_url
-    except RuntimeError as e:
-        proc.terminate()
-        try:
-            stdout_capture, _ = proc.communicate(timeout=10)
-        except TimeoutExpired:
-            proc.kill()
-            stdout_capture, _ = proc.communicate()
-        tail = "\n".join((stdout_capture or "").strip().splitlines()[-20:])
-        pytest.skip(f"Task app failed to start: {e}\n{tail}")
-    finally:
-        if proc.poll() is None:
-            proc.terminate()
-            try:
-                proc.wait(timeout=5)
-            except TimeoutExpired:
-                proc.kill()
-
-
-@pytest.mark.slow
-def test_enron_server_health(enron_server: str) -> None:
-    """Test that the Enron server health endpoint works."""
-    resp = requests.get(f"{enron_server}/health", timeout=5.0)
-    assert resp.status_code in (200, 400), f"Unexpected status: {resp.status_code}"
-
-
-@pytest.mark.slow
-def test_enron_task_info(enron_server: str) -> None:
-    """Test that the Enron server returns valid task_info."""
-    resp = requests.get(f"{enron_server}/task_info", timeout=5.0)
-    assert resp.status_code == 200
-    data = resp.json()
-    assert "task" in data
-    assert data["task"]["id"] == "enron_email_qa"
-
-
-@pytest.mark.slow
-def test_enron_eval_with_groq(enron_server: str) -> None:
-    """Spin up the Enron task app and run a Groq-backed eval."""
-    if not CONFIG_PATH.exists():
-        pytest.skip(f"Config file not found: {CONFIG_PATH}")
-    
-    cmd = [
-        "uv",
-        "run",
-        "-m",
-        "synth_ai",
-        "eval",
-        "grpo-enron",
-        "--config",
-        str(CONFIG_PATH),
-        "--url",
-        enron_server,
-        "--model",
-        "qwen/qwen3-32b",
-        "--seeds",
-        "0",
-    ]
-    result = subprocess.run(
-        cmd,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        text=True,
-        env=os.environ.copy(),
-        check=False,
-        timeout=300,
-    )
-    
-    if result.returncode != 0:
-        pytest.fail(f"Eval failed with return code {result.returncode}:\n{result.stdout}")
-    
-    # Check for success indicators
-    assert "Eval complete" in result.stdout
-    assert "1 ok, 0 failed" in result.stdout or "status=200" in result.stdout
-    
-    # Check that we got a meaningful score
-    assert "official" in result.stdout.lower() or "mean_return" in result.stdout.lower()
-
-
-

examples/task_apps/enron/tests/integration/test_enron_rollout.py

@@ -1,135 +0,0 @@
-"""Integration test for Enron rollouts via /rollout endpoint."""
-import os
-import pytest
-
-requests = pytest.importorskip("requests")
-
-# Use the actual ENVIRONMENT_API_KEY from .env
-AUTH_HEADER = {"Authorization": "Bearer sk_env_30c78a787bac223c716918181209f263"}
-
-
-@pytest.mark.slow
-def test_enron_manual_rollout(enron_server: str) -> None:
-    """Test a manual Enron rollout with explicit search/read/answer actions."""
-    rollout_payload = {
-        "run_id": "test_manual_enron",
-        "env": {"seed": 0},
-        "ops": [
-            {
-                "tool": "search_emails",
-                "args": {
-                    "inbox": "test@enron.com",
-                    "keywords": ["test", "question"],
-                    "max_results": 5,
-                },
-            },
-            {
-                "tool": "answer_question",
-                "args": {"answer": "This is a test answer"},
-            },
-        ],
-        "policy": {
-            "policy_name": "manual",
-            "config": {"provider": "noop"},
-        },
-    }
-    
-    resp = requests.post(
-        f"{enron_server}/rollout",
-        json=rollout_payload,
-        headers=AUTH_HEADER,
-        timeout=60.0,
-    )
-    
-    assert resp.status_code == 200, f"Rollout failed: {resp.status_code} {resp.text}"
-    data = resp.json()
-    
-    # Verify response structure
-    assert "trajectories" in data
-    assert len(data["trajectories"]) > 0
-    assert "metrics" in data
-    assert "trace" in data
-    
-    # Check that trace is present
-    assert data["trace"] is not None
-    assert "session_trace" in data["trace"]
-    
-    trajectory = data["trajectories"][0]
-    assert "steps" in trajectory
-    
-    # Should have at least initial observation
-    assert len(trajectory["steps"]) > 0
-
-
-@pytest.mark.slow
-def test_enron_policy_rollout(enron_server: str) -> None:
-    """Test an Enron rollout using Groq policy."""
-    if "GROQ_API_KEY" not in os.environ:
-        pytest.skip("GROQ_API_KEY required for this test")
-    
-    rollout_payload = {
-        "run_id": "test_policy_enron",
-        "env": {"seed": 0},
-        "ops": [],  # Empty ops means use policy
-        "policy": {
-            "policy_name": "qwen-groq",
-            "config": {
-                "provider": "groq",
-                "model": "qwen/qwen3-32b",
-                "temperature": 0.2,
-                "max_tokens": 1024,
-            },
-        },
-    }
-    
-    resp = requests.post(
-        f"{enron_server}/rollout",
-        json=rollout_payload,
-        headers=AUTH_HEADER,
-        timeout=180.0,  # Enron can be slow with multiple tool calls
-    )
-    
-    assert resp.status_code == 200, f"Rollout failed: {resp.status_code} {resp.text}"
-    data = resp.json()
-    
-    # Verify response structure
-    assert "trajectories" in data
-    assert "metrics" in data
-    assert "trace" in data
-    
-    trajectory = data["trajectories"][0]
-    assert "steps" in trajectory
-    
-    # Check that steps were taken
-    assert len(trajectory["steps"]) > 0
-    
-    # Verify metrics
-    metrics = data["metrics"]
-    assert "episode_returns" in metrics or "mean_return" in metrics
-    
-    # Check that we got some reward (could be negative for search penalty)
-    if "episode_returns" in metrics and len(metrics["episode_returns"]) > 0:
-        # Just verify it's a number
-        assert isinstance(metrics["episode_returns"][0], (int, float))
-
-
-@pytest.mark.fast
-def test_enron_rollout_with_auth(enron_server: str) -> None:
-    """Test that Enron rollout requires proper authentication."""
-    rollout_payload = {
-        "run_id": "test_auth",
-        "env": {"seed": 0},
-        "ops": [],
-        "policy": {"config": {"provider": "noop"}},
-    }
-    
-    # Try without auth header
-    resp = requests.post(
-        f"{enron_server}/rollout",
-        json=rollout_payload,
-        timeout=10.0,
-    )
-    
-    # Should fail without auth (400 or 401)
-    assert resp.status_code in (400, 401, 403), f"Expected auth error, got {resp.status_code}"
-

examples/task_apps/enron/tests/unit/__init__.py

@@ -1,4 +0,0 @@
-# Unit tests for Enron task app
-
-
-