synth-ai 0.2.13.dev1__py3-none-any.whl → 0.2.14__py3-none-any.whl

examples/task_apps/enron/task_app/grpo_enron_task_app.py

@@ -0,0 +1,146 @@
+"""Compatibility wrapper for the GRPO Enron task app.
+
+This mirrors the structure of the Crafter task app wrapper while delegating
+all configuration to the colocated `grpo_enron.py` module. Normal usage should
+prefer invoking `uvx synth-ai serve grpo-enron`, but this module remains for
+direct execution or importing the FastAPI app object.
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from starlette.requests import Request
+from synth_ai.task.apps import ModalDeploymentConfig, registry
+from synth_ai.task.auth import is_api_key_header_authorized, normalize_environment_api_key
+from synth_ai.task.server import TaskAppConfig, create_task_app, run_task_app
+
+from .grpo_enron import build_config
+
+APP_ID = "grpo-enron"
+
+
+def _build_base_config() -> TaskAppConfig:
+    # Lazily construct the base config to avoid heavy work at import time.
+    return build_config()
+
+
+try:
+    _REGISTERED_ENTRY = registry.get(APP_ID)
+except Exception:  # pragma: no cover - registry unavailable in some contexts
+    MODAL_DEPLOYMENT: ModalDeploymentConfig | None = None
+    ENV_FILES: tuple[str, ...] = ()
+else:
+    MODAL_DEPLOYMENT = _REGISTERED_ENTRY.modal
+    ENV_FILES = tuple(_REGISTERED_ENTRY.env_files)
+
+
+def build_task_app_config() -> TaskAppConfig:
+    """Return a fresh TaskAppConfig for this wrapper."""
+    base = _build_base_config()
+    return base.clone()
+
+
+def fastapi_app():
+    """Return the FastAPI application for Modal or other ASGI hosts."""
+
+    app = create_task_app(build_task_app_config())
+
+    # Replace default health endpoints so we can permit soft auth failures and log 422s.
+    filtered_routes = []
+    for route in app.router.routes:
+        path = getattr(route, "path", None)
+        methods = getattr(route, "methods", set()) or set()
+        if path in {"/health", "/health/rollout"} and "GET" in methods:
+            continue
+        filtered_routes.append(route)
+    app.router.routes = filtered_routes
+
+    def _log_env_key_prefix(source: str, env_key: str | None) -> str | None:
+        if not env_key:
+            return None
+        prefix = env_key[: max(1, len(env_key) // 2)]
+        print(f"[{source}] expected ENVIRONMENT_API_KEY prefix: {prefix}")
+        return prefix
+
+    @app.get("/health")
+    async def health(request: Request):
+        env_key = normalize_environment_api_key()
+        if not env_key:
+            return JSONResponse(
+                status_code=503,
+                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
+            )
+        if not is_api_key_header_authorized(request):
+            prefix = _log_env_key_prefix("health", env_key)
+            content = {"status": "healthy", "authorized": False}
+            if prefix:
+                content["expected_api_key_prefix"] = prefix
+            return JSONResponse(status_code=200, content=content)
+        return {"status": "healthy", "authorized": True}
+
+    @app.get("/health/rollout")
+    async def health_rollout(request: Request):
+        env_key = normalize_environment_api_key()
+        if not env_key:
+            return JSONResponse(
+                status_code=503,
+                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
+            )
+        if not is_api_key_header_authorized(request):
+            prefix = _log_env_key_prefix("health/rollout", env_key)
+            content = {"status": "healthy", "authorized": False}
+            if prefix:
+                content["expected_api_key_prefix"] = prefix
+            return JSONResponse(status_code=200, content=content)
+        return {"ok": True, "authorized": True}
+
+    @app.exception_handler(RequestValidationError)
+    async def _on_validation_error(request: Request, exc: RequestValidationError):
+        try:
+            hdr = request.headers
+            snapshot = {
+                "path": str(request.url.path),
+                "have_x_api_key": bool(hdr.get("x-api-key")),
+                "have_x_api_keys": bool(hdr.get("x-api-keys")),
+                "have_authorization": bool(hdr.get("authorization")),
+                "errors": exc.errors()[:5],
+            }
+            print("[422] validation", snapshot, flush=True)
+        except Exception:
+            pass
+        return JSONResponse(
+            status_code=422,
+            content={"status": "invalid", "detail": exc.errors()[:5]},
+        )
+
+    return app
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the Enron task app locally")
+    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=8102)
+    parser.add_argument("--reload", action="store_true", help="Enable uvicorn autoreload")
+    parser.add_argument(
+        "--env-file",
+        action="append",
+        default=[],
+        help="Additional .env files to load before startup",
+    )
+    args = parser.parse_args()
+
+    default_env = Path(__file__).resolve().parents[4] / "backend" / ".env.dev"
+    env_files = [str(default_env)] if default_env.exists() else []
+    env_files.extend(args.env_file or [])
+
+    run_task_app(
+        build_task_app_config,
+        host=args.host,
+        port=args.port,
+        reload=args.reload,
+        env_files=env_files,
+    )

examples/task_apps/enron/tests/__init__.py

@@ -0,0 +1,4 @@
+# Enron task app tests
+
+
+

examples/task_apps/enron/tests/conftest.py

@@ -0,0 +1,115 @@
+"""Shared fixtures for Enron tests."""
+import os
+import socket
+import subprocess
+from subprocess import TimeoutExpired
+import time
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+
+requests = pytest.importorskip("requests")
+
+
+def _which(executable: str) -> bool:
+    return any(
+        (Path(path) / executable).exists()
+        for path in os.getenv("PATH", "").split(os.pathsep)
+    )
+
+
+def _find_free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
+def _wait_for_server(base_url: str, timeout: float = 60.0) -> None:
+    """Wait for the Enron server to become ready."""
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        try:
+            # Try /info first (no auth required if --insecure)
+            resp = requests.get(f"{base_url}/info", timeout=2.0)
+            if resp.status_code == 200:
+                return
+            # If 400/401, server is up but needs auth - that's OK
+            if resp.status_code in (400, 401):
+                return
+        except Exception:
+            time.sleep(0.5)
+    raise RuntimeError(f"Task app at {base_url} did not become ready")
+
+
+@pytest.fixture(scope="module")
+def enron_server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]:
+    """Start the Enron task app server for testing."""
+    if not _which("uv"):
+        pytest.skip("uv executable not found on PATH")
+    if "GROQ_API_KEY" not in os.environ:
+        pytest.skip("GROQ_API_KEY must be set for Groq-backed tests")
+
+    port = _find_free_port()
+    base_url = f"http://127.0.0.1:{port}"
+    tmp_path = tmp_path_factory.mktemp("enron")
+    trace_dir = tmp_path / "traces"
+    trace_dir.mkdir(parents=True, exist_ok=True)
+
+    env = os.environ.copy()
+    cmd = [
+        "uv",
+        "run",
+        "-m",
+        "synth_ai",
+        "task-app",
+        "serve",
+        "grpo-enron",
+        "--port",
+        str(port),
+        "--no-reload",
+    ]
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        env=env,
+        stdin=subprocess.PIPE,
+    )
+    
+    # Send "n" to decline tracing
+    try:
+        if proc.stdin:
+            proc.stdin.write("n\n")
+            proc.stdin.flush()
+    except Exception:
+        pass
+    
+    stdout_capture = ""
+    try:
+        time.sleep(2)
+        if proc.poll() is not None:
+            stdout_capture, _ = proc.communicate(timeout=2)
+            tail = "\n".join(stdout_capture.strip().splitlines()[-20:]) if stdout_capture else ""
+            pytest.skip(f"Task app terminated immediately:\n{tail}")
+        
+        _wait_for_server(base_url)
+        yield base_url
+    except RuntimeError as e:
+        proc.terminate()
+        try:
+            stdout_capture, _ = proc.communicate(timeout=10)
+        except TimeoutExpired:
+            proc.kill()
+            stdout_capture, _ = proc.communicate()
+        tail = "\n".join((stdout_capture or "").strip().splitlines()[-20:])
+        pytest.skip(f"Task app failed to start: {e}\n{tail}")
+    finally:
+        if proc.poll() is None:
+            proc.terminate()
+            try:
+                proc.wait(timeout=5)
+            except TimeoutExpired:
+                proc.kill()
+

examples/task_apps/enron/tests/integration/__init__.py

@@ -0,0 +1,4 @@
+# Integration tests for Enron task app
+
+
+

examples/task_apps/enron/tests/integration/test_enron_eval.py

@@ -0,0 +1,179 @@
+"""Integration tests for Enron task app with Groq evaluation."""
+from __future__ import annotations
+
+import os
+import socket
+import subprocess
+from subprocess import TimeoutExpired
+import time
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+
+requests = pytest.importorskip("requests")
+
+
+HERE = Path(__file__).resolve().parent
+TASK_APP_ROOT = HERE.parents[1]
+CONFIG_PATH = TASK_APP_ROOT / "eval_groq_qwen32.toml"
+
+
+def _which(executable: str) -> bool:
+    return any(
+        (Path(path) / executable).exists()
+        for path in os.getenv("PATH", "").split(os.pathsep)
+    )
+
+
+def _find_free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
+def _wait_for_server(base_url: str, timeout: float = 60.0) -> None:
+    """Wait for the Enron server to become ready."""
+    deadline = time.time() + timeout
+    while time.time() < deadline:
+        try:
+            resp = requests.get(f"{base_url}/info", timeout=2.0)
+            if resp.status_code == 200:
+                return
+        except Exception:
+            time.sleep(0.5)
+    raise RuntimeError(f"Task app at {base_url} did not become ready")
+
+
+@pytest.fixture
+def enron_server(tmp_path: Path) -> Iterator[str]:
+    """Start the Enron task app server for testing."""
+    if not _which("uv"):
+        pytest.skip("uv executable not found on PATH")
+    if "GROQ_API_KEY" not in os.environ:
+        pytest.skip("GROQ_API_KEY must be set for Groq-backed evals")
+
+    port = _find_free_port()
+    base_url = f"http://127.0.0.1:{port}"
+    trace_dir = tmp_path / "traces"
+    trace_dir.mkdir(parents=True, exist_ok=True)
+
+    env = os.environ.copy()
+    cmd = [
+        "uv",
+        "run",
+        "-m",
+        "synth_ai",
+        "task-app",
+        "serve",
+        "grpo-enron",
+        "--port",
+        str(port),
+        "--no-reload",
+    ]
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        env=env,
+        stdin=subprocess.PIPE,
+    )
+    
+    # Send "n" to decline tracing
+    try:
+        if proc.stdin:
+            proc.stdin.write("n\n")
+            proc.stdin.flush()
+    except Exception:
+        pass
+    
+    stdout_capture = ""
+    try:
+        time.sleep(2)
+        if proc.poll() is not None:
+            stdout_capture, _ = proc.communicate(timeout=2)
+            tail = "\n".join(stdout_capture.strip().splitlines()[-20:]) if stdout_capture else ""
+            pytest.skip(f"Task app terminated immediately:\n{tail}")
+        
+        _wait_for_server(base_url)
+        yield base_url
+    except RuntimeError as e:
+        proc.terminate()
+        try:
+            stdout_capture, _ = proc.communicate(timeout=10)
+        except TimeoutExpired:
+            proc.kill()
+            stdout_capture, _ = proc.communicate()
+        tail = "\n".join((stdout_capture or "").strip().splitlines()[-20:])
+        pytest.skip(f"Task app failed to start: {e}\n{tail}")
+    finally:
+        if proc.poll() is None:
+            proc.terminate()
+            try:
+                proc.wait(timeout=5)
+            except TimeoutExpired:
+                proc.kill()
+
+
+@pytest.mark.slow
+def test_enron_server_health(enron_server: str) -> None:
+    """Test that the Enron server health endpoint works."""
+    resp = requests.get(f"{enron_server}/health", timeout=5.0)
+    assert resp.status_code in (200, 400), f"Unexpected status: {resp.status_code}"
+
+
+@pytest.mark.slow
+def test_enron_task_info(enron_server: str) -> None:
+    """Test that the Enron server returns valid task_info."""
+    resp = requests.get(f"{enron_server}/task_info", timeout=5.0)
+    assert resp.status_code == 200
+    data = resp.json()
+    assert "task" in data
+    assert data["task"]["id"] == "enron_email_qa"
+
+
+@pytest.mark.slow
+def test_enron_eval_with_groq(enron_server: str) -> None:
+    """Spin up the Enron task app and run a Groq-backed eval."""
+    if not CONFIG_PATH.exists():
+        pytest.skip(f"Config file not found: {CONFIG_PATH}")
+    
+    cmd = [
+        "uv",
+        "run",
+        "-m",
+        "synth_ai",
+        "eval",
+        "grpo-enron",
+        "--config",
+        str(CONFIG_PATH),
+        "--url",
+        enron_server,
+        "--model",
+        "qwen/qwen3-32b",
+        "--seeds",
+        "0",
+    ]
+    result = subprocess.run(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        env=os.environ.copy(),
+        check=False,
+        timeout=300,
+    )
+    
+    if result.returncode != 0:
+        pytest.fail(f"Eval failed with return code {result.returncode}:\n{result.stdout}")
+    
+    # Check for success indicators
+    assert "Eval complete" in result.stdout
+    assert "1 ok, 0 failed" in result.stdout or "status=200" in result.stdout
+    
+    # Check that we got a meaningful score
+    assert "official" in result.stdout.lower() or "mean_return" in result.stdout.lower()
+
+
+

examples/task_apps/enron/tests/integration/test_enron_rollout.py

@@ -0,0 +1,135 @@
+"""Integration test for Enron rollouts via /rollout endpoint."""
+import os
+import pytest
+
+requests = pytest.importorskip("requests")
+
+# Use the actual ENVIRONMENT_API_KEY from .env
+AUTH_HEADER = {"Authorization": "Bearer sk_env_30c78a787bac223c716918181209f263"}
+
+
+@pytest.mark.slow
+def test_enron_manual_rollout(enron_server: str) -> None:
+    """Test a manual Enron rollout with explicit search/read/answer actions."""
+    rollout_payload = {
+        "run_id": "test_manual_enron",
+        "env": {"seed": 0},
+        "ops": [
+            {
+                "tool": "search_emails",
+                "args": {
+                    "inbox": "test@enron.com",
+                    "keywords": ["test", "question"],
+                    "max_results": 5,
+                },
+            },
+            {
+                "tool": "answer_question",
+                "args": {"answer": "This is a test answer"},
+            },
+        ],
+        "policy": {
+            "policy_name": "manual",
+            "config": {"provider": "noop"},
+        },
+    }
+    
+    resp = requests.post(
+        f"{enron_server}/rollout",
+        json=rollout_payload,
+        headers=AUTH_HEADER,
+        timeout=60.0,
+    )
+    
+    assert resp.status_code == 200, f"Rollout failed: {resp.status_code} {resp.text}"
+    data = resp.json()
+    
+    # Verify response structure
+    assert "trajectories" in data
+    assert len(data["trajectories"]) > 0
+    assert "metrics" in data
+    assert "trace" in data
+    
+    # Check that trace is present
+    assert data["trace"] is not None
+    assert "session_trace" in data["trace"]
+    
+    trajectory = data["trajectories"][0]
+    assert "steps" in trajectory
+    
+    # Should have at least initial observation
+    assert len(trajectory["steps"]) > 0
+
+
+@pytest.mark.slow
+def test_enron_policy_rollout(enron_server: str) -> None:
+    """Test an Enron rollout using Groq policy."""
+    if "GROQ_API_KEY" not in os.environ:
+        pytest.skip("GROQ_API_KEY required for this test")
+    
+    rollout_payload = {
+        "run_id": "test_policy_enron",
+        "env": {"seed": 0},
+        "ops": [],  # Empty ops means use policy
+        "policy": {
+            "policy_name": "qwen-groq",
+            "config": {
+                "provider": "groq",
+                "model": "qwen/qwen3-32b",
+                "temperature": 0.2,
+                "max_tokens": 1024,
+            },
+        },
+    }
+    
+    resp = requests.post(
+        f"{enron_server}/rollout",
+        json=rollout_payload,
+        headers=AUTH_HEADER,
+        timeout=180.0,  # Enron can be slow with multiple tool calls
+    )
+    
+    assert resp.status_code == 200, f"Rollout failed: {resp.status_code} {resp.text}"
+    data = resp.json()
+    
+    # Verify response structure
+    assert "trajectories" in data
+    assert "metrics" in data
+    assert "trace" in data
+    
+    trajectory = data["trajectories"][0]
+    assert "steps" in trajectory
+    
+    # Check that steps were taken
+    assert len(trajectory["steps"]) > 0
+    
+    # Verify metrics
+    metrics = data["metrics"]
+    assert "episode_returns" in metrics or "mean_return" in metrics
+    
+    # Check that we got some reward (could be negative for search penalty)
+    if "episode_returns" in metrics and len(metrics["episode_returns"]) > 0:
+        # Just verify it's a number
+        assert isinstance(metrics["episode_returns"][0], (int, float))
+
+
+@pytest.mark.fast
+def test_enron_rollout_with_auth(enron_server: str) -> None:
+    """Test that Enron rollout requires proper authentication."""
+    rollout_payload = {
+        "run_id": "test_auth",
+        "env": {"seed": 0},
+        "ops": [],
+        "policy": {"config": {"provider": "noop"}},
+    }
+    
+    # Try without auth header
+    resp = requests.post(
+        f"{enron_server}/rollout",
+        json=rollout_payload,
+        timeout=10.0,
+    )
+    
+    # Should fail without auth (400 or 401)
+    assert resp.status_code in (400, 401, 403), f"Expected auth error, got {resp.status_code}"
+

examples/task_apps/enron/tests/unit/__init__.py

@@ -0,0 +1,4 @@
+# Unit tests for Enron task app
+
+
+