synth-ai 0.2.9.dev3__py3-none-any.whl → 0.2.9.dev5__py3-none-any.whl

examples/warming_up_to_rl/task_app/grpo_crafter_task_app.py

@@ -0,0 +1,165 @@
+
+"""Compatibility wrapper for the GRPO Crafter task app.
+
+This module now delegates to the shared TaskAppConfig defined in
+`synth_ai.task.apps.grpo_crafter`. It is kept for legacy usage (running the
+file directly or targeting `fastapi_app` from external tooling). Prefer using
+`uvx synth-ai serve grpo-crafter` for local development and testing.
+"""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from starlette.requests import Request
+
+from synth_ai.task.apps import ModalDeploymentConfig, registry
+from .grpo_crafter import build_config
+from synth_ai.task.auth import is_api_key_header_authorized, normalize_environment_api_key
+from synth_ai.task.server import TaskAppConfig, create_task_app, run_task_app
+
+
+APP_ID = "grpo-crafter"
+
+
+_BASE_CONFIG = build_config()
+TASK_APP_CONFIG = TaskAppConfig(
+    app_id="grpo-crafter",
+    name=_BASE_CONFIG.name,
+    description=_BASE_CONFIG.description,
+    base_task_info=_BASE_CONFIG.base_task_info,
+    describe_taskset=_BASE_CONFIG.describe_taskset,
+    provide_task_instances=_BASE_CONFIG.provide_task_instances,
+    rollout=_BASE_CONFIG.rollout,
+    dataset_registry=_BASE_CONFIG.dataset_registry,
+    rubrics=_BASE_CONFIG.rubrics,
+    proxy=_BASE_CONFIG.proxy,
+    routers=_BASE_CONFIG.routers,
+    middleware=_BASE_CONFIG.middleware,
+    app_state=_BASE_CONFIG.app_state,
+    require_api_key=_BASE_CONFIG.require_api_key,
+    expose_debug_env=_BASE_CONFIG.expose_debug_env,
+    cors_origins=_BASE_CONFIG.cors_origins,
+    startup_hooks=_BASE_CONFIG.startup_hooks,
+    shutdown_hooks=_BASE_CONFIG.shutdown_hooks,
+)
+
+try:
+    _REGISTERED_ENTRY = registry.get(APP_ID)
+except Exception:  # pragma: no cover - registry unavailable in some contexts
+    MODAL_DEPLOYMENT: ModalDeploymentConfig | None = None
+    ENV_FILES: tuple[str, ...] = ()
+else:
+    MODAL_DEPLOYMENT = _REGISTERED_ENTRY.modal
+    ENV_FILES = tuple(_REGISTERED_ENTRY.env_files)
+
+
+def build_task_app_config() -> TaskAppConfig:
+    """Return a fresh TaskAppConfig for this wrapper."""
+
+    return TASK_APP_CONFIG.clone()
+
+
+def fastapi_app():
+    """Return the FastAPI application for Modal or other ASGI hosts."""
+
+    app = create_task_app(build_task_app_config())
+
+    # Replace default health endpoints so we can permit soft auth failures and log 422s.
+    filtered_routes = []
+    for route in app.router.routes:
+        path = getattr(route, "path", None)
+        methods = getattr(route, "methods", set()) or set()
+        if path in {"/health", "/health/rollout"} and "GET" in methods:
+            continue
+        filtered_routes.append(route)
+    app.router.routes = filtered_routes
+
+    def _log_env_key_prefix(source: str, env_key: str | None) -> str | None:
+        if not env_key:
+            return None
+        prefix = env_key[: max(1, len(env_key) // 2)]
+        print(f"[{source}] expected ENVIRONMENT_API_KEY prefix: {prefix}")
+        return prefix
+
+    @app.get("/health")
+    async def health(request: Request):
+        env_key = normalize_environment_api_key()
+        if not env_key:
+            return JSONResponse(
+                status_code=503,
+                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
+            )
+        if not is_api_key_header_authorized(request):
+            prefix = _log_env_key_prefix("health", env_key)
+            content = {"status": "healthy", "authorized": False}
+            if prefix:
+                content["expected_api_key_prefix"] = prefix
+            return JSONResponse(status_code=200, content=content)
+        return {"status": "healthy", "authorized": True}
+
+    @app.get("/health/rollout")
+    async def health_rollout(request: Request):
+        env_key = normalize_environment_api_key()
+        if not env_key:
+            return JSONResponse(
+                status_code=503,
+                content={"status": "unhealthy", "detail": "Missing ENVIRONMENT_API_KEY"},
+            )
+        if not is_api_key_header_authorized(request):
+            prefix = _log_env_key_prefix("health/rollout", env_key)
+            content = {"status": "healthy", "authorized": False}
+            if prefix:
+                content["expected_api_key_prefix"] = prefix
+            return JSONResponse(status_code=200, content=content)
+        return {"ok": True, "authorized": True}
+
+    @app.exception_handler(RequestValidationError)
+    async def _on_validation_error(request: Request, exc: RequestValidationError):
+        try:
+            hdr = request.headers
+            snapshot = {
+                "path": str(getattr(request, "url").path),
+                "have_x_api_key": bool(hdr.get("x-api-key")),
+                "have_x_api_keys": bool(hdr.get("x-api-keys")),
+                "have_authorization": bool(hdr.get("authorization")),
+                "errors": exc.errors()[:5],
+            }
+            print("[422] validation", snapshot, flush=True)
+        except Exception:
+            pass
+        return JSONResponse(
+            status_code=422,
+            content={"status": "invalid", "detail": exc.errors()[:5]},
+        )
+
+    return app
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Run the Crafter task app locally")
+    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=8001)
+    parser.add_argument("--reload", action="store_true", help="Enable uvicorn autoreload")
+    parser.add_argument(
+        "--env-file",
+        action="append",
+        default=[],
+        help="Additional .env files to load before startup",
+    )
+    args = parser.parse_args()
+
+    default_env = Path(__file__).resolve().parents[4] / "backend" / ".env.dev"
+    env_files = [str(default_env)] if default_env.exists() else []
+    env_files.extend(args.env_file or [])
+
+    run_task_app(
+        build_task_app_config,
+        host=args.host,
+        port=args.port,
+        reload=args.reload,
+        env_files=env_files,
+    )

examples/warming_up_to_rl/task_app/synth_envs_hosted/README.md

@@ -0,0 +1,173 @@
+# GRPO Synth Envs Hosted Service
+
+This service provides hosted environment and policy management for GRPO (Group Relative Policy Optimization) training with synthetic environments.
+
+## Architecture
+
+The service implements a FastAPI-based HTTP API that manages:
+- **Environments**: Stateful environment instances (currently Crafter)
+- **Policies**: Thin policy clients that prepare inference requests
+- **Rollouts**: Coordinated execution of environment-policy interaction loops
+- **Snapshots**: State persistence using Modal Volumes
+- **Branching**: Creating multiple copies of environments/policies for exploration
+
+## Key Components
+
+### Core Modules
+- `hosted_app.py`: FastAPI app factory and configuration
+- `registry.py`: In-memory registries for active instances
+- `storage/volume.py`: Modal Volume operations for snapshots
+- `inference/openai_client.py`: OpenAI-compatible inference client
+
+### API Routers
+- `environment_routes.py`: Environment lifecycle endpoints
+- `policy_routes.py`: Policy lifecycle endpoints
+- `rollout.py`: Rollout coordinator and run management
+- `branching.py`: Branching operations
+
+### Environment Implementations
+- `envs/crafter/`: Crafter environment and policy implementations
+
+## API Endpoints
+
+### Service Discovery
+- `GET /info`: Service configuration and endpoints
+- `GET /health`: Health check
+
+### Environment Management
+- `POST /env/create`: Create new environment
+- `POST /env/reset`: Reset environment
+- `POST /env/step`: Execute environment step
+- `POST /env/snapshot`: Save environment state
+- `POST /env/restore`: Restore from snapshot
+- `POST /env/terminate`: Clean up environment
+
+### Policy Management
+- `POST /policy/create`: Create new policy
+- `POST /policy/step`: Generate actions (with optional inference)
+- `POST /policy/snapshot`: Save policy state
+- `POST /policy/restore`: Restore from snapshot
+- `POST /policy/terminate`: Clean up policy
+
+### Coordination
+- `POST /rollout`: Execute coordinated rollout
+- `POST /branch`: Create environment/policy branches
+- `POST /run/abort`: Abort running rollout
+- `GET /run/status/{run_id}`: Check run status
+
+## Local Development
+
+```bash
+# Install dependencies
+pip install fastapi uvicorn httpx pydantic
+
+# Run the service
+python main.py
+
+# Or with uvicorn directly
+uvicorn main:app --reload --port 8000
+```
+
+## Modal Deployment
+
+```bash
+# Deploy to Modal
+modal deploy main.py
+
+# Run once
+modal run main.py
+```
+
+## Environment Variables
+
+- `SERVICE_BASE_URL`: Base URL for this service (default: http://localhost:8000)
+- `VLLM_BASE_URL`: Base URL for vLLM inference service (default: http://localhost:8001)
+- `DEFAULT_MODEL`: Default model name for inference
+
+## Storage
+
+The service uses Modal Volumes for persistent storage:
+- Volume name: `synth-env-state`
+- Mount path: `/data/state`
+- Layout: `/data/state/runs/{rl_run_id}/{kind}/{shard}/{snapshot_id}.tar.gz`
+
+## Example Usage
+
+```python
+import httpx
+
+# Create environment
+env_response = httpx.post(
+    "http://localhost:8000/env/create",
+    json={
+        "env_name": "crafter",
+        "config": {},
+        "seed": 42,
+        "rl_run_id": "test-run-1"
+    }
+)
+env_id = env_response.json()["env_id"]
+
+# Create policy
+policy_response = httpx.post(
+    "http://localhost:8000/policy/create",
+    json={
+        "policy_name": "crafter-react",
+        "config": {"inference_url": "http://vllm:8001"},
+        "rl_run_id": "test-run-1",
+        "bound_env_id": env_id
+    }
+)
+policy_id = policy_response.json()["policy_id"]
+
+# Execute rollout
+rollout_response = httpx.post(
+    "http://localhost:8000/rollout",
+    json={
+        "run_id": "test-run-1",
+        "env": {"env_id": env_id},
+        "policy": {"policy_id": policy_id},
+        "ops": ["agent", "env"] * 10,
+        "on_done": "reset"
+    }
+)
+trajectories = rollout_response.json()["trajectories"]
+```
+
+## Testing
+
+The implementation follows the plan outlined in `plan.md` and decisions in `decisions.md`. Key test areas:
+- Environment create/step/reset lifecycle
+- Policy inference request building
+- Snapshot/restore round trips
+- Rollout coordination with abort support
+- Branching operations
+
+4b
+"aggregate": {
+    "completed": 20,
+    "total": 20,
+    "avg_turns": 10.0,
+    "avg_achievements": 1.3,
+    "achievements_freq": {
+      "collect_wood": 9,
+      "collect_sapling": 8,
+      "collect_drink": 7,
+      "place_plant": 2
+    }
+  }
+
+
+groq qwen/qwen3-32b
+ ],
+  "aggregate": {
+    "completed": 20,
+    "total": 20,
+    "avg_turns": 10.0,
+    "avg_achievements": 1.0,
+    "achievements_freq": {
+      "collect_sapling": 7,
+      "collect_wood": 9,
+      "collect_drink": 4
+    }
+  }

examples/warming_up_to_rl/task_app/synth_envs_hosted/__init__.py

@@ -0,0 +1,5 @@
+"""GRPO Synth Envs Hosted Service."""
+
+from .hosted_app import create_app, TaskApp
+
+__all__ = ["create_app", "TaskApp"]

examples/warming_up_to_rl/task_app/synth_envs_hosted/branching.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import logging
+from typing import Dict, List, Optional
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+
+from .registry import registry
+from .storage.volume import storage
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+class BranchRequest(BaseModel):
+    env_ids: Optional[List[str]] = None
+    policy_ids: Optional[List[str]] = None
+    num_children: int = 1
+    max_branches: int = 10
+
+
+class BranchResponse(BaseModel):
+    env_branches: Dict[str, List[str]]
+    policy_branches: Dict[str, List[str]]
+
+
+@router.post("/branch", response_model=BranchResponse)
+async def create_branches(request: BranchRequest) -> BranchResponse:
+    """Create branches of environments and/or policies."""
+
+    if request.num_children > request.max_branches:
+        raise HTTPException(
+            status_code=422,
+            detail=f"num_children ({request.num_children}) exceeds max_branches ({request.max_branches})",
+        )
+
+    env_branches = {}
+    policy_branches = {}
+
+    try:
+        # Branch environments
+        if request.env_ids:
+            for env_id in request.env_ids:
+                env_handle = registry.get_env(env_id)
+                if not env_handle:
+                    logger.warning(f"Environment {env_id} not found, skipping")
+                    continue
+
+                child_ids = []
+
+                for child_idx in range(request.num_children):
+                    # Create snapshot of parent
+                    from .environment_routes import (
+                        snapshot_environment,
+                        EnvSnapshotRequest,
+                    )
+
+                    snapshot_response = await snapshot_environment(
+                        EnvSnapshotRequest(env_id=env_id)
+                    )
+
+                    # Restore to new environment with modified seed
+                    from .environment_routes import (
+                        restore_environment,
+                        EnvRestoreRequest,
+                    )
+
+                    restore_response = await restore_environment(
+                        EnvRestoreRequest(snapshot_id=snapshot_response.snapshot_id)
+                    )
+
+                    child_id = restore_response.env_id
+                    child_handle = registry.get_env(child_id)
+
+                    # Update child seed for determinism
+                    if child_handle and child_handle.seed is not None:
+                        child_handle.seed = child_handle.seed + child_idx + 1
+                        child_handle.env.seed = child_handle.seed
+
+                    child_ids.append(child_id)
+
+                    # Track parent relationship in snapshot metadata
+                    snapshot_meta = registry.get_snapshot(snapshot_response.snapshot_id)
+                    if snapshot_meta:
+                        snapshot_meta.parent_snapshot_id = env_id
+
+                env_branches[env_id] = child_ids
+
+        # Branch policies
+        if request.policy_ids:
+            for policy_id in request.policy_ids:
+                policy_handle = registry.get_policy(policy_id)
+                if not policy_handle:
+                    logger.warning(f"Policy {policy_id} not found, skipping")
+                    continue
+
+                child_ids = []
+
+                for child_idx in range(request.num_children):
+                    # Create snapshot of parent
+                    from .policy_routes import snapshot_policy, PolicySnapshotRequest
+
+                    snapshot_response = await snapshot_policy(
+                        PolicySnapshotRequest(policy_id=policy_id)
+                    )
+
+                    # Restore to new policy
+                    from .policy_routes import restore_policy, PolicyRestoreRequest
+
+                    restore_response = await restore_policy(
+                        PolicyRestoreRequest(snapshot_id=snapshot_response.snapshot_id)
+                    )
+
+                    child_id = restore_response.policy_id
+                    child_ids.append(child_id)
+
+                    # Copy bound environment if parent had one
+                    child_handle = registry.get_policy(child_id)
+                    if child_handle and policy_handle.bound_env_id:
+                        # If we also branched the env, bind to corresponding child
+                        if policy_handle.bound_env_id in env_branches:
+                            child_envs = env_branches[policy_handle.bound_env_id]
+                            if child_idx < len(child_envs):
+                                child_handle.bound_env_id = child_envs[child_idx]
+                        else:
+                            # Otherwise keep same env binding
+                            child_handle.bound_env_id = policy_handle.bound_env_id
+
+                    # Track parent relationship
+                    snapshot_meta = registry.get_snapshot(snapshot_response.snapshot_id)
+                    if snapshot_meta:
+                        snapshot_meta.parent_snapshot_id = policy_id
+
+                policy_branches[policy_id] = child_ids
+
+        return BranchResponse(
+            env_branches=env_branches,
+            policy_branches=policy_branches,
+        )
+
+    except Exception as e:
+        logger.error(f"Failed to create branches: {e}")
+        raise HTTPException(status_code=500, detail=str(e))