synth-ai 0.2.9.dev7__py3-none-any.whl → 0.2.9.dev8__py3-none-any.whl

examples/rl_old/task_app.py

@@ -1,1131 +0,0 @@
-import modal
-from typing import Any, Optional
-from dataclasses import dataclass, field
-import os as _os
-import sys as _sys
-from pathlib import Path as _Path
-import time
-
-# Make local 'crafter' importable when running locally
-_HERE = _Path(__file__).resolve()
-_LOCAL_CRAFTER_PARENT = _HERE.parent.parent  # points to examples/rl
-if str(_LOCAL_CRAFTER_PARENT) not in _sys.path:
-    _sys.path.insert(0, str(_LOCAL_CRAFTER_PARENT))
-if "/opt" not in _sys.path:
-    _sys.path.insert(0, "/opt")
-
-# Use environment-aware names to avoid collisions across dev/prod
-_env_flag = (
-    (
-        _os.getenv("SYNTH_BACKEND_URL_OVERRIDE", "")
-        or _os.getenv("ENVIRONMENT", "")
-        or _os.getenv("APP_ENVIRONMENT", "")
-    )
-    .strip()
-    .lower()
-)
-_is_prod = _env_flag in ("prod", "production")
-
-# Secret name must be provided explicitly via TASK_APP_SECRET_NAME
-MODAL_SECRET_NAME = _os.getenv("TASK_APP_SECRET_NAME")
-assert MODAL_SECRET_NAME, "TASK_APP_SECRET_NAME must be set before launching the task app"
-
-
-# Modal app name (overridable via TASK_APP_NAME)
-_default_app_name = "grpo-task-service-sdk-prod" if _is_prod else "grpo-task-service-sdk"
-app = modal.App(_os.getenv("TASK_APP_NAME", _default_app_name))
-
-image = (
-    modal.Image.debian_slim(python_version="3.11")
-    .pip_install(
-        [
-            "fastapi",
-            "uvicorn",
-            "pydantic>=2",
-            "httpx",
-            "requests",
-            "tqdm",
-            "urllib3>=2.3.0",
-            "jsonschema>=4.23.0",
-            "typing_extensions>=4.0.0",
-            "numpy",
-            "pandas",
-            "sqlalchemy",
-            "aiosqlite",
-            "asyncpg>=0.30.0",
-            "crafter",
-            "pillow",
-            "imageio",
-            "opensimplex",
-            "ruamel.yaml",
-            "networkx>=3.4.2",
-            "redis>=6.2.0",
-            "duckdb>=1.0.0",
-            "ty>=0.0.1a5",
-            "toml>=0.10.2",
-            "libsql>=0.1.8",
-            "python-dotenv",
-            "anthropic",
-            "openai",
-            "diskcache",
-            "backoff",
-            "groq",
-            "google-genai",
-            "google-generativeai",
-            "google-api-python-client",
-            "google-api-core>=2.25.1",
-            "google-auth",
-            "google-auth-httplib2",
-            "opentelemetry-api>=1.26.0,<1.27.0",
-            "opentelemetry-sdk>=1.26.0,<1.27.0",
-            "opentelemetry-exporter-otlp-proto-http>=1.26.0,<1.27.0",
-            "wrapt",
-            "langfuse>=2.53.9,<3.0.0",
-            "together",
-            "mistralai>=1.9.2",
-            "click>=8.1.0",
-            "textual>=1.1.0",
-            "openai-harmony>=0.0.1",
-            "aiohttp>=3.8.0",
-            "datasets>=4.0.0",
-            "gymnasium>=0.29.1",
-            "minigrid>=2.3.1",
-        ]
-    )
-    # Bundle the crafter module into the image for imports at runtime (absolute path)
-    .add_local_dir(
-        str((_HERE.parent / "crafter_task_app_helpers").resolve()), "/opt/crafter_task_app_helpers"
-    )
-    # Bundle synth_ai package to import full environment implementation.
-    # Resolve repo root robustly (examples/rl/task_app.py -> repo_root = examples/rl/../../..)
-    .add_local_dir(str((_HERE.parent.parent.parent / "synth_ai").resolve()), "/opt/synth_ai")
-)
-
-# --- OpenAI payload sanitizer (local) ---
-OPENAI_MAX_COMPLETION_TOKENS_MIN = 16000
-OPENAI_REMOVE_FIELDS = (
-    "stop_after_tool_calls",
-    "thinking_mode",
-    "thinking_budget",
-    "reasoning",
-)
-OPENAI_REMOVE_SAMPLING_FIELDS = ("temperature", "top_p")
-OPENAI_TOOL_CHOICE_FORCED = {"type": "function", "function": {"name": "interact"}}
-
-
-def prepare_inference_payload_for_model(
-    model: str | None, payload: dict[str, Any]
-) -> dict[str, Any]:
-    """Sanitize payload for OpenAI API.
-
-    - Always strip Synth-specific fields not supported by OpenAI (e.g., stop_after_tool_calls).
-    - For gpt-5 family: map max_tokens->max_completion_tokens, enforce tool_choice, disable parallel tools,
-      and remove vendor-specific sampling fields.
-    """
-    out = dict(payload)
-    # Always remove unsupported fields for OpenAI
-    for k in OPENAI_REMOVE_FIELDS:
-        if k in out:
-            out.pop(k)
-
-    # gpt-5 family specific adjustments
-    if model and "gpt-5" in model:
-        if "max_completion_tokens" not in out and "max_tokens" in out:
-            out["max_completion_tokens"] = out.pop("max_tokens")
-        # Ensure we don't send both
-        if "max_tokens" in out:
-            out.pop("max_tokens")
-        for k in OPENAI_REMOVE_SAMPLING_FIELDS:
-            if k in out:
-                out.pop(k)
-        mct = out.get("max_completion_tokens")
-        if not isinstance(mct, int) or mct < OPENAI_MAX_COMPLETION_TOKENS_MIN:
-            out["max_completion_tokens"] = OPENAI_MAX_COMPLETION_TOKENS_MIN
-        out["tool_choice"] = OPENAI_TOOL_CHOICE_FORCED
-        out["parallel_tool_calls"] = False
-    return out
-
-
-@app.function(
-    image=image,
-    secrets=[modal.Secret.from_name(MODAL_SECRET_NAME)],
-    min_containers=1,
-    max_containers=1,
-)
-@modal.asgi_app()
-def fastapi_app():
-    # Import FastAPI/Pydantic inside the container runtime to avoid local import errors
-    from fastapi import FastAPI, Body, HTTPException, status
-    from starlette.requests import Request
-    from fastapi.responses import JSONResponse
-    from pydantic import BaseModel
-    import logging
-    import sys
-    import os
-    import httpx
-
-    # Logger for debug output
-    logger = logging.getLogger(__name__)
-
-    # Preload synth_ai modules and vendor deps so missing packages surface early
-    if "/opt/synth_ai" not in sys.path:
-        sys.path.insert(0, "/opt/synth_ai")
-    # Ensure tracing DB points to a writable location in the container
-    os.environ.setdefault("TURSO_LOCAL_DB_URL", "sqlite+aiosqlite:////tmp/synth_ai.db")
-
-    import importlib
-
-    preload_modules = [
-        # synth_ai core
-        "synth_ai",
-        "synth_ai.lm",
-        "synth_ai.lm.core.main",
-        "synth_ai.lm.core.main_v3",
-        "synth_ai.lm.core.vendor_clients",
-        "synth_ai.lm.core.all",
-        # vendors
-        "synth_ai.lm.vendors.core.anthropic_api",
-        "synth_ai.lm.vendors.core.openai_api",
-        "synth_ai.lm.vendors.openai_standard",
-        "synth_ai.lm.vendors.core.gemini_api",
-        # environments
-        "synth_ai.environments",
-        "synth_ai.environments.environment.rewards.core",
-        "synth_ai.environments.examples.crafter_classic.environment",
-        # tracing
-        "synth_ai.tracing_v3.turso.models",
-        "synth_ai.tracing_v3.turso.manager",
-        # common 3p libs these modules rely on
-        "anthropic",
-        "openai",
-        "groq",
-        "google.genai",
-        "google.generativeai",
-        "googleapiclient.discovery",
-        "google.auth",
-        "google_auth_httplib2",
-        "requests",
-        "tqdm",
-        "langfuse",
-        "diskcache",
-        "backoff",
-        "together",
-        "dotenv",
-        "grpc",
-    ]
-    for mod in preload_modules:
-        try:
-            importlib.import_module(mod)
-        except Exception as _e:
-            print(f"[task:crafter] preload missing/err: {mod}: {_e}", flush=True)
-
-    # Make packaged local crafter modules importable ahead of site-packages 'crafter'
-    if "/opt/crafter_task_app_helpers" not in sys.path:
-        sys.path.insert(0, "/opt/crafter_task_app_helpers")
-    if "/opt" not in sys.path:
-        sys.path.insert(0, "/opt")
-    if "/opt/synth_ai" not in sys.path:
-        sys.path.insert(0, "/opt/synth_ai")
-    from crafter_task_app_helpers.env import EnvRegistry
-    from crafter_task_app_helpers.rewards import compute_decision_rewards
-    from crafter_task_app_helpers.config import ACTION_SPACE, ENV_NAME
-    from crafter_task_app_helpers.policy import CrafterPolicy
-
-    _registry = EnvRegistry()
-
-    # --- JSON sanitization for responses (convert numpy -> python primitives, arrays -> shapes) ---
-    import numpy as _np
-
-    def _to_jsonable(value):
-        # Numpy types first: scalars vs arrays
-        if isinstance(value, (_np.generic,)):
-            return value.item()
-        if isinstance(value, _np.ndarray):
-            return f"<ndarray shape={tuple(value.shape)} dtype={str(value.dtype)}>"
-        # Basic containers
-        if isinstance(value, dict):
-            return {k: _to_jsonable(v) for k, v in value.items()}
-        if isinstance(value, (list, tuple)):
-            return [_to_jsonable(v) for v in value]
-        # Sets to lists
-        if isinstance(value, set):
-            return [_to_jsonable(v) for v in value]
-        return value
-
-    class InitRequest(BaseModel):
-        env_name: str | None = None
-        env_config: dict[str, Any] | None = None
-
-    class StepRequest(BaseModel):
-        env_id: str
-        action: str
-
-    api = FastAPI(debug=True)
-
-    # Basic root endpoints so HEAD/GET / succeeds for preflight checks
-    @api.head("/")
-    def head_root():  # type: ignore[empty-body]
-        return JSONResponse(status_code=status.HTTP_200_OK, content=None)
-
-    @api.get("/")
-    def get_root():
-        return {"ok": True, "service": "synth-ai task app"}
-
-    @api.get("/health")
-    def health(request: Request):
-        env_key = os.environ.get("ENVIRONMENT_API_KEY")
-        if not env_key:
-            raise HTTPException(
-                status_code=503,
-                detail="Auth not configured: missing ENVIRONMENT_API_KEY in task service environment",
-            )
-        # Authorize using all header variants; avoid typed Header to prevent 422s
-        try:
-            from synth_ai.task.auth import is_api_key_header_authorized
-
-            authorized = is_api_key_header_authorized(request)
-        except Exception:
-            # Fallback: check only x-api-key
-            header_key = request.headers.get("x-api-key")
-            authorized = bool(header_key) and (header_key == env_key)
-        if not authorized:
-            # Soft 200 with authorized flag so CLI preflight can proceed
-            prefix = env_key[: max(1, len(env_key) // 2)]
-            content = {"status": "healthy", "authorized": False, "expected_api_key_prefix": prefix}
-            return JSONResponse(status_code=200, content=content)
-        return {"healthy": True, "authorized": True}
-
-    # Rollout health endpoint used by CLI configure flow
-    @api.get("/health/rollout")
-    def health_rollout(request: Request):
-        expected = os.environ.get("ENVIRONMENT_API_KEY")
-        if not expected:
-            raise HTTPException(
-                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
-                detail="Missing ENVIRONMENT_API_KEY in service env",
-            )
-        try:
-            from synth_ai.task.auth import is_api_key_header_authorized
-
-            authorized = is_api_key_header_authorized(request)
-        except Exception:
-            header_key = request.headers.get("x-api-key")
-            authorized = bool(header_key) and (header_key == expected)
-        if not authorized:
-            prefix = expected[: max(1, len(expected) // 2)]
-            content = {"status": "healthy", "authorized": False, "expected_api_key_prefix": prefix}
-            return JSONResponse(status_code=200, content=content)
-        return {"ok": True, "authorized": True}
-
-    # Log and surface 422 validation errors with header presence
-    from fastapi.exceptions import RequestValidationError
-
-    @api.exception_handler(RequestValidationError)
-    async def _on_validation_error(request: Request, exc: RequestValidationError):
-        try:
-            hdr = request.headers
-            snapshot = {
-                "path": str(getattr(request, "url").path),
-                "have_x_api_key": bool(hdr.get("x-api-key")),
-                "have_x_api_keys": bool(hdr.get("x-api-keys")),
-                "have_authorization": bool(hdr.get("authorization")),
-                "errors": exc.errors()[:5],
-            }
-            print("[422] validation", snapshot, flush=True)
-        except Exception:
-            pass
-        return JSONResponse(
-            status_code=422, content={"status": "invalid", "detail": exc.errors()[:5]}
-        )
-
-    @api.post(f"/env/{ENV_NAME}/initialize")
-    async def initialize(req: InitRequest, request: Request):
-        # Optionally tie the environment to a run_id header so we can guarantee isolation
-        run_id_hdr = request.headers.get("X-Run-Id") or request.headers.get("X-Run-ID")
-        env_id, obs = await _registry.initialize(req.env_config, run_id=run_id_hdr)
-        return {"env_id": env_id, "observation": _to_jsonable(obs)}
-
-    @api.post(f"/env/{ENV_NAME}/step")
-    async def step(req: StepRequest):
-        obs, reward, done, info = await _registry.step(req.env_id, req.action)
-        return {
-            "observation": _to_jsonable(obs),
-            "reward": float(reward) if isinstance(reward, (int, float)) else reward,
-            "done": bool(done),
-            "info": _to_jsonable(info) if info is not None else None,
-        }
-
-    @api.post(f"/env/{ENV_NAME}/terminate")
-    async def terminate(req: dict[str, str] = Body(...)):
-        env_id = str(req.get("env_id"))
-        return await _registry.terminate(env_id)
-
-    @api.get("/actions")
-    def actions():
-        return {"actions": ACTION_SPACE}
-
-    # OpenAI proxy: forward chat/completions to OpenAI using env OPENAI_API_KEY
-    @api.post("/proxy/v1/chat/completions")
-    def proxy_chat_completions(req: dict[str, Any]):
-        openai_key = os.environ.get("OPENAI_API_KEY")
-        if not openai_key:
-            raise HTTPException(
-                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
-                detail="Missing OPENAI_API_KEY in task service environment",
-            )
-        # Sanitize payload for OpenAI models (e.g., gpt-5-*)
-        model = req.get("model")
-        payload = prepare_inference_payload_for_model(model, req)
-        headers = {"Authorization": f"Bearer {openai_key}"}
-        # Increase timeout for proxy calls (models may be slower)
-        with httpx.Client(timeout=120.0) as client:
-            resp = client.post(
-                "https://api.openai.com/v1/chat/completions", json=payload, headers=headers
-            )
-            try:
-                data = resp.json()
-            except Exception:
-                data = {"error": "invalid_json", "raw": resp.text[:800]}
-            if resp.status_code >= 400:
-                return JSONResponse(status_code=resp.status_code, content=data)
-            return data
-
-    # Unified rollout schema imported from SDK task contracts
-    from synth_ai.task.contracts import (
-        RolloutEnvSpec,
-        RolloutPolicySpec,
-        RolloutRecordConfig,
-        RolloutSafetyConfig,
-        RolloutRequest,
-        RolloutStep,
-        RolloutTrajectory,
-        RolloutMetrics,
-        RolloutResponse,
-    )
-
-    @api.post("/rollout", response_model=RolloutResponse)
-    async def rollout(req: RolloutRequest, request: Request):
-        expected = os.environ.get("ENVIRONMENT_API_KEY")
-        if not expected:
-            logger.error("rollout.auth.misconfigured: missing ENVIRONMENT_API_KEY")
-            raise HTTPException(
-                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
-                detail="Auth not configured: missing ENVIRONMENT_API_KEY",
-            )
-        # Compute masked diagnostics (never log full keys)
-        try:
-            exp_len = len(expected)
-            exp_suf = expected[-5:] if exp_len >= 5 else ""  # last 5 chars
-            # Collect candidates from headers: X-API-Key, X-API-Keys (CSV), Authorization: Bearer
-            hdr = request.headers
-            single = hdr.get("x-api-key") or ""
-            multi = [p.strip() for p in (hdr.get("x-api-keys") or "").split(",") if p.strip()]
-            auth = hdr.get("authorization") or ""
-            bearer = auth.split(" ", 1)[1].strip() if auth.lower().startswith("bearer ") else ""
-            candidates = [c for c in [single, bearer, *multi] if c]
-            # Assert server sees ALL keys sent by client
-            if multi:
-                logger.info(
-                    "rollout.auth.candidates: n=%s first15=%s",
-                    len(candidates),
-                    [c[:15] for c in candidates],
-                )
-            got_len = len(single or bearer or "")
-            got_suf = (single or bearer or "")[-5:] if got_len >= 5 else ""
-        except Exception:
-            exp_len = -1
-            exp_suf = ""
-            got_len = -1
-            got_suf = ""
-        # Authorize if ANY candidate matches expected
-        authorized = any(c == expected for c in candidates)
-        if not authorized:
-            logger.warning(
-                "rollout.auth.failed: have_any=%s expect_len=%s expect_last5=%s got_len=%s got_last5=%s",
-                bool(candidates),
-                exp_len,
-                exp_suf,
-                got_len,
-                got_suf,
-            )
-            raise HTTPException(
-                status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid or missing API key"
-            )
-        else:
-            logger.info(
-                "rollout.auth.ok: expect_len=%s expect_last5=%s got_len=%s got_last5=%s",
-                exp_len,
-                exp_suf,
-                got_len,
-                got_suf,
-            )
-
-        # Extract policy config
-        inference_url = req.policy.config["inference_url"]
-        model = req.policy.config.get("model")
-        max_steps = int(req.env.config.get("max_steps_per_episode", 10))
-        policy = CrafterPolicy(inference_url=inference_url, model=model)
-
-        # Debug: request summary
-        print(
-            "[task:crafter] ROLLOUT req: ",
-            {
-                "run_id": req.run_id,
-                "env": req.env.env_name,
-                "seed": req.env.seed,
-                "ops": len(req.ops),
-                "model": model,
-                "inference_url": inference_url,
-                "max_steps": max_steps,
-            },
-            flush=True,
-        )
-
-        # Initialize env (preemptively terminate any prior instance for this run to avoid sharing)
-        cfg = dict(req.env.config or {})
-        if req.env.seed is not None:
-            cfg["seed"] = int(req.env.seed)
-        env_id, observation = await _registry.initialize(cfg, run_id=req.run_id)
-
-        trajectory_steps: list[RolloutStep] = []
-        # Track per-decision achievement flips for stepwise shaping
-        decision_summaries: list[dict[str, Any]] = []
-        prev_ach: dict[str, bool] | None = None
-        total_reward = 0.0
-        ops_executed = 0
-        pending_tool_calls: list[dict[str, Any]] | None = None
-        try:
-            for op in req.ops:
-                if ops_executed >= req.safety.max_ops:
-                    break
-                if op == "agent":
-                    # Format current observation for the prompt
-                    # Cache for mapping semantic ids to names
-                    _id_to_item_cache: list[str] | None = None
-
-                    def _ensure_semantic_mapping() -> list[str] | None:
-                        nonlocal _id_to_item_cache
-                        if _id_to_item_cache is not None:
-                            return _id_to_item_cache
-                        # Build mapping using crafter's internal ids
-                        import itertools as _it
-                        import crafter as _crafter
-
-                        dummy = None
-                        try:
-                            dummy = _crafter.Env()
-                            max_id = (
-                                max(
-                                    max(dummy._world._mat_ids.values()),
-                                    max(dummy._sem_view._obj_ids.values()),
-                                )
-                                + 1
-                            )
-                            id_to_item = ["void"] * max_id
-                            for name, ind in _it.chain(
-                                dummy._world._mat_ids.items(), dummy._sem_view._obj_ids.items()
-                            ):
-                                if name is None:
-                                    clean = "none"
-                                elif hasattr(name, "__name__"):
-                                    clean = name.__name__
-                                else:
-                                    clean = str(name)
-                                id_to_item[ind] = clean.lower()
-                            _id_to_item_cache = id_to_item
-                        finally:
-                            if dummy is not None:
-                                try:
-                                    dummy.close()
-                                except Exception:
-                                    pass
-                        return _id_to_item_cache
-
-                    def _format_obs(obs: dict[str, Any]) -> str:
-                        if not isinstance(obs, dict):
-                            # Avoid dumping raw matrices; encourage exploration to gather context
-                            return "no salient state; explore to gather context"
-                        inv = obs.get("inventory") or {}
-                        pos = obs.get("player_position")
-                        steps = obs.get("num_steps_taken")
-                        direction = obs.get("player_direction")
-                        ach = obs.get("achievements_status") or {}
-                        inv_lines = ", ".join(f"{k}:{v}" for k, v in inv.items() if v)
-                        ach_on = [k for k, v in ach.items() if v]
-                        lines = []
-                        if pos is not None:
-                            px, py = int(pos[0]), int(pos[1])
-                            lines.append(f"position: (x={px}, y={py})")
-                        if direction is not None:
-                            dx, dy = int(direction[0]), int(direction[1])
-                            dir_label = {
-                                (1, 0): "→ east/right",
-                                (-1, 0): "← west/left",
-                                (0, 1): "↓ south/down",
-                                (0, -1): "↑ north/up",
-                                (0, 0): "• idle",
-                            }.get((dx, dy), f"({dx},{dy})")
-                            lines.append(f"direction: {dir_label}")
-                        if steps is not None:
-                            lines.append(f"steps: {int(steps)}")
-                        if inv_lines:
-                            lines.append(f"inventory: {inv_lines}")
-                        if ach:
-                            all_achievements = list(ach.keys())
-                            lines.append(f"achievements_available: {', '.join(all_achievements)}")
-                            lines.append(
-                                f"achievements_unlocked: {', '.join(ach_on)}"
-                                if ach_on
-                                else "achievements_unlocked: "
-                            )
-                            lines.append(
-                                f"achievements_progress: {len(ach_on)}/{len(all_achievements)}"
-                            )
-                        # Local surroundings (7x7) using semantic_map
-                        smap = obs.get("semantic_map")
-                        if smap is not None and pos is not None:
-                            try:
-                                px, py = int(pos[0]), int(pos[1])
-                                view_size = 7
-                                half = view_size // 2
-                                id_to_item = _ensure_semantic_mapping() or []
-                                grid_rows: list[str] = []
-                                # Build matrix centered at player, then transpose for human-friendly view
-                                matrix: list[list[str]] = []
-                                for dy in range(-half, half + 1):
-                                    row: list[str] = []
-                                    for dx in range(-half, half + 1):
-                                        x, y = px + dx, py + dy
-                                        if not (0 <= x < smap.shape[0] and 0 <= y < smap.shape[1]):
-                                            row.append("void")
-                                        elif dx == 0 and dy == 0:
-                                            row.append("player")
-                                        else:
-                                            idx = int(smap[x, y])
-                                            name = (
-                                                id_to_item[idx]
-                                                if 0 <= idx < len(id_to_item)
-                                                else str(idx)
-                                            )
-                                            row.append(name)
-                                    matrix.append(row)
-                                # Transpose to match visual orientation
-                                transposed = list(zip(*matrix))
-                                for row in transposed:
-                                    grid_rows.append(" ".join(row))
-                                if grid_rows:
-                                    lines.append(f"Local Map View (7x7):\n" + "\n".join(grid_rows))
-                            except Exception:
-                                # If any issue occurs, skip map rendering without crashing
-                                pass
-                        if not lines:
-                            lines.append("no salient state; explore to gather context")
-                        return "\n".join(lines)
-
-                    # Build compact context from last few tool calls (gpt-5-nano friendly)
-                    lines: list[str] = []
-                    for rec in reversed(trajectory_steps):
-                        if len(lines) >= 3:
-                            break
-                        tcs = rec.tool_calls
-                        if not tcs:
-                            continue
-                        tc0 = tcs[0] if isinstance(tcs, list) and tcs else None
-                        if not isinstance(tc0, dict):
-                            continue
-                        name = tc0.get("tool_name") or tc0.get("name") or "unknown"
-                        args = tc0.get("arguments")
-                        lines.append(f"- {name}: {args}")
-                    context_text = "Previous tool calls (most recent first):\n" + (
-                        "\n".join(lines) if lines else "- none"
-                    )
-                    obs_text = _format_obs(observation)
-                    combined_text = f"Current observation:\n{obs_text}\n\n{context_text}"
-                    payload = policy.build_inference_request(
-                        combined_text, history=[], turn=len(trajectory_steps)
-                    )
-                    # Debug: print the full prompt content in a stable labeled block for grepability
-                    try:
-                        print("PROMPT_DUMP_BEGIN")
-                        print(combined_text)
-                        print("PROMPT_DUMP_END")
-                    except Exception:
-                        pass
-                    # Debug: print user prompt and achievements unlocked list
-                    try:
-                        _msgs = payload.get("messages", [])
-                        _last_user = None
-                        for _m in reversed(_msgs):
-                            if isinstance(_m, dict) and _m.get("role") == "user":
-                                _last_user = _m
-                                break
-                        if _last_user is not None:
-                            _content = _last_user.get("content")
-                            print("[task:crafter] user prompt:", _content, flush=True)
-                    except Exception:
-                        pass
-                    try:
-                        _ach = (
-                            observation.get("achievements_status")
-                            if isinstance(observation, dict)
-                            else {}
-                        )
-                        _ach_on = [k for k, v in (_ach or {}).items() if v]
-                        print(f"[task:crafter] achievements_unlocked: {_ach_on}", flush=True)
-                    except Exception:
-                        pass
-
-                    # Prepare payload based on model family (OpenAI vs vLLM)
-                    def _prepare_payload(p: dict, mdl: str | None) -> dict:
-                        return prepare_inference_payload_for_model(mdl, p)
-
-                    # Debug: payload shape
-                    print(
-                        "[task:crafter] inference payload: ",
-                        {
-                            "has_model": bool(payload.get("model") is not None),
-                            "messages": payload.get("messages", []),
-                            "tools": isinstance(payload.get("tools"), list),
-                            "tool_choice": payload.get("tool_choice"),
-                            "stop_after_tool_calls": payload.get("stop_after_tool_calls"),
-                        },
-                        flush=True,
-                    )
-                    headers: dict[str, str] = {}
-                    _okey = os.environ.get("OPENAI_API_KEY")
-                    # Configure granular timeouts for slow model/tool runs
-                    _timeouts = httpx.Timeout(connect=10.0, read=180.0, write=60.0, pool=60.0)
-                    with httpx.Client(timeout=_timeouts) as client:
-                        # Decide endpoint: avoid calling our own /proxy inside the same request
-                        _direct = "api.openai.com" in inference_url
-                        if _direct:
-                            # Call OpenAI directly
-                            if _okey:
-                                headers["Authorization"] = f"Bearer {_okey}"
-                            to_send = _prepare_payload(payload, model)
-                            endpoint_base = "https://api.openai.com"
-                        else:
-                            # Non-OpenAI inference endpoint
-                            to_send = payload
-                            endpoint_base = inference_url
-                            # If targeting Synth proxy, attach backend auth
-                            if "/proxy" in endpoint_base:
-                                _skey = os.environ.get("SYNTH_API_KEY")
-                                if _skey:
-                                    headers["Authorization"] = f"Bearer {_skey}"
-
-                        # Debug: outbound request diagnostics
-                        try:
-                            import json as _json
-
-                            _size = len(_json.dumps(to_send))
-                        except Exception:
-                            _size = -1
-                        print(
-                            "[task:crafter] inference dispatch:",
-                            {
-                                "endpoint": f"{endpoint_base.rstrip('/')}/v1/chat/completions",
-                                "direct_openai": bool(_direct),
-                                "timeout": {
-                                    "read": 180.0,
-                                    "connect": 10.0,
-                                    "write": 60.0,
-                                    "pool": 60.0,
-                                },
-                                "payload_bytes": _size,
-                                "has_auth": bool(headers.get("Authorization")),
-                            },
-                            flush=True,
-                        )
-
-                        _t0 = time.time()
-                        try:
-                            resp = client.post(
-                                f"{endpoint_base.rstrip('/')}/v1/chat/completions",
-                                json=to_send,
-                                headers=headers,
-                            )
-                        except httpx.ReadTimeout as rte:
-                            _elapsed = time.time() - _t0
-                            print(
-                                f"[task:crafter][timeout] read timeout after {_elapsed:.1f}s: {rte}",
-                                flush=True,
-                            )
-                            raise
-                        except Exception as re:
-                            _elapsed = time.time() - _t0
-                            print(
-                                f"[task:crafter][error] request failed after {_elapsed:.1f}s: {type(re).__name__}: {re}",
-                                flush=True,
-                            )
-                            raise
-                        _elapsed = time.time() - _t0
-                        print(
-                            f"[task:crafter] inference status= {resp.status_code} elapsed={_elapsed:.2f}s",
-                            flush=True,
-                        )
-                        # Emit a light-weight perf snapshot for visibility
-                        try:
-                            print(
-                                "[metric] perf ",
-                                "tok/s=n/a",
-                                f"decision p50=n/a p95=n/a",
-                                "roll n/a",
-                                flush=True,
-                            )
-                        except Exception:
-                            pass
-                        # Debug: response status and body (on errors)
-                        print("[task:crafter] inference status=", resp.status_code, flush=True)
-                        if resp.status_code >= 400:
-                            body_preview = resp.text[:800]
-                            print("[task:crafter] inference error body:", body_preview, flush=True)
-                        data = resp.json()
-                    print(f"[task:crafter] inference response: {data}")
-                    parsed = CrafterPolicy.parse_response_to_tool_calls(data, use_tools=True) or []
-                    # Debug: parsed tool call summary
-                    print(
-                        "[task:crafter] parsed tool_calls: ",
-                        {
-                            "n": len(parsed),
-                            "first": (parsed[0] if isinstance(parsed, list) and parsed else None),
-                        },
-                        flush=True,
-                    )
-                    # Print full tool call payloads for inspection
-                    try:
-                        import json as _json
-
-                        for _i, _tc in enumerate(parsed):
-                            try:
-                                print(
-                                    f"[task:crafter] tool_call[{_i}]:",
-                                    _json.dumps(_tc, separators=(",", ":")),
-                                    flush=True,
-                                )
-                            except Exception:
-                                print(f"[task:crafter] tool_call[{_i}]: {_tc}", flush=True)
-                    except Exception:
-                        pass
-                    if not parsed:
-                        # Dump compact body preview to understand schema when no tools parsed
-                        try:
-                            import json as _json
-
-                            preview = _json.dumps(data, separators=(",", ":"))
-                            print(
-                                "[task:crafter] body(no_tools) preview:", preview[:800], flush=True
-                            )
-                        except Exception:
-                            pass
-                        # Early terminate the episode to avoid hanging on empty tool calls
-                        print("[task:crafter] NO_TOOL_CALLS: terminating episode early", flush=True)
-                        break
-                    pending_tool_calls = parsed
-                    ops_executed += 1
-                elif op == "env":
-                    if not pending_tool_calls:
-                        print("[task:crafter] no tool_calls; skipping env step", flush=True)
-                        continue
-                    info: dict[str, Any] | None = None
-                    for tc in pending_tool_calls:
-                        name = tc.get("tool_name")
-                        if name == "interact":
-                            # Parse the JSON arguments string
-                            import json
-
-                            args_str = tc.get("arguments", "{}")
-                            try:
-                                args_dict = json.loads(args_str)
-                                actions = args_dict.get("actions", [])
-                                reasoning = args_dict.get("reasoning", "")
-                                print(f"[task:crafter] reasoning: {reasoning}", flush=True)
-                            except (json.JSONDecodeError, TypeError):
-                                print(
-                                    f"[task:crafter] ERROR: Failed to parse arguments: {args_str}",
-                                    flush=True,
-                                )
-                                actions = []
-                                reasoning = "Parse error"
-
-                            print(f"[task:crafter] env actions: {actions}", flush=True)
-                            # Print a compact echo of the current prompt + tool call for easier triage
-                            try:
-                                import json as _json
-
-                                print(
-                                    "TOOLCALL_CONFIG:",
-                                    _json.dumps(
-                                        {
-                                            "policy": req.policy.policy_name,
-                                            "tools_present": True,
-                                            "tool_choice": "required",
-                                            "stop_after": 1,
-                                        }
-                                    ),
-                                )
-                            except Exception:
-                                pass
-
-                            # Execute each action individually
-                            # Reset decision-level flip set for this decision
-                            decision_flips: set[str] = set()
-                            for act in actions:
-                                observation, reward, done, _info = await _registry.step(env_id, act)
-                                total_reward += float(reward)
-                                # Debug: print step outcome (compact)
-                                try:
-                                    ok = (
-                                        list(observation.keys())
-                                        if isinstance(observation, dict)
-                                        else []
-                                    )
-                                    print(
-                                        f"[task:crafter] step => a={act} r={float(reward)} done={bool(done)} obs_keys={ok[:5]}",
-                                        flush=True,
-                                    )
-                                except Exception:
-                                    pass
-                                step = RolloutStep(
-                                    obs=observation,
-                                    tool_calls=pending_tool_calls,
-                                    reward=float(reward),
-                                    done=bool(done),
-                                    truncated=False,
-                                    info=info,
-                                )
-                                trajectory_steps.append(step)
-                                ops_executed += 1
-
-                                # Check for achievement-based termination
-                                if isinstance(observation, dict):
-                                    current_achievements = observation.get(
-                                        "achievements_status", {}
-                                    )
-                                    # Track flips 0→1 within this decision
-                                    try:
-                                        if not isinstance(current_achievements, dict):
-                                            current_achievements = {}
-                                        if prev_ach is None:
-                                            prev_ach = {
-                                                k: bool(v)
-                                                for k, v in (current_achievements or {}).items()
-                                            }
-                                        else:
-                                            for name, on in (current_achievements or {}).items():
-                                                if bool(on) and not bool(prev_ach.get(name, False)):
-                                                    decision_flips.add(str(name))
-                                            # Update prev_ach to latest snapshot
-                                            prev_ach = {
-                                                k: bool(v)
-                                                for k, v in (current_achievements or {}).items()
-                                            }
-                                    except Exception:
-                                        pass
-                                    achieved_count = sum(
-                                        1 for v in current_achievements.values() if v
-                                    )
-                                    total_achievements = len(current_achievements)
-
-                                    # Terminate if we've achieved a significant portion of available achievements
-                                    if total_achievements > 0 and achieved_count >= max(
-                                        3, total_achievements // 2
-                                    ):
-                                        print(
-                                            f"[task:crafter] achievement_termination: {achieved_count}/{total_achievements} achievements reached",
-                                            flush=True,
-                                        )
-                                        print(
-                                            f"[task:crafter] achieved: {[k for k, v in current_achievements.items() if v]}",
-                                            flush=True,
-                                        )
-                                        break
-
-                                if done or len(trajectory_steps) >= max_steps:
-                                    print(
-                                        f"[task:crafter] episode_end: done={bool(done)} steps={len(trajectory_steps)} total_reward={total_reward}",
-                                        flush=True,
-                                    )
-                                    break
-                        elif name == "terminate":
-                            # Handle termination
-                            print("[task:crafter] Agent requested termination", flush=True)
-                            break
-                        else:
-                            # Non-interact tool call: count as a step without env change
-                            print("[task:crafter] non-interact tool_call:", name, flush=True)
-                            step = RolloutStep(
-                                obs=observation,
-                                tool_calls=pending_tool_calls,
-                                reward=None,
-                                done=False,
-                                truncated=False,
-                                info=info,
-                            )
-                            trajectory_steps.append(step)
-                            ops_executed += 1
-                            # End of decision: record indicator_i for shaping
-                            try:
-                                indicator_i = 1 if decision_flips else 0
-                                decision_summaries.append({"indicator_i": indicator_i})
-                            except Exception:
-                                pass
-                            pending_tool_calls = None
-                    if len(trajectory_steps) >= max_steps:
-                        print(
-                            f"[task:crafter] max_steps_reached: steps={len(trajectory_steps)} total_reward={total_reward}",
-                            flush=True,
-                        )
-                        break
-                else:
-                    # Unknown op: skip
-                    continue
-                if len(trajectory_steps) >= max_steps:
-                    break
-        finally:
-            await _registry.terminate(env_id)
-
-        # Sanitize steps for JSON
-        safe_steps = [
-            RolloutStep(
-                obs=_to_jsonable(s.obs),
-                tool_calls=s.tool_calls,
-                reward=float(s.reward) if s.reward is not None else None,
-                done=bool(s.done),
-                truncated=bool(s.truncated) if s.truncated is not None else None,
-                info=_to_jsonable(s.info) if s.info is not None else None,
-            )
-            for s in trajectory_steps
-        ]
-
-        trajectory = RolloutTrajectory(
-            env_id=env_id,
-            policy_id=req.policy.policy_name or "crafter-policy",
-            steps=safe_steps,
-            final={"observation": _to_jsonable(observation)},
-            length=len(safe_steps),
-        )
-        # Calculate achievements for this episode
-        final_obs = observation
-        if isinstance(final_obs, dict):
-            final_achievements = final_obs.get("achievements_status", {})
-        else:
-            # Handle numpy array case - no achievements available
-            final_achievements = {}
-        total_achievements = sum(1 for v in final_achievements.values() if v)
-
-        # Step-reward shaping: compute decision-level rewards if enabled
-        branches: dict[str, Any] = {}
-        try:
-            sr_cfg = (
-                (req.record.config or {}).get("step_rewards")
-                if isinstance(req.record, RolloutRecordConfig)
-                else None
-            )
-        except Exception:
-            sr_cfg = None
-        try:
-            enabled = False
-            mode = None
-            step_beta = 0.0
-            indicator_lambda = 0.0
-            if isinstance(sr_cfg, dict):
-                enabled = bool(sr_cfg.get("enabled", False))
-                mode = (sr_cfg.get("mode") or "off").strip().lower()
-                step_beta = float(sr_cfg.get("step_beta", 0.0))
-                indicator_lambda = float(sr_cfg.get("indicator_lambda", 0.0))
-            # Env overrides
-            import os as _os2
-
-            if _os2.getenv("STEP_BETA"):
-                step_beta = float(_os2.getenv("STEP_BETA"))
-            if _os2.getenv("STEP_LAMBDA"):
-                indicator_lambda = float(_os2.getenv("STEP_LAMBDA"))
-            if enabled and mode == "decision_stepwise" and decision_summaries:
-                dec_rewards = compute_decision_rewards(
-                    decision_summaries=decision_summaries,
-                    total_achievements=total_achievements,
-                    step_beta=step_beta,
-                    indicator_lambda=indicator_lambda,
-                )
-                branches["decision_rewards"] = dec_rewards
-                print(
-                    "[task:crafter] step_rewards: ",
-                    {
-                        "enabled": True,
-                        "mode": mode,
-                        "step_beta": step_beta,
-                        "indicator_lambda": indicator_lambda,
-                        "decisions": len(dec_rewards),
-                    },
-                    flush=True,
-                )
-        except Exception as _e_sr:
-            print(f"[task:crafter] step_rewards_error: {_e_sr}", flush=True)
-
-        # Optional tracing of episode/rewards (gated)
-        try:
-            import os as _os3
-
-            if _os3.getenv("TRACE_RL", "0") == "1":
-                from synth_ai.tracing_v3.session_tracer import SessionTracer  # type: ignore
-
-                tracer = SessionTracer()
-                await tracer.initialize()
-                meta = {
-                    "env": req.env.env_name,
-                    "policy": req.policy.policy_name,
-                    "step_rewards": {
-                        "enabled": bool(sr_cfg.get("enabled", False))
-                        if isinstance(sr_cfg, dict)
-                        else False,
-                        "mode": (sr_cfg.get("mode") if isinstance(sr_cfg, dict) else None),
-                    },
-                }
-                async with tracer.session(metadata=meta):
-                    # Record episode outcome at end
-                    await tracer.record_outcome_reward(
-                        total_reward=int(total_reward),
-                        achievements_count=int(total_achievements),
-                        total_steps=int(len(trajectory_steps)),
-                    )
-        except Exception as _te:
-            print(f"[task:crafter] tracing_error: {_te}", flush=True)
-
-        metrics = RolloutMetrics(
-            episode_returns=[total_reward],
-            mean_return=float(total_achievements),
-            num_steps=len(trajectory_steps),
-            num_episodes=1,
-        )
-        # Debug: print reward and achievement metrics
-        print(
-            f"[task:crafter] Rollout metrics: total_reward={total_reward}, total_achievements={total_achievements}, mean_return={metrics.mean_return}, episode_returns={metrics.episode_returns}",
-            flush=True,
-        )
-        return RolloutResponse(
-            run_id=req.run_id,
-            trajectories=[trajectory],
-            branches=branches,
-            metrics=metrics,
-            aborted=False,
-            ops_executed=ops_executed,
-        )
-
-    @api.get("/test_auth")
-    def test_auth(request: Request):
-        expected = os.environ.get("ENVIRONMENT_API_KEY")
-        if not expected:
-            raise HTTPException(
-                status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
-                detail="Missing ENVIRONMENT_API_KEY in service env",
-            )
-        header_key = request.headers.get("x-api-key") or request.headers.get("X-API-Key")
-        ok = bool(header_key) and (header_key == expected)
-        if not ok:
-            raise HTTPException(
-                status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid or missing API key"
-            )
-        return {"ok": True}
-
-    return api