synth-ai 0.2.9.dev3__py3-none-any.whl → 0.2.9.dev5__py3-none-any.whl

examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/shared.py

@@ -0,0 +1,302 @@
+"""Shared utilities for Crafter environment and policy.
+
+This module formats Crafter observations for the LLM and parses actions.
+It now mirrors the ludic_private implementation for semantic map rendering
+by dynamically deriving the id->name mapping from the actual Crafter env
+when available, with a sensible fallback. This fixes the issue where the
+rendered surroundings appeared only as iron/stone due to a mismatched
+hardcoded mapping.
+"""
+
+from typing import Dict, Any, List, Set
+import numpy as np
+import re
+import itertools
+
+VIEW_SIZE = 5  # Default view size for the map (match eval_rollout_table)
+
+# Action mappings from the game
+CRAFTER_ACTIONS = {
+    "noop": 0,
+    "move_left": 1,
+    "move_right": 2,
+    "move_up": 3,
+    "move_down": 4,
+    "do": 5,
+    "sleep": 6,
+    "place_stone": 7,
+    "place_table": 8,
+    "place_furnace": 9,
+    "place_plant": 10,
+    "make_wood_pickaxe": 11,
+    "make_stone_pickaxe": 12,
+    "make_iron_pickaxe": 13,
+    "make_wood_sword": 14,
+    "make_stone_sword": 15,
+    "make_iron_sword": 16,
+}
+
+# Common action aliases
+ACTION_ALIASES = {
+    # Movement aliases
+    "left": "move_left",
+    "right": "move_right",
+    "up": "move_up",
+    "down": "move_down",
+    # Interaction aliases
+    "interact": "do",
+    "use": "do",
+    "action": "do",
+    # Sleep
+    "rest": "sleep",
+    # Crafting
+    "craft_wood_pickaxe": "make_wood_pickaxe",
+    "craft_stone_pickaxe": "make_stone_pickaxe",
+    "craft_iron_pickaxe": "make_iron_pickaxe",
+    "craft_wood_sword": "make_wood_sword",
+    "craft_stone_sword": "make_stone_sword",
+    "craft_iron_sword": "make_iron_sword",
+}
+
+VALID_PRIMARY_ACTIONS: Set[str] = set(CRAFTER_ACTIONS.keys())
+VALID_ACTION_ALIASES: Set[str] = set(ACTION_ALIASES.keys())
+ALL_VALID_ACTION_STRINGS: Set[str] = VALID_PRIMARY_ACTIONS | VALID_ACTION_ALIASES
+
+
+def validate_action(action: str) -> bool:
+    """Check if an action string is valid."""
+    normalized = action.strip().lower().replace(" ", "_")
+    return normalized in ALL_VALID_ACTION_STRINGS
+
+
+def parse_actions(action_text: str) -> List[str]:
+    """Extract actions from response text.
+    
+    Tries multiple parsing strategies:
+    1. <action>...</action> tags (original format)
+    2. [action]...[/action] or [action]... format
+    3. ACTION: prefix format
+    4. Plain action names if they match valid actions
+    5. Newline-separated actions
+    """
+    import json
+    
+    # First try the original <action> tag format
+    matches = re.findall(r"<action>(.*?)</action>", action_text, re.IGNORECASE)
+    if matches:
+        return [m.strip() for m in matches if validate_action(m.strip())]
+    
+    # Try [action] format
+    matches = re.findall(r"\[action\](.*?)(?:\[/action\]|\n|$)", action_text, re.IGNORECASE)
+    if matches:
+        return [m.strip() for m in matches if validate_action(m.strip())]
+    
+    # If no tags found, try to parse plain text
+    text = action_text.strip()
+    
+    # Check if the entire text is a valid action
+    if validate_action(text):
+        return [text]
+    
+    # Try splitting by newlines and checking each line
+    lines = text.split('\n')
+    actions = []
+    for line in lines:
+        line = line.strip()
+        
+        # Remove various prefixes
+        for prefix in ['ACTION:', 'Action:', 'action:', 'ACTION', '-', '*', '•', '**ACTION:**']:
+            if line.startswith(prefix):
+                line = line[len(prefix):].strip()
+                break
+        
+        # Also handle numbered lists
+        if re.match(r'^\d+\.\s*', line):
+            line = re.sub(r'^\d+\.\s*', '', line)
+        
+        # Split by common separators to handle multiple actions on one line
+        parts = re.split(r'[,;]|\s+and\s+|\s+then\s+', line)
+        
+        for part in parts:
+            part = part.strip()
+            # Remove quotes if present
+            if part.startswith('"') and part.endswith('"'):
+                part = part[1:-1]
+            if part.startswith("'") and part.endswith("'"):
+                part = part[1:-1]
+                
+            # Check if it's a valid action
+            if part and validate_action(part):
+                actions.append(part)
+    
+    return actions
+
+
+def format_observation(obs_data: Dict[str, Any], step_count: int = 0, max_steps: int = 100) -> str:
+    """Format a Crafter observation dictionary into a human-readable string.
+    
+    This is critical for preventing massive token counts when observations
+    contain large numpy arrays or deeply nested structures.
+    """
+    if not obs_data:
+        return ""
+    
+    # Extract key information
+    health = obs_data.get("health") or obs_data.get("inventory", {}).get("health", 0)
+    inventory_dict = obs_data.get("inventory", {})
+    pos = obs_data.get("player_position", [0, 0])
+    direction = obs_data.get("player_direction", [0, 1])
+    achievements = obs_data.get("achievements_status", {})
+
+    # Prefer step/max from observation if provided by the env
+    step_from_obs = (
+        obs_data.get("steps")
+        if obs_data.get("steps") is not None
+        else obs_data.get("num_steps_taken")
+    )
+    if isinstance(step_from_obs, (int, float)) and step_from_obs >= 0:
+        step_count = int(step_from_obs)
+
+    max_steps_from_obs = obs_data.get("max_steps_episode") or obs_data.get("max_steps")
+    if isinstance(max_steps_from_obs, (int, float)) and max_steps_from_obs > 0:
+        max_steps = int(max_steps_from_obs)
+    
+    # Format inventory (skip health as it's shown separately)
+    inv_items = [f"{k}:{v}" for k, v in inventory_dict.items() if v > 0 and k != "health"]
+    inventory_str = ", ".join(inv_items) if inv_items else "empty"
+    
+    # Format achievements
+    achieved_list = [k for k, v in achievements.items() if v]
+    achievements_str = ", ".join(achieved_list) if achieved_list else "none"
+    
+    # Format semantic map view (simplified version)
+    map_view = _format_semantic_map_view(obs_data, VIEW_SIZE)
+    
+    return (
+        f"=== CRAFTER GAME STATE ===\n"
+        f"Step: {step_count}/{max_steps}\n"
+        f"Health: {health}\n"
+        f"Position: {pos}\n"
+        f"Facing: {direction}\n"
+        f"Inventory: {inventory_str}\n"
+        f"Achievements: {achievements_str}\n"
+        f"{map_view}\n\n"
+        f"Choose your next actions.\n"
+    )
+
+def _try_build_dynamic_mapping():
+    """Attempt to build id->name mapping from a real Crafter env.
+
+    Returns a list where index is semantic ID and value is the lowercase name.
+    On failure (crafter not installed or internal API changed), returns None.
+    """
+    try:
+        import crafter  # type: ignore
+    except Exception:
+        return None
+
+    dummyenv = None
+    try:
+        dummyenv = crafter.Env()
+        # Combine material IDs and semantic view object IDs
+        world_ids = getattr(dummyenv, "_world", None)
+        sem_view = getattr(dummyenv, "_sem_view", None)
+        if world_ids is None or sem_view is None:
+            return None
+        mat_ids = getattr(world_ids, "_mat_ids", None)
+        obj_ids = getattr(sem_view, "_obj_ids", None)
+        if not isinstance(mat_ids, dict) or not isinstance(obj_ids, dict):
+            return None
+        max_id = max(max(mat_ids.values()), max(obj_ids.values())) + 1
+        id_to_item = ["void"] * max_id
+        for name, idx in itertools.chain(mat_ids.items(), obj_ids.items()):
+            if name is None:
+                clean = "none"
+            elif hasattr(name, "__name__"):
+                clean = name.__name__.lower()
+            else:
+                clean = str(name).lower()
+            if 0 <= idx < len(id_to_item):
+                id_to_item[idx] = clean
+        return id_to_item
+    except Exception:
+        return None
+    finally:
+        try:
+            if dummyenv is not None:
+                dummyenv.close()
+        except Exception:
+            pass
+
+
+# Build dynamic mapping if possible; otherwise fall back to a basic map
+_ID_TO_NAME = _try_build_dynamic_mapping()
+_FALLBACK_ID_TO_NAME = {
+    0: "none",      # None from materials
+    1: "water",
+    2: "grass",
+    3: "stone",
+    4: "path",
+    5: "sand",
+    6: "tree",
+    7: "lava",
+    8: "coal",
+    9: "iron",
+    10: "diamond",
+    11: "table",
+    12: "furnace",
+    13: "player",
+    14: "cow",
+    15: "zombie",
+    16: "skeleton",
+    17: "arrow",
+    18: "plant",
+}
+
+
+def _format_semantic_map_view(obs_data: Dict[str, Any], view_size: int = VIEW_SIZE) -> str:
+    """Format the semantic map into a text representation using dynamic IDs.
+
+    Shows a local view around the player with nearby objects.
+    """
+    semantic_map = obs_data.get("semantic_map")
+    player_position = obs_data.get("player_position", [0, 0])
+
+    if semantic_map is None:
+        return "Map view unavailable"
+
+    # Convert to numpy array if needed
+    sem_arr = np.asarray(semantic_map)
+    if sem_arr.ndim == 1:
+        # Reshape flat array to 2D
+        side = int(len(sem_arr) ** 0.5)
+        sem_arr = sem_arr.reshape(side, side)
+
+    px, py = map(int, player_position)
+    half = view_size // 2
+
+    # Choose mapping source
+    use_list = isinstance(_ID_TO_NAME, list) and len(_ID_TO_NAME) > 0
+
+    # Build matrix centered at player, then transpose for human-friendly view
+    matrix: List[List[str]] = []
+    for dy in range(-half, half + 1):
+        row_tokens: List[str] = []
+        for dx in range(-half, half + 1):
+            x, y = px + dx, py + dy
+            if not (0 <= x < sem_arr.shape[0] and 0 <= y < sem_arr.shape[1]):
+                row_tokens.append("void")
+            elif dx == 0 and dy == 0:
+                row_tokens.append("player")
+            else:
+                obj_id = int(sem_arr[x, y])
+                if use_list and 0 <= obj_id < len(_ID_TO_NAME):
+                    name = _ID_TO_NAME[obj_id]  # type: ignore[index]
+                else:
+                    name = _FALLBACK_ID_TO_NAME.get(obj_id, str(obj_id))
+                row_tokens.append(name)
+        matrix.append(row_tokens)
+
+    transposed = list(zip(*matrix))
+    grid_rows: List[str] = [" ".join(row) for row in transposed]
+    return "\nLocal Map View (" + str(view_size) + "x" + str(view_size) + "):\n" + "\n".join(grid_rows)

examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/tools.py

@@ -0,0 +1,47 @@
+"""OpenAI tools schema for Crafter, defined in Python."""
+
+# Pass this list directly to OpenAI/vLLM `tools=`
+TOOLS_SCHEMA = [
+    {
+        "type": "function",
+        "function": {
+            "name": "interact_many",
+            "description": "Execute a short sequence of Crafter actions in order (1-8).",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "actions": {
+                        "type": "array",
+                        "description": "List of Crafter actions to execute sequentially.",
+                        "items": {
+                            "type": "string",
+                            "enum": [
+                                "noop",
+                                "move_left",
+                                "move_right",
+                                "move_up",
+                                "move_down",
+                                "do",
+                                "sleep",
+                                "place_stone",
+                                "place_table",
+                                "place_furnace",
+                                "place_plant",
+                                "make_wood_pickaxe",
+                                "make_stone_pickaxe",
+                                "make_iron_pickaxe",
+                                "make_wood_sword",
+                                "make_stone_sword",
+                                "make_iron_sword",
+                            ],
+                        },
+                        "minItems": 1,
+                        "maxItems": 8,
+                    }
+                },
+                "required": ["actions"],
+                "additionalProperties": False,
+            },
+        },
+    }
+]

examples/warming_up_to_rl/task_app/synth_envs_hosted/hosted_app.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import os
+from typing import Optional
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from starlette.requests import Request
+
+
+class TaskApp:
+    """Holds service configuration and shared state."""
+
+    def __init__(
+        self,
+        service_base_url: Optional[str] = None,
+        vllm_base_url: Optional[str] = None,
+        default_model: Optional[str] = None,
+    ) -> None:
+        self.service_base_url = service_base_url or os.getenv(
+            "SERVICE_BASE_URL", "http://localhost:8000"
+        )
+        self.vllm_base_url = vllm_base_url or os.getenv(
+            "VLLM_BASE_URL", "http://localhost:8001"
+        )
+        self.default_model = default_model or os.getenv("DEFAULT_MODEL")
+
+
+class ServiceInfo(BaseModel):
+    """Service discovery response."""
+
+    service: dict
+    inference: dict
+
+
+def create_app(allowed_environments: list[str] = None) -> FastAPI:
+    """FastAPI app factory.
+
+    Args:
+        allowed_environments: List of environment names this service is allowed to handle.
+                            If None, all environments are allowed (for backward compatibility).
+    """
+    env_filter = f" ({', '.join(allowed_environments)})" if allowed_environments else ""
+    app = FastAPI(
+        title=f"GRPO Synth Envs Hosted Service{env_filter}",
+        description=f"Hosted environment and policy service for GRPO training{env_filter}",
+        version="0.1.0",
+    )
+
+    # Add CORS middleware for development
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    # Initialize task app configuration
+    task_app = TaskApp()
+    app.state.task_app = task_app
+    app.state.allowed_environments = allowed_environments
+
+    # Add environment validation middleware
+    if allowed_environments:
+
+        @app.middleware("http")
+        async def validate_environment(request, call_next):
+            # Check if this is an environment-related request
+            if request.url.path.startswith("/env/") or request.url.path.startswith(
+                "/rollout"
+            ):
+                # Extract environment name from request body for POST requests
+                if request.method == "POST":
+                    # We need to read the body to check env_name
+                    body = await request.body()
+                    try:
+                        import json
+
+                        data = json.loads(body) if body else {}
+                        env_name = data.get("env_name", "").lower()
+
+                        # Check if environment is allowed
+                        if env_name and env_name not in [
+                            e.lower() for e in allowed_environments
+                        ]:
+                            from fastapi import HTTPException
+
+                            raise HTTPException(
+                                status_code=403,
+                                detail=f"Environment '{env_name}' not allowed. This service only handles: {allowed_environments}",
+                            )
+                    except json.JSONDecodeError:
+                        pass  # Invalid JSON, let the endpoint handle it
+
+                    # Recreate request with the body we consumed
+                    request._body = body
+
+            response = await call_next(request)
+            return response
+
+    # Mount routers
+    from .environment_routes import router as env_router
+    from .rollout import router as rollout_router
+    from .branching import router as branching_router
+
+    app.include_router(env_router, prefix="/env", tags=["environment"])
+
+    # Policy routes are optional; skip if optional envs are missing in this build
+    try:
+        from .policy_routes import router as policy_router
+        app.include_router(policy_router, prefix="/policy", tags=["policy"])
+    except Exception as _e:
+        # Log lightweight message; policy endpoints will be unavailable
+        try:
+            print(f"[hosted_app] Skipping policy routes: {_e}", flush=True)
+        except Exception:
+            pass
+
+    app.include_router(rollout_router, tags=["rollout"])
+    app.include_router(branching_router, tags=["branching"])
+
+    @app.get("/info", response_model=ServiceInfo)
+    async def get_info() -> ServiceInfo:
+        """Service discovery endpoint."""
+        return ServiceInfo(
+            service={
+                "base_url": task_app.service_base_url,
+                "endpoints": {
+                    "env": "/env/*",
+                    "policy": "/policy/*",
+                    "rollout": "/rollout",
+                    "branch": "/branch",
+                    "run": "/run/*",
+                },
+            },
+            inference={
+                "base_url": task_app.vllm_base_url,
+                "endpoints": {
+                    "chat_completions": "/v1/chat/completions",
+                },
+                "default_model": task_app.default_model,
+            },
+        )
+
+    @app.get("/health")
+    async def health_check(request: Request) -> dict:
+        """Health and auth sanity check.
+
+        - Returns 503 if server missing ENVIRONMENT_API_KEY (misconfigured container).
+        - If X-API-Key header is provided and mismatches, returns 401.
+        - Otherwise returns 200 with basic info.
+        """
+        import os as _os
+
+        # Check if any environment API keys are configured
+        from synth_ai.task.auth import allowed_environment_api_keys
+        allowed_keys = allowed_environment_api_keys()
+        if not allowed_keys:
+            # Server-side misconfiguration; rollout would fail with 503
+            return JSONResponse(
+                status_code=503,
+                content={
+                    "status": "unhealthy",
+                    "detail": "Auth not configured: missing ENVIRONMENT_API_KEY in task service environment",
+                },
+            )
+        
+        # Authorize using all header variants without typed Header params (avoid 422s)
+        from synth_ai.task.auth import is_api_key_header_authorized
+        authorized = is_api_key_header_authorized(request)
+        if not authorized:
+            # Soft-pass 200 with authorized=False to avoid failing CLI preflight
+            primary_key = list(allowed_keys)[0] if allowed_keys else None
+            prefix = (primary_key[: max(1, len(primary_key) // 2)] if primary_key else None)
+            content = {"status": "healthy", "authorized": False}
+            if prefix:
+                content["expected_api_key_prefix"] = prefix
+            return JSONResponse(status_code=200, content=content)
+        return {"status": "healthy", "authorized": True, "service": {"base_url": task_app.service_base_url}}
+
+    # Log and surface 422 validation errors with header presence
+    from fastapi.exceptions import RequestValidationError
+    @app.exception_handler(RequestValidationError)
+    async def _on_validation_error(request: Request, exc: RequestValidationError):
+        try:
+            hdr = request.headers
+            snapshot = {
+                "path": str(getattr(request, "url").path),
+                "have_x_api_key": bool(hdr.get("x-api-key")),
+                "have_x_api_keys": bool(hdr.get("x-api-keys")),
+                "have_authorization": bool(hdr.get("authorization")),
+                "errors": exc.errors()[:5],
+            }
+            print("[422] validation", snapshot, flush=True)
+        except Exception:
+            pass
+        return JSONResponse(status_code=422, content={"status": "invalid", "detail": exc.errors()[:5]})
+
+    return app

examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/__init__.py

@@ -0,0 +1,5 @@
+"""Inference module for OpenAI-compatible API clients."""
+
+from .openai_client import OpenAIClient, create_inference_client
+
+__all__ = ["OpenAIClient", "create_inference_client"]