synth-ai 0.2.9.dev5__py3-none-any.whl → 0.2.9.dev6__py3-none-any.whl

examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/__init__.py

@@ -1 +1 @@
-"""Environment implementations."""
+"""Environment implementations."""

examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/__init__.py

@@ -3,4 +3,4 @@
 from .environment import CrafterEnvironmentWrapper
 from .policy import CrafterPolicy
 
-__all__ = ["CrafterEnvironmentWrapper", "CrafterPolicy"]
+__all__ = ["CrafterEnvironmentWrapper", "CrafterPolicy"]

examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/app.py

@@ -1 +1 @@
- # wraps hosted app
+# wraps hosted app

examples/warming_up_to_rl/task_app/synth_envs_hosted/envs/crafter/environment.py

@@ -1,19 +1,56 @@
 from __future__ import annotations
 
-from typing import Any, Dict, List, Optional
+import base64
 import logging
+from io import BytesIO
+from typing import Any
 
-from synth_ai.environments.stateful.core import StatefulEnvironment
+import numpy as np
+from PIL import Image
 from synth_ai.environments.environment.tools import EnvToolCall
+from synth_ai.environments.stateful.core import StatefulEnvironment
 
 from ...utils import convert_numpy_to_python
-from .tools import TOOLS_SCHEMA
 from .shared import CRAFTER_ACTIONS, _format_semantic_map_view
-
+from .tools import TOOLS_SCHEMA
 
 logger = logging.getLogger(__name__)
 
 
+def _encode_image_to_base64(image_array: Any) -> dict[str, Any] | None:
+    """Encode an RGB ndarray into a base64 PNG payload with metadata."""
+
+    if not isinstance(image_array, np.ndarray):
+        return None
+    if image_array.ndim != 3 or image_array.shape[-1] not in (1, 3, 4):
+        return None
+    try:
+        # Ensure uint8 for PIL compatibility
+        array_uint8 = (
+            image_array.astype("uint8")
+            if image_array.dtype != np.uint8
+            else image_array  # pragma: no cover - fast path
+        )
+        mode = "L" if array_uint8.shape[-1] == 1 else "RGB"
+        if array_uint8.shape[-1] == 4:
+            mode = "RGBA"
+        img = Image.fromarray(array_uint8, mode=mode)
+        buffer = BytesIO()
+        img.save(buffer, format="PNG")
+        encoded = base64.b64encode(buffer.getvalue()).decode("ascii")
+        width = int(array_uint8.shape[1])
+        height = int(array_uint8.shape[0])
+        return {
+            "format": "png",
+            "width": width,
+            "height": height,
+            "data": encoded,
+            "data_url": f"data:image/png;base64,{encoded}",
+        }
+    except Exception:
+        return None
+
+
 class CrafterEnvironmentWrapper:
     """Host-side environment wrapper matching the sketch contract.
 
@@ -25,20 +62,20 @@ class CrafterEnvironmentWrapper:
       - snapshot()/restore() handled at route level; this wrapper exposes checkpoint via synth-ai
     """
 
-    def __init__(self, env: StatefulEnvironment, seed: Optional[int] = None) -> None:
+    def __init__(self, env: StatefulEnvironment, seed: int | None = None) -> None:
         self.env = env
         self.seed = seed
         self.step_idx = 0
-        self.last_observation: Optional[Dict[str, Any]] = None
-        self.last_info: Optional[Dict[str, Any]] = None
+        self.last_observation: dict[str, Any] | None = None
+        self.last_info: dict[str, Any] | None = None
 
-    async def initialize(self) -> Dict[str, Any]:
+    async def initialize(self) -> dict[str, Any]:
         obs = await self.env.initialize()
         # synth-ai InternalObservation expected to expose .observation (dict-like)
         self.step_idx = 0
         self.last_observation = getattr(obs, "observation", obs)  # tolerate dict-like
         self.last_info = getattr(obs, "info", None)
-        out_obs: Dict[str, Any] = convert_numpy_to_python(self.last_observation) or {}
+        out_obs = self._prepare_observation(self.last_observation)
         # Attach a 7x7 semantic map patch centered on player for client-side rendering
         try:
             pub = self.env.engine._get_public_state_from_env()  # type: ignore[attr-defined]
@@ -47,13 +84,13 @@ class CrafterEnvironmentWrapper:
             size = 7
             half = size // 2
             patch = []
-            H = len(sem) if hasattr(sem, "__len__") else 0
-            W = len(sem[0]) if H and hasattr(sem[0], "__len__") else 0
+            height = len(sem) if hasattr(sem, "__len__") else 0
+            width = len(sem[0]) if height and hasattr(sem[0], "__len__") else 0
             for dy in range(-half, half + 1):
                 row = []
                 for dx in range(-half, half + 1):
                     x, y = int(px) + dx, int(py) + dy
-                    if 0 <= x < H and 0 <= y < W:
+                    if 0 <= x < height and 0 <= y < width:
                         row.append(int(sem[x][y]))
                     else:
                         row.append(0)
@@ -68,7 +105,7 @@ class CrafterEnvironmentWrapper:
             "step_idx": self.step_idx,
         }
 
-    async def step(self, tool_calls: List[Dict[str, Any]] | List[EnvToolCall]) -> Dict[str, Any]:
+    async def step(self, tool_calls: list[dict[str, Any]] | list[EnvToolCall]) -> dict[str, Any]:
         # Normalize JSON tool_calls into EnvToolCall instances if needed
         # Underlying synth-ai environment expects only tool="interact" with args={"action": <action_name>}.
         # LLM may emit:
@@ -79,9 +116,9 @@ class CrafterEnvironmentWrapper:
         allowed_actions = set(
             TOOLS_SCHEMA[0]["function"]["parameters"]["properties"]["actions"]["items"]["enum"]
         )
-        normalized: List[EnvToolCall] = []
+        normalized: list[EnvToolCall] = []
 
-        def _action_to_int(action: Any) -> Optional[int]:
+        def _action_to_int(action: Any) -> int | None:
             # Handle invalid actions gracefully instead of failing
             if isinstance(action, int):
                 return action
@@ -90,6 +127,7 @@ class CrafterEnvironmentWrapper:
                 logger.warning("Unknown Crafter action: %s - ignoring", action_str)
                 return None  # Signal to skip this action
             return CRAFTER_ACTIONS[action_str]
+
         for tc in tool_calls:
             if isinstance(tc, EnvToolCall):
                 # Expand interact_many; otherwise coerce non-interact tools into interact(action=tool)
@@ -103,12 +141,12 @@ class CrafterEnvironmentWrapper:
                             )
                 elif tc.tool != "interact":
                     candidate_action = tc.args.get("action") if isinstance(tc.args, dict) else None
-                    resolved_action = candidate_action if candidate_action in allowed_actions else tc.tool
+                    resolved_action = (
+                        candidate_action if candidate_action in allowed_actions else tc.tool
+                    )
                     action_int = _action_to_int(resolved_action)
                     if action_int is not None:  # Skip invalid actions
-                        normalized.append(
-                            EnvToolCall(tool="interact", args={"action": action_int})
-                        )
+                        normalized.append(EnvToolCall(tool="interact", args={"action": action_int}))
                 else:
                     normalized.append(tc)
             else:
@@ -120,13 +158,14 @@ class CrafterEnvironmentWrapper:
                 args = tc.get("arguments") or tc.get("args") or {}
                 if isinstance(args, str):
                     import json as _json
+
                     try:
                         args = _json.loads(args)
                     except Exception:
                         args = {}
                 # Expand interact_many into multiple interacts
                 if tool_name == "interact_many":
-                    for action in (args.get("actions") or []):
+                    for action in args.get("actions") or []:
                         action_int = _action_to_int(action)
                         if action_int is not None:  # Skip invalid actions
                             normalized.append(
@@ -135,26 +174,35 @@ class CrafterEnvironmentWrapper:
                 else:
                     # For any non-interact tool, resolve to an interact action.
                     # Support a packed list of actions under 'actions' for convenience.
-                    if isinstance(args, dict) and isinstance(args.get("actions"), list) and args.get("actions"):
+                    if (
+                        isinstance(args, dict)
+                        and isinstance(args.get("actions"), list)
+                        and args.get("actions")
+                    ):
                         for action in args.get("actions"):
                             action_int = _action_to_int(action)
                             if action_int is not None:
-                                normalized.append(EnvToolCall(tool="interact", args={"action": action_int}))
+                                normalized.append(
+                                    EnvToolCall(tool="interact", args={"action": action_int})
+                                )
                     else:
                         candidate_action = None
                         if isinstance(args, dict) and "action" in args:
                             candidate_action = args["action"]
                         # If the caller provided a numeric action id, accept it directly
-                        action_int: Optional[int]
-                        if isinstance(candidate_action, int):
-                            action_int = _action_to_int(candidate_action)
-                        elif isinstance(candidate_action, str) and candidate_action in allowed_actions:
+                        action_int: int | None
+                        if isinstance(candidate_action, int) or (
+                            isinstance(candidate_action, str)
+                            and candidate_action in allowed_actions
+                        ):
                             action_int = _action_to_int(candidate_action)
                         else:
                             # Fallback: interpret the tool name itself as the action label
                             action_int = _action_to_int(tool_name)
                         if action_int is not None:
-                            normalized.append(EnvToolCall(tool="interact", args={"action": action_int}))
+                            normalized.append(
+                                EnvToolCall(tool="interact", args={"action": action_int})
+                            )
 
         # Ensure we have at least one valid action; default to noop if none provided
         if not normalized:
@@ -162,7 +210,7 @@ class CrafterEnvironmentWrapper:
             normalized.append(EnvToolCall(tool="interact", args={"action": 0}))  # noop action
 
         # Pre-step logging: capture current public state and print concise summary
-        before_state: Optional[Dict[str, Any]] = None
+        before_state: dict[str, Any] | None = None
         try:
             pub_before = self.env.engine._get_public_state_from_env()  # type: ignore[attr-defined]
             before_state = {
@@ -173,7 +221,9 @@ class CrafterEnvironmentWrapper:
                 "semantic_map": pub_before.semantic_map,
             }
             actions_printable = [
-                (tc.args.get("action") if isinstance(tc.args, dict) else None) if isinstance(tc, EnvToolCall) else None
+                (tc.args.get("action") if isinstance(tc.args, dict) else None)
+                if isinstance(tc, EnvToolCall)
+                else None
                 for tc in normalized
             ]
             logger.info(
@@ -185,7 +235,11 @@ class CrafterEnvironmentWrapper:
                 [k for k, v in before_state["achievements_status"].items() if v],
                 actions_printable,
             )
-            logger.info("Surroundings BEFORE (seed=%s):\n%s", str(self.seed), _format_semantic_map_view(before_state))
+            logger.info(
+                "Surroundings BEFORE (seed=%s):\n%s",
+                str(self.seed),
+                _format_semantic_map_view(before_state),
+            )
         except Exception as _:
             # Logging should not interfere with stepping; fail-fast elsewhere
             pass
@@ -212,7 +266,7 @@ class CrafterEnvironmentWrapper:
         ach_added_latest: list[str] | None = None
         try:
             pub_after = self.env.engine._get_public_state_from_env()  # type: ignore[attr-defined]
-            after_dict: Dict[str, Any] = {
+            after_dict: dict[str, Any] = {
                 "inventory": pub_after.inventory,
                 "achievements_status": pub_after.achievements_status,
                 "player_position": list(pub_after.player_position),
@@ -236,8 +290,8 @@ class CrafterEnvironmentWrapper:
                     # Position delta
                     pb = before_state.get("player_position", [0, 0])
                     pa = after_dict.get("player_position", [0, 0])
-                    pb_t = (int(pb[0]), int(pb[1])) if isinstance(pb, (list, tuple)) else (0, 0)
-                    pa_t = (int(pa[0]), int(pa[1])) if isinstance(pa, (list, tuple)) else (0, 0)
+                    pb_t = (int(pb[0]), int(pb[1])) if isinstance(pb, list | tuple) else (0, 0)
+                    pa_t = (int(pa[0]), int(pa[1])) if isinstance(pa, list | tuple) else (0, 0)
                     delta = (pa_t[0] - pb_t[0], pa_t[1] - pb_t[1])
 
                     # Inventory changes
@@ -253,11 +307,17 @@ class CrafterEnvironmentWrapper:
                     inv_changes = ", ".join(changed_items) if changed_items else "none"
 
                     # Achievements gained/lost
-                    ach_b = {k for k, v in (before_state.get("achievements_status", {}) or {}).items() if v}
-                    ach_a = {k for k, v in (after_dict.get("achievements_status", {}) or {}).items() if v}
-                    ach_added = sorted(list(ach_a - ach_b))
+                    ach_b = {
+                        k
+                        for k, v in (before_state.get("achievements_status", {}) or {}).items()
+                        if v
+                    }
+                    ach_a = {
+                        k for k, v in (after_dict.get("achievements_status", {}) or {}).items() if v
+                    }
+                    ach_added = sorted(ach_a - ach_b)
                     ach_added_latest = ach_added
-                    ach_removed = sorted(list(ach_b - ach_a))
+                    ach_removed = sorted(ach_b - ach_a)
 
                     logger.info(
                         "Changes: pos %s->%s Δ=%s | inv %s | ach +%s -%s",
@@ -272,16 +332,23 @@ class CrafterEnvironmentWrapper:
                     if reward is None and ach_added_latest:
                         try:
                             reward = float(len(ach_added_latest))
-                            logger.info("Reward shaping applied: +%s (achievements added)", len(ach_added_latest))
+                            logger.info(
+                                "Reward shaping applied: +%s (achievements added)",
+                                len(ach_added_latest),
+                            )
                         except Exception:
                             pass
                 except Exception:
                     pass
-            logger.info("Surroundings AFTER (seed=%s):\n%s", str(self.seed), _format_semantic_map_view(after_dict))
+            logger.info(
+                "Surroundings AFTER (seed=%s):\n%s",
+                str(self.seed),
+                _format_semantic_map_view(after_dict),
+            )
         except Exception as _:
             pass
-        result: Dict[str, Any] = {
-            "observation": convert_numpy_to_python(observation),
+        result: dict[str, Any] = {
+            "observation": self._prepare_observation(observation),
             "step_idx": self.step_idx,
             "done": bool(done) if done is not None else False,  # Ensure boolean
         }
@@ -293,13 +360,13 @@ class CrafterEnvironmentWrapper:
             size = 7
             half = size // 2
             patch = []
-            H = len(sem) if hasattr(sem, "__len__") else 0
-            W = len(sem[0]) if H and hasattr(sem[0], "__len__") else 0
+            height = len(sem) if hasattr(sem, "__len__") else 0
+            width = len(sem[0]) if height and hasattr(sem[0], "__len__") else 0
             for dy in range(-half, half + 1):
                 row = []
                 for dx in range(-half, half + 1):
                     x, y = px + dx, py + dy
-                    if 0 <= x < H and 0 <= y < W:
+                    if 0 <= x < height and 0 <= y < width:
                         row.append(int(sem[x][y]))
                     else:
                         row.append(0)
@@ -309,10 +376,7 @@ class CrafterEnvironmentWrapper:
                 obs_out["semantic_map_patch7"] = patch
         except Exception:
             pass
-        if info is not None:
-            result_info = convert_numpy_to_python(info)
-        else:
-            result_info = {}
+        result_info = convert_numpy_to_python(info) if info is not None else {}
         # Attach achievements delta for downstream metrics if useful
         if ach_added_latest is not None:
             try:
@@ -340,6 +404,7 @@ class CrafterEnvironmentWrapper:
             # Build reverse action map for readability
             int_to_action = {v: k for k, v in CRAFTER_ACTIONS.items()}
             from collections import Counter
+
             action_ids = []
             for tc in normalized:
                 if isinstance(tc, EnvToolCall) and isinstance(tc.args, dict):
@@ -371,29 +436,57 @@ class CrafterEnvironmentWrapper:
             )
         except Exception:
             pass
+
         return result
 
-    async def checkpoint(self) -> Dict[str, Any]:
+    def _prepare_observation(self, observation: Any) -> dict[str, Any]:
+        """Convert raw observation into a JSON-serializable dict with encoded image."""
+
+        obs_dict: dict[str, Any]
+        image_payload: dict[str, Any] | None = None
+
+        if isinstance(observation, dict):
+            image_payload = _encode_image_to_base64(observation.get("observation_image"))
+            # Work on a shallow copy to avoid mutating engine state
+            sanitized = dict(observation)
+            sanitized.pop("observation_image", None)
+            obs_dict = convert_numpy_to_python(sanitized) or {}
+        else:
+            obs_dict = convert_numpy_to_python(observation) or {}
+
+        if not isinstance(obs_dict, dict):
+            obs_dict = {"value": obs_dict}
+
+        if image_payload:
+            obs_dict["observation_image_base64"] = image_payload["data"]
+            obs_dict["observation_image_format"] = image_payload["format"]
+            obs_dict["observation_image_width"] = image_payload["width"]
+            obs_dict["observation_image_height"] = image_payload["height"]
+            obs_dict["observation_image_data_url"] = image_payload["data_url"]
+
+        return obs_dict
+
+    async def checkpoint(self) -> dict[str, Any]:
         obs = await self.env.checkpoint()
         observation = getattr(obs, "observation", obs)
         info = getattr(obs, "info", None)
         return {
             "observation": convert_numpy_to_python(observation),
             "info": convert_numpy_to_python(info) if info else None,
-            "step_idx": self.step_idx
+            "step_idx": self.step_idx,
         }
 
-    async def terminate(self) -> Dict[str, Any]:
+    async def terminate(self) -> dict[str, Any]:
         obs = await self.env.terminate()
         observation = getattr(obs, "observation", obs)
         info = getattr(obs, "info", None)
         return {
             "observation": convert_numpy_to_python(observation),
             "info": convert_numpy_to_python(info) if info else None,
-            "step_idx": self.step_idx
+            "step_idx": self.step_idx,
         }
 
-    def state_dict(self) -> Dict[str, Any]:
+    def state_dict(self) -> dict[str, Any]:
         return {
             "seed": self.seed,
             "step_idx": self.step_idx,
@@ -401,13 +494,13 @@ class CrafterEnvironmentWrapper:
             "last_info": self.last_info,
         }
 
-    def load_state_dict(self, state: Dict[str, Any]) -> None:
+    def load_state_dict(self, state: dict[str, Any]) -> None:
         self.seed = state["seed"]
         self.step_idx = int(state["step_idx"])
         self.last_observation = state["last_observation"]
         self.last_info = state["last_info"]
 
-    async def serialize(self) -> Dict[str, Any]:
+    async def serialize(self) -> dict[str, Any]:
         return {
             "name": "crafter",
             "config": {"seed": self.seed},
@@ -417,9 +510,9 @@ class CrafterEnvironmentWrapper:
     @classmethod
     async def deserialize(
         cls,
-        payload: Dict[str, Any],
+        payload: dict[str, Any],
         env: StatefulEnvironment,
-    ) -> "CrafterEnvironmentWrapper":
+    ) -> CrafterEnvironmentWrapper:
         seed = payload["config"]["seed"]
         wrapper = cls(env=env, seed=seed)
         wrapper.load_state_dict(payload["state"])