synth-ai 0.2.14__py3-none-any.whl → 0.2.17__py3-none-any.whl

examples/task_apps/crafter/task_app/synth_envs_hosted/policy_routes.py

@@ -417,8 +417,6 @@ async def step_policy(
             inf_req = meta["inference_request"]
             msgs = inf_req["messages"]
             model_name = inf_req.get("model") or getattr(policy, "model", None) or ""
-            system_messages: list[str] = []
-            user_messages: list[str] = []
             if msgs and len(msgs) > 0 and msgs[0]["role"] == "system":
                 sys_text = msgs[0]["content"]
                 policy_name = getattr(policy, "name", "") or type(policy).__name__.lower()
@@ -464,6 +462,8 @@ async def step_policy(
                 )
 
             # Emit full system/user prompts for observability (no secrets included)
+            system_prompt_records: list[dict[str, Any]] = []
+            user_prompt_records: list[dict[str, Any]] = []
             try:
 
                 def _as_text(content: object) -> str:
@@ -483,8 +483,6 @@ async def step_policy(
                         return "".join(parts)
                     return str(content)
 
-                system_prompt_records: list[dict[str, Any]] = []
-                user_prompt_records: list[dict[str, Any]] = []
                 for message in msgs:
                     role = message.get("role")
                     raw_content = message.get("content")
@@ -527,6 +525,11 @@ async def step_policy(
 
             if tracing_context is not None:
                 try:
+                    logger.info(
+                        "[TRACE_DEBUG] record_policy_prompts sys=%s user=%s",
+                        len(system_prompt_records),
+                        len(user_prompt_records),
+                    )
                     await tracing_context.record_policy_prompts(
                         system_prompt_records, user_prompt_records
                     )
@@ -782,9 +785,10 @@ async def step_policy(
                 "sokoban-react",
                 "crafter-react",
             ) and getattr(policy, "use_tools", True):
-                req_tools = meta["inference_request"]["tools"]
-                req_tool_choice = meta["inference_request"]["tool_choice"]
-                req_stop_after = meta["inference_request"]["stop_after_tool_calls"]
+                inf_req = meta.get("inference_request", {})
+                req_tools = inf_req.get("tools")
+                req_tool_choice = inf_req.get("tool_choice")
+                req_stop_after = inf_req.get("stop_after_tool_calls")
                 logger.info(
                     f"TOOLCALL_CONFIG: policy={policy_name} tools_present={bool(req_tools)} tool_choice={req_tool_choice} stop_after={req_stop_after}"
                 )
@@ -793,6 +797,8 @@ async def step_policy(
                         status_code=500,
                         detail=f"TOOLCALL_ASSERTION_FAIL: Missing tools or tool_choice!=required for policy {policy_name}",
                     )
+                if req_stop_after is None:
+                    inf_req["stop_after_tool_calls"] = 1
 
             # Call inference service with retries for Flash cold-start (503)
             import time as _t
@@ -901,38 +907,71 @@ async def step_policy(
                             req_body["temperature"] = 0.1
                         meta["inference_request"] = req_body
 
-                # Strip image parts: Crafter policy currently only uses text prompts.
-                # Some providers reject image_url payloads entirely, so always flatten to plain text.
-                req_body2 = meta.get("inference_request", {})
-                if isinstance(req_body2, dict):
-                    msgs = req_body2.get("messages")
-                    if isinstance(msgs, list):
-                        new_msgs = []
-                        changed = False
-                        for m in msgs:
-                            try:
-                                if isinstance(m, dict):
-                                    content = m.get("content")
-                                    if isinstance(content, list):
-                                        parts: list[str] = []
-                                        for seg in content:
-                                            if isinstance(seg, dict):
-                                                txt = seg.get("text") or seg.get("content")
-                                                if isinstance(txt, str) and txt:
-                                                    parts.append(txt)
-                                        m2 = dict(m)
-                                        m2["content"] = "\n".join(parts)
-                                        new_msgs.append(m2)
-                                        changed = True
+                # Message flattening: Convert multimodal content to text-only for non-vision models.
+                # SKIP message flattening for vision models to preserve image_url parts!
+                # The old code here was flattening multimodal content (list) to text-only (str),
+                # which strips out image_url parts. This breaks vision models.
+                # Only flatten for non-vision models that can't handle multimodal format.
+                is_vision_model = False
+                try:
+                    # Check if the policy is a vision-capable policy
+                    if isinstance(policy, CrafterPolicy):
+                        is_vision_model = getattr(policy, "use_vision", False)
+                except Exception:
+                    pass
+                
+                logger.debug(f"🔊 [POLICY_ROUTES] is_vision_model={is_vision_model}, will_flatten={not is_vision_model}")
+                
+                if not is_vision_model:
+                    # Only flatten for non-vision models (backward compatibility)
+                    req_body2 = meta.get("inference_request", {})
+                    if isinstance(req_body2, dict):
+                        msgs = req_body2.get("messages")
+                        if isinstance(msgs, list):
+                            new_msgs = []
+                            changed = False
+                            for m in msgs:
+                                try:
+                                    if isinstance(m, dict):
+                                        content = m.get("content")
+                                        if isinstance(content, list):
+                                            parts: list[str] = []
+                                            for seg in content:
+                                                if isinstance(seg, dict):
+                                                    txt = seg.get("text") or seg.get("content")
+                                                    if isinstance(txt, str) and txt:
+                                                        parts.append(txt)
+                                            m2 = dict(m)
+                                            m2["content"] = "\n".join(parts)
+                                            new_msgs.append(m2)
+                                            changed = True
+                                        else:
+                                            new_msgs.append(m)
                                     else:
                                         new_msgs.append(m)
-                                else:
+                                except Exception:
                                     new_msgs.append(m)
-                            except Exception:
-                                new_msgs.append(m)
-                        if changed:
-                            req_body2["messages"] = new_msgs
-                            meta["inference_request"] = req_body2
+                            if changed:
+                                req_body2["messages"] = new_msgs
+                                meta["inference_request"] = req_body2
+                                logger.debug(f"🔊 [POLICY_ROUTES] Flattened messages for non-vision model")
+                else:
+                    logger.debug(f"🔊 [POLICY_ROUTES] Preserving multimodal content for vision model")
+                
+                # DEBUG: Log final message structure before calling inference
+                final_req = meta.get("inference_request", {})
+                if isinstance(final_req, dict):
+                    final_msgs = final_req.get("messages", [])
+                    logger.debug(f"🔊 [POLICY_ROUTES_FINAL] Sending {len(final_msgs)} messages to inference")
+                    for idx, msg in enumerate(final_msgs):
+                        if isinstance(msg, dict):
+                            content = msg.get("content")
+                            logger.debug(f"🔊 [POLICY_ROUTES_FINAL] Message[{idx}]: type={type(content).__name__}, is_list={isinstance(content, list)}")
+                            if isinstance(content, list):
+                                logger.debug(f"🔊 [POLICY_ROUTES_FINAL]   Content list has {len(content)} items")
+                                for part_idx, part in enumerate(content[:3]):  # Show first 3 items
+                                    if isinstance(part, dict):
+                                        logger.debug(f"🔊 [POLICY_ROUTES_FINAL]     Part[{part_idx}]: type={part.get('type')}")
 
             _t_start = _t.time()
             call_started_at = datetime.utcnow()

examples/task_apps/crafter/task_app/synth_envs_hosted/rollout.py

@@ -491,6 +491,11 @@ class RolloutTracingContext:
             getattr(request.record, "trace_format", "compact") or "compact"
         ).lower()
         self.return_trace = bool(getattr(request.record, "return_trace", False))
+        logger.warning(
+            "[TRACE_DEBUG] RolloutTracingContext init: trace_format=%s return_trace=%s",
+            self.trace_format,
+            self.return_trace,
+        )
         self.sft_output_dir = getattr(fastapi_request.app.state, "sft_output_dir", None)
         self.session_trace = None
         self.metadata_updates: dict[str, Any] = {}
@@ -590,7 +595,7 @@ class RolloutTracingContext:
         # Debug: Check message count
         if self.tracer and self.tracer._current_trace:
             msg_count = len(self.tracer._current_trace.markov_blanket_message_history)
-            logger.info(f"[TRACE_DEBUG] After record_policy_prompts: {msg_count} messages in trace")
+            logger.warning("[TRACE_DEBUG] After record_policy_prompts: %s messages", msg_count)
 
     def _content_to_text(self, content: Any) -> str:
         if isinstance(content, str):
@@ -669,6 +674,11 @@ class RolloutTracingContext:
                     message_type="assistant",  # Map to standard assistant message type
                     metadata={**self._message_metadata(), "is_tool_call": True},
                 )
+                if self.tracer._current_trace:
+                    logger.warning(
+                        "[TRACE_DEBUG] After tool invocation: messages=%s",
+                        len(self.tracer._current_trace.markov_blanket_message_history),
+                    )
             except Exception as exc:
                 logger.debug("TRACING_TOOL_MSG_FAIL: %s", exc)
 
@@ -985,10 +995,19 @@ class RolloutTracingContext:
     def build_trace_payload(self, session_trace: Any) -> dict[str, Any] | None:
         if not self.return_trace or session_trace is None:
             return None
-        if self.trace_format == "full":
+        
+        # For both "full" and "structured" formats, return the complete session trace
+        # The CLI (synth-ai eval) expects this for proper trace storage
+        if self.trace_format in ("full", "structured"):
             payload = session_trace.to_dict()
             payload.setdefault("metadata", {}).update(self.metadata_updates)
+            logger.warning(
+                "[TRACE_DEBUG] build_trace_payload returning structured trace with messages=%s",
+                len(payload.get("markov_blanket_message_history") or []),
+            )
             return payload
+        
+        # For "compact" format, return only summary stats
         metadata = dict(session_trace.metadata)
         metadata.update(self.metadata_updates)
         return {
@@ -1173,14 +1192,6 @@ async def execute_rollout(
             logger.debug(f"TRACER_FACTORY_FAIL: {exc}")
     tracing_context = RolloutTracingContext(tracer_instance, request, req)
     await tracing_context.start_session()
-    # Print whether tracing is active for this rollout
-    try:
-        print(
-            f"[rollout] tracing enabled={bool(tracing_context.enabled)} run_id={request.run_id}",
-            flush=True,
-        )
-    except Exception:
-        pass
 
     # Register run
     registry.register_run(request.run_id)
@@ -1625,16 +1636,21 @@ async def execute_rollout(
 
             elif op == "env":
                 if not pending_tool_calls:
+                    # Instead of failing, inject a no-op action to keep the rollout going
                     with contextlib.suppress(Exception):
                         logger.warning(
-                            "POLICY_STEP_FAIL: missing tool_calls; failing rollout run_id=%s op_idx=%s",
+                            "POLICY_STEP_NOOP: missing tool_calls; injecting noop action run_id=%s op_idx=%s",
                             request.run_id,
                             str(op_idx),
                         )
-                    raise HTTPException(
-                        status_code=500,
-                        detail="policy_step_failed: missing tool_calls (no_tool_calls)",
-                    )
+                    # Create a noop tool call in the format expected by the environment
+                    pending_tool_calls = [
+                        {
+                            "id": f"noop_{op_idx}",
+                            "tool": "interact",
+                            "arguments": {"action": "noop"},
+                        }
+                    ]
 
                 # Environment step
                 from .environment_routes import EnvStepRequest, step_environment

examples/task_apps/enron/__init__.py

@@ -1 +1,2 @@
 
+

examples/task_apps/enron/task_app/grpo_enron_task_app.py

@@ -2,7 +2,7 @@
 
 This mirrors the structure of the Crafter task app wrapper while delegating
 all configuration to the colocated `grpo_enron.py` module. Normal usage should
-prefer invoking `uvx synth-ai serve grpo-enron`, but this module remains for
+prefer invoking `uvx synth-ai deploy --runtime uvicorn grpo-enron`, but this module remains for
 direct execution or importing the FastAPI app object.
 """
 

examples/task_apps/math/README.md

@@ -3,7 +3,7 @@
 This directory hosts the legacy entrypoint for the math single-step task app. Prefer starting the app via:
 
 ```bash
-uvx synth-ai serve math-single-step --env-file examples/rl/.env --port 8101
+uvx synth-ai deploy --runtime uvicorn math-single-step --env-file examples/rl/.env --port 8101
 ```
 
 If you need to run it directly (e.g., for Modal `modal deploy` compatibility), use:
@@ -19,4 +19,3 @@ Environment variables:
 - `MATH_DATASET_DEFAULT_SPLIT`, `MATH_DATASET_VALIDATION_SPLIT`, `MATH_DATASET_TEST_SPLIT`
 
 The task app enforces a single `math_submit` tool call per episode, enabling RL to reward correct final answers and penalise missing or malformed submissions.
-

examples/task_apps/pokemon_red/README.md

@@ -17,7 +17,7 @@ A reinforcement learning environment for Pokémon Red using PyBoy emulation with
 
 ```bash
 # From synth-ai root
-uv run -m synth_ai task-app serve pokemon_red --port 8913
+uv run -m synth_ai task-app deploy --runtime uvicorn pokemon_red --port 8913
 ```
 
 ### 2. Run a Random Rollout
@@ -232,7 +232,7 @@ uv add pyboy
 lsof -ti :8913 | xargs -r kill -9
 
 # Or use a different port
-uv run -m synth_ai task-app serve pokemon_red --port 8914
+uv run -m synth_ai task-app deploy --runtime uvicorn pokemon_red --port 8914
 ```
 
 ## Examples
@@ -249,7 +249,7 @@ cd /Users/joshpurtell/Documents/GitHub/synth-ai
 echo "OPENAI_API_KEY=sk-..." >> .env
 
 # 2. Start the task app server (in background)
-nohup sh -c 'printf "n\n" | uv run -m synth_ai task-app serve pokemon_red --port 8913 --no-reload' > nohup_pokemon.log 2>&1 &
+nohup sh -c 'printf "n\n" | uv run -m synth_ai task-app deploy --runtime uvicorn pokemon_red --port 8913 --no-reload' > nohup_pokemon.log 2>&1 &
 
 # Wait for startup
 sleep 8
@@ -354,4 +354,3 @@ TOTAL REWARD: 705 points
 - **PyBoy**: Game Boy emulator - https://github.com/Baekalfen/PyBoy
 - **Pokémon Red Disassembly**: RAM map reference - https://github.com/pret/pokered
 - **Datacrystal.org**: Memory address documentation
-

examples/task_apps/pokemon_red/eval_image_only_gpt4o.toml

@@ -1,11 +1,12 @@
-# Evaluation config for Pokemon Red with image-only input
+# Evaluation config for Pokemon Red with image-only input and NEW REWARD SYSTEM
 # This config uses GPT-4o mini with only image data (no text observations)
+# Uses the comprehensive reward system with deterministic progress milestones
 
 [eval]
 app_id = "pokemon_red"
 model = "gpt-4o-mini-2024-07-18"
-seeds = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
-max_turns = 10
+seeds = [0, 1, 2, 3, 4]  # Test with fewer seeds for quick results
+max_turns = 20  # Allow more turns to see progress
 concurrency = 1  # Keep low initially to avoid issues
 env_name = "pokemon_red"
 policy_name = "pokemon_red_policy"
@@ -13,7 +14,7 @@ trace_format = "full"
 return_trace = true
 
 [eval.env_config]
-max_steps_per_episode = 10
+max_steps_per_episode = 20
 
 [eval.policy_config]
 provider = "openai"
@@ -24,6 +25,6 @@ top_p = 0.95
 max_tokens = 512
 use_vision = true
 image_only_mode = true
-max_llm_calls = 10
+max_llm_calls = 20
 
 

examples/task_apps/pokemon_red/eval_pokemon_red_policy.py

@@ -129,7 +129,7 @@ async def main():
             print("✓ Server is healthy")
         except Exception as e:
             print(f"❌ Server not responding: {e}")
-            print(f"   Start it with: uv run -m synth_ai task-app serve pokemon_red --port 8913")
+            print(f"   Start it with: uv run -m synth_ai task-app deploy --runtime uvicorn pokemon_red --port 8913")
             return
         
         # Check API key
@@ -222,4 +222,3 @@ async def main():
 
 if __name__ == "__main__":
     asyncio.run(main())
-

examples/task_apps/pokemon_red/task_app.py

@@ -12,7 +12,7 @@ from synth_ai.environments.examples.red.taskset import INSTANCE as RED_DEFAULT_I
 from synth_ai.environments.examples.red.engine_helpers.reward_library.pallet_town_progression import (
     PalletTownProgressionCompositeReward,
 )
-from synth_ai.task.apps import TaskAppEntry, register_task_app
+from synth_ai.task.apps import ModalDeploymentConfig, TaskAppEntry, register_task_app
 from synth_ai.task.contracts import (
     RolloutMetrics,
     RolloutRequest,
@@ -260,8 +260,10 @@ async def rollout_executor(request: RolloutRequest, fastapi_request: Request) ->
             {
                 "role": "system",
                 "content": (
-                    "You are controlling Pokémon Red. Respond with a single tool call named 'press_button' "
-                    "with JSON arguments {button: 'A|B|UP|DOWN|LEFT|RIGHT|START|SELECT', frames: 1-120}."
+                    "You are controlling Pokémon Red, a classic Game Boy game. You can see the game screen in the images provided. "
+                    "Your goal is to make progress in the game. Use the execute_sequence tool to press buttons. "
+                    "Choose appropriate button presses based on what you see in the game screen. "
+                    "Always respond with exactly one tool call in the format: <tool_call>{\"name\": \"execute_sequence\", \"arguments\": {...}}</tool_call>"
                 ),
             },
             {
@@ -788,11 +790,40 @@ def build_config() -> TaskAppConfig:
 register_task_app(
     entry=TaskAppEntry(
         app_id="pokemon_red",
-        description="Pokémon Red demo task app",
+        description="Pokémon Red demo task app with vision support",
         config_factory=build_config,
         aliases=("pokemon_red_demo",),
         env_files=(),
-        modal=None,
+        modal=ModalDeploymentConfig(
+            app_name="pokemon-red-vision-task-app",
+            python_version="3.11",
+            pip_packages=(
+                "fastapi>=0.100.0",
+                "uvicorn>=0.23.0",
+                "pydantic>=2.0.0",
+                "numpy>=1.24.0",
+                "aiohttp>=3.8.0",
+                "httpx>=0.24.0",
+                "python-dotenv>=1.0.1",
+                # Tracing/DB runtime deps
+                "sqlalchemy>=2.0.42",
+                "aiosqlite>=0.21.0",
+                "greenlet>=3.2.3",
+                # Pokemon Red environment
+                "pyboy>=2.0.0",
+                "pillow>=9.0.0",
+            ),
+            extra_local_dirs=(
+                # Mount repo root so local modules resolve when deployed on Modal
+                ("/Users/joshpurtell/Documents/GitHub/synth-ai", "/opt/synth_ai_repo"),
+                ("/Users/joshpurtell/Documents/GitHub/synth-ai/synth_ai", "/opt/synth_ai_repo/synth_ai"),
+                ("/Users/joshpurtell/Documents/GitHub/synth-ai/examples/task_apps/pokemon_red", "/opt/synth_ai_repo/examples/task_apps/pokemon_red"),
+            ),
+            secret_names=("openai-api-key", "groq-api-key"),
+            memory=16384,
+            cpu=4.0,
+            max_containers=10,
+        ),
     )
 )
 

examples/task_apps/sokoban/README.md

@@ -20,7 +20,7 @@ Sokoban is a classic puzzle game where the player must push boxes onto target lo
 cd /path/to/synth-ai
 
 # Start the Sokoban task app on port 8911
-uvx synth-ai task-app serve sokoban --port 8911
+uvx synth-ai task-app deploy --runtime uvicorn sokoban --port 8911
 ```
 
 The server will be available at `http://localhost:8911`.
@@ -283,7 +283,7 @@ lsof -i :8911
 kill -9 $(lsof -ti :8911)
 
 # Restart
-uvx synth-ai task-app serve sokoban --port 8911
+uvx synth-ai task-app deploy --runtime uvicorn sokoban --port 8911
 ```
 
 ## Examples
@@ -304,4 +304,3 @@ To add new features:
 ## License
 
 MIT
-

examples/task_apps/verilog/eval_groq_qwen32b.toml

@@ -1,24 +1,22 @@
 # Verilog Eval Config for Groq Qwen3-32B
-# Quick eval to test Verilog task app before RL training
-
-[task_app]
-# Update this with your Modal URL after deployment
-url = "https://synth-laboratories--grpo-verilog-task-app-fastapi-app-dev.modal.run"
+# Quick eval to test the Verilog task app before RL training
 
 [eval]
-num_episodes = 3  # Quick test with 3 seeds
+app_id = "grpo-verilog"
+task_app_url = "https://synth-laboratories--grpo-verilog-task-app-fastapi-app-dev.modal.run"
+model = "groq:qwen3-32b"
 seeds = [0, 1, 2]
-max_steps = 15    # More steps for Verilog compilation chains
+max_turns = 15
+concurrency = 1
+return_trace = true
+trace_format = "structured"
+
+[eval.env_config]
+difficulty = "medium"
 
-[policy]
+[eval.policy_config]
 provider = "groq"
 model = "qwen/qwen3-32b"
 temperature = 0.2
 max_tokens = 768
 inference_url = "https://api.groq.com/openai/v1/chat/completions"
-
-[env]
-difficulty = "medium"  # Can be "easy", "medium", or "hard"
-
-
-

examples/task_apps/verilog/task_app/grpo_verilog_task_app.py

@@ -1,7 +1,7 @@
 """Compatibility wrapper for the GRPO Verilog task app.
 
 This mirrors the Crafter task app wrapper while delegating configuration to
-`grpo_verilog.py`. Normal usage should prefer `uvx synth-ai serve grpo-verilog`,
+`grpo_verilog.py`. Normal usage should prefer `uvx synth-ai deploy --runtime uvicorn grpo-verilog`,
 but the module remains for direct execution or importing the FastAPI app.
 """
 

examples/vlm/README.md

@@ -21,8 +21,8 @@ plumbing with lightweight utilities for dataset curation and training.
 3. **Export multimodal SFT rows**  
    ```
    uv run python examples/warming_up_to_rl/export_trace_sft.py \
-       --db traces/v3/synth_ai.db \
-       --output examples/vlm/output/crafter_traces_full.jsonl
+      --db traces/v3/task_app_traces_<timestamp>.db \
+       --output examples/vlm/output/crafter_sft_full.jsonl
    ```
    The exporter now emits `metadata.has_image`, `metadata.user_has_image`, and
    `metadata.assistant_has_image` flags per turn.
@@ -30,7 +30,7 @@ plumbing with lightweight utilities for dataset curation and training.
 4. **Filter to image-rich turns**  
    ```
    uv run python examples/vlm/filter_image_rows.py \
-       --input examples/vlm/output/crafter_traces_full.jsonl \
+       --input examples/vlm/output/crafter_sft_full.jsonl \
        --output examples/vlm/output/crafter_vlm_dataset.jsonl
    ```
 

examples/vlm/configs/crafter_vlm_gpt4o.toml

@@ -1,3 +1,8 @@
+[algorithm]
+type = "offline"
+method = "sft"
+variety = "fft"
+
 [job]
 model = "openai/gpt-4o-mini-2024-07-18"
 modalities = ["text", "image"]

examples/vlm/crafter_openai_vlm_agent.py

@@ -24,6 +24,7 @@ import asyncio
 import base64
 import json
 import os
+from contextlib import suppress
 from pathlib import Path
 from typing import Any
 from uuid import uuid4
@@ -62,7 +63,7 @@ class EpisodeResult:
                 if unlocked:
                     self.achievements.add(str(name))
         reward = obs.get("reward_last_step")
-        if isinstance(reward, (int, float)):
+        if isinstance(reward, int | float):
             self.total_reward += float(reward)
 
 
@@ -107,11 +108,8 @@ def _decode_and_save_image(observation: dict[str, Any], path: Path) -> None:
     if not isinstance(base64_data, str) or not base64_data:
         return
     path.parent.mkdir(parents=True, exist_ok=True)
-    try:
+    with suppress(Exception):
         path.write_bytes(base64.b64decode(base64_data))
-    except Exception:
-        # Best-effort; corrupted frames should not halt rollout
-        pass
 
 
 def _normalise_openai_request(payload: dict[str, Any], model: str, temperature: float) -> dict[str, Any]:

examples/vlm/filter_image_rows.py

@@ -8,7 +8,7 @@ output now that each record's metadata includes `has_image`, `user_has_image`, a
 
 Usage:
     uv run python examples/vlm/filter_image_rows.py \
-        --input examples/sft/ft_data/crafter_traces.jsonl \
+        --input examples/sft/ft_data/crafter_sft.jsonl \
         --output examples/vlm/output/crafter_vlm_dataset.jsonl
 """
 

examples/vlm/run_crafter_vlm_benchmark.py

@@ -224,7 +224,7 @@ async def _run_episode(
                         if unlocked:
                             achievements.add(str(name))
                 reward = obs.get("reward_last_step")
-                if isinstance(reward, (int, float)):
+                if isinstance(reward, int | float):
                     total_reward += float(reward)
 
             _save_observation_frame(env_response, frames_dir / f"step_{step_idx + 1:03d}.png")
@@ -263,7 +263,7 @@ def _summarise(results: list[EpisodeResult]) -> dict[str, Any]:
             "mean_steps": round(mean_steps, 2),
             "mean_achievements": round(mean_achievements, 2),
             "total_tool_calls": sum(r.tool_calls for r in mode_results),
-            "achievements": {name: count for name, count in sorted(achievement_counts.items())},
+            "achievements": dict(sorted(achievement_counts.items())),
         }
     return summary