synth-ai 0.2.13.dev2__py3-none-any.whl → 0.2.16__py3-none-any.whl

examples/task_apps/crafter/task_app/grpo_crafter.py

@@ -2,20 +2,23 @@
 
 from __future__ import annotations
 
+import json
 import logging
 import os
 import sys
 from collections.abc import Iterable, Sequence
+from contextlib import suppress
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
 from synth_ai.task.apps import ModalDeploymentConfig, TaskAppEntry, register_task_app
-from synth_ai.task.contracts import RolloutMetrics, RolloutRequest, RolloutResponse, TaskInfo
+from synth_ai.task.contracts import RolloutMetrics, RolloutMode, RolloutRequest, RolloutResponse, TaskInfo
 from synth_ai.task.datasets import TaskDatasetRegistry, TaskDatasetSpec
 from synth_ai.task.json import to_jsonable  # noqa: F401  (imported for side-effect compatibility)
 from synth_ai.task.rubrics import load_rubric
 from synth_ai.task.server import ProxyConfig, RubricBundle, TaskAppConfig
+from synth_ai.task.validators import normalize_inference_url
 from synth_ai.task.tracing_utils import (
     build_tracer_factory,
     resolve_sft_output_dir,
@@ -24,6 +27,18 @@ from synth_ai.task.tracing_utils import (
 )
 from synth_ai.tracing_v3.session_tracer import SessionTracer
 
+try:
+    from .synth_envs_hosted.utils import (
+        ensure_chat_completions_url,
+        extract_trace_correlation_id,
+    )
+except Exception:  # pragma: no cover - utils unavailable if optional deps missing
+    def ensure_chat_completions_url(raw_url, mode=None):
+        """Fallback to shared utility for URL normalization."""
+        return normalize_inference_url(raw_url) if raw_url else raw_url
+
+    def extract_trace_correlation_id(_raw_url):
+        return None
 logger = logging.getLogger(__name__)
 
 DEFAULT_ALIAS_OPS: list[str] = ["agent", "env"] * 10
@@ -95,6 +110,110 @@ SYNTH_ENVS_HOSTED_ROOT = (TASK_APP_ROOT / "synth_envs_hosted").resolve()
 EXAMPLES_ROOT = (REPO_ROOT / "examples").resolve()
 RUBRICS_ROOT = (EXAMPLES_ROOT / "multi_step" / "rubrics").resolve()
 
+DEFAULT_OUTCOME_RUBRIC_DATA: dict[str, Any] = {
+    "version": "1",
+    "goal_text": (
+        "Reward episodes that climb the Crafter achievement ladder, stockpile key resources "
+        "(especially wood), and finish alive with clear understanding of any failure."
+    ),
+    "aggregation": "weighted_sum",
+    "criteria": [
+        {
+            "id": "achievement_progression",
+            "description": (
+                "Weigh achievements by tier: late-game unlocks (iron tools, furnace, armor) earn "
+                "the most, mid-tier crafting (stone tools, furnace prep) gets partial credit, early "
+                "tasks (collecting saplings/wood tools) only lightly scored."
+            ),
+            "weight": 0.35,
+        },
+        {
+            "id": "resource_stockpile",
+            "description": (
+                "Assess resource totals with emphasis on wood stores; high scores require abundant "
+                "wood plus supporting materials (stone, coal, iron) that signal readiness for "
+                "crafting."
+            ),
+            "weight": 0.2,
+        },
+        {
+            "id": "survival_state",
+            "description": (
+                "Reward finishing alive with healthy food/drink bars and safe positioning; penalize "
+                "deaths, low vitals, or lingering hazards at episode end."
+            ),
+            "weight": 0.2,
+        },
+        {
+            "id": "failure_analysis",
+            "description": (
+                "If the run ends in death or timeout, clearly identify the cause and deduct unless "
+                "the agent mitigated risk; highlight when the agent survives despite danger."
+            ),
+            "weight": 0.15,
+        },
+        {
+            "id": "future_readiness",
+            "description": (
+                "Describe how prepared the agent is for the next objectives (tools crafted, shelters, "
+                "furnaces, smelted materials) and whether the inventory supports further progress."
+            ),
+            "weight": 0.1,
+        },
+    ],
+}
+
+DEFAULT_EVENTS_RUBRIC_DATA: dict[str, Any] = {
+    "version": "1",
+    "goal_text": (
+        "Score each decision in proportion to the concrete Crafter achievement progress it "
+        "delivers, topping out the scale when the log shows a fresh achievement unlock and keeping "
+        "routine upkeep near zero."
+    ),
+    "aggregation": "weighted_sum",
+    "criteria": [
+        {
+            "id": "achievement_unlocks",
+            "description": (
+                "Assign 0.9-1.0 when the decision explicitly unlocks a new Crafter achievement (look "
+                'for "Achievement unlocked" messages or equivalent deterministic completions such as '
+                "placing a furnace that immediately crafts ingots). Cap the score at 0.4 when no new "
+                "achievement fires, and drop to <=0.1 if the turn repeats known actions without "
+                "measurable progress."
+            ),
+            "weight": 0.55,
+        },
+        {
+            "id": "milestone_setup",
+            "description": (
+                "Give 0.5-0.7 when the action completes the last prerequisite for a specific upcoming "
+                "achievement (e.g., gathering the final ore before smelting, crafting sticks right "
+                "before a tool). Keep the score <=0.3 if the progress is speculative or still several "
+                "steps away."
+            ),
+            "weight": 0.2,
+        },
+        {
+            "id": "inventory_depth",
+            "description": (
+                "Reward 0.3-0.5 for pulls that clearly deepen critical buffers (fuel, food, ore) and "
+                "immediately unblock the next milestone. If resources are already plentiful or the "
+                "haul is generic filler, stay at <=0.2."
+            ),
+            "weight": 0.15,
+        },
+        {
+            "id": "execution_quality",
+            "description": (
+                "Only add up to 0.1 for clean, legal execution that avoids wasted turns; drop to 0.0 "
+                "whenever the agent idles, repeats failed moves, or takes damage without compensating "
+                "progress."
+            ),
+            "weight": 0.1,
+        },
+    ],
+}
+
 for path in (REPO_ROOT, TASK_APP_ROOT, SYNTH_ENVS_HOSTED_ROOT, EXAMPLES_ROOT):
     try:
         resolved = path.resolve()
@@ -115,6 +234,28 @@ try:
 except Exception:
     pass
 
+def _load_rubric_with_fallback(filename: str, fallback: dict[str, Any]):
+    """Load rubric from JSON file when available, otherwise use bundled fallback."""
+
+    search_paths = [RUBRICS_ROOT / filename, TASK_APP_ROOT / "rubrics" / filename]
+    for path in search_paths:
+        try:
+            if path.exists():
+                logger.debug("Loading rubric from %s", path)
+                return load_rubric(str(path))
+        except Exception as exc:
+            logger.warning("Failed to load rubric %s from %s: %s", filename, path, exc)
+
+    logger.warning("Falling back to inline rubric %s: file not available", filename)
+    try:
+        materialized = search_paths[0]
+        materialized.parent.mkdir(parents=True, exist_ok=True)
+        materialized.write_text(json.dumps(fallback, indent=2), encoding="utf-8")
+    except Exception:
+        logger.debug("Unable to materialize inline rubric %s", filename, exc_info=True)
+    return load_rubric(fallback)
+
+
 HAS_HOSTED = True
 try:
     import crafter  # type: ignore
@@ -343,9 +484,13 @@ def _base_task_info(dataset: CrafterDataset) -> TaskInfo:
     )
 
 
-OUTCOME_RUBRIC = load_rubric(str(RUBRICS_ROOT / "crafter_outcome_rubric.json"))
+OUTCOME_RUBRIC = _load_rubric_with_fallback(
+    "crafter_outcome_rubric.json", DEFAULT_OUTCOME_RUBRIC_DATA
+)
 
-EVENTS_RUBRIC = load_rubric(str(RUBRICS_ROOT / "crafter_events_rubric.json"))
+EVENTS_RUBRIC = _load_rubric_with_fallback(
+    "crafter_events_rubric.json", DEFAULT_EVENTS_RUBRIC_DATA
+)
 
 
 def describe_taskset(dataset: CrafterDataset) -> dict[str, Any]:
@@ -470,16 +615,14 @@ def _coerce_math_to_crafter(request: RolloutRequest) -> RolloutRequest:
 
     coerced = request.model_copy(update={"env": updated_env, "policy": updated_policy, "ops": ops_override})
 
-    try:
+    with suppress(Exception):
         print(
             "[rollout] remapped math request -> crafter "
             f"(env={request.env.env_name!r}→{coerced.env.env_name!r}, "
             f"policy={request.policy.policy_name!r}→{coerced.policy.policy_name!r})",
             flush=True,
         )
-    except Exception:
-        pass
-    try:
+    with suppress(Exception):
         logger.info(
             "ROLLOUT_ALIAS: remapped math env/policy to crafter (env=%s→%s, policy=%s→%s)",
             request.env.env_name,
@@ -487,15 +630,98 @@ def _coerce_math_to_crafter(request: RolloutRequest) -> RolloutRequest:
             request.policy.policy_name,
             coerced.policy.policy_name,
         )
-    except Exception:
-        pass
 
     return coerced
 
 
+def _resolve_trace_correlation_id(policy_cfg: dict[str, Any], mode: Any = None) -> str | None:
+    """Best-effort extraction of the trace correlation identifier."""
+    candidates: list[Any] = [
+        policy_cfg.get("trace_correlation_id"),
+        policy_cfg.get("trace"),
+    ]
+    logger.debug(
+        "_resolve_trace_correlation_id: inspecting policy_cfg keys=%s candidates=%s",
+        sorted(policy_cfg.keys()),
+        candidates,
+    )
+    for candidate in candidates:
+        if isinstance(candidate, str):
+            stripped = candidate.strip()
+            if stripped:
+                return stripped
+
+    return extract_trace_correlation_id(policy_cfg.get("inference_url"))
+
+
 async def rollout_executor(request: RolloutRequest, fastapi_request) -> RolloutResponse:
+    request = _coerce_math_to_crafter(request)
+
+    policy_cfg = dict(request.policy.config or {})
+    logger.info(
+        "ROLLOUT_EXEC: incoming policy config keys=%s inference_url=%s run_id=%s mode=%s",
+        sorted(policy_cfg.keys()),
+        policy_cfg.get("inference_url"),
+        request.run_id,
+        request.mode,
+    )
+    inferred_url = ensure_chat_completions_url(policy_cfg.get("inference_url"), mode=request.mode)
+    if isinstance(inferred_url, str) and inferred_url:
+        if inferred_url != policy_cfg.get("inference_url"):
+            logger.warning(
+                "ROLLOUT_EXEC: normalized inference_url run_id=%s from %s to %s",
+                request.run_id,
+                policy_cfg.get("inference_url"),
+                inferred_url,
+            )
+        policy_cfg["inference_url"] = inferred_url
+    else:
+        logger.warning(
+            "ROLLOUT_EXEC: inference_url missing or not normalized run_id=%s raw=%s",
+            request.run_id,
+            policy_cfg.get("inference_url"),
+        )
+
+    trace_correlation_id = _resolve_trace_correlation_id(policy_cfg, mode=request.mode)
+    
+    # ASSERTION: trace_correlation_id MUST be present for RL mode (but not EVAL mode)
+    if request.mode == RolloutMode.RL:
+        assert trace_correlation_id is not None, (
+            f"FATAL: trace_correlation_id extraction failed for run_id={request.run_id}. "
+            f"policy_cfg_keys={sorted(policy_cfg.keys())} "
+            f"inference_url={policy_cfg.get('inference_url')}"
+        )
+        assert isinstance(trace_correlation_id, str) and trace_correlation_id.strip(), (
+            f"FATAL: trace_correlation_id is empty for run_id={request.run_id}. "
+            f"Got: {trace_correlation_id!r}"
+        )
+    
+    if trace_correlation_id:
+        policy_cfg["trace_correlation_id"] = trace_correlation_id
+    logger.info(
+        "ROLLOUT_EXEC: resolved trace_correlation_id=%s run_id=%s",
+        trace_correlation_id,
+        request.run_id,
+    )
+
+    pipeline_metadata: dict[str, Any] = {}
+    if trace_correlation_id:
+        pipeline_metadata["trace_correlation_id"] = trace_correlation_id
+    if isinstance(policy_cfg.get("inference_url"), str) and policy_cfg["inference_url"]:
+        pipeline_metadata.setdefault("inference_url", policy_cfg["inference_url"])
+    logger.info(
+        "ROLLOUT_EXEC: pipeline metadata prepared run_id=%s metadata=%s",
+        request.run_id,
+        pipeline_metadata,
+    )
+
     # If hosted env service code is not bundled, return a no-op rollout response compatible with contracts
     if not HAS_HOSTED:
+        logger.warning(
+            "ROLLOUT_EXEC: HAS_HOSTED disabled, returning stub response run_id=%s metadata=%s",
+            request.run_id,
+            pipeline_metadata,
+        )
         return RolloutResponse(
             run_id=request.run_id,
             trajectories=[],
@@ -510,11 +736,10 @@ async def rollout_executor(request: RolloutRequest, fastapi_request) -> RolloutR
             aborted=False,
             ops_executed=0,
             trace=None,
+            trace_correlation_id=trace_correlation_id or f"trace_{request.run_id}",
+            pipeline_metadata=pipeline_metadata,
         )
 
-    request = _coerce_math_to_crafter(request)
-
-    policy_cfg = dict(request.policy.config or {})
     try:
         max_llm_calls = int(policy_cfg.get("max_llm_calls") or 10)
     except Exception:
@@ -545,6 +770,7 @@ async def rollout_executor(request: RolloutRequest, fastapi_request) -> RolloutR
         converted_ops = converted_ops[:max_ops_allowed]
     legacy_request = LegacyRolloutRequest(
         run_id=request.run_id,
+        mode=request.mode,  # Preserve mode for nested requests
         env=LegacyRolloutEnvSpec(
             env_id=request.env.env_id,
             env_name=request.env.env_name,
@@ -568,12 +794,79 @@ async def rollout_executor(request: RolloutRequest, fastapi_request) -> RolloutR
     legacy_response: LegacyRolloutResponse = await legacy_execute_rollout(
         legacy_request, fastapi_request
     )
+    logger.info(
+        "ROLLOUT_EXEC: legacy rollout completed run_id=%s trace_id=%s",
+        request.run_id,
+        trace_correlation_id,
+    )
     data = legacy_response.model_dump()
     metrics = data.get("metrics", {}) or {}
     metrics.setdefault("outcome_score", None)
     metrics.setdefault("events_score", None)
     metrics.setdefault("details", {})
     data["metrics"] = metrics
+    
+    # Add trace_correlation_id at TOP-LEVEL (REQUIRED for RL training pipeline)
+    # Use fallback if somehow missing
+    data["trace_correlation_id"] = trace_correlation_id or f"trace_{request.run_id}"
+    
+    # Add trace_correlation_id to pipeline_metadata
+    existing_meta = data.get("pipeline_metadata")
+    if not isinstance(existing_meta, dict):
+        existing_meta = {}
+    # ALWAYS set trace_correlation_id (use fallback if needed)
+    final_cid = trace_correlation_id or f"trace_{request.run_id}"
+    existing_meta["trace_correlation_id"] = final_cid
+    if isinstance(policy_cfg.get("inference_url"), str) and policy_cfg["inference_url"]:
+        existing_meta.setdefault("inference_url", policy_cfg["inference_url"])
+    data["pipeline_metadata"] = existing_meta
+    
+    # Add trace_correlation_id to each trajectory (required for RL training pipeline)
+    if "trajectories" in data:
+        for traj in data.get("trajectories", []):
+            if isinstance(traj, dict):
+                traj["trace_correlation_id"] = final_cid
+    logger.info(
+        "ROLLOUT_EXEC: final pipeline metadata run_id=%s metadata=%s",
+        request.run_id,
+        existing_meta,
+    )
+    if trace_correlation_id and existing_meta.get("trace_correlation_id") != trace_correlation_id:
+        logger.error(
+            "ROLLOUT_EXEC: metadata trace mismatch run_id=%s expected=%s actual=%s",
+            request.run_id,
+            trace_correlation_id,
+            existing_meta.get("trace_correlation_id"),
+        )
+    if not existing_meta.get("trace_correlation_id"):
+        logger.error(
+            "ROLLOUT_EXEC: final metadata missing trace_correlation_id run_id=%s metadata=%s",
+            request.run_id,
+            existing_meta,
+        )
+    
+    # ASSERTION: Verify trace_correlation_id is present in response at all required levels
+    assert "trace_correlation_id" in data, (
+        f"FATAL: trace_correlation_id missing from top-level response data for run_id={request.run_id}. "
+        f"Keys: {list(data.keys())}"
+    )
+    assert data["trace_correlation_id"] == final_cid, (
+        f"FATAL: trace_correlation_id mismatch in response for run_id={request.run_id}. "
+        f"Expected: {final_cid!r}, Got: {data.get('trace_correlation_id')!r}"
+    )
+    assert "pipeline_metadata" in data, (
+        f"FATAL: pipeline_metadata missing from response for run_id={request.run_id}"
+    )
+    assert data["pipeline_metadata"].get("trace_correlation_id") == final_cid, (
+        f"FATAL: trace_correlation_id missing or mismatched in pipeline_metadata for run_id={request.run_id}. "
+        f"Expected: {final_cid!r}, Got: {data['pipeline_metadata'].get('trace_correlation_id')!r}"
+    )
+    logger.info(
+        "ROLLOUT_EXEC: assertions passed - trace_correlation_id present in response run_id=%s cid=%s",
+        request.run_id,
+        final_cid,
+    )
+    
     return RolloutResponse.model_validate(data)
 
 
@@ -617,7 +910,7 @@ def build_config() -> TaskAppConfig:
     routers: tuple = (environment_router, policy_router, branching_router) if HAS_HOSTED else ()
 
     config = TaskAppConfig(
-        app_id="grpo-crafter",
+        app_id="grpo-crafter-task-app",
         name="GRPO Crafter Task App",
         description="Crafter Classic environment with GRPO task endpoints and LLM proxies.",
         base_task_info=base_info,
@@ -638,7 +931,7 @@ def build_config() -> TaskAppConfig:
 
 register_task_app(
     entry=TaskAppEntry(
-        app_id="grpo-crafter",
+        app_id="grpo-crafter-task-app",
         description="Crafter Classic task app with rollout + proxy endpoints",
         config_factory=build_config,
         aliases=("crafter", "crafter-task"),
@@ -665,6 +958,8 @@ register_task_app(
                 (str(REPO_ROOT), "/opt/synth_ai_repo"),
                 (str(REPO_ROOT / "synth_ai"), "/opt/synth_ai_repo/synth_ai"),
                 (str(TASK_APP_ROOT), "/opt/synth_ai_repo/examples/task_apps/crafter/task_app"),
+                # Explicitly mount rubrics directory
+                (str(RUBRICS_ROOT), "/opt/synth_ai_repo/examples/multi_step/rubrics"),
             ),
             secret_names=("groq-api-key", "openai-api-key"),
             memory=16384,

examples/task_apps/crafter/task_app/synth_envs_hosted/envs/crafter/environment.py

@@ -209,6 +209,16 @@ class CrafterEnvironmentWrapper:
             logger.info("No valid actions provided, defaulting to noop")
             normalized.append(EnvToolCall(tool="interact", args={"action": 0}))  # noop action
 
+        # Limit to first 20 actions to prevent spam from overly long tool calls
+        MAX_ACTIONS_PER_STEP = 20
+        if len(normalized) > MAX_ACTIONS_PER_STEP:
+            logger.warning(
+                "Tool call contained %d actions, limiting to first %d to prevent spam",
+                len(normalized),
+                MAX_ACTIONS_PER_STEP,
+            )
+            normalized = normalized[:MAX_ACTIONS_PER_STEP]
+
         # Pre-step logging: capture current public state and print concise summary
         before_state: dict[str, Any] | None = None
         try:

examples/task_apps/crafter/task_app/synth_envs_hosted/envs/crafter/policy.py

@@ -45,6 +45,7 @@ class CrafterPolicy(Policy):
         self.model = model
         self.use_tools = True
         self.use_vision = False  # Enable vision for VLMs
+        self.image_only_mode = False  # If True, only send images without text observations
         # Sampling parameters (populated via initialize(config))
         self.temperature: float | None = None
         self.top_p: float | None = None
@@ -58,6 +59,13 @@ class CrafterPolicy(Policy):
         self.trajectory_history: list[dict[str, Any]] = []  # env/policy step records
 
     async def initialize(self, config: dict[str, Any]) -> None:
+        # DEBUG: Log the incoming config
+        import logging
+        _logger = logging.getLogger(__name__)
+        _logger.debug(f"🔊 [POLICY_INIT] Received config keys: {list(config.keys())}")
+        _logger.debug(f"🔊 [POLICY_INIT] use_vision in config: {'use_vision' in config}, value: {config.get('use_vision')}")
+        _logger.debug(f"🔊 [POLICY_INIT] image_only_mode in config: {'image_only_mode' in config}, value: {config.get('image_only_mode')}")
+        
         if "inference_url" in config:
             self.inference_url = config["inference_url"]
         if "model" in config:
@@ -66,6 +74,12 @@ class CrafterPolicy(Policy):
             self.use_tools = bool(config["use_tools"])
         if "use_vision" in config:
             self.use_vision = bool(config["use_vision"])
+            _logger.debug(f"🔊 [POLICY_INIT] Set use_vision={self.use_vision} from config")
+        if "image_only_mode" in config:
+            self.image_only_mode = bool(config["image_only_mode"])
+            # If image_only_mode is enabled, automatically enable vision
+            if self.image_only_mode:
+                self.use_vision = True
         # Auto-detect vision capability from model name if not explicitly set
         if "use_vision" not in config and self.model:
             self.use_vision = self._is_vision_model(self.model)
@@ -91,6 +105,9 @@ class CrafterPolicy(Policy):
         self.history_messages = []
         self.turn_index = 0
         self.trajectory_history = []
+        
+        # DEBUG: Log final state
+        _logger.debug(f"🔊 [POLICY_INIT] FINAL STATE: use_vision={self.use_vision}, image_only_mode={self.image_only_mode}, model={self.model}")
 
     def _append_user_observation(self, observation_text: str) -> None:
         self.history_messages.append({"role": "user", "content": observation_text})
@@ -125,10 +142,36 @@ class CrafterPolicy(Policy):
             history=history,
             turn=turn,
             image_parts=image_parts,
+            image_only_mode=self.image_only_mode,
         )
+        
+        # DEBUG: Log message structure
+        import logging
+        _logger = logging.getLogger(__name__)
+        _logger.debug(f"🔊 [BUILD_REQUEST] Built {len(messages)} messages")
+        for idx, msg in enumerate(messages):
+            role = msg.get("role")
+            content = msg.get("content")
+            if isinstance(content, list):
+                _logger.debug(f"🔊 [BUILD_REQUEST] Message[{idx}] role={role}, content=list[{len(content)}]")
+                for part_idx, part in enumerate(content):
+                    if isinstance(part, dict):
+                        part_type = part.get("type")
+                        _logger.debug(f"🔊 [BUILD_REQUEST]   Part[{part_idx}]: type={part_type}")
+            else:
+                content_len = len(str(content)) if content else 0
+                _logger.debug(f"🔊 [BUILD_REQUEST] Message[{idx}] role={role}, content_len={content_len}")
+        
         payload: dict[str, Any] = {
             "messages": messages,
         }
+        
+        # DEBUG: Verify messages are in payload correctly
+        _logger.debug(f"🔊 [BUILD_REQUEST_PAYLOAD] Created payload with {len(payload['messages'])} messages")
+        for idx, msg in enumerate(payload["messages"]):
+            content = msg.get("content")
+            _logger.debug(f"🔊 [BUILD_REQUEST_PAYLOAD] Payload message[{idx}]: type={type(content).__name__}, is_list={isinstance(content, list)}, len={len(content) if isinstance(content, list) else len(str(content)) if content else 0}")
+        
         if self.model is not None:
             payload["model"] = self.model
         # Thinking controls
@@ -354,7 +397,18 @@ class CrafterPolicy(Policy):
             raw_candidate = metadata.get("raw_observation")
             if isinstance(raw_candidate, dict):
                 raw_observation = raw_candidate
+        
+        # DEBUG: Log image extraction
+        import logging
+        _logger = logging.getLogger(__name__)
+        _logger.debug(f"🔊 [POLICY] use_vision={self.use_vision}, has_raw_obs={raw_observation is not None}")
+        if raw_observation:
+            obs = raw_observation.get("observation", raw_observation)
+            data_url = obs.get("observation_image_data_url") if isinstance(obs, dict) else None
+            _logger.debug(f"🔊 [POLICY] has_data_url={data_url is not None}, url_preview={data_url[:50] if data_url else 'NONE'}...")
+        
         image_parts = self._extract_image_parts(raw_observation)
+        _logger.debug(f"🔊 [POLICY] Extracted {len(image_parts)} image parts")
 
         payload = self.build_inference_request(
             combined_text,
@@ -362,7 +416,17 @@ class CrafterPolicy(Policy):
             turn=self.turn_index,
             image_parts=image_parts,
         )
-        # print("Debugging only:; ", payload)
+        
+        # DEBUG: Verify payload before returning
+        _logger.debug(f"🔊 [POLICY_STEP_RETURN] About to return payload with {len(payload.get('messages', []))} messages")
+        for idx, msg in enumerate(payload.get("messages", [])):
+            content = msg.get("content")
+            _logger.debug(f"🔊 [POLICY_STEP_RETURN] Return message[{idx}]: type={type(content).__name__}, is_list={isinstance(content, list)}")
+            if isinstance(content, list):
+                _logger.debug(f"🔊 [POLICY_STEP_RETURN]   Content list has {len(content)} items")
+                # Add assertion to catch corruption early
+                assert len(content) > 0, f"Message content list is empty! This should contain images."
+        
         meta_out = {
             "inference_url": self.inference_url,
             "inference_request": payload,
@@ -417,14 +481,21 @@ class CrafterPolicy(Policy):
         """Prepare an inference request (implementing abstract method)."""
         # Format observation with rich contextual information
         observation_text = self._format_observation_for_llm(observation)
-        image_parts = self._extract_image_parts(observation)
-
-        # Build messages (observation_text already formatted; no raw matrices)
+        
+        # Extract image parts based on vision settings
+        if self.use_vision:
+            image_parts = self._extract_image_parts(observation)
+        else:
+            # Text-only mode: don't include any images
+            image_parts = []
+
+        # Build messages with appropriate mode
         messages = CrafterReActAgent.build_messages(
             observation=observation_text,
             history=history,
             turn=self.turn_index,
             image_parts=image_parts,
+            image_only_mode=self.image_only_mode,
         )
 
         # Return messages and tools schema

examples/task_apps/crafter/task_app/synth_envs_hosted/envs/crafter/react_agent.py

@@ -85,8 +85,17 @@ class CrafterReActAgent:
         history: list[dict[str, Any]] | None = None,
         turn: int | None = None,
         image_parts: list[dict[str, Any]] | None = None,
+        image_only_mode: bool = False,
     ) -> list[dict[str, Any]]:
-        """Construct OpenAI-style messages list for vLLM generation."""
+        """Construct OpenAI-style messages list for vLLM generation.
+        
+        Args:
+            observation: Text observation to include
+            history: Previous conversation history
+            turn: Current turn number
+            image_parts: Image content parts in OpenAI format
+            image_only_mode: If True, only include images without text observation
+        """
         msgs: list[dict[str, Any]] = [
             {"role": "system", "content": CrafterReActAgent.get_system_prompt()}
         ]
@@ -94,8 +103,14 @@ class CrafterReActAgent:
             msgs.extend(history)
         user_content: Any
         if image_parts:
-            user_content = [{"type": "text", "text": observation}] + list(image_parts)
+            # Image-only mode: send only images without text observation
+            if image_only_mode:
+                user_content = list(image_parts)
+            else:
+                # Normal vision mode: send both text and images
+                user_content = [{"type": "text", "text": observation}] + list(image_parts)
         else:
+            # Text-only mode (default): no images
             user_content = observation
         msgs.append({"role": "user", "content": user_content})
         return msgs