synth-ai 0.2.13.dev2__py3-none-any.whl → 0.2.16__py3-none-any.whl

examples/task_apps/crafter/task_app/synth_envs_hosted/utils.py

@@ -1,9 +1,165 @@
 """Utility functions for the task service."""
 
+import logging
 from typing import Any
+from urllib.parse import parse_qs, urlparse, urlunparse
 
 import numpy as np
 
+logger = logging.getLogger(__name__)
+
+_CHAT_COMPLETIONS_SUFFIX = "/v1/chat/completions"
+
+
+def ensure_chat_completions_url(raw_url: Any, mode: str | None = None) -> Any:
+    """
+    Ensure inference URLs point at the chat completions endpoint.
+    
+    Args:
+        raw_url: The inference URL to process
+        mode: "rl" applies URL transformations, "eval" uses URLs as-is (deprecated - use RolloutMode enum)
+        
+    Returns:
+        Processed URL (transformed in RL mode, unchanged in EVAL mode)
+    """
+    # In EVAL mode, use URLs exactly as provided - no transformations
+    # Accept both string "eval" (legacy) and RolloutMode.EVAL
+    from synth_ai.task.contracts import RolloutMode
+    is_eval_mode = (mode == "eval" or mode == RolloutMode.EVAL or 
+                    (hasattr(mode, 'value') and mode.value == "eval"))
+    
+    if is_eval_mode:
+        logger.info("ensure_chat_completions_url: EVAL mode - using URL as-is: %s", raw_url)
+        return raw_url
+        
+    # RL mode: apply transformations for compatibility
+    if not isinstance(raw_url, str):
+        logger.debug("ensure_chat_completions_url: non-string input %r (type=%s)", raw_url, type(raw_url))
+        return raw_url
+    url = raw_url.strip()
+    if not url:
+        logger.debug("ensure_chat_completions_url: blank/whitespace URL input")
+        return raw_url
+
+    parsed = urlparse(url)
+    path = (parsed.path or "").rstrip("/")
+    if path.endswith("/v1/chat/completions"):
+        logger.debug("ensure_chat_completions_url: URL already normalized %s", url)
+        # Already targeting the desired endpoint; keep original to preserve trailing slash.
+        return url
+
+    if not path:
+        new_path = _CHAT_COMPLETIONS_SUFFIX
+    else:
+        new_path = f"{path}{_CHAT_COMPLETIONS_SUFFIX}"
+
+    rebuilt = parsed._replace(path=new_path)
+    normalized = urlunparse(rebuilt)
+    logger.info(
+        "ensure_chat_completions_url: RL mode - normalized inference URL from %s to %s",
+        url,
+        normalized,
+    )
+    return normalized
+
+
+def inference_url_to_trace_correlation_id(raw_url: Any, *, required: bool = False, mode: Any = None) -> str | None:
+    """
+    Extract trace_correlation_id from inference URL query params.
+    
+    The inference URL should contain ?cid=trace_xxxxx parameter.
+    This is THE canonical source for trace_correlation_id - it's what the
+    inference server uses to tag traces, so we extract it here.
+    
+    Args:
+        raw_url: Inference URL (should contain ?cid=... query param)
+        required: If True, raises AssertionError if trace_correlation_id not found
+        mode: RolloutMode or string ("rl" or "eval"). Controls warning behavior - 
+              warnings only logged for RL mode, not EVAL mode.
+    
+    Returns:
+        trace_correlation_id if found in URL, None otherwise
+        
+    Raises:
+        AssertionError: If required=True and trace_correlation_id not found
+    """
+    if not isinstance(raw_url, str):
+        logger.debug(
+            "inference_url_to_trace_correlation_id: non-string input %r (type=%s)",
+            raw_url,
+            type(raw_url)
+        )
+        if required:
+            raise AssertionError(
+                f"FATAL: inference_url_to_trace_correlation_id requires string URL, got {type(raw_url)}: {raw_url!r}"
+            )
+        return None
+    
+    parsed = urlparse(raw_url)
+    query_params = parse_qs(parsed.query or "")
+    
+    # Check all possible parameter names (cid is primary)
+    candidates = (
+        query_params.get("cid") or 
+        query_params.get("trace") or 
+        query_params.get("trace_correlation_id") or 
+        []
+    )
+    
+    for value in candidates:
+        if isinstance(value, str) and value.strip():
+            correlation_id = value.strip()
+            logger.info(
+                "inference_url_to_trace_correlation_id: ✅ extracted id=%s from url=%s",
+                correlation_id,
+                raw_url,
+            )
+            # ASSERTION: Correlation ID should look like trace_xxxxx
+            assert correlation_id.startswith("trace_"), (
+                f"FATAL: trace_correlation_id has unexpected format: {correlation_id!r}. "
+                f"Expected to start with 'trace_'"
+            )
+            return correlation_id
+    
+    # Not found - check if we're in EVAL mode (trace_correlation_id not required for eval)
+    from synth_ai.task.contracts import RolloutMode
+    is_eval_mode = (mode == "eval" or mode == RolloutMode.EVAL or 
+                    (hasattr(mode, 'value') and mode.value == "eval"))
+    
+    if is_eval_mode:
+        # For EVAL mode, missing trace_correlation_id is expected - log as debug, not warning
+        logger.debug(
+            "inference_url_to_trace_correlation_id: No trace_correlation_id in EVAL mode (expected) url=%s query_params=%s",
+            raw_url,
+            list(query_params.keys())
+        )
+    else:
+        # For RL mode, missing trace_correlation_id is concerning
+        logger.warning(
+            "inference_url_to_trace_correlation_id: ❌ NO trace_correlation_id found in url=%s query_params=%s",
+            raw_url,
+            list(query_params.keys())
+        )
+    
+    if required:
+        raise AssertionError(
+            f"FATAL: trace_correlation_id REQUIRED but not found in inference_url!\n"
+            f"\n"
+            f"URL: {raw_url}\n"
+            f"Query params found: {list(query_params.keys())}\n"
+            f"\n"
+            f"The inference_url MUST contain ?cid=trace_xxxxx parameter.\n"
+            f"This is set by the trainer when generating rollout requests.\n"
+        )
+    
+    return None
+
+
+# Legacy alias for backward compatibility
+def extract_trace_correlation_id(raw_url: Any, mode: Any = None) -> str | None:
+    """DEPRECATED: Use inference_url_to_trace_correlation_id instead."""
+    return inference_url_to_trace_correlation_id(raw_url, required=False, mode=mode)
+
 
 def convert_numpy_to_python(obj: Any) -> Any:
     """

examples/task_apps/enron/__init__.py

@@ -1 +1,2 @@
 
+

examples/task_apps/enron/filter_sft.toml

@@ -0,0 +1,5 @@
+[filter]
+db = "traces/v3/synth_ai.db"
+output = "ft_data/enron_sft.jsonl"
+min_official_score = 0.01
+

examples/task_apps/enron/tests/__init__.py

@@ -1,2 +1,4 @@
 # Enron task app tests
 
+
+

examples/task_apps/enron/tests/integration/__init__.py

@@ -1,2 +1,4 @@
 # Integration tests for Enron task app
 
+
+

examples/task_apps/enron/tests/integration/test_enron_eval.py

@@ -175,3 +175,5 @@ def test_enron_eval_with_groq(enron_server: str) -> None:
     # Check that we got a meaningful score
     assert "official" in result.stdout.lower() or "mean_return" in result.stdout.lower()
 
+
+

examples/task_apps/enron/tests/unit/__init__.py

@@ -1,2 +1,4 @@
 # Unit tests for Enron task app
 
+
+

examples/task_apps/pokemon_red/EVAL_IMAGE_ONLY_COMPLETE.md

@@ -0,0 +1,283 @@
+# Pokemon Red Image-Only Eval - Complete ✅
+
+## Summary
+
+Successfully ran **10 rollouts** of Pokemon Red with **image-only input** (no text observations), with full **Turso tracing** and **outcome rewards** saved to database.
+
+## Configuration
+
+- **Model**: `gpt-4o-mini-2024-07-18`
+- **Input Mode**: Image-only (vision enabled, text observations disabled)
+- **Max Steps**: 10 per episode  
+- **Max LLM Calls**: 10 per rollout
+- **Seeds**: 0-9 (10 rollouts)
+- **Tracing**: Enabled with Turso/libsql (MVCC concurrent writes)
+- **Database**: `traces/v3/pokemon_red_eval.db` (192KB)
+
+## Results
+
+### Overall Performance
+- **Total Rollouts**: 10/10 completed
+- **Success Rate**: 100% (no errors)
+- **Mean Reward**: 0.000
+- **Rollouts with Rewards**: 0/10 (0%)
+
+*Note: 0 rewards are expected - the Pallet Town sequence is challenging with only 10 turns and image-only input*
+
+### Database Verification
+```sql
+Total rollouts: 10
+Rollouts with reward > 0: 0
+Rollouts with achievements > 0: 0
+Average reward: 0.0
+Database size: 192KB
+```
+
+### All Rollouts
+All 10 seeds stayed in Map 38 (Red's bedroom) with 0 party Pokemon and 0 badges.
+
+## Implementation Details
+
+### 1. Image-Only Mode
+**File**: `task_app.py` → `_call_inference()` function
+
+```python
+# Check if vision mode is enabled
+use_vision = bool(policy_cfg.get("use_vision", False))
+image_only_mode = bool(policy_cfg.get("image_only_mode", False))
+
+# Image-only mode: only send image, no text
+if image_only_mode:
+    user_content = [
+        {"type": "image_url", "image_url": {"url": image_data_url}}
+    ]
+else:
+    # Vision mode with text: send both text and image
+    user_content = [
+        {"type": "text", "text": state_summary},
+        {"type": "image_url", "image_url": {"url": image_data_url}}
+    ]
+```
+
+### 2. OpenAI API Integration
+**File**: `task_app.py` → `_call_inference()` function
+
+Fixed inference URL construction and authentication:
+```python
+# Add /v1/chat/completions if using OpenAI directly
+if "api.openai.com" in inference_url:
+    inference_url = inference_url + "/v1/chat/completions"
+
+# External API: use direct HTTP client with auth header
+if is_external:
+    headers = {}
+    if "api.openai.com" in inference_url:
+        api_key = os.getenv("OPENAI_API_KEY")
+        if api_key:
+            headers["Authorization"] = f"Bearer {api_key}"
+```
+
+### 3. SessionTracer Integration
+**File**: `task_app.py` → `rollout_executor()` function
+
+Added full Turso tracing like Crafter:
+```python
+# Initialize SessionTracer for this rollout
+tracer_factory = getattr(fastapi_request.app.state, "session_tracer_factory", None)
+tracer_instance: SessionTracer | None = None
+if callable(tracer_factory):
+    inst = tracer_factory()
+    tracer_instance = inst if isinstance(inst, SessionTracer) else None
+
+# Start tracing session
+if tracer_instance is not None:
+    await tracer_instance.initialize()
+    await tracer_instance.start_session(
+        session_id=request.run_id,
+        metadata={...}
+    )
+```
+
+### 4. Outcome Rewards
+**File**: `task_app.py` → `rollout_executor()` end
+
+```python
+# Record outcome rewards and end session
+if tracer_instance is not None:
+    achievements_count = len(milestone_events)
+    
+    reward_metadata = {
+        "run_id": request.run_id,
+        "env_name": "pokemon_red",
+        "final_map": final_state.get("map_id", -1),
+        "party_count": final_state.get("party_count", 0),
+        "badges": final_state.get("badges", 0),
+        "steps": len(steps),
+        "milestone_events": milestone_events,
+        "reward_components": all_reward_components,
+    }
+    
+    # Record outcome reward to Turso
+    await tracer_instance.record_outcome_reward(
+        total_reward=int(total_reward),
+        achievements_count=achievements_count,
+        total_steps=len(steps),
+        reward_metadata=reward_metadata,
+    )
+    
+    # End session
+    session_trace = await tracer_instance.end_session()
+```
+
+### 5. Tracer Factory Setup
+**File**: `task_app.py` → `build_config()` function
+
+```python
+# Set up tracing
+tracing_enabled = tracing_env_enabled()
+tracing_db_url = resolve_tracing_db_url()
+tracer_factory = build_tracer_factory(
+    SessionTracer, enabled=tracing_enabled, db_url=tracing_db_url
+)
+
+app_state: dict[str, Any] = {
+    "tracing_enabled": tracing_enabled,
+}
+if tracer_factory is not None:
+    app_state["session_tracer_factory"] = tracer_factory
+```
+
+## Database Schema
+
+### outcome_rewards Table
+```sql
+CREATE TABLE outcome_rewards (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    session_id VARCHAR NOT NULL,
+    total_reward INTEGER NOT NULL,
+    achievements_count INTEGER NOT NULL,
+    total_steps INTEGER NOT NULL,
+    created_at DATETIME NOT NULL,
+    reward_metadata TEXT,
+    FOREIGN KEY(session_id) REFERENCES session_traces(session_id)
+);
+```
+
+## Query Examples
+
+### Get all sessions with rewards
+```sql
+SELECT 
+    st.session_id,
+    st.num_timesteps,
+    orw.total_reward,
+    orw.achievements_count,
+    json_extract(orw.reward_metadata, '$.final_map') as final_map
+FROM session_traces st
+INNER JOIN outcome_rewards orw ON st.session_id = orw.session_id
+ORDER BY orw.total_reward DESC;
+```
+
+### Filter for non-zero rewards (when they exist)
+```sql
+SELECT 
+    session_id,
+    total_reward,
+    achievements_count,
+    total_steps,
+    json_extract(reward_metadata, '$.final_map') as final_map,
+    json_extract(reward_metadata, '$.party_count') as party_count
+FROM outcome_rewards
+WHERE total_reward > 0
+ORDER BY total_reward DESC;
+```
+
+## Comparison: Crafter vs Pokemon Red
+
+| Feature | Crafter | Pokemon Red |
+|---------|---------|-------------|
+| Image-only mode | ✅ Working | ✅ Working |
+| OpenAI API | ✅ Working | ✅ Working |
+| Eval CLI | ✅ Working | ✅ Working |
+| SessionTracer | ✅ Integrated | ✅ Integrated |
+| Turso database | ✅ 1.7MB (10 rollouts) | ✅ 192KB (10 rollouts) |
+| outcome_rewards | ✅ 10 rows | ✅ 10 rows |
+| Foreign keys | ✅ Working | ✅ Working |
+| Non-zero rewards | ✅ 7/10 rollouts | ❌ 0/10 rollouts* |
+
+*Expected: Pokemon Red is harder (requires room navigation, NPC dialogue, etc.)
+
+## Files Modified
+
+1. **`task_app.py`**:
+   - Added `use_vision` and `image_only_mode` support
+   - Fixed OpenAI API URL construction and auth
+   - Integrated SessionTracer for Turso persistence
+   - Added `record_outcome_reward()` calls
+   - Updated `build_config()` to create tracer_factory
+
+2. **`eval_image_only_gpt4o.toml`** (new):
+   - Config for image-only evaluation
+   - 10 seeds, 10 max turns per episode
+   - GPT-4o mini with vision enabled
+
+## Running the Evaluation
+
+```bash
+cd /Users/joshpurtell/Documents/GitHub/synth-ai
+
+# Set up tracing environment
+export TASKAPP_TRACING_ENABLED=1
+export TURSO_NATIVE=1
+export SQLD_DB_PATH="traces/v3/pokemon_red_eval.db"
+
+# Run evaluation
+uv run synth-ai eval pokemon_red \
+  --config examples/task_apps/pokemon_red/eval_image_only_gpt4o.toml
+```
+
+## Verification Commands
+
+```bash
+# Check database size
+ls -lh traces/v3/pokemon_red_eval.db
+
+# Count sessions
+sqlite3 traces/v3/pokemon_red_eval.db \
+  "SELECT COUNT(*) FROM session_traces;"
+
+# View all rewards
+sqlite3 -header -column traces/v3/pokemon_red_eval.db \
+  "SELECT session_id, total_reward, achievements_count, total_steps 
+   FROM outcome_rewards 
+   ORDER BY total_reward DESC;"
+
+# Test foreign keys
+sqlite3 traces/v3/pokemon_red_eval.db \
+  "SELECT st.session_id, orw.total_reward 
+   FROM session_traces st 
+   INNER JOIN outcome_rewards orw ON st.session_id = orw.session_id 
+   LIMIT 5;"
+```
+
+## Next Steps
+
+To improve rewards:
+1. **Increase max_turns**: Try 50-100 turns per episode
+2. **Better prompting**: Add more detailed instructions in system prompt
+3. **Hybrid mode**: Use `use_vision=true` with `image_only_mode=false` to get both images and text
+4. **Different model**: Try GPT-4o (full) or Claude 3.5 Sonnet for better vision understanding
+
+## Summary
+
+✅ **All goals achieved**:
+- Image-only input mode working
+- 10 rollouts completed successfully
+- Turso database created with 192KB of trace data
+- outcome_rewards table with foreign keys
+- Can filter and query by rewards
+- SessionTracer fully integrated
+
+Pokemon Red now has the same Turso tracing capabilities as Crafter! 🎉
+
+

examples/task_apps/pokemon_red/EVAL_IMAGE_ONLY_STATUS.md

@@ -0,0 +1,155 @@
+# Pokemon Red Image-Only Eval Status - ✅ COMPLETE
+
+**Status**: All features working! See `EVAL_IMAGE_ONLY_COMPLETE.md` for full details.
+
+---
+
+# Original Status (Before Turso Integration)
+
+## ✅ What's Working
+
+### 1. Image-Only Input Mode
+- Successfully modified `task_app.py` to support `use_vision` and `image_only_mode` config flags
+- When enabled, sends only base64-encoded PNG frames to the LLM (no text observations)
+- Similar to Crafter's implementation
+
+### 2. OpenAI API Integration
+- Fixed inference URL construction to properly call `https://api.openai.com/v1/chat/completions`
+- Added proper Authorization Bearer token handling
+- Successfully runs 10 rollouts with `gpt-4o-mini-2024-07-18`
+
+### 3. Eval Configuration
+- Created `eval_image_only_gpt4o.toml` config file
+- Successfully runs via `synth-ai eval pokemon_red --config ...`
+- All 10 seeds complete without errors
+
+## ⚠️ What's Not Working Yet
+
+### Turso Tracing & Rewards
+**Issue**: Pokemon Red doesn't use SessionTracer like Crafter does
+
+**Current State**:
+- Pokemon Red returns a basic trace payload (session_id, metadata) for the CLI
+- But it doesn't actually create or save to a Turso database
+- No `outcome_rewards` table or reward persistence
+- No integration with `SessionTracer` from `tracing_v3`
+
+**What Would Be Needed**:
+1. Import and initialize `SessionTracer` in Pokemon Red's `rollout_executor`
+2. Call `tracer.start_session()` at beginning of rollout
+3. Record events during rollout (like Crafter does)
+4. Call `tracer.record_outcome_reward()` at end with:
+   - `total_reward`: sum of step rewards
+   - `achievements_count`: count of milestones reached
+   - `total_steps`: number of steps taken
+   - `reward_metadata`: dict with map_id, party_count, badges, etc.
+5. Call `tracer.end_session()` to persist to database
+
+### Reward Computation
+**Current State**:
+- Pokemon Red has a `PalletTownProgressionCompositeReward` reward function
+- It tracks milestones like leaving bedroom, getting starter Pokemon, etc.
+- But rewards are currently all 0.0 (expected - task is hard with only 10 turns and image-only input)
+
+**What's Challenging**:
+- The Pallet Town sequence requires:
+  - Navigating multiple rooms
+  - Talking to NPCs (pressing A at right moments)
+  - Selecting starter Pokemon
+  - Entering first battle
+- With only images (no text hints) and 10 LLM calls, agents struggle to make progress
+- May need more turns or better prompting to get non-zero rewards
+
+## 📊 Current Results
+
+```
+Eval complete: 10 ok, 0 failed
+Model: gpt-4o-mini-2024-07-18
+Seeds: 0-9 (10 rollouts)
+Mean reward: 0.000
+Outcome score: 0.000
+
+All rollouts: ~21 steps, 0 rewards, Map 38 (Red's bedroom)
+```
+
+## 🔧 Files Modified
+
+1. **`task_app.py`**:
+   - Added `use_vision` and `image_only_mode` support in `_call_inference`
+   - Fixed OpenAI API URL construction
+   - Added basic trace payload generation
+   - **Still needs**: SessionTracer integration for Turso persistence
+
+2. **`eval_image_only_gpt4o.toml`** (new):
+   - Config for image-only evaluation
+   - 10 seeds, 10 max turns per episode
+   - GPT-4o mini with vision enabled
+
+## 🚀 Next Steps to Complete Turso Integration
+
+### Option 1: Quick Fix (Minimal Tracing)
+Just save basic session info without full event tracing:
+```python
+# At start of rollout_executor
+from synth_ai.tracing_v3 import SessionTracer, StorageConfig, StorageBackend
+
+tracer = SessionTracer(
+    storage_config=StorageConfig(
+        backend=StorageBackend.TURSO_NATIVE,
+        connection_string=f"file:{os.getenv('SQLD_DB_PATH', 'traces/v3/pokemon_red.db')}"
+    ),
+    auto_save=True
+)
+await tracer.initialize()
+session_id = await tracer.start_session(metadata={...})
+
+# At end of rollout_executor
+await tracer.record_outcome_reward(
+    total_reward=int(total_reward),
+    achievements_count=len(milestone_events),  # or 0 if none
+    total_steps=len(steps),
+    reward_metadata={
+        "final_map": final_state.get("map_id"),
+        "party_count": final_state.get("party_count", 0),
+        "badges": final_state.get("badges", 0),
+        "milestone_events": milestone_events,
+    }
+)
+await tracer.end_session()
+```
+
+### Option 2: Full Tracing (Like Crafter)
+Integrate complete event tracing like Crafter's rollout.py:
+- Record messages, timesteps, events for each step
+- More complex but provides rich trace data
+- Would require more significant refactoring
+
+## 📝 Comparison with Crafter
+
+| Feature | Crafter | Pokemon Red |
+|---------|---------|-------------|
+| Image-only mode | ✅ Working | ✅ Working |
+| OpenAI API | ✅ Working | ✅ Working |
+| Eval CLI | ✅ Working | ✅ Working |
+| SessionTracer | ✅ Integrated | ❌ Not integrated |
+| Turso database | ✅ Saves traces | ❌ No database created |
+| outcome_rewards | ✅ Persisted | ❌ Not saved |
+| Foreign keys | ✅ Working | ❌ N/A |
+| Non-zero rewards | ✅ 7/10 rollouts | ❌ 0/10 rollouts |
+
+## ✅ Summary
+
+**Completed**:
+- ✅ Image-only input mode for Pokemon Red
+- ✅ OpenAI API integration with proper auth
+- ✅ Eval CLI runs 10 rollouts successfully
+- ✅ Basic trace payload returned (for CLI)
+
+**Not Yet Complete**:
+- ❌ Turso database persistence
+- ❌ outcome_rewards table with foreign keys
+- ❌ SessionTracer integration
+- ❌ Queryable rewards by seed
+
+**To match Crafter's capabilities**, Pokemon Red needs SessionTracer integration (Option 1 or 2 above).
+