synth-ai 0.2.9.dev3__tar.gz → 0.2.9.dev5__tar.gz

{synth_ai-0.2.9.dev3 → synth_ai-0.2.9.dev5}/MANIFEST.in

@@ -4,6 +4,11 @@ include synth_ai/install_sqld.sh
 recursive-include synth_ai *.py
 # Only include lightweight config/data files from core packages
 recursive-include synth_ai *.toml
+# Include examples for task app discovery
+recursive-include examples *.py
+recursive-include examples *.toml
+recursive-include examples *.md
+recursive-include examples *.sh
 
 # Prune heavy example/demo/test/data trees from the sdist (keep code)
 prune tests

{synth_ai-0.2.9.dev3/synth_ai.egg-info → synth_ai-0.2.9.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synth-ai
-Version: 0.2.9.dev3
+Version: 0.2.9.dev5
 Summary: RL as a service SDK - Core AI functionality and tracing
 Author-email: Synth AI <josh@usesynth.ai>
 License-Expression: MIT

synth_ai-0.2.9.dev5/examples/analyze_semantic_words.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Run Crafter agent and analyze semantic map word distribution
+# This script demonstrates semantic analysis of agent observations
+# Output: Markdown tables and JSON data (no plotting dependencies)
+
+echo "🔍 Analyzing semantic map words from Crafter agent..."
+echo "Make sure the synth-ai service is running: uvx synth-ai serve"
+echo ""
+
+cd synth_ai/environments/examples/crafter_classic/agent_demos/
+
+# Run the semantic analysis (markdown output only)
+python analyze_semantic_words_markdown.py --model gemini-1.5-flash --episodes 3 --max-turns 30
+
+echo ""
+echo "✅ Analysis complete! Check the generated markdown report and JSON files."

synth_ai-0.2.9.dev5/examples/common_old/backend.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from synth_ai.config.base_url import get_backend_from_env, PROD_BASE_URL_DEFAULT
+
+
+DEFAULT_PROD_BACKEND = f"{PROD_BASE_URL_DEFAULT.rstrip('/')}/api"
+
+
+def resolve_backend_url() -> str:
+    """Resolve backend base URL honoring BACKEND_OVERRIDE and env overrides.
+
+    Always returns a URL ending with /api.
+    """
+    base, _ = get_backend_from_env()
+    base = base.rstrip("/")
+    return base if base.endswith("/api") else f"{base}/api"
+
+
+if __name__ == "__main__":
+    print(resolve_backend_url())
+

synth_ai-0.2.9.dev5/examples/crafter_debug_render.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""
+Quick local Crafter observation inspector.
+
+- Initializes a CrafterClassic env via local service (default http://localhost:8901)
+- Fetches one observation
+- Renders a 7x7 semantic view around the player with best-effort item names
+- Prints status (health/food/energy), inventory, and achievements
+
+Run:
+  uv run python examples/crafter_debug_render.py --base-url http://localhost:8901 --seed 1
+"""
+import argparse
+import math
+import os
+from typing import Any, Dict, List
+
+import httpx
+
+
+def try_import_crafter_mapping():
+    try:
+        import crafter  # type: ignore
+
+        env = crafter.Env()
+        try:
+            max_id = (
+                max(max(env._world._mat_ids.values()), max(env._sem_view._obj_ids.values())) + 1
+            )
+            id_to_item = ["void"] * max_id
+            for name, ind in env._world._mat_ids.items():
+                label = name.__name__ if hasattr(name, "__name__") else str(name)
+                id_to_item[ind] = label.lower()
+            for name, ind in env._sem_view._obj_ids.items():
+                label = name.__name__ if hasattr(name, "__name__") else str(name)
+                id_to_item[ind] = label.lower()
+            return id_to_item
+        finally:
+            try:
+                env.close()
+            except Exception:
+                pass
+    except Exception:
+        return None
+
+
+def format_semantic_map_view(obs: Dict[str, Any], view_size: int = 7) -> str:
+    sem = obs.get("semantic_map") or obs.get("sem_map") or obs.get("map")
+    if sem is None:
+        return "No semantic map available"
+
+    # Normalize to 2D grid
+    grid: List[List[int]]
+    if isinstance(sem, list) and sem and isinstance(sem[0], list):
+        grid = sem
+    elif isinstance(sem, list):
+        try:
+            n = int(math.sqrt(len(sem)))
+            if n * n != len(sem) or n == 0:
+                return "Semantic map format not recognized"
+            grid = [sem[i * n : (i + 1) * n] for i in range(n)]
+        except Exception:
+            return "Semantic map format not recognized"
+    else:
+        return "Semantic map format not recognized"
+
+    rows = len(grid)
+    cols = len(grid[0]) if rows > 0 else 0
+    if rows == 0 or cols == 0:
+        return "Empty semantic map"
+
+    # Resolve item mapping if available
+    id_to_item = try_import_crafter_mapping()
+
+    # Player position if provided; otherwise center
+    ppos = obs.get("player_position") or [rows // 2, cols // 2]
+    try:
+        px = int(ppos[0])
+        py = int(ppos[1])
+    except Exception:
+        px, py = rows // 2, cols // 2
+
+    half = max(1, view_size // 2)
+    lines: List[str] = []
+    visible: set[str] = set()
+    for dy in range(-half, half + 1):
+        row_cells: List[str] = []
+        for dx in range(-half, half + 1):
+            x = px + dx
+            y = py + dy
+            if dx == 0 and dy == 0:
+                row_cells.append("you")
+            elif 0 <= x < rows and 0 <= y < cols:
+                try:
+                    val = int(grid[x][y])
+                except Exception:
+                    val = -1
+                if id_to_item and 0 <= val < len(id_to_item):
+                    name = id_to_item[val]
+                else:
+                    # Fallback: simple mapping for common ids (best-effort)
+                    name = {
+                        0: "grass",
+                        1: "stone",
+                        2: "stone",
+                        3: "tree",
+                        4: "coal",
+                        5: "iron",
+                        6: "water",
+                        7: "zombie",
+                        14: "wood",
+                    }.get(val, str(val))
+                row_cells.append(name)
+                if name not in {"grass", "you", "0"}:
+                    visible.add(name)
+            else:
+                row_cells.append("void")
+        lines.append(" ".join(row_cells))
+
+    legend = f"Visible items: {', '.join(sorted(visible))}" if visible else "No notable items visible"
+    return "\n".join(lines) + "\n" + legend
+
+
+async def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base-url", default=os.getenv("CRAFTER_BASE_URL", "http://localhost:8901"))
+    parser.add_argument("--seed", type=int, default=1)
+    args = parser.parse_args()
+
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        init = await client.post(
+            f"{args.base_url}/env/CrafterClassic/initialize",
+            json={"config": {"difficulty": "easy", "seed": args.seed}},
+        )
+        init.raise_for_status()
+        data = init.json()
+        env_id = data["env_id"]
+        obs = data["observation"]
+
+        print("=== INITIAL OBSERVATION ===")
+        print(format_semantic_map_view(obs, view_size=7))
+        inv = obs.get("inventory", {})
+        ach = obs.get("achievements_status", {})
+        print("\n=== STATUS ===")
+        print(f"Health: {obs.get('health', 10)}/10")
+        print(f"Hunger: {obs.get('food', 10)}/10")
+        print(f"Energy: {obs.get('energy', 10)}/10")
+        inv_items = ", ".join([f"{k}: {v}" for k, v in inv.items() if v]) if isinstance(inv, dict) else str(inv)
+        print(f"Inventory: {inv_items if inv_items else 'empty'}")
+        if isinstance(ach, dict):
+            unlocked = sum(1 for v in ach.values() if v)
+            print(f"Achievements: {unlocked}/{len(ach)} unlocked")
+
+        # Take one step right to get a new obs
+        step = await client.post(
+            f"{args.base_url}/env/CrafterClassic/step",
+            json={
+                "env_id": env_id,
+                "action": {"tool_calls": [{"tool": "interact", "args": {"action": 2}}]},
+            },
+        )
+        step.raise_for_status()
+        sdata = step.json()
+        sobs = sdata["observation"]
+        print("\n=== NEXT OBSERVATION (after move_right) ===")
+        print(format_semantic_map_view(sobs, view_size=7))
+
+        # Cleanup
+        try:
+            await client.post(f"{args.base_url}/env/CrafterClassic/terminate", json={"env_id": env_id})
+        except Exception:
+            pass
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())
+
+

synth_ai-0.2.9.dev5/examples/evals_old/README.md

@@ -0,0 +1,98 @@
+# Crafter Model Comparison Cookbook
+
+This cookbook demonstrates how to run parallel experiments comparing different language models on the Crafter environment, with robust timeout handling and performance analysis.
+
+## Overview
+
+This cookbook runs episodes of the Crafter game environment with different language models (e.g., gpt-5-nano and Qwen/Qwen3-32B-Instruct) in parallel, collecting performance metrics and analyzing the results.
+
+## Features
+
+- **Parallel episode execution**: Runs multiple episodes simultaneously for faster experimentation
+- **Timeout handling**: 
+  - Turn-level timeout (20s per LLM call)
+  - Episode-level timeout (180s total)
+  - Action execution timeout (5s)
+- **Progress tracking**: Real-time progress bars showing steps across all episodes
+- **Performance comparison**: Analyzes achievements, invalid action rates, and model usage statistics
+- **Deterministic seeding**: Uses consecutive seeds for reproducible experiments
+
+## Prerequisites
+
+1. Ensure the Crafter environment service is running:
+   ```bash
+   cd synth-ai/
+   bash serve.sh
+   ```
+
+2. Set up your API keys (Synth/OpenAI or provider as needed):
+   ```bash
+   export OPENAI_API_KEY="your-api-key"
+   ```
+
+## Usage
+
+Run the comparison script (gpt-5-nano vs Qwen/Qwen3-32B-Instruct):
+```bash
+uvpm examples.evals.compare_models --episodes 5 --max-turns 100 --difficulty easy \
+  --models "gpt-5-nano" "Qwen/Qwen3-32B-Instruct"
+```
+
+Or with custom parameters:
+```bash
+python compare_models.py \
+    --episodes 10 \
+    --max-turns 100 \
+    --difficulty easy \
+    --models "gpt-5-nano" "Qwen/Qwen3-32B-Instruct" \
+    --base-seed 1000 \
+    --turn-timeout 30.0 \
+    --episode-timeout 300.0
+```
+
+## Parameters
+
+- `--episodes`: Number of episodes per model (default: 5)
+- `--max-turns`: Maximum turns per episode (default: 50)
+- `--difficulty`: Game difficulty - easy, medium, hard (default: easy)
+- `--models`: Models to test (default: gpt-4o-mini gpt-4.1-mini)
+- `--base-seed`: Starting seed for episodes (default: 1000)
+- `--turn-timeout`: Timeout per turn in seconds (default: 20.0)
+- `--episode-timeout`: Total timeout per episode in seconds (default: 180.0)
+
+## Output
+
+The script produces:
+1. Real-time progress bars showing episode execution
+2. Performance summary table comparing models
+3. Achievement frequency analysis
+4. Model usage statistics (filtered to current experiment only)
+5. JSON file with detailed results
+
+## Example Output (abridged)
+
+```
+📊 Analysis Results:
+================================================================================
+
+📈 Model Performance Summary:
+Model                Avg Achievements   Max Achievements   Invalid Rate    Success Rate   
+--------------------------------------------------------------------------------------
+gpt-5-nano             1.60 ± 1.10                    4            1.20%          100.00%
+Qwen/Qwen3-32B-Inst    1.40 ± 1.05                    3            1.80%          100.00%
+
+🏆 Achievement Frequencies:
+Achievement                 gpt-5-na   qwen3-32
+-----------------------------------------------
+collect_drink               1/5   ( 20%)   3/5   ( 60%)
+collect_sapling             2/5   ( 40%)   2/5   ( 40%)
+collect_wood                4/5   ( 80%)   2/5   ( 40%)
+```
+
+## Implementation Details
+
+The comparison uses:
+- Async/await for parallel episode execution
+- Session-based tracing with v3 architecture
+- Structured output tools for consistent LLM interactions
+- SQLite database for tracking model usage and costs

synth_ai-0.2.9.dev5/examples/evals_old/__init__.py

@@ -0,0 +1,6 @@
+"""
+Crafter Model Comparison Cookbook
+
+A self-contained example for comparing language model performance on the Crafter environment
+with parallel execution and robust timeout handling.
+"""