synth-ai 0.2.4.dev8__py3-none-any.whl → 0.2.4.dev9__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/analyze_semantic_words_markdown.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+"""
+Run Crafter agent and analyze semantic map words - output as markdown tables only.
+
+This script:
+1. Runs a Crafter agent for multiple episodes
+2. Extracts all unique words from the semantic map observations
+3. Outputs analysis as markdown tables (no plotting dependencies)
+
+Usage:
+    python analyze_semantic_words_markdown.py --model gemini-1.5-flash --episodes 3
+"""
+
+import argparse
+import asyncio
+import json
+import re
+
+# Import the Crafter agent
+import sys
+from collections import Counter
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Set
+
+sys.path.append(str(Path(__file__).parent))
+from test_crafter_react_agent import run_crafter_episodes
+
+
+def extract_words_from_semantic_map(observation: str) -> set[str]:
+    """Extract meaningful words from a semantic map observation string."""
+    if not observation or "semantic_map" not in observation.lower():
+        return set()
+    
+    # Look for patterns like object names in the semantic map
+    # Common Crafter objects/entities
+    crafter_words = {
+        # Resources
+        'wood', 'stone', 'coal', 'iron', 'diamond', 'water',
+        # Animals  
+        'cow', 'pig', 'skeleton', 'zombie',
+        # Structures/Objects
+        'tree', 'grass', 'furnace', 'table', 'bed', 'chest',
+        'house', 'fence', 'door', 'wall',
+        # Tools
+        'axe', 'pickaxe', 'sword', 'shovel',
+        # Food
+        'bread', 'meat', 'apple',
+        # Environment
+        'mountain', 'river', 'forest', 'desert', 'cave',
+        'lava', 'sand', 'dirt', 'path',
+        # Actions/States
+        'crafting', 'mining', 'building', 'farming',
+        'health', 'hunger', 'energy'
+    }
+    
+    # Extract words using regex - look for alphabetic words
+    words = re.findall(r'\b[a-zA-Z]{3,}\b', observation.lower())
+    
+    # Filter to keep only meaningful Crafter-related words
+    found_words = set()
+    for word in words:
+        if word in crafter_words or any(cw in word for cw in crafter_words):
+            found_words.add(word)
+    return found_words
+
+def analyze_episode_traces(traces_data: list[dict]) -> dict[str, int]:
+    """Analyze traces to extract semantic map words."""
+    word_counter = Counter()
+    
+    for episode_data in traces_data:
+        if 'observations' in episode_data:
+            for obs in episode_data['observations']:
+                if isinstance(obs, dict):
+                    # Look for semantic map in observation
+                    obs_str = str(obs)
+                    words = extract_words_from_semantic_map(obs_str)
+                    word_counter.update(words)
+                elif isinstance(obs, str):
+                    words = extract_words_from_semantic_map(obs)
+                    word_counter.update(words)
+    
+    return dict(word_counter)
+
+def generate_markdown_report(word_counts: dict[str, int], model: str, episodes: int) -> str:
+    """Generate a markdown report of the semantic map analysis."""
+    if not word_counts:
+        return "# Semantic Map Analysis\n\n**No words found in semantic maps!**\n"
+    
+    total_words = sum(word_counts.values())
+    unique_words = len(word_counts)
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    
+    # Sort words by frequency
+    sorted_words = sorted(word_counts.items(), key=lambda x: x[1], reverse=True)
+    
+    # Generate markdown
+    md = f"""# Semantic Map Word Analysis
+    
+**Model:** {model}  
+**Episodes:** {episodes}  
+**Generated:** {timestamp}  
+
+## Summary
+
+- **Total word occurrences:** {total_words}
+- **Unique words discovered:** {unique_words}
+- **Average occurrences per word:** {total_words/unique_words:.1f}
+
+## Top Words by Frequency
+
+| Rank | Word | Count | Percentage |
+|------|------|-------|------------|
+"""
+    
+    # Top 15 words table
+    for i, (word, count) in enumerate(sorted_words[:15], 1):
+        percentage = (count / total_words) * 100
+        md += f"| {i:2d} | {word} | {count} | {percentage:.1f}% |\n"
+    
+    # Word categories
+    categories = {
+        "Resources": ['wood', 'stone', 'coal', 'iron', 'diamond', 'water'],
+        "Animals": ['cow', 'pig', 'skeleton', 'zombie'],
+        "Structures": ['tree', 'furnace', 'table', 'house', 'chest', 'fence', 'door'],
+        "Tools": ['axe', 'pickaxe', 'sword', 'shovel'],
+        "Environment": ['mountain', 'river', 'forest', 'desert', 'cave', 'lava', 'grass'],
+        "Food": ['bread', 'meat', 'apple']
+    }
+    
+    md += "\n## Words by Category\n\n"
+    
+    for category, words in categories.items():
+        found_words = [(w, word_counts[w]) for w in words if w in word_counts]
+        if found_words:
+            md += f"### {category}\n\n"
+            md += "| Word | Count |\n|------|-------|\n"
+            for word, count in sorted(found_words, key=lambda x: x[1], reverse=True):
+                md += f"| {word} | {count} |\n"
+            md += "\n"
+    
+    # Frequency distribution
+    freq_counts = Counter(word_counts.values())
+    md += "## Frequency Distribution\n\n"
+    md += "| Frequency | Number of Words |\n|-----------|----------------|\n"
+    for freq in sorted(freq_counts.keys(), reverse=True):
+        md += f"| {freq} | {freq_counts[freq]} |\n"
+    
+    # All words alphabetically
+    md += "\n## All Words (Alphabetical)\n\n"
+    md += "| Word | Count |\n|------|-------|\n"
+    for word in sorted(word_counts.keys()):
+        md += f"| {word} | {word_counts[word]} |\n"
+    
+    return md
+
+async def main():
+    parser = argparse.ArgumentParser(description="Analyze semantic map words - markdown output only")
+    parser.add_argument("--model", default="gemini-1.5-flash", 
+                       help="Model to use for agent (default: gemini-1.5-flash)")
+    parser.add_argument("--episodes", type=int, default=3,
+                       help="Number of episodes to run (default: 3)")
+    parser.add_argument("--max-turns", type=int, default=50,
+                       help="Maximum turns per episode (default: 50)")
+    parser.add_argument("--output-dir", default="semantic_analysis",
+                       help="Directory to save analysis results")
+    
+    args = parser.parse_args()
+    
+    print(f"🚀 Running {args.episodes} episodes with {args.model}")
+    print("📊 Will analyze semantic map words and generate markdown report")
+    
+    # Create output directory
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(exist_ok=True)
+    
+    # Run the agent episodes
+    try:
+        print("\n🎮 Starting Crafter episodes...")
+        traces_result = await run_crafter_episodes(
+            model_name=args.model,
+            num_episodes=args.episodes,
+            max_turns=args.max_turns,
+            difficulty="easy",
+            base_seed=1000
+        )
+        
+        print(f"✅ Completed {args.episodes} episodes")
+        
+        # Analyze semantic map words
+        print("\n🔍 Analyzing semantic map words...")
+        word_counts = analyze_episode_traces(traces_result)
+        
+        # Generate markdown report
+        print("\n📝 Generating markdown report...")
+        markdown_report = generate_markdown_report(word_counts, args.model, args.episodes)
+        
+        # Save markdown report
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        report_file = output_dir / f"semantic_analysis_{args.model}_{timestamp}.md"
+        
+        with open(report_file, 'w') as f:
+            f.write(markdown_report)
+        
+        print(f"💾 Markdown report saved to: {report_file}")
+        
+        # Also save raw data as JSON
+        analysis_data = {
+            "model": args.model,
+            "episodes": args.episodes,
+            "timestamp": timestamp,
+            "word_counts": word_counts,
+            "total_unique_words": len(word_counts),
+            "total_word_occurrences": sum(word_counts.values())
+        }
+        
+        json_file = output_dir / f"word_data_{args.model}_{timestamp}.json"
+        with open(json_file, 'w') as f:
+            json.dump(analysis_data, f, indent=2)
+        
+        print(f"💾 Raw data saved to: {json_file}")
+        
+        # Print summary to console
+        print("\n" + "="*60)
+        print("SEMANTIC MAP WORD ANALYSIS SUMMARY")
+        print("="*60)
+        
+        if word_counts:
+            total_words = sum(word_counts.values())
+            unique_words = len(word_counts)
+            print(f"Total word occurrences: {total_words}")
+            print(f"Unique words discovered: {unique_words}")
+            
+            # Top 10 most common words
+            sorted_words = sorted(word_counts.items(), key=lambda x: x[1], reverse=True)
+            print("\nTop 10 most frequent words:")
+            for i, (word, count) in enumerate(sorted_words[:10], 1):
+                print(f"{i:2d}. {word:<12} ({count} times)")
+        else:
+            print("No semantic map words found!")
+        
+        print(f"\n📄 Full analysis available in: {report_file}")
+        print("\n🎉 Analysis complete!")
+        
+    except Exception as e:
+        print(f"❌ Error during analysis: {e}")
+        raise
+
+if __name__ == "__main__":
+    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_comprehensive_evaluation.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+"""
+Run script for Full Enchilada Crafter Evaluation
+"""
+
+import argparse
+import asyncio
+
+from src.synth_env.examples.crafter_classic.agent_demos.full_enchilada import (
+    run_full_enchilada_eval,
+)
+
+
+async def main():
+    parser = argparse.ArgumentParser(description="Run Full Enchilada Crafter Evaluation")
+    parser.add_argument(
+        "--models", nargs="+", default=["gpt-4o-mini"], help="Model names to evaluate"
+    )
+    parser.add_argument(
+        "--difficulties",
+        nargs="+",
+        default=["easy", "hard"],
+        help="Difficulty levels to test",
+    )
+    parser.add_argument(
+        "--num-trajectories",
+        type=int,
+        default=3,
+        help="Number of trajectories per condition",
+    )
+    parser.add_argument("--max-turns", type=int, default=30, help="Maximum turns per trajectory")
+    parser.add_argument("--no-images", action="store_true", help="Disable image capture")
+    parser.add_argument(
+        "--no-viewer",
+        action="store_true",
+        help="Don't launch the viewer after evaluation",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=None,
+        help="Output directory (default: src/evals/crafter/run_TIMESTAMP)",
+    )
+
+    args = parser.parse_args()
+
+    await run_full_enchilada_eval(
+        model_names=args.models,
+        difficulties=args.difficulties,
+        num_trajectories=args.num_trajectories,
+        max_turns=args.max_turns,
+        capture_images=not args.no_images,
+        launch_viewer=not args.no_viewer,
+        output_dir=args.output_dir,
+    )
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_evaluation_browser.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""
+Browse existing Crafter evaluations and launch viewer for a selected run.
+"""
+
+import argparse
+import asyncio
+import json
+from datetime import datetime
+from pathlib import Path
+
+import uvicorn
+from fastapi.staticfiles import StaticFiles
+from src.synth_env.examples.crafter_classic.agent_demos.full_enchilada import (
+    app,
+    set_current_eval_dir,
+)
+from tabulate import tabulate
+
+
+def list_evaluations(evals_dir: Path = Path("src/evals/crafter")):
+    """List all available evaluations with summary info."""
+    if not evals_dir.exists():
+        print(f"No evaluations found at {evals_dir}")
+        return []
+
+    evaluations = []
+    for run_dir in sorted(evals_dir.glob("run_*"), reverse=True):
+        if run_dir.is_dir():
+            summary_file = run_dir / "evaluation_summary.json"
+            if summary_file.exists():
+                with open(summary_file) as f:
+                    summary = json.load(f)
+
+                eval_info = {
+                    "run_id": run_dir.name,
+                    "timestamp": summary["evaluation_metadata"]["timestamp"],
+                    "models": ", ".join(summary["models_evaluated"]),
+                    "difficulties": ", ".join(summary["difficulties_evaluated"]),
+                    "num_trajectories": summary["evaluation_metadata"]["num_trajectories"],
+                    "path": run_dir,
+                }
+                evaluations.append(eval_info)
+
+    return evaluations
+
+
+async def view_evaluation(eval_dir: Path):
+    """Launch viewer for a specific evaluation."""
+    if not eval_dir.exists():
+        print(f"Evaluation directory not found: {eval_dir}")
+        return
+
+    viewer_dir = eval_dir / "viewer"
+    if not viewer_dir.exists():
+        print(f"Viewer files not found in {eval_dir}")
+        return
+
+    print(f"\n📁 Viewing evaluation: {eval_dir}")
+    print("🌐 Launching viewer at http://localhost:8000")
+    print("   Press Ctrl+C to stop the viewer")
+
+    # Set the current eval directory for the viewer
+    set_current_eval_dir(eval_dir)
+
+    # Mount static files from the viewer directory
+    app.mount("/", StaticFiles(directory=str(viewer_dir), html=True), name="viewer")
+
+    # Run viewer
+    config = uvicorn.Config(app, host="0.0.0.0", port=8000, log_level="error")
+    server = uvicorn.Server(config)
+    await server.serve()
+
+
+async def main():
+    parser = argparse.ArgumentParser(description="Browse Crafter evaluations")
+    parser.add_argument(
+        "--eval-dir",
+        type=str,
+        default="src/evals/crafter",
+        help="Base directory for evaluations",
+    )
+    parser.add_argument(
+        "--run-id", type=str, help="Specific run ID to view (e.g., run_20240115_143022)"
+    )
+    parser.add_argument("--latest", action="store_true", help="View the latest evaluation")
+
+    args = parser.parse_args()
+    evals_dir = Path(args.eval_dir)
+
+    # List evaluations
+    evaluations = list_evaluations(evals_dir)
+
+    if not evaluations:
+        return
+
+    # Display table of evaluations
+    if not args.run_id and not args.latest:
+        print("\n📊 Available Crafter Evaluations:")
+        table_data = []
+        for i, eval_info in enumerate(evaluations):
+            # Parse timestamp for cleaner display
+            try:
+                ts = datetime.fromisoformat(eval_info["timestamp"])
+                ts_str = ts.strftime("%Y-%m-%d %H:%M:%S")
+            except Exception:
+                ts_str = eval_info["timestamp"]
+
+            table_data.append(
+                [
+                    i + 1,
+                    eval_info["run_id"],
+                    ts_str,
+                    eval_info["models"],
+                    eval_info["difficulties"],
+                    eval_info["num_trajectories"],
+                ]
+            )
+
+        headers = ["#", "Run ID", "Timestamp", "Models", "Difficulties", "Trajectories"]
+        print(tabulate(table_data, headers=headers, tablefmt="grid"))
+
+        # Ask user to select
+        print("\nEnter the number of the evaluation to view (or 'q' to quit): ", end="")
+        choice = input().strip()
+
+        if choice.lower() == "q":
+            return
+
+        try:
+            idx = int(choice) - 1
+            if 0 <= idx < len(evaluations):
+                selected_eval = evaluations[idx]
+                await view_evaluation(selected_eval["path"])
+            else:
+                print("Invalid selection")
+        except ValueError:
+            print("Invalid input")
+
+    # View specific run
+    elif args.run_id:
+        eval_path = evals_dir / args.run_id
+        await view_evaluation(eval_path)
+
+    # View latest
+    elif args.latest and evaluations:
+        latest_eval = evaluations[0]
+        await view_evaluation(latest_eval["path"])
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_evaluation_config.toml

@@ -0,0 +1,24 @@
+[evaluation]
+# Maximum number of turns per agent
+max_turns = 100
+
+# Number of trajectories per model-difficulty combination  
+trajectories_per_condition = 10
+
+# Difficulty modes to test
+difficulties = ["easy"]
+
+# Models to evaluate
+models = [
+    "gpt-4.1-nano",
+    "gpt-4o-mini"
+]
+
+# Parallel execution settings
+parallel_episodes = true
+timeout_seconds = 300
+
+# Output settings
+show_progress_bars = true
+show_detailed_logging = false
+show_final_table = true