synth-ai 0.2.8.dev2__py3-none-any.whl → 0.4.3__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/analyze_semantic_words_markdown.py

@@ -1,250 +0,0 @@
-#!/usr/bin/env python3
-"""
-Run Crafter agent and analyze semantic map words - output as markdown tables only.
-
-This script:
-1. Runs a Crafter agent for multiple episodes
-2. Extracts all unique words from the semantic map observations
-3. Outputs analysis as markdown tables (no plotting dependencies)
-
-Usage:
-    python analyze_semantic_words_markdown.py --model gemini-1.5-flash --episodes 3
-"""
-
-import argparse
-import asyncio
-import json
-import re
-
-# Import the Crafter agent
-import sys
-from collections import Counter
-from datetime import datetime
-from pathlib import Path
-from typing import Dict, List, Set
-
-sys.path.append(str(Path(__file__).parent))
-from test_crafter_react_agent import run_crafter_episodes
-
-
-def extract_words_from_semantic_map(observation: str) -> set[str]:
-    """Extract meaningful words from a semantic map observation string."""
-    if not observation or "semantic_map" not in observation.lower():
-        return set()
-    
-    # Look for patterns like object names in the semantic map
-    # Common Crafter objects/entities
-    crafter_words = {
-        # Resources
-        'wood', 'stone', 'coal', 'iron', 'diamond', 'water',
-        # Animals  
-        'cow', 'pig', 'skeleton', 'zombie',
-        # Structures/Objects
-        'tree', 'grass', 'furnace', 'table', 'bed', 'chest',
-        'house', 'fence', 'door', 'wall',
-        # Tools
-        'axe', 'pickaxe', 'sword', 'shovel',
-        # Food
-        'bread', 'meat', 'apple',
-        # Environment
-        'mountain', 'river', 'forest', 'desert', 'cave',
-        'lava', 'sand', 'dirt', 'path',
-        # Actions/States
-        'crafting', 'mining', 'building', 'farming',
-        'health', 'hunger', 'energy'
-    }
-    
-    # Extract words using regex - look for alphabetic words
-    words = re.findall(r'\b[a-zA-Z]{3,}\b', observation.lower())
-    
-    # Filter to keep only meaningful Crafter-related words
-    found_words = set()
-    for word in words:
-        if word in crafter_words or any(cw in word for cw in crafter_words):
-            found_words.add(word)
-    return found_words
-
-def analyze_episode_traces(traces_data: list[dict]) -> dict[str, int]:
-    """Analyze traces to extract semantic map words."""
-    word_counter = Counter()
-    
-    for episode_data in traces_data:
-        if 'observations' in episode_data:
-            for obs in episode_data['observations']:
-                if isinstance(obs, dict):
-                    # Look for semantic map in observation
-                    obs_str = str(obs)
-                    words = extract_words_from_semantic_map(obs_str)
-                    word_counter.update(words)
-                elif isinstance(obs, str):
-                    words = extract_words_from_semantic_map(obs)
-                    word_counter.update(words)
-    
-    return dict(word_counter)
-
-def generate_markdown_report(word_counts: dict[str, int], model: str, episodes: int) -> str:
-    """Generate a markdown report of the semantic map analysis."""
-    if not word_counts:
-        return "# Semantic Map Analysis\n\n**No words found in semantic maps!**\n"
-    
-    total_words = sum(word_counts.values())
-    unique_words = len(word_counts)
-    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    
-    # Sort words by frequency
-    sorted_words = sorted(word_counts.items(), key=lambda x: x[1], reverse=True)
-    
-    # Generate markdown
-    md = f"""# Semantic Map Word Analysis
-    
-**Model:** {model}  
-**Episodes:** {episodes}  
-**Generated:** {timestamp}  
-
-## Summary
-
-- **Total word occurrences:** {total_words}
-- **Unique words discovered:** {unique_words}
-- **Average occurrences per word:** {total_words/unique_words:.1f}
-
-## Top Words by Frequency
-
-| Rank | Word | Count | Percentage |
-|------|------|-------|------------|
-"""
-    
-    # Top 15 words table
-    for i, (word, count) in enumerate(sorted_words[:15], 1):
-        percentage = (count / total_words) * 100
-        md += f"| {i:2d} | {word} | {count} | {percentage:.1f}% |\n"
-    
-    # Word categories
-    categories = {
-        "Resources": ['wood', 'stone', 'coal', 'iron', 'diamond', 'water'],
-        "Animals": ['cow', 'pig', 'skeleton', 'zombie'],
-        "Structures": ['tree', 'furnace', 'table', 'house', 'chest', 'fence', 'door'],
-        "Tools": ['axe', 'pickaxe', 'sword', 'shovel'],
-        "Environment": ['mountain', 'river', 'forest', 'desert', 'cave', 'lava', 'grass'],
-        "Food": ['bread', 'meat', 'apple']
-    }
-    
-    md += "\n## Words by Category\n\n"
-    
-    for category, words in categories.items():
-        found_words = [(w, word_counts[w]) for w in words if w in word_counts]
-        if found_words:
-            md += f"### {category}\n\n"
-            md += "| Word | Count |\n|------|-------|\n"
-            for word, count in sorted(found_words, key=lambda x: x[1], reverse=True):
-                md += f"| {word} | {count} |\n"
-            md += "\n"
-    
-    # Frequency distribution
-    freq_counts = Counter(word_counts.values())
-    md += "## Frequency Distribution\n\n"
-    md += "| Frequency | Number of Words |\n|-----------|----------------|\n"
-    for freq in sorted(freq_counts.keys(), reverse=True):
-        md += f"| {freq} | {freq_counts[freq]} |\n"
-    
-    # All words alphabetically
-    md += "\n## All Words (Alphabetical)\n\n"
-    md += "| Word | Count |\n|------|-------|\n"
-    for word in sorted(word_counts.keys()):
-        md += f"| {word} | {word_counts[word]} |\n"
-    
-    return md
-
-async def main():
-    parser = argparse.ArgumentParser(description="Analyze semantic map words - markdown output only")
-    parser.add_argument("--model", default="gemini-1.5-flash", 
-                       help="Model to use for agent (default: gemini-1.5-flash)")
-    parser.add_argument("--episodes", type=int, default=3,
-                       help="Number of episodes to run (default: 3)")
-    parser.add_argument("--max-turns", type=int, default=50,
-                       help="Maximum turns per episode (default: 50)")
-    parser.add_argument("--output-dir", default="semantic_analysis",
-                       help="Directory to save analysis results")
-    
-    args = parser.parse_args()
-    
-    print(f"🚀 Running {args.episodes} episodes with {args.model}")
-    print("📊 Will analyze semantic map words and generate markdown report")
-    
-    # Create output directory
-    output_dir = Path(args.output_dir)
-    output_dir.mkdir(exist_ok=True)
-    
-    # Run the agent episodes
-    try:
-        print("\n🎮 Starting Crafter episodes...")
-        traces_result = await run_crafter_episodes(
-            model_name=args.model,
-            num_episodes=args.episodes,
-            max_turns=args.max_turns,
-            difficulty="easy",
-            base_seed=1000
-        )
-        
-        print(f"✅ Completed {args.episodes} episodes")
-        
-        # Analyze semantic map words
-        print("\n🔍 Analyzing semantic map words...")
-        word_counts = analyze_episode_traces(traces_result)
-        
-        # Generate markdown report
-        print("\n📝 Generating markdown report...")
-        markdown_report = generate_markdown_report(word_counts, args.model, args.episodes)
-        
-        # Save markdown report
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        report_file = output_dir / f"semantic_analysis_{args.model}_{timestamp}.md"
-        
-        with open(report_file, 'w') as f:
-            f.write(markdown_report)
-        
-        print(f"💾 Markdown report saved to: {report_file}")
-        
-        # Also save raw data as JSON
-        analysis_data = {
-            "model": args.model,
-            "episodes": args.episodes,
-            "timestamp": timestamp,
-            "word_counts": word_counts,
-            "total_unique_words": len(word_counts),
-            "total_word_occurrences": sum(word_counts.values())
-        }
-        
-        json_file = output_dir / f"word_data_{args.model}_{timestamp}.json"
-        with open(json_file, 'w') as f:
-            json.dump(analysis_data, f, indent=2)
-        
-        print(f"💾 Raw data saved to: {json_file}")
-        
-        # Print summary to console
-        print("\n" + "="*60)
-        print("SEMANTIC MAP WORD ANALYSIS SUMMARY")
-        print("="*60)
-        
-        if word_counts:
-            total_words = sum(word_counts.values())
-            unique_words = len(word_counts)
-            print(f"Total word occurrences: {total_words}")
-            print(f"Unique words discovered: {unique_words}")
-            
-            # Top 10 most common words
-            sorted_words = sorted(word_counts.items(), key=lambda x: x[1], reverse=True)
-            print("\nTop 10 most frequent words:")
-            for i, (word, count) in enumerate(sorted_words[:10], 1):
-                print(f"{i:2d}. {word:<12} ({count} times)")
-        else:
-            print("No semantic map words found!")
-        
-        print(f"\n📄 Full analysis available in: {report_file}")
-        print("\n🎉 Analysis complete!")
-        
-    except Exception as e:
-        print(f"❌ Error during analysis: {e}")
-        raise
-
-if __name__ == "__main__":
-    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_comprehensive_evaluation.py

@@ -1,59 +0,0 @@
-#!/usr/bin/env python3
-"""
-Run script for Full Enchilada Crafter Evaluation
-"""
-
-import argparse
-import asyncio
-
-from src.synth_env.examples.crafter_classic.agent_demos.full_enchilada import (
-    run_full_enchilada_eval,
-)
-
-
-async def main():
-    parser = argparse.ArgumentParser(description="Run Full Enchilada Crafter Evaluation")
-    parser.add_argument(
-        "--models", nargs="+", default=["gpt-4o-mini"], help="Model names to evaluate"
-    )
-    parser.add_argument(
-        "--difficulties",
-        nargs="+",
-        default=["easy", "hard"],
-        help="Difficulty levels to test",
-    )
-    parser.add_argument(
-        "--num-trajectories",
-        type=int,
-        default=3,
-        help="Number of trajectories per condition",
-    )
-    parser.add_argument("--max-turns", type=int, default=30, help="Maximum turns per trajectory")
-    parser.add_argument("--no-images", action="store_true", help="Disable image capture")
-    parser.add_argument(
-        "--no-viewer",
-        action="store_true",
-        help="Don't launch the viewer after evaluation",
-    )
-    parser.add_argument(
-        "--output-dir",
-        type=str,
-        default=None,
-        help="Output directory (default: src/evals/crafter/run_TIMESTAMP)",
-    )
-
-    args = parser.parse_args()
-
-    await run_full_enchilada_eval(
-        model_names=args.models,
-        difficulties=args.difficulties,
-        num_trajectories=args.num_trajectories,
-        max_turns=args.max_turns,
-        capture_images=not args.no_images,
-        launch_viewer=not args.no_viewer,
-        output_dir=args.output_dir,
-    )
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_evaluation_browser.py

@@ -1,152 +0,0 @@
-#!/usr/bin/env python3
-"""
-Browse existing Crafter evaluations and launch viewer for a selected run.
-"""
-
-import argparse
-import asyncio
-import json
-from datetime import datetime
-from pathlib import Path
-
-import uvicorn
-from fastapi.staticfiles import StaticFiles
-from src.synth_env.examples.crafter_classic.agent_demos.full_enchilada import (
-    app,
-    set_current_eval_dir,
-)
-from tabulate import tabulate
-
-
-def list_evaluations(evals_dir: Path = Path("src/evals/crafter")):
-    """List all available evaluations with summary info."""
-    if not evals_dir.exists():
-        print(f"No evaluations found at {evals_dir}")
-        return []
-
-    evaluations = []
-    for run_dir in sorted(evals_dir.glob("run_*"), reverse=True):
-        if run_dir.is_dir():
-            summary_file = run_dir / "evaluation_summary.json"
-            if summary_file.exists():
-                with open(summary_file) as f:
-                    summary = json.load(f)
-
-                eval_info = {
-                    "run_id": run_dir.name,
-                    "timestamp": summary["evaluation_metadata"]["timestamp"],
-                    "models": ", ".join(summary["models_evaluated"]),
-                    "difficulties": ", ".join(summary["difficulties_evaluated"]),
-                    "num_trajectories": summary["evaluation_metadata"]["num_trajectories"],
-                    "path": run_dir,
-                }
-                evaluations.append(eval_info)
-
-    return evaluations
-
-
-async def view_evaluation(eval_dir: Path):
-    """Launch viewer for a specific evaluation."""
-    if not eval_dir.exists():
-        print(f"Evaluation directory not found: {eval_dir}")
-        return
-
-    viewer_dir = eval_dir / "viewer"
-    if not viewer_dir.exists():
-        print(f"Viewer files not found in {eval_dir}")
-        return
-
-    print(f"\n📁 Viewing evaluation: {eval_dir}")
-    print("🌐 Launching viewer at http://localhost:8000")
-    print("   Press Ctrl+C to stop the viewer")
-
-    # Set the current eval directory for the viewer
-    set_current_eval_dir(eval_dir)
-
-    # Mount static files from the viewer directory
-    app.mount("/", StaticFiles(directory=str(viewer_dir), html=True), name="viewer")
-
-    # Run viewer
-    config = uvicorn.Config(app, host="0.0.0.0", port=8000, log_level="error")
-    server = uvicorn.Server(config)
-    await server.serve()
-
-
-async def main():
-    parser = argparse.ArgumentParser(description="Browse Crafter evaluations")
-    parser.add_argument(
-        "--eval-dir",
-        type=str,
-        default="src/evals/crafter",
-        help="Base directory for evaluations",
-    )
-    parser.add_argument(
-        "--run-id", type=str, help="Specific run ID to view (e.g., run_20240115_143022)"
-    )
-    parser.add_argument("--latest", action="store_true", help="View the latest evaluation")
-
-    args = parser.parse_args()
-    evals_dir = Path(args.eval_dir)
-
-    # List evaluations
-    evaluations = list_evaluations(evals_dir)
-
-    if not evaluations:
-        return
-
-    # Display table of evaluations
-    if not args.run_id and not args.latest:
-        print("\n📊 Available Crafter Evaluations:")
-        table_data = []
-        for i, eval_info in enumerate(evaluations):
-            # Parse timestamp for cleaner display
-            try:
-                ts = datetime.fromisoformat(eval_info["timestamp"])
-                ts_str = ts.strftime("%Y-%m-%d %H:%M:%S")
-            except Exception:
-                ts_str = eval_info["timestamp"]
-
-            table_data.append(
-                [
-                    i + 1,
-                    eval_info["run_id"],
-                    ts_str,
-                    eval_info["models"],
-                    eval_info["difficulties"],
-                    eval_info["num_trajectories"],
-                ]
-            )
-
-        headers = ["#", "Run ID", "Timestamp", "Models", "Difficulties", "Trajectories"]
-        print(tabulate(table_data, headers=headers, tablefmt="grid"))
-
-        # Ask user to select
-        print("\nEnter the number of the evaluation to view (or 'q' to quit): ", end="")
-        choice = input().strip()
-
-        if choice.lower() == "q":
-            return
-
-        try:
-            idx = int(choice) - 1
-            if 0 <= idx < len(evaluations):
-                selected_eval = evaluations[idx]
-                await view_evaluation(selected_eval["path"])
-            else:
-                print("Invalid selection")
-        except ValueError:
-            print("Invalid input")
-
-    # View specific run
-    elif args.run_id:
-        eval_path = evals_dir / args.run_id
-        await view_evaluation(eval_path)
-
-    # View latest
-    elif args.latest and evaluations:
-        latest_eval = evaluations[0]
-        await view_evaluation(latest_eval["path"])
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_evaluation_config.toml

@@ -1,24 +0,0 @@
-[evaluation]
-# Maximum number of turns per agent
-max_turns = 100
-
-# Number of trajectories per model-difficulty combination  
-trajectories_per_condition = 10
-
-# Difficulty modes to test
-difficulties = ["easy"]
-
-# Models to evaluate
-models = [
-    "gpt-4.1-nano",
-    "gpt-4o-mini"
-]
-
-# Parallel execution settings
-parallel_episodes = true
-timeout_seconds = 300
-
-# Output settings
-show_progress_bars = true
-show_detailed_logging = false
-show_final_table = true