synth-ai 0.2.2.dev0__py3-none-any.whl → 0.2.4.dev2__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/analyze_semantic_words_markdown.py

@@ -0,0 +1,252 @@
+#!/usr/bin/env python3
+"""
+Run Crafter agent and analyze semantic map words - output as markdown tables only.
+
+This script:
+1. Runs a Crafter agent for multiple episodes
+2. Extracts all unique words from the semantic map observations
+3. Outputs analysis as markdown tables (no plotting dependencies)
+
+Usage:
+    python analyze_semantic_words_markdown.py --model gemini-1.5-flash --episodes 3
+"""
+
+import asyncio
+import argparse
+import json
+import re
+from collections import Counter
+from pathlib import Path
+from typing import Dict, List, Set
+from datetime import datetime
+
+# Import the Crafter agent
+import sys
+sys.path.append(str(Path(__file__).parent))
+from test_crafter_react_agent import run_crafter_episodes
+
+def extract_words_from_semantic_map(observation: str) -> Set[str]:
+    """Extract meaningful words from a semantic map observation string."""
+    if not observation or "semantic_map" not in observation.lower():
+        return set()
+    
+    # Look for patterns like object names in the semantic map
+    # Common Crafter objects/entities
+    crafter_words = {
+        # Resources
+        'wood', 'stone', 'coal', 'iron', 'diamond', 'water',
+        # Animals  
+        'cow', 'pig', 'skeleton', 'zombie',
+        # Structures/Objects
+        'tree', 'grass', 'furnace', 'table', 'bed', 'chest',
+        'house', 'fence', 'door', 'wall',
+        # Tools
+        'axe', 'pickaxe', 'sword', 'shovel',
+        # Food
+        'bread', 'meat', 'apple',
+        # Environment
+        'mountain', 'river', 'forest', 'desert', 'cave',
+        'lava', 'sand', 'dirt', 'path',
+        # Actions/States
+        'crafting', 'mining', 'building', 'farming',
+        'health', 'hunger', 'energy'
+    }
+    
+    # Extract words using regex - look for alphabetic words
+    words = re.findall(r'\b[a-zA-Z]{3,}\b', observation.lower())
+    
+    # Filter to keep only meaningful Crafter-related words
+    found_words = set()
+    for word in words:
+        if word in crafter_words:
+            found_words.add(word)
+        # Also check for partial matches for compound words
+        elif any(cw in word for cw in crafter_words):
+            found_words.add(word)
+    
+    return found_words
+
+def analyze_episode_traces(traces_data: List[Dict]) -> Dict[str, int]:
+    """Analyze traces to extract semantic map words."""
+    word_counter = Counter()
+    
+    for episode_data in traces_data:
+        if 'observations' in episode_data:
+            for obs in episode_data['observations']:
+                if isinstance(obs, dict):
+                    # Look for semantic map in observation
+                    obs_str = str(obs)
+                    words = extract_words_from_semantic_map(obs_str)
+                    word_counter.update(words)
+                elif isinstance(obs, str):
+                    words = extract_words_from_semantic_map(obs)
+                    word_counter.update(words)
+    
+    return dict(word_counter)
+
+def generate_markdown_report(word_counts: Dict[str, int], model: str, episodes: int) -> str:
+    """Generate a markdown report of the semantic map analysis."""
+    if not word_counts:
+        return "# Semantic Map Analysis\n\n**No words found in semantic maps!**\n"
+    
+    total_words = sum(word_counts.values())
+    unique_words = len(word_counts)
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    
+    # Sort words by frequency
+    sorted_words = sorted(word_counts.items(), key=lambda x: x[1], reverse=True)
+    
+    # Generate markdown
+    md = f"""# Semantic Map Word Analysis
+    
+**Model:** {model}  
+**Episodes:** {episodes}  
+**Generated:** {timestamp}  
+
+## Summary
+
+- **Total word occurrences:** {total_words}
+- **Unique words discovered:** {unique_words}
+- **Average occurrences per word:** {total_words/unique_words:.1f}
+
+## Top Words by Frequency
+
+| Rank | Word | Count | Percentage |
+|------|------|-------|------------|
+"""
+    
+    # Top 15 words table
+    for i, (word, count) in enumerate(sorted_words[:15], 1):
+        percentage = (count / total_words) * 100
+        md += f"| {i:2d} | {word} | {count} | {percentage:.1f}% |\n"
+    
+    # Word categories
+    categories = {
+        "Resources": ['wood', 'stone', 'coal', 'iron', 'diamond', 'water'],
+        "Animals": ['cow', 'pig', 'skeleton', 'zombie'],
+        "Structures": ['tree', 'furnace', 'table', 'house', 'chest', 'fence', 'door'],
+        "Tools": ['axe', 'pickaxe', 'sword', 'shovel'],
+        "Environment": ['mountain', 'river', 'forest', 'desert', 'cave', 'lava', 'grass'],
+        "Food": ['bread', 'meat', 'apple']
+    }
+    
+    md += "\n## Words by Category\n\n"
+    
+    for category, words in categories.items():
+        found_words = [(w, word_counts[w]) for w in words if w in word_counts]
+        if found_words:
+            md += f"### {category}\n\n"
+            md += "| Word | Count |\n|------|-------|\n"
+            for word, count in sorted(found_words, key=lambda x: x[1], reverse=True):
+                md += f"| {word} | {count} |\n"
+            md += "\n"
+    
+    # Frequency distribution
+    freq_counts = Counter(word_counts.values())
+    md += "## Frequency Distribution\n\n"
+    md += "| Frequency | Number of Words |\n|-----------|----------------|\n"
+    for freq in sorted(freq_counts.keys(), reverse=True):
+        md += f"| {freq} | {freq_counts[freq]} |\n"
+    
+    # All words alphabetically
+    md += "\n## All Words (Alphabetical)\n\n"
+    md += "| Word | Count |\n|------|-------|\n"
+    for word in sorted(word_counts.keys()):
+        md += f"| {word} | {word_counts[word]} |\n"
+    
+    return md
+
+async def main():
+    parser = argparse.ArgumentParser(description="Analyze semantic map words - markdown output only")
+    parser.add_argument("--model", default="gemini-1.5-flash", 
+                       help="Model to use for agent (default: gemini-1.5-flash)")
+    parser.add_argument("--episodes", type=int, default=3,
+                       help="Number of episodes to run (default: 3)")
+    parser.add_argument("--max-turns", type=int, default=50,
+                       help="Maximum turns per episode (default: 50)")
+    parser.add_argument("--output-dir", default="semantic_analysis",
+                       help="Directory to save analysis results")
+    
+    args = parser.parse_args()
+    
+    print(f"🚀 Running {args.episodes} episodes with {args.model}")
+    print(f"📊 Will analyze semantic map words and generate markdown report")
+    
+    # Create output directory
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(exist_ok=True)
+    
+    # Run the agent episodes
+    try:
+        print("\n🎮 Starting Crafter episodes...")
+        traces_result = await run_crafter_episodes(
+            model_name=args.model,
+            num_episodes=args.episodes,
+            max_turns=args.max_turns,
+            difficulty="easy",
+            base_seed=1000
+        )
+        
+        print(f"✅ Completed {args.episodes} episodes")
+        
+        # Analyze semantic map words
+        print("\n🔍 Analyzing semantic map words...")
+        word_counts = analyze_episode_traces(traces_result)
+        
+        # Generate markdown report
+        print("\n📝 Generating markdown report...")
+        markdown_report = generate_markdown_report(word_counts, args.model, args.episodes)
+        
+        # Save markdown report
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        report_file = output_dir / f"semantic_analysis_{args.model}_{timestamp}.md"
+        
+        with open(report_file, 'w') as f:
+            f.write(markdown_report)
+        
+        print(f"💾 Markdown report saved to: {report_file}")
+        
+        # Also save raw data as JSON
+        analysis_data = {
+            "model": args.model,
+            "episodes": args.episodes,
+            "timestamp": timestamp,
+            "word_counts": word_counts,
+            "total_unique_words": len(word_counts),
+            "total_word_occurrences": sum(word_counts.values())
+        }
+        
+        json_file = output_dir / f"word_data_{args.model}_{timestamp}.json"
+        with open(json_file, 'w') as f:
+            json.dump(analysis_data, f, indent=2)
+        
+        print(f"💾 Raw data saved to: {json_file}")
+        
+        # Print summary to console
+        print("\n" + "="*60)
+        print("SEMANTIC MAP WORD ANALYSIS SUMMARY")
+        print("="*60)
+        
+        if word_counts:
+            total_words = sum(word_counts.values())
+            unique_words = len(word_counts)
+            print(f"Total word occurrences: {total_words}")
+            print(f"Unique words discovered: {unique_words}")
+            
+            # Top 10 most common words
+            sorted_words = sorted(word_counts.items(), key=lambda x: x[1], reverse=True)
+            print(f"\nTop 10 most frequent words:")
+            for i, (word, count) in enumerate(sorted_words[:10], 1):
+                print(f"{i:2d}. {word:<12} ({count} times)")
+        else:
+            print("No semantic map words found!")
+        
+        print(f"\n📄 Full analysis available in: {report_file}")
+        print("\n🎉 Analysis complete!")
+        
+    except Exception as e:
+        print(f"❌ Error during analysis: {e}")
+        raise
+
+if __name__ == "__main__":
+    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/crafter_modal_ft/filter_traces_sft_duckdb_v2_backup.py

@@ -0,0 +1,413 @@
+#!/usr/bin/env python3
+"""
+Filter traces from DuckDB to create Modal/Synth SFT-ready .jsonl files
+Supports two modes:
+1. Trajectory-level filtering: Include entire trajectories above a score threshold
+2. Window-based filtering: Extract high-scoring windows of actions
+
+This is identical to the OpenAI version but configured for Modal/Synth fine-tuning.
+"""
+
+import json
+import argparse
+from pathlib import Path
+from typing import List, Dict, Any, Tuple, Optional
+from collections import defaultdict
+import numpy as np
+import os
+import sys
+import toml
+
+# Add synth_ai to path
+sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent.parent.parent))
+
+from synth_ai.tracing_v2.duckdb.ft_utils import FinetuningDataExtractor
+from synth_ai.tracing_v2.duckdb.manager import DuckDBTraceManager
+
+
+def create_histogram(data: List[float], bins: int = 20, width: int = 60, height: int = 15, 
+                    title: str = "", x_label: str = "", y_label: str = "") -> str:
+    """Create a beautiful ASCII histogram."""
+    if not data:
+        return "No data to display"
+    
+    # Create histogram
+    counts, edges = np.histogram(data, bins=bins)
+    max_count = max(counts) if len(counts) > 0 else 1
+    
+    # Normalize heights
+    if max_count > 0:
+        heights = [int(c * height / max_count) for c in counts]
+    else:
+        heights = [0] * len(counts)
+    
+    # Build the plot
+    lines = []
+    
+    # Title
+    if title:
+        lines.append(f"\n{title.center(width + 10)}")
+        lines.append("=" * (width + 10))
+    
+    # Y-axis label
+    if y_label:
+        lines.append(f"{y_label}")
+    
+    # Plot area with y-axis
+    for y in range(height, 0, -1):
+        # Y-axis value
+        y_val = int(max_count * y / height)
+        line = f"{y_val:>6} │"
+        
+        # Bars
+        for h in heights:
+            if h >= y:
+                line += "█"
+            else:
+                line += " "
+        
+        lines.append(line)
+    
+    # X-axis
+    lines.append(f"{'':>6} └" + "─" * len(heights))
+    
+    # X-axis labels
+    x_labels_line = " " * 8
+    min_val, max_val = min(data), max(data)
+    
+    # Add labels at key positions
+    label_positions = [0, len(heights)//4, len(heights)//2, 3*len(heights)//4, len(heights)-1]
+    for i, pos in enumerate(label_positions):
+        if pos < len(edges) - 1:
+            val = edges[pos]
+            label = f"{val:.1f}"
+            # Calculate position
+            target_pos = 8 + pos
+            if i == 0:
+                x_labels_line = label + x_labels_line[len(label):]
+            elif i == len(label_positions) - 1:
+                start = max(0, target_pos - len(label))
+                x_labels_line = x_labels_line[:start] + label
+            else:
+                start = max(0, target_pos - len(label)//2)
+                end = min(len(x_labels_line), start + len(label))
+                if start < len(x_labels_line):
+                    x_labels_line = x_labels_line[:start] + label[:end-start] + x_labels_line[end:]
+    
+    lines.append(x_labels_line)
+    
+    # X-axis label
+    if x_label:
+        lines.append(f"\n{x_label.center(width + 10)}")
+    
+    return "\n".join(lines)
+
+
+def create_bar_chart(categories: List[str], values: List[int], width: int = 60, 
+                     title: str = "", show_values: bool = True) -> str:
+    """Create a horizontal bar chart."""
+    if not categories or not values:
+        return "No data to display"
+    
+    max_val = max(values) if values else 1
+    lines = []
+    
+    # Title
+    if title:
+        lines.append(f"\n{title}")
+        lines.append("=" * (width + 20))
+    
+    # Find longest category name for alignment
+    max_cat_len = max(len(cat) for cat in categories)
+    
+    # Create bars
+    for cat, val in zip(categories, values):
+        # Normalize bar length
+        bar_len = int(val * width / max_val) if max_val > 0 else 0
+        bar = "█" * bar_len
+        
+        # Format line
+        if show_values:
+            line = f"{cat:<{max_cat_len}} │ {bar} {val}"
+        else:
+            line = f"{cat:<{max_cat_len}} │ {bar}"
+        
+        lines.append(line)
+    
+    return "\n".join(lines)
+
+
+def filter_traces_from_duckdb(
+    db_path: str,
+    output_path: str,
+    config: Dict[str, Any]
+) -> Tuple[int, Dict[str, Any]]:
+    """
+    Filter traces from DuckDB based on configuration.
+    
+    Returns:
+        Tuple of (num_examples, statistics_dict)
+    """
+    mode = config.get("mode", "trajectory")
+    filters = config.get("filters", {})
+    
+    # Extract filtering parameters
+    min_reward = filters.get("min_total_reward", 0.0)
+    min_achievements = filters.get("min_achievements", 0)
+    max_cost = filters.get("max_cost", float('inf'))
+    max_tokens = filters.get("max_tokens", float('inf'))
+    
+    # Modal/Synth specific: filter by model if specified
+    target_models = filters.get("models", [])
+    
+    statistics = {
+        "total_sessions": 0,
+        "filtered_sessions": 0,
+        "total_examples": 0,
+        "reward_distribution": [],
+        "token_distribution": [],
+        "cost_distribution": [],
+        "model_distribution": defaultdict(int)
+    }
+    
+    with FinetuningDataExtractor(db_path) as extractor:
+        # Get all sessions
+        all_sessions_query = "SELECT session_id FROM session_traces"
+        all_sessions = extractor.db_manager.query_traces(all_sessions_query)
+        statistics["total_sessions"] = len(all_sessions)
+        
+        # Filter sessions based on criteria
+        filtered_sessions = []
+        
+        for session_id in all_sessions['session_id']:
+            metrics = extractor.get_session_metrics(session_id)
+            
+            # Apply filters
+            if metrics['total_reward'] < min_reward:
+                continue
+            if metrics['total_cost'] > max_cost:
+                continue
+            if metrics['total_tokens'] > max_tokens:
+                continue
+            
+            # Check achievements if required
+            if min_achievements > 0:
+                achievement_sessions = extractor.filter_by_achievements(min_achievements)
+                if session_id not in achievement_sessions:
+                    continue
+            
+            # Check model filter if specified
+            if target_models:
+                model_query = f"""
+                    SELECT DISTINCT model_name 
+                    FROM events 
+                    WHERE session_id = '{session_id}' 
+                    AND event_type = 'cais' 
+                    AND model_name IS NOT NULL
+                """
+                session_models = extractor.db_manager.query_traces(model_query)
+                if not any(model in target_models for model in session_models['model_name']):
+                    continue
+            
+            filtered_sessions.append(session_id)
+            
+            # Collect statistics
+            statistics["reward_distribution"].append(metrics['total_reward'])
+            statistics["token_distribution"].append(metrics['total_tokens'])
+            statistics["cost_distribution"].append(metrics['total_cost'])
+        
+        statistics["filtered_sessions"] = len(filtered_sessions)
+        
+        # Extract training data
+        if mode == "trajectory":
+            training_data = extractor.extract_openai_format(
+                session_ids=filtered_sessions,
+                min_reward=min_reward
+            )
+        else:  # window mode
+            # For window mode, we need to implement window extraction
+            # For now, use trajectory mode
+            training_data = extractor.extract_openai_format(
+                session_ids=filtered_sessions,
+                min_reward=min_reward
+            )
+        
+        statistics["total_examples"] = len(training_data)
+        
+        # Write to output file
+        output_file = Path(output_path)
+        output_file.parent.mkdir(exist_ok=True)
+        
+        with open(output_file, 'w') as f:
+            for example in training_data:
+                f.write(json.dumps(example) + '\n')
+        
+        # Get model distribution
+        model_query = """
+            SELECT model_name, COUNT(*) as count
+            FROM events
+            WHERE event_type = 'cais' AND model_name IS NOT NULL
+            GROUP BY model_name
+        """
+        model_stats = extractor.db_manager.query_traces(model_query)
+        for _, row in model_stats.iterrows():
+            statistics["model_distribution"][row['model_name']] = row['count']
+    
+    return len(training_data), statistics
+
+
+def print_statistics(stats: Dict[str, Any]):
+    """Print filtering statistics with visualizations."""
+    print("\n" + "="*80)
+    print("FILTERING STATISTICS (Modal/Synth)")
+    print("="*80)
+    
+    # Basic counts
+    print(f"\nTotal sessions in database: {stats['total_sessions']}")
+    print(f"Sessions after filtering: {stats['filtered_sessions']}")
+    print(f"Training examples generated: {stats['total_examples']}")
+    
+    filter_rate = (stats['filtered_sessions'] / stats['total_sessions'] * 100) if stats['total_sessions'] > 0 else 0
+    print(f"Filter pass rate: {filter_rate:.1f}%")
+    
+    # Reward distribution
+    if stats['reward_distribution'] and any(not np.isnan(x) for x in stats['reward_distribution']):
+        valid_rewards = [x for x in stats['reward_distribution'] if not np.isnan(x)]
+        if valid_rewards:
+            print(create_histogram(
+                valid_rewards,
+                bins=20,
+                title="Reward Distribution",
+                x_label="Total Reward",
+                y_label="Count"
+            ))
+            
+            print(f"\nReward statistics:")
+            print(f"  Min: {min(valid_rewards):.2f}")
+            print(f"  Max: {max(valid_rewards):.2f}")
+            print(f"  Mean: {np.mean(valid_rewards):.2f}")
+            print(f"  Median: {np.median(valid_rewards):.2f}")
+    else:
+        print("\nNo valid reward data to display.")
+    
+    # Token distribution
+    if stats['token_distribution'] and any(not np.isnan(x) for x in stats['token_distribution']):
+        valid_tokens = [x for x in stats['token_distribution'] if not np.isnan(x)]
+        if valid_tokens:
+            print(create_histogram(
+                valid_tokens,
+                bins=20,
+                title="Token Usage Distribution",
+                x_label="Total Tokens",
+                y_label="Count"
+            ))
+    
+    # Model distribution
+    if stats['model_distribution']:
+        models = list(stats['model_distribution'].keys())
+        counts = list(stats['model_distribution'].values())
+        print(create_bar_chart(
+            models,
+            counts,
+            title="Model Usage",
+            show_values=True
+        ))
+    
+    print("\n" + "="*80)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Filter traces from DuckDB for Modal/Synth fine-tuning",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Example usage:
+  # Use default config
+  python filter_traces_sft_duckdb.py -d crafter_traces.duckdb -o ft_data/training.jsonl
+  
+  # Use custom config file
+  python filter_traces_sft_duckdb.py -d crafter_traces.duckdb -c filter_config.toml
+  
+  # Override config parameters
+  python filter_traces_sft_duckdb.py -d crafter_traces.duckdb --min-reward 5.0 --max-cost 0.1
+  
+  # Filter by model
+  python filter_traces_sft_duckdb.py -d crafter_traces.duckdb --models "Qwen/Qwen2.5-7B-Instruct"
+        """
+    )
+    
+    parser.add_argument('-d', '--database', required=True, help='Path to DuckDB database')
+    parser.add_argument('-o', '--output', default='ft_data/training_modal.jsonl', help='Output JSONL file')
+    parser.add_argument('-c', '--config', help='Configuration TOML file')
+    
+    # Filter overrides
+    parser.add_argument('--mode', choices=['trajectory', 'window'], help='Filtering mode')
+    parser.add_argument('--min-reward', type=float, help='Minimum total reward')
+    parser.add_argument('--min-achievements', type=int, help='Minimum achievements')
+    parser.add_argument('--max-cost', type=float, help='Maximum cost')
+    parser.add_argument('--max-tokens', type=int, help='Maximum tokens')
+    parser.add_argument('--models', nargs='+', help='Filter by model names (e.g., Qwen/Qwen2.5-7B-Instruct)')
+    
+    parser.add_argument('--dry-run', action='store_true', help='Show statistics without writing output')
+    
+    args = parser.parse_args()
+    
+    # Load config
+    config = {
+        "mode": "trajectory",
+        "filters": {
+            "min_total_reward": 1.0,
+            "min_achievements": 0,
+            "max_cost": 10.0,
+            "max_tokens": 100000,
+            "models": []  # Empty means all models
+        }
+    }
+    
+    if args.config:
+        with open(args.config, 'r') as f:
+            loaded_config = toml.load(f)
+            config.update(loaded_config)
+    
+    # Apply command-line overrides
+    if args.mode:
+        config["mode"] = args.mode
+    if args.min_reward is not None:
+        config["filters"]["min_total_reward"] = args.min_reward
+    if args.min_achievements is not None:
+        config["filters"]["min_achievements"] = args.min_achievements
+    if args.max_cost is not None:
+        config["filters"]["max_cost"] = args.max_cost
+    if args.max_tokens is not None:
+        config["filters"]["max_tokens"] = args.max_tokens
+    if args.models:
+        config["filters"]["models"] = args.models
+    
+    print(f"🤖 Modal/Synth Fine-Tuning Data Filter")
+    print(f"Using database: {args.database}")
+    print(f"Output file: {args.output}")
+    print(f"Mode: {config['mode']}")
+    print(f"Filters: {json.dumps(config['filters'], indent=2)}")
+    
+    if args.dry_run:
+        print("\n🔍 DRY RUN - No output will be written")
+    
+    # Run filtering
+    num_examples, stats = filter_traces_from_duckdb(
+        args.database,
+        args.output if not args.dry_run else "/dev/null",
+        config
+    )
+    
+    # Print statistics
+    print_statistics(stats)
+    
+    if not args.dry_run:
+        print(f"\n✅ Successfully wrote {num_examples} training examples to {args.output}")
+        print(f"   Ready for Modal/Synth fine-tuning!")
+    else:
+        print(f"\n✅ Would write {num_examples} training examples (dry run)")
+
+
+if __name__ == "__main__":
+    main()