synth-ai 0.2.4.dev8__py3-none-any.whl → 0.2.4.dev9__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/old/generate_ft_data_modal.py

@@ -0,0 +1,386 @@
+#!/usr/bin/env python3
+"""
+Script to generate and filter fine-tuning data from Crafter rollouts
+Applies quality filters and formats data for Modal fine-tuning
+"""
+
+import argparse
+import json
+import os
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+import numpy as np
+
+
+class CrafterDataFilter:
+    """Filter and process Crafter rollout data for fine-tuning."""
+    
+    def __init__(
+        self,
+        min_achievements: int = 3,
+        min_reward: float = 0.0,
+        max_steps: int = 100,
+        min_steps: int = 10,
+        achievement_weights: Optional[Dict[str, float]] = None
+    ):
+        self.min_achievements = min_achievements
+        self.min_reward = min_reward
+        self.max_steps = max_steps
+        self.min_steps = min_steps
+        
+        # Default achievement weights (higher = more valuable)
+        self.achievement_weights = achievement_weights or {
+            # Basic resources
+            "collect_wood": 1.0,
+            "collect_stone": 1.5,
+            "collect_coal": 3.0,
+            "collect_iron": 5.0,
+            "collect_diamond": 10.0,
+            
+            # Crafting
+            "place_table": 2.0,
+            "place_furnace": 3.0,
+            "make_wood_pickaxe": 2.5,
+            "make_stone_pickaxe": 3.5,
+            "make_iron_pickaxe": 6.0,
+            "make_wood_sword": 2.5,
+            "make_stone_sword": 3.5,
+            "make_iron_sword": 6.0,
+            
+            # Survival
+            "eat_cow": 2.0,
+            "eat_plant": 1.0,
+            "collect_drink": 1.0,
+            "sleep": 1.5,
+            
+            # Combat
+            "defeat_zombie": 3.0,
+            "defeat_skeleton": 4.0,
+        }
+    
+    def calculate_episode_score(self, episode: dict) -> float:
+        """Calculate a quality score for an episode."""
+        score = 0.0
+        
+        # Achievement score
+        achievements = episode.get("achievements", [])
+        for ach in achievements:
+            score += self.achievement_weights.get(ach, 1.0)
+        
+        # Efficiency bonus (achievements per step)
+        num_steps = len(episode.get("steps", []))
+        if num_steps > 0:
+            efficiency = len(achievements) / num_steps
+            score += efficiency * 10
+        
+        # Reward contribution
+        total_reward = episode.get("total_reward", 0)
+        score += total_reward * 0.5
+        
+        # Diversity bonus (unique actions)
+        unique_actions = len(set(step["action"] for step in episode.get("steps", [])))
+        score += unique_actions * 0.5
+        
+        return score
+    
+    def filter_episode(self, episode: dict) -> Tuple[bool, str]:
+        """
+        Check if an episode passes quality filters.
+        Returns (passes, reason).
+        """
+        # Check achievements
+        achievements = episode.get("achievements", [])
+        if len(achievements) < self.min_achievements:
+            return False, f"Too few achievements: {len(achievements)} < {self.min_achievements}"
+        
+        # Check reward
+        total_reward = episode.get("total_reward", 0)
+        if total_reward < self.min_reward:
+            return False, f"Low reward: {total_reward} < {self.min_reward}"
+        
+        # Check step count
+        num_steps = len(episode.get("steps", []))
+        if num_steps < self.min_steps:
+            return False, f"Too few steps: {num_steps} < {self.min_steps}"
+        if num_steps > self.max_steps:
+            return False, f"Too many steps: {num_steps} > {self.max_steps}"
+        
+        # Check for errors
+        if episode.get("termination_reason", "").startswith("error"):
+            return False, "Episode ended with error"
+        
+        return True, "Passed all filters"
+    
+    def optimize_conversation(self, messages: List[dict]) -> List[dict]:
+        """
+        Optimize conversation for fine-tuning by:
+        - Removing redundant information
+        - Condensing observations
+        - Ensuring proper format
+        """
+        optimized = []
+        
+        for msg in messages:
+            role = msg.get("role")
+            content = msg.get("content", "")
+            
+            if role == "system":
+                # Keep system message as-is
+                optimized.append(msg)
+            
+            elif role == "user":
+                # Condense user observations
+                if len(content) > 1000:
+                    # Extract key information
+                    lines = content.split("\n")
+                    key_lines = []
+                    
+                    for line in lines:
+                        # Keep important lines
+                        if any(keyword in line.lower() for keyword in [
+                            "health:", "hunger:", "thirst:", "inventory:",
+                            "achievements:", "map", "recent"
+                        ]):
+                            key_lines.append(line)
+                    
+                    # Keep reasoning prompt
+                    if "Think step by step:" in content:
+                        idx = content.index("Think step by step:")
+                        key_lines.append(content[idx:])
+                    
+                    content = "\n".join(key_lines)
+                
+                optimized.append({"role": role, "content": content})
+            
+            elif role == "assistant":
+                # Keep assistant responses but trim if very long
+                if len(content) > 500:
+                    # Keep first part (reasoning) and action
+                    lines = content.split("\n")
+                    kept_lines = []
+                    
+                    # Keep reasoning (usually first few lines)
+                    for i, line in enumerate(lines[:5]):
+                        kept_lines.append(line)
+                    
+                    # Find and keep action
+                    for line in lines:
+                        if any(action in line for action in [
+                            "move_", "do", "sleep", "place_", "make_"
+                        ]):
+                            if line not in kept_lines:
+                                kept_lines.append(line)
+                    
+                    content = "\n".join(kept_lines)
+                
+                optimized.append({"role": role, "content": content})
+        
+        return optimized
+    
+    def create_training_example(self, episode: dict) -> dict:
+        """Create a training example from a filtered episode."""
+        messages = episode.get("messages", [])
+        
+        # Optimize conversation
+        optimized_messages = self.optimize_conversation(messages)
+        
+        # Add metadata as comment in system message
+        metadata_comment = (
+            f"# Episode stats: {len(episode['achievements'])} achievements, "
+            f"{episode['total_reward']:.1f} reward, {len(episode['steps'])} steps"
+        )
+        
+        if optimized_messages and optimized_messages[0]["role"] == "system":
+            optimized_messages[0]["content"] += f"\n{metadata_comment}"
+        
+        return {"messages": optimized_messages}
+
+
+def analyze_rollout_directory(rollout_dir: Path) -> Dict[str, any]:
+    """Analyze a rollout directory and return statistics."""
+    stats = {
+        "total_episodes": 0,
+        "total_achievements": defaultdict(int),
+        "reward_distribution": [],
+        "step_distribution": [],
+        "achievement_distribution": [],
+    }
+    
+    # Find rollout data files
+    rollout_files = list(rollout_dir.glob("**/rollout_data.json"))
+    
+    for rollout_file in rollout_files:
+        with open(rollout_file, "r") as f:
+            data = json.load(f)
+        
+        episodes = data.get("episodes", [])
+        stats["total_episodes"] += len(episodes)
+        
+        for ep in episodes:
+            # Track achievements
+            for ach in ep.get("achievements", []):
+                stats["total_achievements"][ach] += 1
+            
+            # Track distributions
+            stats["reward_distribution"].append(ep.get("total_reward", 0))
+            stats["step_distribution"].append(len(ep.get("steps", [])))
+            stats["achievement_distribution"].append(len(ep.get("achievements", [])))
+    
+    return stats
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate fine-tuning data from Crafter rollouts")
+    parser.add_argument("rollout_dir", type=str, help="Directory containing rollout data")
+    parser.add_argument("--output", type=str, default=None,
+                       help="Output file (default: rollout_dir/fine_tuning_filtered.jsonl)")
+    parser.add_argument("--min-achievements", type=int, default=3,
+                       help="Minimum achievements required")
+    parser.add_argument("--min-reward", type=float, default=0.0,
+                       help="Minimum total reward required")
+    parser.add_argument("--max-steps", type=int, default=100,
+                       help="Maximum steps allowed")
+    parser.add_argument("--min-steps", type=int, default=10,
+                       help="Minimum steps required")
+    parser.add_argument("--top-k", type=int, default=None,
+                       help="Only keep top K episodes by score")
+    parser.add_argument("--analyze-only", action="store_true",
+                       help="Only analyze data, don't generate output")
+    parser.add_argument("--verbose", action="store_true",
+                       help="Show detailed filtering information")
+    
+    args = parser.parse_args()
+    
+    rollout_dir = Path(args.rollout_dir)
+    if not rollout_dir.exists():
+        print(f"❌ Rollout directory not found: {rollout_dir}")
+        return 1
+    
+    # Analyze rollout data
+    print(f"📊 Analyzing rollout data in {rollout_dir}...")
+    stats = analyze_rollout_directory(rollout_dir)
+    
+    print(f"\n📈 Rollout Statistics:")
+    print(f"   Total episodes: {stats['total_episodes']}")
+    
+    if stats['total_episodes'] == 0:
+        print("❌ No episodes found in rollout directory")
+        return 1
+    
+    print(f"   Avg achievements: {np.mean(stats['achievement_distribution']):.1f}")
+    print(f"   Avg reward: {np.mean(stats['reward_distribution']):.1f}")
+    print(f"   Avg steps: {np.mean(stats['step_distribution']):.1f}")
+    
+    print(f"\n🏆 Top achievements:")
+    for ach, count in sorted(stats['total_achievements'].items(), 
+                            key=lambda x: x[1], reverse=True)[:10]:
+        pct = count / stats['total_episodes'] * 100
+        print(f"   {ach}: {count} ({pct:.1f}%)")
+    
+    if args.analyze_only:
+        return 0
+    
+    # Create filter
+    filter_config = CrafterDataFilter(
+        min_achievements=args.min_achievements,
+        min_reward=args.min_reward,
+        max_steps=args.max_steps,
+        min_steps=args.min_steps
+    )
+    
+    # Process episodes
+    print(f"\n🔍 Filtering episodes...")
+    all_episodes = []
+    filtered_episodes = []
+    filter_reasons = defaultdict(int)
+    
+    # Load all episodes
+    rollout_files = list(rollout_dir.glob("**/rollout_data.json"))
+    for rollout_file in rollout_files:
+        with open(rollout_file, "r") as f:
+            data = json.load(f)
+        
+        episodes = data.get("episodes", [])
+        all_episodes.extend(episodes)
+    
+    # Filter episodes
+    for ep in all_episodes:
+        passes, reason = filter_config.filter_episode(ep)
+        if passes:
+            # Calculate score for ranking
+            score = filter_config.calculate_episode_score(ep)
+            filtered_episodes.append((score, ep))
+        else:
+            filter_reasons[reason] += 1
+            if args.verbose:
+                print(f"   Filtered out: {reason}")
+    
+    print(f"\n📋 Filtering Results:")
+    print(f"   Total episodes: {len(all_episodes)}")
+    print(f"   Passed filters: {len(filtered_episodes)}")
+    print(f"   Pass rate: {len(filtered_episodes)/len(all_episodes)*100:.1f}%")
+    
+    if filter_reasons:
+        print(f"\n❌ Filter reasons:")
+        for reason, count in sorted(filter_reasons.items(), 
+                                   key=lambda x: x[1], reverse=True):
+            print(f"   {reason}: {count}")
+    
+    if not filtered_episodes:
+        print("\n❌ No episodes passed the filters!")
+        return 1
+    
+    # Sort by score and apply top-k if specified
+    filtered_episodes.sort(key=lambda x: x[0], reverse=True)
+    
+    if args.top_k and args.top_k < len(filtered_episodes):
+        print(f"\n🎯 Selecting top {args.top_k} episodes by score")
+        filtered_episodes = filtered_episodes[:args.top_k]
+    
+    # Generate training examples
+    print(f"\n✍️  Generating training examples...")
+    training_examples = []
+    
+    for score, episode in filtered_episodes:
+        example = filter_config.create_training_example(episode)
+        training_examples.append(example)
+        
+        if args.verbose:
+            achievements = episode.get("achievements", [])
+            print(f"   Score: {score:.1f}, Achievements: {achievements}")
+    
+    # Save output
+    output_file = args.output
+    if not output_file:
+        output_file = rollout_dir / "fine_tuning_filtered.jsonl"
+    else:
+        output_file = Path(output_file)
+    
+    with open(output_file, "w") as f:
+        for example in training_examples:
+            f.write(json.dumps(example) + "\n")
+    
+    print(f"\n✅ Generated {len(training_examples)} training examples")
+    print(f"📁 Saved to: {output_file}")
+    
+    # Calculate token estimate
+    total_chars = sum(len(json.dumps(ex)) for ex in training_examples)
+    est_tokens = total_chars // 4
+    print(f"📊 Estimated tokens: {est_tokens:,}")
+    
+    # Show sample
+    print(f"\n📝 Sample training example:")
+    sample = training_examples[0]["messages"]
+    for msg in sample[:3]:  # Show first 3 messages
+        role = msg["role"]
+        content = msg["content"][:100] + "..." if len(msg["content"]) > 100 else msg["content"]
+        print(f"   [{role}] {content}")
+    
+    return 0
+
+
+if __name__ == "__main__":
+    exit(main())

synth_ai/environments/examples/crafter_classic/agent_demos/old/generate_ft_metadata.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+"""
+Generate metadata for fine-tuning datasets
+"""
+
+import json
+from pathlib import Path
+from collections import defaultdict
+import sys
+
+sys.path.append(str(Path(__file__).parent))
+from filter_traces_sft import load_trace, extract_trajectory_score, extract_llm_calls, calculate_window_score
+
+
+def analyze_trajectory_dataset(traces_dir: Path, threshold: float = 2.0):
+    """Analyze trajectory-based filtering results."""
+    trace_files = sorted(traces_dir.glob("*.json"))
+    
+    included_traces = []
+    excluded_traces = []
+    score_distribution = defaultdict(int)
+    achievement_counts = defaultdict(int)
+    total_llm_calls = 0
+    
+    for trace_file in trace_files:
+        trace = load_trace(trace_file)
+        score = extract_trajectory_score(trace)
+        score_distribution[int(score)] += 1
+        
+        # Get achievements
+        metadata = trace.get('session_metadata', [])
+        if isinstance(metadata, list):
+            for item in metadata:
+                if isinstance(item, dict) and item.get('metadata_type') == 'episode_results':
+                    episode_results = item.get('data', {})
+                    achievements = episode_results.get('achievements', {})
+                    for ach, unlocked in achievements.items():
+                        if unlocked:
+                            achievement_counts[ach] += 1
+                    break
+        
+        # Count LLM calls
+        llm_calls = extract_llm_calls(trace)
+        
+        if score >= threshold:
+            included_traces.append({
+                'trace_file': trace_file.name,
+                'score': score,
+                'num_llm_calls': len(llm_calls),
+                'achievements': [k for k, v in achievements.items() if v] if 'achievements' in locals() else []
+            })
+            total_llm_calls += len(llm_calls)
+        else:
+            excluded_traces.append({
+                'trace_file': trace_file.name,
+                'score': score
+            })
+    
+    return {
+        'threshold': threshold,
+        'total_traces': len(trace_files),
+        'included_traces': len(included_traces),
+        'excluded_traces': len(excluded_traces),
+        'yield_rate': len(included_traces) / len(trace_files) * 100,
+        'total_examples': total_llm_calls,
+        'avg_examples_per_trace': total_llm_calls / len(included_traces) if included_traces else 0,
+        'score_distribution': dict(score_distribution),
+        'achievement_distribution': dict(achievement_counts),
+        'included_trace_details': included_traces
+    }
+
+
+def analyze_window_dataset(traces_dir: Path, window_size: int = 5, threshold: float = 1.0):
+    """Analyze window-based filtering results with greedy extraction."""
+    trace_files = sorted(traces_dir.glob("*.json"))
+    
+    window_scores = defaultdict(int)
+    traces_with_windows = 0
+    total_windows = 0
+    total_examples = 0
+    window_details = []
+    
+    for trace_file in trace_files:
+        trace = load_trace(trace_file)
+        llm_calls = extract_llm_calls(trace)
+        
+        if not llm_calls:
+            continue
+            
+        # Get max turn
+        max_turn = max(turn for turn, _ in llm_calls)
+        trace_has_window = False
+        used_turns = set()
+        
+        # Greedy extraction - same as in filter_traces_sft.py
+        for start in range(0, max_turn - window_size + 2):
+            end = start + window_size - 1
+            
+            # Skip if any turn in window already used
+            if any(t in used_turns for t in range(start, end + 1)):
+                continue
+            
+            score = calculate_window_score(trace, start, end)
+            
+            if score >= threshold:
+                window_scores[int(score)] += 1
+                total_windows += 1
+                trace_has_window = True
+                
+                # Mark turns as used
+                for t in range(start, end + 1):
+                    used_turns.add(t)
+                
+                # Count examples in window
+                window_llm_calls = [llm for turn, llm in llm_calls if start <= turn <= end]
+                total_examples += len(window_llm_calls)
+                
+                window_details.append({
+                    'trace_file': trace_file.name,
+                    'window': f"[{start}-{end}]",
+                    'score': score,
+                    'num_examples': len(window_llm_calls)
+                })
+        
+        if trace_has_window:
+            traces_with_windows += 1
+    
+    return {
+        'window_size': window_size,
+        'threshold': threshold,
+        'total_traces': len(trace_files),
+        'traces_with_qualifying_windows': traces_with_windows,
+        'total_windows_extracted': total_windows,
+        'total_examples': total_examples,
+        'avg_examples_per_window': total_examples / total_windows if total_windows else 0,
+        'window_score_distribution': dict(window_scores),
+        'window_details': window_details[:20]  # First 20 for brevity
+    }
+
+
+def main():
+    traces_dir = Path("traces")
+    ft_dir = Path("ft_dataset")
+    
+    # Analyze trajectory dataset
+    print("Analyzing trajectory-based dataset...")
+    traj_metadata = analyze_trajectory_dataset(traces_dir, threshold=2.0)
+    
+    # Analyze window dataset
+    print("Analyzing window-based dataset...")
+    window_metadata = analyze_window_dataset(traces_dir, window_size=5, threshold=1.0)
+    
+    # Create combined metadata
+    combined_metadata = {
+        'dataset_creation': {
+            'source_traces_dir': str(traces_dir),
+            'num_source_traces': traj_metadata['total_traces'],
+            'filtering_methods': ['trajectory_score', 'window_score']
+        },
+        'trajectory_filtering': traj_metadata,
+        'window_filtering': window_metadata,
+        'comparison': {
+            'trajectory_examples': traj_metadata['total_examples'],
+            'window_examples': window_metadata['total_examples'],
+            'trajectory_yield_rate': f"{traj_metadata['yield_rate']:.1f}%",
+            'window_trace_coverage': f"{window_metadata['traces_with_qualifying_windows'] / window_metadata['total_traces'] * 100:.1f}%"
+        }
+    }
+    
+    # Save metadata
+    with open(ft_dir / "metadata.json", 'w') as f:
+        json.dump(combined_metadata, f, indent=2)
+    
+    # Save trajectory-specific metadata
+    with open(ft_dir / "trajectory_score_metadata.json", 'w') as f:
+        json.dump(traj_metadata, f, indent=2)
+    
+    # Save window-specific metadata
+    with open(ft_dir / "window_score_metadata.json", 'w') as f:
+        json.dump(window_metadata, f, indent=2)
+    
+    # Print summary
+    print("\n" + "="*60)
+    print("FINE-TUNING DATASET SUMMARY")
+    print("="*60)
+    print(f"Source traces: {traj_metadata['total_traces']}")
+    print(f"\nTrajectory-based filtering (score >= 2.0):")
+    print(f"  - Included traces: {traj_metadata['included_traces']} ({traj_metadata['yield_rate']:.1f}%)")
+    print(f"  - Total examples: {traj_metadata['total_examples']}")
+    print(f"  - Avg examples/trace: {traj_metadata['avg_examples_per_trace']:.1f}")
+    
+    print(f"\nWindow-based filtering (window_size=5, score >= 1.0):")
+    print(f"  - Traces with windows: {window_metadata['traces_with_qualifying_windows']} ({window_metadata['traces_with_qualifying_windows'] / window_metadata['total_traces'] * 100:.1f}%)")
+    print(f"  - Total windows: {window_metadata['total_windows_extracted']}")
+    print(f"  - Total examples: {window_metadata['total_examples']}")
+    print(f"  - Avg examples/window: {window_metadata['avg_examples_per_window']:.1f}")
+    
+    print(f"\nWhy so many examples?")
+    print(f"  - Each trace has multiple LLM calls (turns)")
+    print(f"  - Trajectory method: {traj_metadata['included_traces']} traces × {traj_metadata['avg_examples_per_trace']:.1f} turns = {traj_metadata['total_examples']} examples")
+    print(f"  - Window method: {window_metadata['total_windows_extracted']} windows × {window_metadata['avg_examples_per_window']:.1f} turns = {window_metadata['total_examples']} examples")
+
+
+if __name__ == "__main__":
+    main()