synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/old/trace_eval_OLD.py

@@ -1,202 +0,0 @@
-#!/usr/bin/env python3
-"""
-Trace evaluation functions for Crafter episodes.
-Scores traces based on achievements and invalid actions.
-"""
-
-from typing import Dict, List, Any, Tuple
-from pathlib import Path
-import json
-from collections import defaultdict
-
-# Scoring weights
-WEIGHTS = {
-    'easy_achievement': 1.0,      # Easy achievement (e.g., collect_wood)
-    'medium_achievement': 2.5,    # Medium achievement (e.g., make_wood_pickaxe)
-    'hard_achievement': 5.0,      # Hard achievement (e.g., make_iron_sword)
-    'invalid_action': -0.05,      # Invalid action penalty (50 invalid = -1 medium achievement)
-}
-
-# Map hook names to scoring categories
-HOOK_TO_SCORE_TYPE = {
-    'easy_achievement': 'easy_achievement',
-    'medium_achievement': 'medium_achievement', 
-    'hard_achievement': 'hard_achievement',
-    'invalid_action': 'invalid_action'
-}
-
-
-def evaluate_event(event: Dict[str, Any]) -> Tuple[float, str]:
-    """
-    Evaluate a single event based on its hooks.
-    Returns: (score, symbol) where symbol is '+', '-', or '0'
-    """
-    score = 0.0
-    symbol = '0'
-    
-    # Check if event has metadata from hooks
-    event_metadata = event.get('event_metadata', [])
-    
-    for metadata in event_metadata:
-        hook_name = metadata.get('hook_name', '')
-        
-        if hook_name in HOOK_TO_SCORE_TYPE:
-            score_type = HOOK_TO_SCORE_TYPE[hook_name]
-            weight = WEIGHTS[score_type]
-            score += weight
-            
-            # Determine symbol
-            if weight > 0:
-                symbol = '+'
-            elif weight < 0:
-                symbol = '-'
-    
-    return score, symbol
-
-
-def evaluate_trace(trace_path: Path) -> Dict[str, Any]:
-    """
-    Evaluate an entire trace file.
-    Returns detailed scoring breakdown and trajectory visualization.
-    """
-    with open(trace_path, 'r') as f:
-        trace_data = json.load(f)
-    
-    # Track counts
-    counts = defaultdict(int)
-    total_score = 0.0
-    trajectory_symbols = []
-    
-    # Process event history
-    event_history = trace_data.get('event_history', [])
-    
-    for event in event_history:
-        event_score, symbol = evaluate_event(event)
-        total_score += event_score
-        
-        # Only add symbol if score is non-zero
-        if event_score != 0:
-            trajectory_symbols.append(symbol)
-        
-        # Count hook types
-        for metadata in event.get('event_metadata', []):
-            hook_name = metadata.get('hook_name', '')
-            if hook_name in HOOK_TO_SCORE_TYPE:
-                score_type = HOOK_TO_SCORE_TYPE[hook_name]
-                counts[score_type] += 1
-    
-    # Create trajectory string
-    trajectory_str = ''.join(trajectory_symbols) if trajectory_symbols else '(no scored events)'
-    
-    return {
-        'total_score': total_score,
-        'counts': dict(counts),
-        'trajectory': trajectory_str,
-        'num_events': len(event_history),
-        'trace_file': trace_path.name
-    }
-
-
-def print_trace_evaluation(eval_result: Dict[str, Any]):
-    """Print a formatted evaluation result for a single trace."""
-    print(f"\n📊 Trace: {eval_result['trace_file']}")
-    print(f"   Score: {eval_result['total_score']:.2f}")
-    print(f"   Events: {eval_result['num_events']}")
-    
-    counts = eval_result['counts']
-    if counts:
-        print("   Breakdown:")
-        if 'easy_achievement' in counts:
-            print(f"     Easy achievements: {counts['easy_achievement']} × {WEIGHTS['easy_achievement']} = {counts['easy_achievement'] * WEIGHTS['easy_achievement']:.2f}")
-        if 'medium_achievement' in counts:
-            print(f"     Medium achievements: {counts['medium_achievement']} × {WEIGHTS['medium_achievement']} = {counts['medium_achievement'] * WEIGHTS['medium_achievement']:.2f}")
-        if 'hard_achievement' in counts:
-            print(f"     Hard achievements: {counts['hard_achievement']} × {WEIGHTS['hard_achievement']} = {counts['hard_achievement'] * WEIGHTS['hard_achievement']:.2f}")
-        if 'invalid_action' in counts:
-            print(f"     Invalid actions: {counts['invalid_action']} × {WEIGHTS['invalid_action']} = {counts['invalid_action'] * WEIGHTS['invalid_action']:.2f}")
-    
-    print(f"   Trajectory: {eval_result['trajectory']}")
-
-
-def evaluate_all_traces(trace_dir: Path, pattern: str = "*.json") -> List[Dict[str, Any]]:
-    """
-    Evaluate all trace files in a directory.
-    Returns list of evaluation results sorted by score.
-    """
-    trace_files = list(trace_dir.glob(pattern))
-    results = []
-    
-    for trace_file in trace_files:
-        try:
-            result = evaluate_trace(trace_file)
-            results.append(result)
-        except Exception as e:
-            print(f"⚠️  Error evaluating {trace_file.name}: {e}")
-    
-    # Sort by score (descending)
-    results.sort(key=lambda x: x['total_score'], reverse=True)
-    
-    return results
-
-
-def print_evaluation_summary(results: List[Dict[str, Any]]):
-    """Print a summary of all trace evaluations."""
-    if not results:
-        print("No traces to evaluate.")
-        return
-    
-    print("\n" + "=" * 80)
-    print("📈 TRACE EVALUATION SUMMARY")
-    print("=" * 80)
-    print(f"{'Rank':<6} {'Score':<10} {'Trajectory':<50} {'File':<30}")
-    print("-" * 80)
-    
-    for i, result in enumerate(results, 1):
-        trajectory = result['trajectory']
-        if len(trajectory) > 50:
-            trajectory = trajectory[:47] + "..."
-        print(f"{i:<6} {result['total_score']:<10.2f} {trajectory:<50} {result['trace_file'][:30]:<30}")
-    
-    print("-" * 80)
-    
-    # Summary statistics
-    scores = [r['total_score'] for r in results]
-    avg_score = sum(scores) / len(scores) if scores else 0
-    max_score = max(scores) if scores else 0
-    min_score = min(scores) if scores else 0
-    
-    print(f"Average Score: {avg_score:.2f}")
-    print(f"Best Score: {max_score:.2f}")
-    print(f"Worst Score: {min_score:.2f}")
-    print("=" * 80)
-
-
-if __name__ == "__main__":
-    import argparse
-    
-    parser = argparse.ArgumentParser(description="Evaluate Crafter trace files")
-    parser.add_argument("trace_path", type=str, help="Directory containing trace files or single trace file")
-    parser.add_argument("--pattern", type=str, default="*.json", help="File pattern to match (for directories)")
-    parser.add_argument("--verbose", action="store_true", help="Show detailed evaluation for each trace")
-    
-    args = parser.parse_args()
-    
-    trace_path = Path(args.trace_path)
-    if not trace_path.exists():
-        print(f"❌ Path not found: {trace_path}")
-        exit(1)
-    
-    # Check if it's a file or directory
-    if trace_path.is_file():
-        print(f"🔍 Evaluating single trace: {trace_path}")
-        result = evaluate_trace(trace_path)
-        print_trace_evaluation(result)
-    else:
-        print(f"🔍 Evaluating traces in: {trace_path}")
-        results = evaluate_all_traces(trace_path, args.pattern)
-        
-        if args.verbose:
-            for result in results:
-                print_trace_evaluation(result)
-        
-        print_evaluation_summary(results)

synth_ai/environments/examples/crafter_classic/agent_demos/old/validate_openai_format.py

@@ -1,166 +0,0 @@
-#!/usr/bin/env python3
-"""
-Validate that JSONL files are compatible with OpenAI fine-tuning format
-"""
-
-import json
-import sys
-from pathlib import Path
-from typing import List, Dict, Any
-
-
-def validate_openai_format(file_path: Path) -> tuple[bool, List[str]]:
-    """
-    Validate JSONL file for OpenAI fine-tuning compatibility.
-    Returns (is_valid, list_of_errors)
-    """
-    errors = []
-    line_count = 0
-    
-    try:
-        with open(file_path, 'r') as f:
-            for line_num, line in enumerate(f, 1):
-                line_count += 1
-                
-                # Parse JSON
-                try:
-                    data = json.loads(line.strip())
-                except json.JSONDecodeError as e:
-                    errors.append(f"Line {line_num}: Invalid JSON - {e}")
-                    continue
-                
-                # Check required structure
-                if not isinstance(data, dict):
-                    errors.append(f"Line {line_num}: Root must be a dictionary")
-                    continue
-                
-                # Check for messages key
-                if 'messages' not in data:
-                    errors.append(f"Line {line_num}: Missing 'messages' key")
-                    continue
-                
-                messages = data['messages']
-                if not isinstance(messages, list):
-                    errors.append(f"Line {line_num}: 'messages' must be a list")
-                    continue
-                
-                if len(messages) < 2:
-                    errors.append(f"Line {line_num}: Need at least 2 messages (system/user + assistant)")
-                    continue
-                
-                # Validate each message
-                roles_seen = []
-                for msg_idx, msg in enumerate(messages):
-                    if not isinstance(msg, dict):
-                        errors.append(f"Line {line_num}, message {msg_idx}: Message must be a dictionary")
-                        continue
-                    
-                    # Check required fields
-                    if 'role' not in msg:
-                        errors.append(f"Line {line_num}, message {msg_idx}: Missing 'role'")
-                        continue
-                    
-                    role = msg['role']
-                    roles_seen.append(role)
-                    
-                    if role not in ['system', 'user', 'assistant', 'tool']:
-                        errors.append(f"Line {line_num}, message {msg_idx}: Invalid role '{role}'")
-                    
-                    # Check content or tool_calls
-                    if role == 'assistant':
-                        # Assistant messages can have content, tool_calls, or both
-                        if 'content' not in msg and 'tool_calls' not in msg:
-                            errors.append(f"Line {line_num}, message {msg_idx}: Assistant message needs 'content' or 'tool_calls'")
-                        
-                        # Validate tool_calls structure
-                        if 'tool_calls' in msg:
-                            tool_calls = msg['tool_calls']
-                            if not isinstance(tool_calls, list):
-                                errors.append(f"Line {line_num}, message {msg_idx}: 'tool_calls' must be a list")
-                            else:
-                                for tc_idx, tc in enumerate(tool_calls):
-                                    if not isinstance(tc, dict):
-                                        errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: Must be a dict")
-                                        continue
-                                    
-                                    # Check tool call structure
-                                    if 'id' not in tc:
-                                        errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: Missing 'id'")
-                                    if 'type' not in tc:
-                                        errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: Missing 'type'")
-                                    if 'function' not in tc:
-                                        errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: Missing 'function'")
-                                    else:
-                                        func = tc['function']
-                                        if not isinstance(func, dict):
-                                            errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: 'function' must be a dict")
-                                        else:
-                                            if 'name' not in func:
-                                                errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: Missing function 'name'")
-                                            if 'arguments' not in func:
-                                                errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: Missing function 'arguments'")
-                    else:
-                        # Other roles must have content
-                        if 'content' not in msg:
-                            errors.append(f"Line {line_num}, message {msg_idx}: {role} message missing 'content'")
-                
-                # Check message order
-                if roles_seen[-1] != 'assistant':
-                    errors.append(f"Line {line_num}: Last message must be from assistant (found {roles_seen[-1]})")
-                
-    except Exception as e:
-        errors.append(f"Error reading file: {e}")
-        return False, errors
-    
-    print(f"Validated {line_count} examples")
-    return len(errors) == 0, errors
-
-
-def main():
-    if len(sys.argv) < 2:
-        print("Usage: python validate_openai_format.py <jsonl_file>")
-        sys.exit(1)
-    
-    file_path = Path(sys.argv[1])
-    if not file_path.exists():
-        print(f"File not found: {file_path}")
-        sys.exit(1)
-    
-    print(f"Validating OpenAI format for: {file_path}")
-    print("=" * 60)
-    
-    is_valid, errors = validate_openai_format(file_path)
-    
-    if is_valid:
-        print("✅ File is valid for OpenAI fine-tuning!")
-        
-        # Show sample
-        with open(file_path, 'r') as f:
-            first_line = f.readline()
-            example = json.loads(first_line)
-            
-        print("\nSample example structure:")
-        print(f"- Number of messages: {len(example['messages'])}")
-        print(f"- Message roles: {[msg['role'] for msg in example['messages']]}")
-        
-        # Check if assistant has tool calls
-        last_msg = example['messages'][-1]
-        if 'tool_calls' in last_msg:
-            print(f"- Assistant uses tool calls: Yes")
-            print(f"- Number of tool calls: {len(last_msg['tool_calls'])}")
-            if last_msg['tool_calls']:
-                print(f"- First tool: {last_msg['tool_calls'][0]['function']['name']}")
-        else:
-            print(f"- Assistant uses tool calls: No")
-    else:
-        print(f"❌ File has {len(errors)} validation errors:")
-        for i, error in enumerate(errors[:10]):  # Show first 10 errors
-            print(f"  {i+1}. {error}")
-        if len(errors) > 10:
-            print(f"  ... and {len(errors) - 10} more errors")
-    
-    return 0 if is_valid else 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())

synth_ai/environments/examples/crafter_classic/config_logging.py

@@ -1,111 +0,0 @@
-"""
-Logging configuration for Craftax environment.
-Suppresses obnoxious JAX debug messages and sets appropriate log levels.
-"""
-
-import logging
-import os
-import warnings
-
-
-def configure_logging():
-    """Configure logging to suppress noisy debug messages."""
-
-    # Suppress JAX debug logging by setting appropriate log levels
-    jax_loggers = [
-        "jax._src.cache_key",
-        "jax._src.compilation_cache",
-        "jax._src.compiler",
-        "jax._src.dispatch",
-        "jax",
-        "jaxlib",
-    ]
-
-    for logger_name in jax_loggers:
-        logger = logging.getLogger(logger_name)
-        logger.setLevel(logging.WARNING)
-        logger.propagate = False
-
-    # Set JAX platform to CPU to avoid GPU-related logging
-    os.environ.setdefault("JAX_PLATFORMS", "cpu")
-
-    # Suppress JAX warnings and compilation messages
-    os.environ.setdefault("JAX_ENABLE_X64", "False")
-    os.environ.setdefault("JAX_LOG_COMPILES", "0")
-    os.environ.setdefault("JAX_COMPILATION_CACHE_DIR", "/tmp/jax_cache")
-
-    # Configure root logger to INFO level (but don't override if already configured)
-    if not logging.getLogger().handlers:
-        logging.basicConfig(level=logging.INFO, format="%(levelname)s:%(name)s:%(message)s")
-
-    # Suppress other noisy libraries
-    logging.getLogger("matplotlib").setLevel(logging.WARNING)
-    logging.getLogger("PIL").setLevel(logging.WARNING)
-    logging.getLogger("urllib3").setLevel(logging.WARNING)
-
-    # Filter out specific warnings
-    warnings.filterwarnings("ignore", category=UserWarning, module="jax")
-    warnings.filterwarnings("ignore", category=FutureWarning, module="jax")
-
-
-def safe_compare(left, right, operation="<"):
-    """
-    Safely compare two values, handling string vs int comparison errors.
-
-    Args:
-        left: Left operand
-        right: Right operand
-        operation: Comparison operation ('>', '<', '>=', '<=', '==', '!=')
-
-    Returns:
-        bool: Result of comparison, or False if types are incompatible
-    """
-    try:
-        # If both are strings, try to convert to numbers
-        if isinstance(left, str) and isinstance(right, str):
-            try:
-                left = float(left)
-                right = float(right)
-            except ValueError:
-                # If conversion fails, compare as strings
-                pass
-        # If one is string and one is number, try to convert string to number
-        elif isinstance(left, str) and isinstance(right, (int, float)):
-            try:
-                left = type(right)(left)
-            except ValueError:
-                logging.warning(f"Cannot compare string '{left}' with number {right}")
-                return False
-        elif isinstance(left, (int, float)) and isinstance(right, str):
-            try:
-                right = type(left)(right)
-            except ValueError:
-                logging.warning(f"Cannot compare number {left} with string '{right}'")
-                return False
-
-        # Perform the comparison
-        if operation == "<":
-            return left < right
-        elif operation == ">":
-            return left > right
-        elif operation == "<=":
-            return left <= right
-        elif operation == ">=":
-            return left >= right
-        elif operation == "==":
-            return left == right
-        elif operation == "!=":
-            return left != right
-        else:
-            raise ValueError(f"Unsupported operation: {operation}")
-
-    except TypeError as e:
-        logging.error(f"Type error in comparison: {left} {operation} {right} - {e}")
-        return False
-    except Exception as e:
-        logging.error(f"Unexpected error in comparison: {left} {operation} {right} - {e}")
-        return False
-
-
-# Configure logging when module is imported
-configure_logging()