synth-ai 0.2.4.dev8__py3-none-any.whl → 0.2.5__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/old/trace_eval_OLD.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""
+Trace evaluation functions for Crafter episodes.
+Scores traces based on achievements and invalid actions.
+"""
+
+from typing import Dict, List, Any, Tuple
+from pathlib import Path
+import json
+from collections import defaultdict
+
+# Scoring weights
+WEIGHTS = {
+    'easy_achievement': 1.0,      # Easy achievement (e.g., collect_wood)
+    'medium_achievement': 2.5,    # Medium achievement (e.g., make_wood_pickaxe)
+    'hard_achievement': 5.0,      # Hard achievement (e.g., make_iron_sword)
+    'invalid_action': -0.05,      # Invalid action penalty (50 invalid = -1 medium achievement)
+}
+
+# Map hook names to scoring categories
+HOOK_TO_SCORE_TYPE = {
+    'easy_achievement': 'easy_achievement',
+    'medium_achievement': 'medium_achievement', 
+    'hard_achievement': 'hard_achievement',
+    'invalid_action': 'invalid_action'
+}
+
+
+def evaluate_event(event: Dict[str, Any]) -> Tuple[float, str]:
+    """
+    Evaluate a single event based on its hooks.
+    Returns: (score, symbol) where symbol is '+', '-', or '0'
+    """
+    score = 0.0
+    symbol = '0'
+    
+    # Check if event has metadata from hooks
+    event_metadata = event.get('event_metadata', [])
+    
+    for metadata in event_metadata:
+        hook_name = metadata.get('hook_name', '')
+        
+        if hook_name in HOOK_TO_SCORE_TYPE:
+            score_type = HOOK_TO_SCORE_TYPE[hook_name]
+            weight = WEIGHTS[score_type]
+            score += weight
+            
+            # Determine symbol
+            if weight > 0:
+                symbol = '+'
+            elif weight < 0:
+                symbol = '-'
+    
+    return score, symbol
+
+
+def evaluate_trace(trace_path: Path) -> Dict[str, Any]:
+    """
+    Evaluate an entire trace file.
+    Returns detailed scoring breakdown and trajectory visualization.
+    """
+    with open(trace_path, 'r') as f:
+        trace_data = json.load(f)
+    
+    # Track counts
+    counts = defaultdict(int)
+    total_score = 0.0
+    trajectory_symbols = []
+    
+    # Process event history
+    event_history = trace_data.get('event_history', [])
+    
+    for event in event_history:
+        event_score, symbol = evaluate_event(event)
+        total_score += event_score
+        
+        # Only add symbol if score is non-zero
+        if event_score != 0:
+            trajectory_symbols.append(symbol)
+        
+        # Count hook types
+        for metadata in event.get('event_metadata', []):
+            hook_name = metadata.get('hook_name', '')
+            if hook_name in HOOK_TO_SCORE_TYPE:
+                score_type = HOOK_TO_SCORE_TYPE[hook_name]
+                counts[score_type] += 1
+    
+    # Create trajectory string
+    trajectory_str = ''.join(trajectory_symbols) if trajectory_symbols else '(no scored events)'
+    
+    return {
+        'total_score': total_score,
+        'counts': dict(counts),
+        'trajectory': trajectory_str,
+        'num_events': len(event_history),
+        'trace_file': trace_path.name
+    }
+
+
+def print_trace_evaluation(eval_result: Dict[str, Any]):
+    """Print a formatted evaluation result for a single trace."""
+    print(f"\n📊 Trace: {eval_result['trace_file']}")
+    print(f"   Score: {eval_result['total_score']:.2f}")
+    print(f"   Events: {eval_result['num_events']}")
+    
+    counts = eval_result['counts']
+    if counts:
+        print("   Breakdown:")
+        if 'easy_achievement' in counts:
+            print(f"     Easy achievements: {counts['easy_achievement']} × {WEIGHTS['easy_achievement']} = {counts['easy_achievement'] * WEIGHTS['easy_achievement']:.2f}")
+        if 'medium_achievement' in counts:
+            print(f"     Medium achievements: {counts['medium_achievement']} × {WEIGHTS['medium_achievement']} = {counts['medium_achievement'] * WEIGHTS['medium_achievement']:.2f}")
+        if 'hard_achievement' in counts:
+            print(f"     Hard achievements: {counts['hard_achievement']} × {WEIGHTS['hard_achievement']} = {counts['hard_achievement'] * WEIGHTS['hard_achievement']:.2f}")
+        if 'invalid_action' in counts:
+            print(f"     Invalid actions: {counts['invalid_action']} × {WEIGHTS['invalid_action']} = {counts['invalid_action'] * WEIGHTS['invalid_action']:.2f}")
+    
+    print(f"   Trajectory: {eval_result['trajectory']}")
+
+
+def evaluate_all_traces(trace_dir: Path, pattern: str = "*.json") -> List[Dict[str, Any]]:
+    """
+    Evaluate all trace files in a directory.
+    Returns list of evaluation results sorted by score.
+    """
+    trace_files = list(trace_dir.glob(pattern))
+    results = []
+    
+    for trace_file in trace_files:
+        try:
+            result = evaluate_trace(trace_file)
+            results.append(result)
+        except Exception as e:
+            print(f"⚠️  Error evaluating {trace_file.name}: {e}")
+    
+    # Sort by score (descending)
+    results.sort(key=lambda x: x['total_score'], reverse=True)
+    
+    return results
+
+
+def print_evaluation_summary(results: List[Dict[str, Any]]):
+    """Print a summary of all trace evaluations."""
+    if not results:
+        print("No traces to evaluate.")
+        return
+    
+    print("\n" + "=" * 80)
+    print("📈 TRACE EVALUATION SUMMARY")
+    print("=" * 80)
+    print(f"{'Rank':<6} {'Score':<10} {'Trajectory':<50} {'File':<30}")
+    print("-" * 80)
+    
+    for i, result in enumerate(results, 1):
+        trajectory = result['trajectory']
+        if len(trajectory) > 50:
+            trajectory = trajectory[:47] + "..."
+        print(f"{i:<6} {result['total_score']:<10.2f} {trajectory:<50} {result['trace_file'][:30]:<30}")
+    
+    print("-" * 80)
+    
+    # Summary statistics
+    scores = [r['total_score'] for r in results]
+    avg_score = sum(scores) / len(scores) if scores else 0
+    max_score = max(scores) if scores else 0
+    min_score = min(scores) if scores else 0
+    
+    print(f"Average Score: {avg_score:.2f}")
+    print(f"Best Score: {max_score:.2f}")
+    print(f"Worst Score: {min_score:.2f}")
+    print("=" * 80)
+
+
+if __name__ == "__main__":
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Evaluate Crafter trace files")
+    parser.add_argument("trace_path", type=str, help="Directory containing trace files or single trace file")
+    parser.add_argument("--pattern", type=str, default="*.json", help="File pattern to match (for directories)")
+    parser.add_argument("--verbose", action="store_true", help="Show detailed evaluation for each trace")
+    
+    args = parser.parse_args()
+    
+    trace_path = Path(args.trace_path)
+    if not trace_path.exists():
+        print(f"❌ Path not found: {trace_path}")
+        exit(1)
+    
+    # Check if it's a file or directory
+    if trace_path.is_file():
+        print(f"🔍 Evaluating single trace: {trace_path}")
+        result = evaluate_trace(trace_path)
+        print_trace_evaluation(result)
+    else:
+        print(f"🔍 Evaluating traces in: {trace_path}")
+        results = evaluate_all_traces(trace_path, args.pattern)
+        
+        if args.verbose:
+            for result in results:
+                print_trace_evaluation(result)
+        
+        print_evaluation_summary(results)

synth_ai/environments/examples/crafter_classic/agent_demos/old/validate_openai_format.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""
+Validate that JSONL files are compatible with OpenAI fine-tuning format
+"""
+
+import json
+import sys
+from pathlib import Path
+from typing import List, Dict, Any
+
+
+def validate_openai_format(file_path: Path) -> tuple[bool, List[str]]:
+    """
+    Validate JSONL file for OpenAI fine-tuning compatibility.
+    Returns (is_valid, list_of_errors)
+    """
+    errors = []
+    line_count = 0
+    
+    try:
+        with open(file_path, 'r') as f:
+            for line_num, line in enumerate(f, 1):
+                line_count += 1
+                
+                # Parse JSON
+                try:
+                    data = json.loads(line.strip())
+                except json.JSONDecodeError as e:
+                    errors.append(f"Line {line_num}: Invalid JSON - {e}")
+                    continue
+                
+                # Check required structure
+                if not isinstance(data, dict):
+                    errors.append(f"Line {line_num}: Root must be a dictionary")
+                    continue
+                
+                # Check for messages key
+                if 'messages' not in data:
+                    errors.append(f"Line {line_num}: Missing 'messages' key")
+                    continue
+                
+                messages = data['messages']
+                if not isinstance(messages, list):
+                    errors.append(f"Line {line_num}: 'messages' must be a list")
+                    continue
+                
+                if len(messages) < 2:
+                    errors.append(f"Line {line_num}: Need at least 2 messages (system/user + assistant)")
+                    continue
+                
+                # Validate each message
+                roles_seen = []
+                for msg_idx, msg in enumerate(messages):
+                    if not isinstance(msg, dict):
+                        errors.append(f"Line {line_num}, message {msg_idx}: Message must be a dictionary")
+                        continue
+                    
+                    # Check required fields
+                    if 'role' not in msg:
+                        errors.append(f"Line {line_num}, message {msg_idx}: Missing 'role'")
+                        continue
+                    
+                    role = msg['role']
+                    roles_seen.append(role)
+                    
+                    if role not in ['system', 'user', 'assistant', 'tool']:
+                        errors.append(f"Line {line_num}, message {msg_idx}: Invalid role '{role}'")
+                    
+                    # Check content or tool_calls
+                    if role == 'assistant':
+                        # Assistant messages can have content, tool_calls, or both
+                        if 'content' not in msg and 'tool_calls' not in msg:
+                            errors.append(f"Line {line_num}, message {msg_idx}: Assistant message needs 'content' or 'tool_calls'")
+                        
+                        # Validate tool_calls structure
+                        if 'tool_calls' in msg:
+                            tool_calls = msg['tool_calls']
+                            if not isinstance(tool_calls, list):
+                                errors.append(f"Line {line_num}, message {msg_idx}: 'tool_calls' must be a list")
+                            else:
+                                for tc_idx, tc in enumerate(tool_calls):
+                                    if not isinstance(tc, dict):
+                                        errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: Must be a dict")
+                                        continue
+                                    
+                                    # Check tool call structure
+                                    if 'id' not in tc:
+                                        errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: Missing 'id'")
+                                    if 'type' not in tc:
+                                        errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: Missing 'type'")
+                                    if 'function' not in tc:
+                                        errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: Missing 'function'")
+                                    else:
+                                        func = tc['function']
+                                        if not isinstance(func, dict):
+                                            errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: 'function' must be a dict")
+                                        else:
+                                            if 'name' not in func:
+                                                errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: Missing function 'name'")
+                                            if 'arguments' not in func:
+                                                errors.append(f"Line {line_num}, message {msg_idx}, tool_call {tc_idx}: Missing function 'arguments'")
+                    else:
+                        # Other roles must have content
+                        if 'content' not in msg:
+                            errors.append(f"Line {line_num}, message {msg_idx}: {role} message missing 'content'")
+                
+                # Check message order
+                if roles_seen[-1] != 'assistant':
+                    errors.append(f"Line {line_num}: Last message must be from assistant (found {roles_seen[-1]})")
+                
+    except Exception as e:
+        errors.append(f"Error reading file: {e}")
+        return False, errors
+    
+    print(f"Validated {line_count} examples")
+    return len(errors) == 0, errors
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: python validate_openai_format.py <jsonl_file>")
+        sys.exit(1)
+    
+    file_path = Path(sys.argv[1])
+    if not file_path.exists():
+        print(f"File not found: {file_path}")
+        sys.exit(1)
+    
+    print(f"Validating OpenAI format for: {file_path}")
+    print("=" * 60)
+    
+    is_valid, errors = validate_openai_format(file_path)
+    
+    if is_valid:
+        print("✅ File is valid for OpenAI fine-tuning!")
+        
+        # Show sample
+        with open(file_path, 'r') as f:
+            first_line = f.readline()
+            example = json.loads(first_line)
+            
+        print("\nSample example structure:")
+        print(f"- Number of messages: {len(example['messages'])}")
+        print(f"- Message roles: {[msg['role'] for msg in example['messages']]}")
+        
+        # Check if assistant has tool calls
+        last_msg = example['messages'][-1]
+        if 'tool_calls' in last_msg:
+            print(f"- Assistant uses tool calls: Yes")
+            print(f"- Number of tool calls: {len(last_msg['tool_calls'])}")
+            if last_msg['tool_calls']:
+                print(f"- First tool: {last_msg['tool_calls'][0]['function']['name']}")
+        else:
+            print(f"- Assistant uses tool calls: No")
+    else:
+        print(f"❌ File has {len(errors)} validation errors:")
+        for i, error in enumerate(errors[:10]):  # Show first 10 errors
+            print(f"  {i+1}. {error}")
+        if len(errors) > 10:
+            print(f"  ... and {len(errors) - 10} more errors")
+    
+    return 0 if is_valid else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

synth_ai/environments/examples/crafter_classic/environment.py

@@ -119,11 +119,20 @@ class CrafterInteractTool(AbstractTool):
 
 # Default observation callable (can be customized via __init__)
 class SynthCrafterObservationCallable(GetObservationCallable):
+    """Default observation: public state dict + per-step reward/flags.
+
+    Additionally computes a small local semantic patch centered on the player
+    to simplify visualization on the client. The patch is exposed under the
+    key `semantic_map_patch7` as a list-of-lists of ints (7x7 unless the
+    semantic map is smaller, in which case it is cropped at edges).
+    """
+
+    def __init__(self, view_size: int = 7) -> None:
+        self.view_size = max(1, int(view_size))
+
     async def get_observation(
         self, pub: CrafterPublicState, priv: CrafterPrivateState
     ) -> InternalObservation:
-        # Example: return a dictionary combining public and selected private info
-        # Actual observation structure depends on agent's needs.
         obs_dict: Dict[str, Any] = dataclasses.asdict(pub)  # type: ignore
         obs_dict["reward_last_step"] = priv.reward_last_step
         obs_dict["total_reward_episode"] = priv.total_reward_episode
@@ -131,6 +140,36 @@ class SynthCrafterObservationCallable(GetObservationCallable):
         obs_dict["truncated"] = priv.truncated
         if pub.error_info:
             obs_dict["tool_error"] = pub.error_info
+
+        # Derive a simple local semantic patch around the player for easy rendering
+        try:
+            sem = pub.semantic_map
+            if sem is not None:
+                rows = int(getattr(sem, "shape", [0, 0])[0])  # type: ignore
+                cols = int(getattr(sem, "shape", [0, 0])[1])  # type: ignore
+                if rows > 0 and cols > 0:
+                    px, py = int(pub.player_position[0]), int(pub.player_position[1])
+                    half = max(1, self.view_size // 2)
+                    x0, y0 = px - half, py - half
+                    x1, y1 = px + half, py + half
+                    patch: list[list[int]] = []
+                    for gy in range(y0, y1 + 1):
+                        row_vals: list[int] = []
+                        for gx in range(x0, x1 + 1):
+                            if 0 <= gy < rows and 0 <= gx < cols:
+                                try:
+                                    val = int(sem[gy, gx])  # type: ignore[index]
+                                except Exception:
+                                    val = 0
+                            else:
+                                val = 0
+                            row_vals.append(val)
+                        patch.append(row_vals)
+                    obs_dict["semantic_map_patch7"] = patch
+        except Exception:
+            # Best-effort; omit patch on error
+            pass
+
         return obs_dict
 
 

synth_ai/environments/examples/crafter_custom/agent_demos/__init__.py

@@ -0,0 +1 @@
+"""Agent demonstrations for custom Crafter environments."""

synth_ai/environments/examples/crafter_custom/agent_demos/trace_eval.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""
+Trace evaluation functions for Crafter episodes.
+Scores traces based on achievements and invalid actions.
+"""
+
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import Any, Dict, List, Tuple
+
+# Scoring weights
+WEIGHTS = {
+    'easy_achievement': 1.0,      # Easy achievement (e.g., collect_wood)
+    'medium_achievement': 2.5,    # Medium achievement (e.g., make_wood_pickaxe)
+    'hard_achievement': 5.0,      # Hard achievement (e.g., make_iron_sword)
+    'invalid_action': -0.05,      # Invalid action penalty (50 invalid = -1 medium achievement)
+}
+
+# Map hook names to scoring categories
+HOOK_TO_SCORE_TYPE = {
+    'easy_achievement': 'easy_achievement',
+    'medium_achievement': 'medium_achievement', 
+    'hard_achievement': 'hard_achievement',
+    'invalid_action': 'invalid_action'
+}
+
+
+def evaluate_event(event: Dict[str, Any]) -> Tuple[float, str]:
+    """
+    Evaluate a single event based on its hooks.
+    Returns: (score, symbol) where symbol is '+', '-', or '0'
+    """
+    score = 0.0
+    symbol = '0'
+    
+    # Check if event has metadata from hooks
+    event_metadata = event.get('event_metadata', [])
+    
+    for metadata in event_metadata:
+        hook_name = metadata.get('hook_name', '')
+        
+        if hook_name in HOOK_TO_SCORE_TYPE:
+            score_type = HOOK_TO_SCORE_TYPE[hook_name]
+            weight = WEIGHTS[score_type]
+            score += weight
+            
+            # Determine symbol
+            if weight > 0:
+                symbol = '+'
+            elif weight < 0:
+                symbol = '-'
+    
+    return score, symbol
+
+
+def evaluate_trace(trace_path: Path) -> Dict[str, Any]:
+    """
+    Evaluate an entire trace file.
+    Returns detailed scoring breakdown and trajectory visualization.
+    """
+    with open(trace_path, 'r') as f:
+        trace_data = json.load(f)
+    
+    # Track counts
+    counts = defaultdict(int)
+    total_score = 0.0
+    trajectory_symbols = []
+    
+    # Process event history
+    event_history = trace_data.get('event_history', [])
+    
+    for event in event_history:
+        event_score, symbol = evaluate_event(event)
+        total_score += event_score
+        
+        # Only add symbol if score is non-zero
+        if event_score != 0:
+            trajectory_symbols.append(symbol)
+        
+        # Count hook types
+        for metadata in event.get('event_metadata', []):
+            hook_name = metadata.get('hook_name', '')
+            if hook_name in HOOK_TO_SCORE_TYPE:
+                score_type = HOOK_TO_SCORE_TYPE[hook_name]
+                counts[score_type] += 1
+    
+    # Create trajectory string
+    trajectory_str = ''.join(trajectory_symbols) if trajectory_symbols else '(no scored events)'
+    
+    return {
+        'total_score': total_score,
+        'counts': dict(counts),
+        'trajectory': trajectory_str,
+        'num_events': len(event_history),
+        'trace_file': trace_path.name
+    }
+
+
+def print_trace_evaluation(eval_result: Dict[str, Any]):
+    """Print a formatted evaluation result for a single trace."""
+    print(f"\n📊 Trace: {eval_result['trace_file']}")
+    print(f"   Score: {eval_result['total_score']:.2f}")
+    print(f"   Events: {eval_result['num_events']}")
+    
+    counts = eval_result['counts']
+    if counts:
+        print("   Breakdown:")
+        if 'easy_achievement' in counts:
+            print(f"     Easy achievements: {counts['easy_achievement']} × {WEIGHTS['easy_achievement']} = {counts['easy_achievement'] * WEIGHTS['easy_achievement']:.2f}")
+        if 'medium_achievement' in counts:
+            print(f"     Medium achievements: {counts['medium_achievement']} × {WEIGHTS['medium_achievement']} = {counts['medium_achievement'] * WEIGHTS['medium_achievement']:.2f}")
+        if 'hard_achievement' in counts:
+            print(f"     Hard achievements: {counts['hard_achievement']} × {WEIGHTS['hard_achievement']} = {counts['hard_achievement'] * WEIGHTS['hard_achievement']:.2f}")
+        if 'invalid_action' in counts:
+            print(f"     Invalid actions: {counts['invalid_action']} × {WEIGHTS['invalid_action']} = {counts['invalid_action'] * WEIGHTS['invalid_action']:.2f}")
+    
+    print(f"   Trajectory: {eval_result['trajectory']}")
+
+
+def evaluate_all_traces(trace_dir: Path, pattern: str = "*.json") -> List[Dict[str, Any]]:
+    """
+    Evaluate all trace files in a directory.
+    Returns list of evaluation results sorted by score.
+    """
+    trace_files = list(trace_dir.glob(pattern))
+    results = []
+    
+    for trace_file in trace_files:
+        try:
+            result = evaluate_trace(trace_file)
+            results.append(result)
+        except Exception as e:
+            print(f"⚠️  Error evaluating {trace_file.name}: {e}")
+    
+    # Sort by score (descending)
+    results.sort(key=lambda x: x['total_score'], reverse=True)
+    
+    return results
+
+
+def print_evaluation_summary(results: List[Dict[str, Any]]):
+    """Print a summary of all trace evaluations."""
+    if not results:
+        print("No traces to evaluate.")
+        return
+    
+    print("\n" + "=" * 80)
+    print("📈 TRACE EVALUATION SUMMARY")
+    print("=" * 80)
+    print(f"{'Rank':<6} {'Score':<10} {'Trajectory':<50} {'File':<30}")
+    print("-" * 80)
+    
+    for i, result in enumerate(results, 1):
+        trajectory = result['trajectory']
+        if len(trajectory) > 50:
+            trajectory = trajectory[:47] + "..."
+        print(f"{i:<6} {result['total_score']:<10.2f} {trajectory:<50} {result['trace_file'][:30]:<30}")
+    
+    print("-" * 80)
+    
+    # Summary statistics
+    scores = [r['total_score'] for r in results]
+    avg_score = sum(scores) / len(scores) if scores else 0
+    max_score = max(scores) if scores else 0
+    min_score = min(scores) if scores else 0
+    
+    print(f"Average Score: {avg_score:.2f}")
+    print(f"Best Score: {max_score:.2f}")
+    print(f"Worst Score: {min_score:.2f}")
+    print("=" * 80)
+
+
+if __name__ == "__main__":
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Evaluate Crafter trace files")
+    parser.add_argument("trace_path", type=str, help="Directory containing trace files or single trace file")
+    parser.add_argument("--pattern", type=str, default="*.json", help="File pattern to match (for directories)")
+    parser.add_argument("--verbose", action="store_true", help="Show detailed evaluation for each trace")
+    
+    args = parser.parse_args()
+    
+    trace_path = Path(args.trace_path)
+    if not trace_path.exists():
+        print(f"❌ Path not found: {trace_path}")
+        exit(1)
+    
+    # Check if it's a file or directory
+    if trace_path.is_file():
+        print(f"🔍 Evaluating single trace: {trace_path}")
+        result = evaluate_trace(trace_path)
+        print_trace_evaluation(result)
+    else:
+        print(f"🔍 Evaluating traces in: {trace_path}")
+        results = evaluate_all_traces(trace_path, args.pattern)
+        
+        if args.verbose:
+            for result in results:
+                print_trace_evaluation(result)
+        
+        print_evaluation_summary(results)