synth-ai 0.2.14__py3-none-any.whl → 0.4.4__py3-none-any.whl

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_map_ground_truth_comparison.py

@@ -1,468 +0,0 @@
-#!/usr/bin/env python3
-"""
-Pytest to create output maps and compare to ground truth for both direct and server
-"""
-
-import pytest
-import os
-import time
-import subprocess
-import requests
-import tempfile
-from pathlib import Path
-from pokemon_env.emulator import EmeraldEmulator
-
-
-class TestMapGroundTruthComparison:
-    """Test suite for comparing map outputs to ground truth"""
-    
-    @pytest.fixture(scope="class")
-    def output_dir(self):
-        """Create output directory for test results"""
-        output_path = Path("test_outputs/pytest_maps")
-        output_path.mkdir(parents=True, exist_ok=True)
-        return output_path
-    
-    @pytest.fixture(scope="class")
-    def ground_truth_dir(self):
-        """Path to ground truth files"""
-        return Path("tests/ground_truth")
-    
-    def format_map_for_comparison(self, tiles, title, location, position):
-        """Format map tiles for comparison with ground truth format"""
-        if not tiles:
-            return f"=== {title} ===\nNo tiles available\n"
-        
-        output = []
-        output.append(f"=== {title} ===")
-        output.append(f"Format: (MetatileID, Behavior, X, Y)")
-        output.append(f"Map dimensions: {len(tiles)}x{len(tiles[0]) if tiles else 0}")
-        output.append("")
-        output.append("--- TRAVERSABILITY MAP ---")
-        
-        # Header with column numbers
-        header = "      " + "  ".join(f"{i:2}" for i in range(len(tiles[0]) if tiles else 0))
-        output.append(header)
-        output.append("    " + "-" * (len(header) - 4))
-        
-        # Map rows
-        for row_idx, row in enumerate(tiles):
-            traversability_row = []
-            for col_idx, tile in enumerate(row):
-                if len(tile) >= 4:
-                    tile_id, behavior, collision, elevation = tile
-                    behavior_val = behavior if not hasattr(behavior, 'value') else behavior.value
-                    
-                    # Convert to traversability symbol
-                    if behavior_val == 0:  # NORMAL
-                        symbol = "." if collision == 0 else "#"
-                    elif behavior_val == 1:  # SECRET_BASE_WALL
-                        symbol = "#"
-                    elif behavior_val == 51:  # IMPASSABLE_SOUTH
-                        symbol = "IM"
-                    elif behavior_val == 96:  # NON_ANIMATED_DOOR
-                        symbol = "D"
-                    elif behavior_val == 101:  # SOUTH_ARROW_WARP
-                        symbol = "SO"
-                    elif behavior_val == 105:  # ANIMATED_DOOR
-                        symbol = "D"
-                    elif behavior_val == 134:  # TELEVISION
-                        symbol = "TE"
-                    else:
-                        symbol = "."  # Default to walkable for other behaviors
-                    
-                    # Mark player position
-                    if position and len(position) >= 2:
-                        # Calculate if this tile is player position
-                        # Player is at center of 15x15 map (position 7,7)
-                        if row_idx == 7 and col_idx == 7:
-                            symbol = "P"
-                    
-                    traversability_row.append(symbol)
-                else:
-                    traversability_row.append("?")
-            
-            # Format row with row number
-            row_str = f"{row_idx:2}: " + " ".join(f"{symbol:1}" for symbol in traversability_row)
-            output.append(row_str)
-        
-        return "\n".join(output)
-    
-    def save_map_output(self, tiles, output_file, title, location, position):
-        """Save map output to file"""
-        formatted_output = self.format_map_for_comparison(tiles, title, location, position)
-        
-        os.makedirs(os.path.dirname(output_file), exist_ok=True)
-        with open(output_file, 'w') as f:
-            f.write(formatted_output)
-        
-        return formatted_output
-    
-    def compare_with_ground_truth(self, output_content, ground_truth_file):
-        """Compare output with ground truth file"""
-        if not os.path.exists(ground_truth_file):
-            return False, f"Ground truth file not found: {ground_truth_file}"
-        
-        with open(ground_truth_file, 'r') as f:
-            ground_truth_content = f.read()
-        
-        # Extract just the traversability map for comparison
-        def extract_traversability_map(content):
-            lines = content.split('\n')
-            map_lines = []
-            in_map_section = False
-            
-            for line in lines:
-                if "--- TRAVERSABILITY MAP ---" in line:
-                    in_map_section = True
-                    continue
-                elif in_map_section and line.strip() and not line.startswith('='):
-                    if line.strip().startswith('---') or 'Map dimensions' in line:
-                        continue
-                    if ':' in line:  # Map row
-                        map_lines.append(line)
-            
-            return '\n'.join(map_lines)
-        
-        output_map = extract_traversability_map(output_content)
-        ground_truth_map = extract_traversability_map(ground_truth_content)
-        
-        lines_match = output_map.strip() == ground_truth_map.strip()
-        
-        if not lines_match:
-            # Calculate similarity metrics
-            output_lines = output_map.strip().split('\n')
-            gt_lines = ground_truth_map.strip().split('\n')
-            
-            matching_lines = 0
-            total_lines = max(len(output_lines), len(gt_lines))
-            
-            for i in range(min(len(output_lines), len(gt_lines))):
-                if output_lines[i] == gt_lines[i]:
-                    matching_lines += 1
-            
-            similarity = (matching_lines / total_lines * 100) if total_lines > 0 else 0
-            
-            return False, f"Maps don't match (similarity: {similarity:.1f}%)\nExpected:\n{ground_truth_map}\nActual:\n{output_map}"
-        
-        return True, "Maps match ground truth perfectly"
-    
-    def test_direct_emulator_house_map(self, output_dir, ground_truth_dir):
-        """Test direct emulator house map against ground truth"""
-        # Initialize direct emulator
-        emu = EmeraldEmulator('Emerald-GBAdvance/rom.gba', headless=True, sound=False)
-        emu.initialize()
-        emu.load_state('tests/states/house.state')
-        
-        try:
-            # Get house map
-            state = emu.memory_reader.get_comprehensive_state()
-            location = state['player']['location']
-            position = state['player']['position']
-            tiles = state['map']['tiles']
-            
-            # Save output
-            output_file = output_dir / "direct_emulator_house.txt"
-            output_content = self.save_map_output(
-                tiles, output_file, 
-                f"House - {location}", location, position
-            )
-            
-            # Compare with ground truth
-            ground_truth_file = ground_truth_dir / "house_direct_emulator.txt"
-            matches, message = self.compare_with_ground_truth(output_content, ground_truth_file)
-            
-            print(f"Direct emulator house map saved to: {output_file}")
-            print(f"Comparison result: {message}")
-            
-            # Allow test to pass even if ground truth doesn't exist yet
-            if not os.path.exists(ground_truth_file):
-                pytest.skip(f"Ground truth file not found: {ground_truth_file}")
-            
-            assert matches, f"Direct emulator house map doesn't match ground truth: {message}"
-            
-        finally:
-            emu.stop()
-    
-    def test_direct_emulator_outside_map(self, output_dir, ground_truth_dir):
-        """Test direct emulator outside map against ground truth"""
-        # Initialize direct emulator
-        emu = EmeraldEmulator('Emerald-GBAdvance/rom.gba', headless=True, sound=False)
-        emu.initialize()
-        emu.load_state('tests/states/house.state')
-        
-        try:
-            # Move outside
-            for i in range(3):
-                emu.press_buttons(['down'], hold_frames=25, release_frames=25)
-                time.sleep(0.2)
-            
-            # Wait for transition to complete
-            time.sleep(0.5)
-            
-            # Get outside map
-            state = emu.memory_reader.get_comprehensive_state()
-            location = state['player']['location']
-            position = state['player']['position']
-            tiles = state['map']['tiles']
-            
-            # Save output
-            output_file = output_dir / "direct_emulator_outside.txt"
-            output_content = self.save_map_output(
-                tiles, output_file,
-                f"Outside - {location}", location, position
-            )
-            
-            # Compare with ground truth
-            ground_truth_file = ground_truth_dir / "outside_direct_emulator.txt"
-            matches, message = self.compare_with_ground_truth(output_content, ground_truth_file)
-            
-            print(f"Direct emulator outside map saved to: {output_file}")
-            print(f"Comparison result: {message}")
-            
-            # Allow test to pass even if ground truth doesn't exist yet
-            if not os.path.exists(ground_truth_file):
-                pytest.skip(f"Ground truth file not found: {ground_truth_file}")
-            
-            assert matches, f"Direct emulator outside map doesn't match ground truth: {message}"
-            
-        finally:
-            emu.stop()
-    
-    def test_server_house_map(self, output_dir, ground_truth_dir):
-        """Test server house map against ground truth"""
-        # Kill any existing server
-        os.system("pkill -f 'server.app' 2>/dev/null")
-        time.sleep(2)
-        
-        # Start server
-        server_cmd = ["python", "-m", "server.app", "--load-state", "tests/states/house.state", "--port", "8101", "--manual"]
-        server_process = subprocess.Popen(server_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-        server_url = "http://127.0.0.1:8101"
-        
-        try:
-            # Wait for server startup
-            for i in range(20):
-                try:
-                    response = requests.get(f"{server_url}/status", timeout=2)
-                    if response.status_code == 200:
-                        break
-                except requests.exceptions.RequestException:
-                    time.sleep(1)
-            else:
-                pytest.fail("Server failed to start")
-            
-            # Get house map
-            response = requests.get(f"{server_url}/state", timeout=5)
-            state = response.json()
-            
-            location = state['player']['location']
-            position = (state['player']['position']['x'], state['player']['position']['y'])
-            tiles = state['map']['tiles']
-            
-            # Save output
-            output_file = output_dir / "server_house.txt"
-            output_content = self.save_map_output(
-                tiles, output_file,
-                f"House - {location}", location, position
-            )
-            
-            # Compare with ground truth
-            ground_truth_file = ground_truth_dir / "house_server.txt"
-            matches, message = self.compare_with_ground_truth(output_content, ground_truth_file)
-            
-            print(f"Server house map saved to: {output_file}")
-            print(f"Comparison result: {message}")
-            
-            # Allow test to pass even if ground truth doesn't exist yet
-            if not os.path.exists(ground_truth_file):
-                pytest.skip(f"Ground truth file not found: {ground_truth_file}")
-            
-            assert matches, f"Server house map doesn't match ground truth: {message}"
-            
-        finally:
-            server_process.terminate()
-            try:
-                server_process.wait(timeout=3)
-            except subprocess.TimeoutExpired:
-                server_process.kill()
-    
-    def test_server_outside_map(self, output_dir, ground_truth_dir):
-        """Test server outside map against ground truth"""
-        # Kill any existing server
-        # os.system("pkill -f 'server.app' 2>/dev/null")
-        # time.sleep(2)
-        
-        # Start server
-        # server_cmd = ["python", "-m", "server.app", "--load-state", "tests/states/house.state", "--port", "8102", "--manual"]
-        # server_process = subprocess.Popen(server_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
-        server_url = "http://127.0.0.1:8000"
-        
-        try:
-            # Wait for server startup
-            for i in range(20):
-                try:
-                    response = requests.get(f"{server_url}/status", timeout=2)
-                    if response.status_code == 200:
-                        break
-                except requests.exceptions.RequestException:
-                    time.sleep(1)
-            else:
-                pytest.fail("Server failed to start")
-            
-            # Enhanced movement to reach position (5,11) like direct emulator
-            target_pos = (5, 11)
-            max_moves = 6
-            for move_num in range(max_moves):
-                try:
-                    # Check current position
-                    response = requests.get(f"{server_url}/state", timeout=10)
-                    state = response.json()
-                    current_pos = (state['player']['position']['x'], state['player']['position']['y'])
-                    current_location = state['player']['location']
-                    
-                    # If we've reached the target position, stop
-                    if current_pos == target_pos:
-                        break
-                    
-                    # If we're in outdoor area but not at target Y, keep moving down
-                    if "LITTLEROOT TOWN" in current_location and "HOUSE" not in current_location:
-                        if current_pos[1] < target_pos[1]:
-                            print('Posting action down')
-                            requests.post(f"{server_url}/action", json={"buttons": ["DOWN"]}, timeout=5)
-                            time.sleep(0.2)
-                            continue
-                        else:
-                            break
-                    else:
-                        # Still in house, keep moving down
-                        requests.post(f"{server_url}/action", json={"buttons": ["DOWN"]}, timeout=5)
-                        time.sleep(0.2)
-                    
-                except Exception:
-                    time.sleep(0.5)
-            
-            # Enhanced buffer synchronization
-            # for i in range(3):
-            #     try:
-            #         requests.post(f"{server_url}/debug/clear_cache", json={}, timeout=5)
-            #         time.sleep(0.2)
-            #     except:
-            #         pass
-            
-            # try:
-            #     requests.post(f"{server_url}/debug/force_buffer_redetection", json={}, timeout=5)
-            #     time.sleep(1.0)
-            # except:
-            #     pass
-            
-            # Get outside map
-            response = requests.get(f"{server_url}/state", timeout=15)
-            state = response.json()
-            
-            location = state['player']['location']
-            position = (state['player']['position']['x'], state['player']['position']['y'])
-            tiles = state['map']['tiles']
-            
-            # Save output
-            output_file = output_dir / "server_outside.txt"
-            output_content = self.save_map_output(
-                tiles, output_file,
-                f"Outside - {location}", location, position
-            )
-            
-            # Compare with ground truth
-            ground_truth_file = ground_truth_dir / "outside_server.txt"
-            matches, message = self.compare_with_ground_truth(output_content, ground_truth_file)
-            
-            print(f"Server outside map saved to: {output_file}")
-            print(f"Comparison result: {message}")
-            
-            # Allow test to pass even if ground truth doesn't exist yet
-            if not os.path.exists(ground_truth_file):
-                pytest.skip(f"Ground truth file not found: {ground_truth_file}")
-            
-            assert matches, f"Server outside map doesn't match ground truth: {message}"
-            
-        finally:
-            server_process.terminate()
-            try:
-                server_process.wait(timeout=3)
-            except subprocess.TimeoutExpired:
-                server_process.kill()
-    
-    def test_cross_comparison_house(self, output_dir):
-        """Test that direct emulator and server produce identical house maps"""
-        # This test runs after the individual tests and compares their outputs
-        direct_file = output_dir / "direct_emulator_house.txt"
-        server_file = output_dir / "server_house.txt"
-        
-        if not direct_file.exists() or not server_file.exists():
-            pytest.skip("Individual map tests must run first")
-        
-        with open(direct_file, 'r') as f:
-            direct_content = f.read()
-        
-        with open(server_file, 'r') as f:
-            server_content = f.read()
-        
-        # Compare the traversability maps
-        def extract_traversability_map(content):
-            lines = content.split('\n')
-            map_lines = []
-            in_map_section = False
-            
-            for line in lines:
-                if "--- TRAVERSABILITY MAP ---" in line:
-                    in_map_section = True
-                    continue
-                elif in_map_section and line.strip() and ':' in line:
-                    map_lines.append(line)
-            
-            return '\n'.join(map_lines)
-        
-        direct_map = extract_traversability_map(direct_content)
-        server_map = extract_traversability_map(server_content)
-        
-        assert direct_map == server_map, f"House maps don't match between direct emulator and server:\nDirect:\n{direct_map}\nServer:\n{server_map}"
-    
-    def test_cross_comparison_outside(self, output_dir):
-        """Test that direct emulator and server produce identical outside maps"""
-        # This test runs after the individual tests and compares their outputs
-        direct_file = output_dir / "direct_emulator_outside.txt"
-        server_file = output_dir / "server_outside.txt"
-        
-        if not direct_file.exists() or not server_file.exists():
-            pytest.skip("Individual map tests must run first")
-        
-        with open(direct_file, 'r') as f:
-            direct_content = f.read()
-        
-        with open(server_file, 'r') as f:
-            server_content = f.read()
-        
-        # Compare the traversability maps
-        def extract_traversability_map(content):
-            lines = content.split('\n')
-            map_lines = []
-            in_map_section = False
-            
-            for line in lines:
-                if "--- TRAVERSABILITY MAP ---" in line:
-                    in_map_section = True
-                    continue
-                elif in_map_section and line.strip() and ':' in line:
-                    map_lines.append(line)
-            
-            return '\n'.join(map_lines)
-        
-        direct_map = extract_traversability_map(direct_content)
-        server_map = extract_traversability_map(server_content)
-        
-        assert direct_map == server_map, f"Outside maps don't match between direct emulator and server:\nDirect:\n{direct_map}\nServer:\n{server_map}"
-
-
-if __name__ == "__main__":
-    # Allow running as script for manual testing
-    import sys
-    sys.exit(pytest.main([__file__, "-v", "-s"]))