synth-ai 0.2.12__py3-none-any.whl → 0.2.13.dev2__py3-none-any.whl

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_battle_state_formatting.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""
+Test battle state formatting - separate presentation for battle vs normal gameplay.
+"""
+
+import pytest
+from utils.state_formatter import format_state_for_llm
+from pokemon_env.enums import MetatileBehavior
+
+
+def test_battle_mode_hides_map():
+    """Test that battle mode doesn't show map information."""
+    battle_state = {
+        'player': {
+            'name': 'Red',
+            'position': {'x': 10, 'y': 10},
+            'party': [
+                {'species_name': 'Pikachu', 'level': 25, 'current_hp': 50, 'max_hp': 75, 'status': 'Normal'}
+            ]
+        },
+        'game': {
+            'is_in_battle': True,
+            'battle_info': {
+                'player_pokemon': {'species': 'Pikachu', 'level': 25, 'current_hp': 50, 'max_hp': 75},
+                'opponent_pokemon': {'species': 'Zubat', 'level': 10, 'current_hp': 20, 'max_hp': 30}
+            }
+        },
+        'map': {
+            'tiles': [[(1, MetatileBehavior.NORMAL, 0, 0)] * 3] * 3,
+            'current_map': 'Route 1'
+        }
+    }
+    
+    formatted = format_state_for_llm(battle_state)
+    
+    # Should show battle mode indicator
+    assert "=== BATTLE MODE ===" in formatted
+    assert "Currently in battle" in formatted
+    
+    # Should show battle status
+    assert "=== BATTLE STATUS ===" in formatted
+    assert "Your Pokemon: Pikachu" in formatted
+    assert "Opponent: Zubat" in formatted
+    
+    # Should NOT show map
+    assert "LOCATION & MAP INFO" not in formatted
+    assert "TRAVERSABILITY MAP" not in formatted
+    assert "Route 1" not in formatted
+
+
+def test_battle_mode_hides_dialogue():
+    """Test that battle mode doesn't show dialogue information."""
+    battle_state = {
+        'player': {'name': 'Red'},
+        'game': {
+            'is_in_battle': True,
+            'dialog_text': 'Trainer wants to battle!',  # This might be residual
+            'battle_info': {
+                'player_pokemon': {'species': 'Charmander', 'level': 5, 'current_hp': 18, 'max_hp': 20},
+                'opponent_pokemon': {'species': 'Rattata', 'level': 3, 'current_hp': 10, 'max_hp': 15}
+            }
+        }
+    }
+    
+    formatted = format_state_for_llm(battle_state)
+    
+    # Should show battle info
+    assert "Charmander" in formatted
+    assert "Rattata" in formatted
+    
+    # Should NOT show dialogue
+    assert "--- DIALOGUE ---" not in formatted
+    assert "Trainer wants to battle" not in formatted
+    assert "RESIDUAL TEXT" not in formatted
+
+
+def test_normal_mode_shows_everything():
+    """Test that normal (non-battle) mode shows all information."""
+    normal_state = {
+        'player': {
+            'name': 'Red',
+            'position': {'x': 10, 'y': 10},
+            'facing': 'North',
+            'party': [
+                {'species_name': 'Squirtle', 'level': 10, 'current_hp': 30, 'max_hp': 35, 'status': 'Normal'}
+            ]
+        },
+        'game': {
+            'is_in_battle': False,
+            'dialog_text': 'Welcome to the Pokemon Center!',
+            'dialogue_detected': {'has_dialogue': True, 'confidence': 0.9}
+        },
+        'map': {
+            'tiles': [[(1, MetatileBehavior.NORMAL, 0, 0)] * 3] * 3,
+            'current_map': 'Pokemon Center'
+        }
+    }
+    
+    formatted = format_state_for_llm(normal_state)
+    
+    # Should show normal player info
+    assert "=== PLAYER INFO ===" in formatted
+    assert "Position: X=10, Y=10" in formatted
+    assert "Facing: North" in formatted
+    
+    # Should show map
+    assert "LOCATION & MAP INFO" in formatted
+    
+    # Should show dialogue
+    assert "--- DIALOGUE ---" in formatted
+    assert "Welcome to the Pokemon Center" in formatted
+    assert "Detection confidence: 90.0%" in formatted
+    
+    # Should NOT show battle mode indicator
+    assert "=== BATTLE MODE ===" not in formatted
+
+
+def test_battle_party_information():
+    """Test that battle mode shows full party for switching decisions."""
+    battle_state = {
+        'player': {
+            'name': 'Red',
+            'party': [
+                {'species_name': 'Venusaur', 'level': 50, 'current_hp': 0, 'max_hp': 200, 'status': 'Fainted'},
+                {'species_name': 'Charizard', 'level': 50, 'current_hp': 180, 'max_hp': 185, 'status': 'Normal'},
+                {'species_name': 'Blastoise', 'level': 50, 'current_hp': 100, 'max_hp': 190, 'status': 'Poisoned'}
+            ]
+        },
+        'game': {
+            'in_battle': True,  # Alternative key
+            'battle_info': {
+                'player_pokemon': {'species': 'Venusaur', 'level': 50, 'current_hp': 0, 'max_hp': 200},
+                'opponent_pokemon': {'species': 'Alakazam', 'level': 55, 'current_hp': 150, 'max_hp': 160}
+            }
+        }
+    }
+    
+    formatted = format_state_for_llm(battle_state)
+    
+    # Should show party status section
+    assert "=== PARTY STATUS ===" in formatted
+    
+    # Should list all party members
+    assert "Venusaur" in formatted
+    assert "Charizard" in formatted
+    assert "Blastoise" in formatted
+    
+    # Should show status conditions
+    assert "Fainted" in formatted or "0/200" in formatted  # Venusaur fainted
+    assert "Poisoned" in formatted  # Blastoise poisoned
+
+
+def test_battle_mode_detection_variants():
+    """Test that both is_in_battle and in_battle keys trigger battle mode."""
+    # Test with is_in_battle
+    state1 = {
+        'player': {'name': 'Red'},
+        'game': {
+            'is_in_battle': True,
+            'battle_info': {'player_pokemon': {'species': 'Mew'}}
+        },
+        'map': {'current_map': 'Should not appear'}
+    }
+    
+    formatted1 = format_state_for_llm(state1)
+    assert "=== BATTLE MODE ===" in formatted1
+    assert "Should not appear" not in formatted1
+    
+    # Test with in_battle
+    state2 = {
+        'player': {'name': 'Blue'},
+        'game': {
+            'in_battle': True,
+            'battle_info': {'player_pokemon': {'species': 'Mewtwo'}}
+        },
+        'map': {'current_map': 'Also should not appear'}
+    }
+    
+    formatted2 = format_state_for_llm(state2)
+    assert "=== BATTLE MODE ===" in formatted2
+    assert "Also should not appear" not in formatted2
+
+
+def test_empty_battle_info():
+    """Test handling of battle mode with missing battle info."""
+    state = {
+        'player': {'name': 'Red'},
+        'game': {
+            'is_in_battle': True,
+            # No battle_info provided
+        }
+    }
+    
+    formatted = format_state_for_llm(state)
+    
+    # Should still enter battle mode
+    assert "=== BATTLE MODE ===" in formatted
+    
+    # Should handle missing battle info gracefully
+    assert "=== PARTY STATUS ===" in formatted  # Still shows party section
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_dialogue_detection.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""
+Test dialogue detection functionality.
+"""
+
+import pytest
+import numpy as np
+from utils.state_formatter import detect_dialogue_on_frame, format_state_for_llm
+
+
+def test_dialogue_detection_with_blue_box():
+    """Test dialogue detection with typical blue dialogue box."""
+    # Create a mock frame with dialogue box characteristics (240x160 GBA resolution)
+    frame = np.zeros((160, 240, 3), dtype=np.uint8)
+    
+    # Add blue dialogue box in bottom 50 pixels
+    # Blue color (R, G, B) where blue is dominant
+    frame[110:160, :] = [50, 70, 150]  # Bluish background
+    
+    # Add some white text areas
+    frame[120:130, 20:220] = [220, 220, 220]  # White text area
+    
+    # Add border lines
+    frame[110:112, :] = [100, 100, 100]  # Top border
+    frame[158:160, :] = [100, 100, 100]  # Bottom border
+    
+    result = detect_dialogue_on_frame(frame_array=frame)
+    
+    assert result['has_dialogue'] == True
+    assert result['confidence'] > 0.5
+    assert 'blue dialogue box' in result['reason'].lower()
+
+
+def test_no_dialogue_detection():
+    """Test that normal gameplay doesn't trigger dialogue detection."""
+    # Create a mock frame with varied gameplay content (no dialogue)
+    frame = np.random.randint(0, 255, (160, 240, 3), dtype=np.uint8)
+    
+    # Make it less random - add some structure but not dialogue-like
+    frame[0:80, :] = [100, 150, 100]  # Greenish top (grass/trees)
+    frame[80:160, :] = [150, 130, 100]  # Brownish bottom (ground)
+    
+    result = detect_dialogue_on_frame(frame_array=frame)
+    
+    assert result['has_dialogue'] == False
+    assert result['confidence'] < 0.5
+
+
+def test_dialogue_detection_grayscale():
+    """Test dialogue detection with grayscale input."""
+    # Create a grayscale frame
+    frame = np.zeros((160, 240), dtype=np.uint8)
+    
+    # Add high contrast pattern in dialogue area
+    frame[110:160, :] = 50  # Dark background
+    frame[120:130, 20:220] = 200  # Light text area
+    
+    # Add horizontal edges
+    frame[110:112, :] = 150
+    frame[158:160, :] = 150
+    
+    result = detect_dialogue_on_frame(frame_array=frame)
+    
+    # Should detect based on contrast and structure
+    assert 'text contrast' in result['reason'].lower() or 'borders' in result['reason'].lower()
+
+
+def test_dialogue_validation_in_state():
+    """Test that dialogue validation works in state formatting."""
+    # State with dialogue text but no frame detection
+    state_no_detection = {
+        'player': {'name': 'Red', 'position': {'x': 10, 'y': 10}},
+        'game': {
+            'dialog_text': 'Hello trainer! Would you like to battle?',
+            'dialogue_detected': {'has_dialogue': True, 'confidence': 0.8}
+        },
+        'map': {}
+    }
+    
+    formatted = format_state_for_llm(state_no_detection)
+    assert 'DIALOGUE' in formatted
+    assert 'Hello trainer' in formatted
+    assert 'Detection confidence: 80.0%' in formatted
+    
+    # State with dialogue text but frame says no dialogue visible
+    state_no_visible = {
+        'player': {'name': 'Red', 'position': {'x': 10, 'y': 10}},
+        'game': {
+            'dialog_text': 'Hello trainer! Would you like to battle?',
+            'dialogue_detected': {'has_dialogue': False, 'confidence': 0.1}
+        },
+        'map': {}
+    }
+    
+    formatted = format_state_for_llm(state_no_visible)
+    assert 'RESIDUAL TEXT' in formatted
+    assert 'not visible' in formatted
+    
+    # State with no dialogue detection info (backwards compatibility)
+    state_legacy = {
+        'player': {'name': 'Red', 'position': {'x': 10, 'y': 10}},
+        'game': {
+            'dialog_text': 'Hello trainer! Would you like to battle?'
+        },
+        'map': {}
+    }
+    
+    formatted = format_state_for_llm(state_legacy)
+    assert 'Hello trainer' in formatted  # Should still show dialogue
+
+
+def test_dialogue_detection_edge_cases():
+    """Test edge cases for dialogue detection."""
+    # Test with None
+    result = detect_dialogue_on_frame(frame_array=None)
+    assert result['has_dialogue'] == False
+    assert 'No frame data' in result['reason']
+    
+    # Test with wrong shape
+    small_frame = np.zeros((10, 10, 3), dtype=np.uint8)
+    result = detect_dialogue_on_frame(frame_array=small_frame)
+    # Should still work but likely no dialogue detected
+    assert 'has_dialogue' in result
+    assert 'confidence' in result
+    
+    # Test with very small dialogue region
+    tiny_frame = np.zeros((50, 240, 3), dtype=np.uint8)
+    result = detect_dialogue_on_frame(frame_array=tiny_frame)
+    assert 'has_dialogue' in result
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

examples/task_apps/dev/pokemon_emerald/external/pokeagent-speedrun/tests/test_dialogue_detection_comprehensive.py

@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+"""
+Comprehensive pytest for dialogue detection system across all states
+"""
+
+import pytest
+import sys
+import os
+import io
+import subprocess
+import time
+import requests
+import json
+import base64
+from pathlib import Path
+from PIL import Image
+
+# Add parent directory to path for imports
+sys.path.append(str(Path(__file__).parent.parent))
+
+from utils.ocr_dialogue import create_ocr_detector
+
+class TestDialogueDetection:
+    """Test dialogue detection accuracy across all provided states"""
+    
+    @pytest.fixture(autouse=True)
+    def setup(self):
+        """Setup for each test"""
+        self.detector = create_ocr_detector()
+        self.agent_port = 8000
+        assert self.detector is not None, "Could not create OCR detector"
+        
+        # Kill any existing agent_direct processes
+        subprocess.run(["pkill", "-f", "agent_direct.py"], capture_output=True)
+        time.sleep(1)
+    
+    def teardown_method(self):
+        """Cleanup after each test"""
+        subprocess.run(["pkill", "-f", "agent_direct.py"], capture_output=True)
+        time.sleep(0.5)
+    
+    def _test_state_file(self, state_file, expected_dialogue, description=""):
+        """Helper to test a single state file"""
+        print(f"\n🧪 Testing: {state_file}")
+        print(f"   Expected dialogue: {expected_dialogue}")
+        print(f"   Description: {description}")
+        
+        # Start agent_direct with this state
+        cmd = [
+            "/home/milkkarten/anaconda3/envs/mgba/bin/python", 
+            "agent_direct.py", 
+            "--load-state", state_file,
+            "--backend", "gemini", 
+            "--manual"
+        ]
+        
+        # Start agent_direct
+        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        
+        try:
+            # Wait for startup
+            time.sleep(3)
+            
+            # Test server responsiveness
+            for attempt in range(5):
+                try:
+                    response = requests.get(f"http://localhost:{self.agent_port}/status", timeout=2)
+                    if response.status_code == 200:
+                        break
+                    time.sleep(1)
+                except:
+                    time.sleep(1)
+            else:
+                pytest.fail(f"Agent_direct failed to start for {state_file}")
+            
+            # Get screenshot
+            frame_response = requests.get(f"http://localhost:{self.agent_port}/api/frame", timeout=5)
+            assert frame_response.status_code == 200, f"Failed to get screenshot for {state_file}"
+            
+            frame_data = frame_response.json()
+            assert frame_data.get('frame'), f"No frame data for {state_file}"
+            
+            # Decode screenshot
+            image_data = base64.b64decode(frame_data['frame'])
+            screenshot = Image.open(io.BytesIO(image_data))
+            
+            # Test dialogue detection
+            box_detected = self.detector.is_dialogue_box_visible(screenshot)
+            ocr_text = self.detector.detect_dialogue_from_screenshot(screenshot)
+            
+            print(f"   📦 Box detected: {box_detected}")
+            print(f"   👁️ OCR text: '{ocr_text}'")
+            
+            # Get memory reading for comparison
+            try:
+                state_response = requests.get(f"http://localhost:{self.agent_port}/state", timeout=3)
+                if state_response.status_code == 200:
+                    state_data = state_response.json()
+                    memory_text = state_data.get('game', {}).get('dialog_text', None)
+                    print(f"   💾 Memory text: '{memory_text}'")
+                else:
+                    memory_text = "N/A"
+            except:
+                memory_text = "N/A"
+            
+            # Verify detection accuracy
+            assert box_detected == expected_dialogue, (
+                f"Detection mismatch for {state_file}: expected {expected_dialogue}, got {box_detected}"
+            )
+            
+            return {
+                'state_file': state_file,
+                'expected_dialogue': expected_dialogue,
+                'box_detected': box_detected,
+                'ocr_text': ocr_text,
+                'memory_text': memory_text,
+                'description': description
+            }
+            
+        finally:
+            process.terminate()
+            time.sleep(1)
+    
+    def test_coordinate_tightness(self):
+        """Test that OCR coordinates are properly tight around text area"""
+        dialogue_coords = self.detector.DIALOGUE_BOX_COORDS
+        ocr_coords = self.detector.OCR_TEXT_COORDS
+        
+        # Calculate margins
+        left_margin = ocr_coords['x'] - dialogue_coords['x']
+        top_margin = ocr_coords['y'] - dialogue_coords['y']
+        right_margin = (dialogue_coords['x'] + dialogue_coords['width']) - (ocr_coords['x'] + ocr_coords['width'])
+        bottom_margin = (dialogue_coords['y'] + dialogue_coords['height']) - (ocr_coords['y'] + ocr_coords['height'])
+        
+        print(f"📏 Margins - Left: {left_margin}px, Top: {top_margin}px, Right: {right_margin}px, Bottom: {bottom_margin}px")
+        
+        # Verify margins are reasonable (4-16 pixels to avoid borders but not cut text)
+        assert 4 <= left_margin <= 16, f"Left margin {left_margin}px outside acceptable range (4-16px)"
+        assert 4 <= top_margin <= 16, f"Top margin {top_margin}px outside acceptable range (4-16px)"
+        assert 4 <= right_margin <= 16, f"Right margin {right_margin}px outside acceptable range (4-16px)"
+        assert 4 <= bottom_margin <= 16, f"Bottom margin {bottom_margin}px outside acceptable range (4-16px)"
+    
+    def test_no_dialog_states(self):
+        """Test states that should NOT have dialogue"""
+        no_dialog_states = [
+            ("tests/states/no_dialog1.state", "No dialogue state 1"),
+            ("tests/states/no_dialog2.state", "No dialogue state 2"),
+            ("tests/states/no_dialog3.state", "No dialogue state 3"),
+        ]
+        
+        for state_file, description in no_dialog_states:
+            if os.path.exists(state_file):
+                result = self._test_state_file(state_file, False, description)
+                assert result['box_detected'] == False, f"False positive detected in {state_file}"
+            else:
+                pytest.skip(f"State file not found: {state_file}")
+    
+    def test_dialog_states(self):
+        """Test states that SHOULD have dialogue"""
+        dialog_states = [
+            ("tests/states/dialog.state", "Original dialogue state"),
+            ("tests/states/dialog2.state", "Second dialogue state"),
+            ("tests/states/dialog3.state", "New dialogue state 3"),
+        ]
+        
+        for state_file, description in dialog_states:
+            if os.path.exists(state_file):
+                result = self._test_state_file(state_file, True, description)
+                assert result['box_detected'] == True, f"Failed to detect dialogue in {state_file}"
+            else:
+                pytest.skip(f"State file not found: {state_file}")
+    
+    def test_static_image_detection(self):
+        """Test detection on static images"""
+        # Test known dialogue frame
+        if os.path.exists("dialog_frame.png"):
+            image = Image.open("dialog_frame.png")
+            box_detected = self.detector.is_dialogue_box_visible(image)
+            assert box_detected == True, "Failed to detect dialogue in known dialogue frame"
+        
+        # Test emerald.png (should be no dialogue)
+        if os.path.exists("emerald.png"):
+            image = Image.open("emerald.png")
+            box_detected = self.detector.is_dialogue_box_visible(image)
+            assert box_detected == False, "False positive detected in emerald.png"
+    
+    def test_ocr_preprocessing_quality(self):
+        """Test that OCR preprocessing produces high-quality black/white output"""
+        if os.path.exists("dialog_frame.png"):
+            image = Image.open("dialog_frame.png")
+            image_np = np.array(image)
+            
+            # Extract OCR region
+            ocr_coords = self.detector.OCR_TEXT_COORDS
+            ocr_region = image_np[
+                ocr_coords['y']:ocr_coords['y'] + ocr_coords['height'],
+                ocr_coords['x']:ocr_coords['x'] + ocr_coords['width']
+            ]
+            
+            # Test preprocessing
+            processed = self.detector._preprocess_for_ocr(ocr_region)
+            
+            # Verify it's binary (only 0 and 255 values)
+            unique_values = np.unique(processed)
+            assert len(unique_values) <= 2, f"Processed image should be binary, found {len(unique_values)} unique values"
+            
+            # Should have both black and white pixels (text and background)
+            if len(unique_values) == 2:
+                assert 0 in unique_values and 255 in unique_values, "Should have pure black (0) and white (255) pixels"
+
+class TestDialogueIntegration:
+    """Test integration with LLM agent comprehensive state"""
+    
+    def test_comprehensive_state_includes_dialog(self):
+        """Test that comprehensive state includes dialogue reading"""
+        # This test verifies the integration works but doesn't need to run agent_direct
+        # Just verify the OCR detector can be imported and works
+        detector = create_ocr_detector()
+        assert detector is not None, "OCR detector should be available for comprehensive state"
+        
+        # Verify key methods exist
+        assert hasattr(detector, 'is_dialogue_box_visible'), "Detector should have dialogue box detection"
+        assert hasattr(detector, 'detect_dialogue_from_screenshot'), "Detector should have text detection"
+        assert hasattr(detector, 'read_dialog_with_ocr_fallback'), "Detector should have smart fallback logic"
+
+if __name__ == "__main__":
+    # Allow running as script for debugging
+    import numpy as np
+    pytest.main([__file__, "-v", "-s"])