synth-ai 0.2.4.dev8__py3-none-any.whl → 0.2.4.dev9__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/old/debug_player_loss.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""
+Debug exactly where player reference is lost
+============================================
+"""
+
+import asyncio
+from uuid import uuid4
+import gc
+
+from synth_ai.environments.examples.crafter_classic.environment import CrafterClassicEnvironment
+from synth_ai.environments.examples.crafter_classic.taskset import CrafterTaskInstance, CrafterTaskInstanceMetadata
+from synth_ai.environments.environment.tools import EnvToolCall
+from synth_ai.environments.tasks.core import Impetus, Intent
+
+
+async def debug_player_loss():
+    """Find exactly where player reference is lost."""
+    task = CrafterTaskInstance(
+        id=uuid4(),
+        impetus=Impetus(instructions="Test"),
+        intent=Intent(rubric={"goal": "Test"}, gold_trajectories=None, gold_state_diff={}),
+        metadata=CrafterTaskInstanceMetadata(
+            difficulty="easy",
+            seed=600,
+            num_trees_radius=5,
+            num_cows_radius=2,
+            num_hostiles_radius=0
+        ),
+        is_reproducible=True,
+        initial_engine_snapshot=None
+    )
+    
+    # Create and initialize environment
+    env = CrafterClassicEnvironment(task)
+    await env.initialize()
+    
+    # Take some actions
+    for action in [1, 2, 5, 3]:
+        await env.step(EnvToolCall(tool="interact", args={"action": action}))
+    
+    print("BEFORE SERIALIZATION:")
+    print(f"  Player id: {id(env.engine.env._player)}")
+    print(f"  Player: {env.engine.env._player}")
+    print(f"  Player pos: {env.engine.env._player.pos}")
+    
+    # Serialize
+    snapshot = await env._serialize_engine()
+    
+    # Check snapshot contents
+    print("\nSNAPSHOT CONTENTS:")
+    print(f"  env_raw_state keys: {list(snapshot.env_raw_state.keys())}")
+    print(f"  player_idx in snapshot: {snapshot.env_raw_state.get('player_idx')}")
+    
+    # Deserialize
+    restored_env = await CrafterClassicEnvironment._deserialize_engine(snapshot, task)
+    
+    print("\nAFTER DESERIALIZATION:")
+    print(f"  Player id: {id(restored_env.engine.env._player)}")
+    print(f"  Player: {restored_env.engine.env._player}")
+    print(f"  Player pos: {restored_env.engine.env._player.pos if restored_env.engine.env._player else 'None'}")
+    
+    # Check if player is in objects list
+    player_in_objects = False
+    for i, obj in enumerate(restored_env.engine.env._world._objects):
+        if obj is restored_env.engine.env._player:
+            player_in_objects = True
+            print(f"  Player found in objects at index {i}")
+            break
+    if not player_in_objects:
+        print("  Player NOT in objects list!")
+    
+    # Now let's trace through a step
+    print("\nTRACING THROUGH STEP:")
+    
+    # Check before _step_engine
+    print("1. Before _step_engine:")
+    print(f"   Player: {restored_env.engine.env._player}")
+    
+    # Check at start of _step_engine (manually)
+    print("\n2. Inside _step_engine (manual check):")
+    print(f"   self.env._player: {restored_env.engine.env._player}")
+    
+    # Try render directly
+    print("\n3. Testing render directly:")
+    try:
+        img = restored_env.engine.env.render()
+        print(f"   Render succeeded")
+        print(f"   Player after render: {restored_env.engine.env._player}")
+    except Exception as e:
+        print(f"   Render failed: {e}")
+        print(f"   Player after failed render: {restored_env.engine.env._player}")
+    
+    # Now try actual step
+    print("\n4. Trying actual step:")
+    try:
+        result = await restored_env.step(EnvToolCall(tool="interact", args={"action": 5}))
+        print(f"   Step succeeded, reward: {result.get('reward_last_step')}")
+    except Exception as e:
+        print(f"   Step failed: {e}")
+    
+    # Check player again
+    print(f"\n5. Player after step attempt: {restored_env.engine.env._player}")
+    
+    # Check if garbage collection is involved
+    print("\n6. Checking garbage collection:")
+    gc.collect()
+    print(f"   Player after gc: {restored_env.engine.env._player}")
+
+
+if __name__ == "__main__":
+    asyncio.run(debug_player_loss())

synth_ai/environments/examples/crafter_classic/agent_demos/old/diagnose_service.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""
+Diagnostic script to test Crafter service performance directly.
+This will help identify if the slowdown is in the service or the client.
+"""
+
+import asyncio
+import time
+import httpx
+import uuid
+from typing import List, Dict, Any
+
+async def test_service_performance():
+    """Test the Crafter service performance directly."""
+    
+    service_url = "http://localhost:8901"
+    
+    print("🔍 Crafter Service Performance Diagnostic")
+    print("=" * 60)
+    
+    # Test 1: Health check
+    print("\n1️⃣ Testing health check...")
+    async with httpx.AsyncClient(timeout=5.0) as client:
+        start_time = time.time()
+        try:
+            response = await client.get(f"{service_url}/health")
+            end_time = time.time()
+            print(f"   ✅ Health check: {response.status_code} in {(end_time - start_time)*1000:.1f}ms")
+            print(f"   📄 Response size: {len(response.content)} bytes")
+        except Exception as e:
+            print(f"   ❌ Health check failed: {e}")
+            return
+    
+    # Test 2: Environment creation
+    print("\n2️⃣ Testing environment creation...")
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        start_time = time.time()
+        try:
+            # Create proper task instance format
+            task_instance = {
+                "id": str(uuid.uuid4()),
+                "metadata": {
+                    "difficulty": "easy",
+                    "seed": 42,
+                    "num_trees_radius": 5,
+                    "num_cows_radius": 2,
+                    "num_hostiles_radius": 0
+                },
+                "impetus": {
+                    "instructions": "Survive and unlock achievements in easy environment."
+                },
+                "intent": {
+                    "rubric": {},
+                    "gold_trajectories": None,
+                    "gold_state_diff": {}
+                },
+                "is_reproducible": True,
+                "initial_engine_snapshot": None,
+                "config": {"world_config": "easy"}
+            }
+            
+            response = await client.post(
+                f"{service_url}/env/CrafterClassic/initialize",
+                json={"task_instance": task_instance}
+            )
+            end_time = time.time()
+            print(f"   ✅ Environment creation: {response.status_code} in {(end_time - start_time)*1000:.1f}ms")
+            
+            if response.status_code != 200:
+                print(f"   ❌ Error response: {response.text[:200]}")
+            
+            if response.status_code == 200:
+                data = response.json()
+                obs_size = len(str(data.get("observation", {})))
+                print(f"   📄 Observation size: {obs_size} characters")
+                print(f"   🆔 Environment ID: {data.get('env_id', 'N/A')}")
+                
+                env_id = data.get("env_id")
+                
+                # Test 3: Step execution
+                if env_id:
+                    print(f"\n3️⃣ Testing step execution (env_id: {env_id})...")
+                    step_start = time.time()
+                    step_response = await client.post(
+                        f"{service_url}/env/CrafterClassic/step",
+                        json={
+                            "env_id": env_id,
+                            "request_id": str(uuid.uuid4()),
+                            "action": {
+                                "tool_calls": [{"tool": "interact", "args": {"action": 5}}]
+                            }
+                        }
+                    )
+                    
+                    if step_response.status_code != 200:
+                        print(f"   ❌ Step error: {step_response.text[:200]}")
+                    step_end = time.time()
+                    print(f"   ✅ Step execution: {step_response.status_code} in {(step_end - step_start)*1000:.1f}ms")
+                    
+                    if step_response.status_code == 200:
+                        step_data = step_response.json()
+                        step_obs_size = len(str(step_data.get("observation", {})))
+                        print(f"   📄 Step observation size: {step_obs_size} characters")
+                        
+                        # Test 4: Multiple rapid steps
+                        print(f"\n4️⃣ Testing multiple rapid steps...")
+                        step_times = []
+                        for i in range(5):
+                            step_start = time.time()
+                            rapid_response = await client.post(
+                                f"{service_url}/env/CrafterClassic/step",
+                                json={
+                                    "env_id": env_id,
+                                    "request_id": str(uuid.uuid4()),
+                                    "action": {
+                                        "tool_calls": [{"tool": "interact", "args": {"action": 5}}]
+                                    }
+                                }
+                            )
+                            step_end = time.time()
+                            step_time = (step_end - step_start) * 1000
+                            step_times.append(step_time)
+                            print(f"   Step {i+1}: {rapid_response.status_code} in {step_time:.1f}ms")
+                        
+                        avg_step_time = sum(step_times) / len(step_times)
+                        print(f"   📊 Average step time: {avg_step_time:.1f}ms")
+                        print(f"   📊 Min step time: {min(step_times):.1f}ms")
+                        print(f"   📊 Max step time: {max(step_times):.1f}ms")
+                        
+                        # Test 5: Close environment
+                        print(f"\n5️⃣ Testing environment cleanup...")
+                        cleanup_start = time.time()
+                        cleanup_response = await client.post(f"{service_url}/env/{env_id}/close")
+                        cleanup_end = time.time()
+                        print(f"   ✅ Cleanup: {cleanup_response.status_code} in {(cleanup_end - cleanup_start)*1000:.1f}ms")
+            
+        except Exception as e:
+            print(f"   ❌ Environment test failed: {e}")
+    
+    # Test 6: Concurrent requests
+    print(f"\n6️⃣ Testing concurrent requests...")
+    async with httpx.AsyncClient(timeout=10.0) as client:
+        async def create_and_step():
+            try:
+                # Create proper task instance format
+                task_instance = {
+                    "id": str(uuid.uuid4()),
+                    "metadata": {
+                        "difficulty": "easy",
+                        "seed": 42,
+                        "num_trees_radius": 5,
+                        "num_cows_radius": 2,
+                        "num_hostiles_radius": 0
+                    },
+                    "impetus": {
+                        "instructions": "Survive and unlock achievements in easy environment."
+                    },
+                    "intent": {
+                        "rubric": {},
+                        "gold_trajectories": None,
+                        "gold_state_diff": {}
+                    },
+                    "is_reproducible": True,
+                    "initial_engine_snapshot": None,
+                    "config": {"world_config": "easy"}
+                }
+                
+                # Create environment
+                create_response = await client.post(
+                    f"{service_url}/env/CrafterClassic/initialize",
+                    json={"task_instance": task_instance}
+                )
+                if create_response.status_code == 200:
+                    env_id = create_response.json()["env_id"]
+                    # Take one step
+                    step_response = await client.post(
+                        f"{service_url}/env/CrafterClassic/step",
+                        json={
+                            "env_id": env_id,
+                            "request_id": str(uuid.uuid4()),
+                            "action": {
+                                "tool_calls": [{"tool": "interact", "args": {"action": 5}}]
+                            }
+                        }
+                    )
+                    # Cleanup
+                    await client.post(f"{service_url}/env/{env_id}/close")
+                    return step_response.status_code == 200
+                return False
+            except:
+                return False
+        
+        # Run 3 concurrent requests
+        start_time = time.time()
+        results = await asyncio.gather(*[create_and_step() for _ in range(3)])
+        end_time = time.time()
+        
+        success_count = sum(results)
+        print(f"   ✅ Concurrent test: {success_count}/3 successful in {(end_time - start_time)*1000:.1f}ms")
+        print(f"   📊 Average per request: {(end_time - start_time)*1000/3:.1f}ms")
+
+if __name__ == "__main__":
+    asyncio.run(test_service_performance()) 

synth_ai/environments/examples/crafter_classic/agent_demos/old/diagnose_slowness.py

@@ -0,0 +1,305 @@
+#!/usr/bin/env python3
+"""Diagnose Crafter environment slowness by profiling different components."""
+
+import time
+import logging
+import crafter
+import numpy as np
+import asyncio
+from typing import Dict, List, Any
+import statistics
+import cProfile
+import pstats
+import io
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# Import patches
+import synth_ai.environments.examples.crafter_classic.engine_deterministic_patch
+import synth_ai.environments.examples.crafter_classic.engine_serialization_patch_v3
+import synth_ai.environments.examples.crafter_classic.world_config_patch_simple
+
+
+class CrafterProfiler:
+    """Profile different aspects of Crafter environment."""
+    
+    def __init__(self):
+        self.timings: Dict[str, List[float]] = {
+            "env_creation": [],
+            "env_reset": [],
+            "env_step": [],
+            "env_render": [],
+            "state_extraction": [],
+            "serialization": [],
+        }
+        
+    def profile_raw_crafter(self, num_episodes: int = 5, steps_per_episode: int = 50):
+        """Profile the raw Crafter environment without any wrappers."""
+        logger.info("=" * 60)
+        logger.info("PROFILING RAW CRAFTER ENVIRONMENT")
+        logger.info("=" * 60)
+        
+        for episode in range(num_episodes):
+            logger.info(f"\nEpisode {episode + 1}/{num_episodes}")
+            
+            # Time environment creation
+            start = time.time()
+            env = crafter.Env(area=(64, 64), length=1000)
+            self.timings["env_creation"].append(time.time() - start)
+            logger.info(f"  Environment created in {self.timings['env_creation'][-1]:.3f}s")
+            
+            # Time reset
+            start = time.time()
+            obs = env.reset()
+            self.timings["env_reset"].append(time.time() - start)
+            logger.info(f"  Environment reset in {self.timings['env_reset'][-1]:.3f}s")
+            
+            # Time steps
+            step_times = []
+            render_times = []
+            
+            for step in range(steps_per_episode):
+                # Time step
+                action = np.random.randint(0, env.action_space.n)
+                start = time.time()
+                obs, reward, done, info = env.step(action)
+                step_time = time.time() - start
+                step_times.append(step_time)
+                
+                # Time render
+                start = time.time()
+                rendered = env.render()
+                render_time = time.time() - start
+                render_times.append(render_time)
+                
+                if done:
+                    logger.info(f"  Episode ended at step {step + 1}")
+                    break
+                    
+            self.timings["env_step"].extend(step_times)
+            self.timings["env_render"].extend(render_times)
+            
+            mean_step = statistics.mean(step_times) if step_times else 0
+            mean_render = statistics.mean(render_times) if render_times else 0
+            logger.info(f"  Mean step time: {mean_step:.3f}s")
+            logger.info(f"  Mean render time: {mean_render:.3f}s")
+            
+    def profile_with_patches(self, num_episodes: int = 3, steps_per_episode: int = 20):
+        """Profile Crafter with all patches applied."""
+        logger.info("\n" + "=" * 60)
+        logger.info("PROFILING CRAFTER WITH PATCHES")
+        logger.info("=" * 60)
+        
+        for episode in range(num_episodes):
+            logger.info(f"\nEpisode {episode + 1}/{num_episodes}")
+            
+            # Create environment
+            start = time.time()
+            env = crafter.Env(area=(64, 64), length=1000)
+            self.timings["env_creation"].append(time.time() - start)
+            logger.info(f"  Environment created in {self.timings['env_creation'][-1]:.3f}s")
+            
+            # Reset
+            start = time.time()
+            obs = env.reset()
+            self.timings["env_reset"].append(time.time() - start)
+            logger.info(f"  Environment reset in {self.timings['env_reset'][-1]:.3f}s")
+            
+            # Test serialization
+            if hasattr(env, 'save'):
+                start = time.time()
+                state = env.save()
+                save_time = time.time() - start
+                self.timings["serialization"].append(save_time)
+                logger.info(f"  State saved in {save_time:.3f}s")
+                
+                # Test loading
+                start = time.time()
+                env.load(state)
+                load_time = time.time() - start
+                logger.info(f"  State loaded in {load_time:.3f}s")
+            
+            # Run steps
+            for step in range(steps_per_episode):
+                action = np.random.randint(0, env.action_space.n)
+                start = time.time()
+                obs, reward, done, info = env.step(action)
+                self.timings["env_step"].append(time.time() - start)
+                
+                if done:
+                    break
+                    
+    def profile_specific_operations(self):
+        """Profile specific operations that might be slow."""
+        logger.info("\n" + "=" * 60)
+        logger.info("PROFILING SPECIFIC OPERATIONS")
+        logger.info("=" * 60)
+        
+        env = crafter.Env(area=(64, 64), length=1000)
+        env.reset()
+        
+        # Profile world generation
+        logger.info("\n1. World Generation:")
+        start = time.time()
+        # Access internal world generation if available
+        if hasattr(env, '_world'):
+            logger.info(f"  World object exists: {env._world is not None}")
+            logger.info(f"  World size: {env._area}")
+        elapsed = time.time() - start
+        logger.info(f"  Time to access world: {elapsed:.3f}s")
+        
+        # Profile rendering operations
+        logger.info("\n2. Rendering Operations:")
+        render_times = []
+        for i in range(10):
+            start = time.time()
+            obs = env.render()
+            render_times.append(time.time() - start)
+        logger.info(f"  Mean render time: {statistics.mean(render_times):.3f}s")
+        logger.info(f"  Render output shape: {obs.shape if hasattr(obs, 'shape') else 'N/A'}")
+        
+        # Profile achievement checking
+        logger.info("\n3. Achievement System:")
+        if hasattr(env, '_achievements'):
+            start = time.time()
+            # Simulate achievement checking
+            for _ in range(10):
+                env.step(0)  # No-op action
+            elapsed = time.time() - start
+            logger.info(f"  10 steps with achievement checking: {elapsed:.3f}s")
+            
+    def run_cprofile_analysis(self):
+        """Run cProfile analysis on a typical episode."""
+        logger.info("\n" + "=" * 60)
+        logger.info("RUNNING CPROFILE ANALYSIS")
+        logger.info("=" * 60)
+        
+        def run_episode():
+            env = crafter.Env(area=(64, 64), length=1000)
+            env.reset()
+            for _ in range(50):
+                action = np.random.randint(0, env.action_space.n)
+                obs, reward, done, info = env.step(action)
+                if done:
+                    break
+                    
+        # Profile the episode
+        profiler = cProfile.Profile()
+        profiler.enable()
+        run_episode()
+        profiler.disable()
+        
+        # Print stats
+        s = io.StringIO()
+        ps = pstats.Stats(profiler, stream=s).sort_stats('cumulative')
+        ps.print_stats(20)  # Top 20 functions
+        logger.info("\nTop 20 time-consuming functions:")
+        logger.info(s.getvalue())
+        
+    def print_summary(self):
+        """Print timing summary."""
+        logger.info("\n" + "=" * 60)
+        logger.info("TIMING SUMMARY")
+        logger.info("=" * 60)
+        
+        for operation, times in self.timings.items():
+            if times:
+                logger.info(f"\n{operation.upper()}:")
+                logger.info(f"  Count: {len(times)}")
+                logger.info(f"  Mean: {statistics.mean(times):.3f}s")
+                logger.info(f"  Median: {statistics.median(times):.3f}s")
+                logger.info(f"  Min: {min(times):.3f}s")
+                logger.info(f"  Max: {max(times):.3f}s")
+                if len(times) > 1:
+                    logger.info(f"  Std Dev: {statistics.stdev(times):.3f}s")
+
+
+async def check_import_time():
+    """Check how long it takes to import Crafter environment."""
+    logger.info("\n" + "=" * 60)
+    logger.info("CHECKING IMPORT TIMES")
+    logger.info("=" * 60)
+    
+    # Time importing the environment module
+    start = time.time()
+    import synth_ai.environments.examples.crafter_classic.environment
+    env_import_time = time.time() - start
+    logger.info(f"Importing crafter_classic.environment: {env_import_time:.3f}s")
+    
+    # Time importing the engine
+    start = time.time()
+    import synth_ai.environments.examples.crafter_classic.engine
+    engine_import_time = time.time() - start
+    logger.info(f"Importing crafter_classic.engine: {engine_import_time:.3f}s")
+    
+    # Time creating environment wrapper
+    from synth_ai.environments.examples.crafter_classic.environment import CrafterClassicEnvironment
+    from types import SimpleNamespace
+    
+    start = time.time()
+    task = SimpleNamespace(initial_engine_snapshot={})
+    env_wrapper = CrafterClassicEnvironment(task)
+    wrapper_creation_time = time.time() - start
+    logger.info(f"Creating CrafterClassicEnvironment wrapper: {wrapper_creation_time:.3f}s")
+    
+    # Time initialization
+    start = time.time()
+    await env_wrapper.initialize()
+    init_time = time.time() - start
+    logger.info(f"Initializing environment: {init_time:.3f}s")
+
+
+def main():
+    """Run all profiling tests."""
+    profiler = CrafterProfiler()
+    
+    # Test 1: Profile raw Crafter
+    profiler.profile_raw_crafter(num_episodes=2, steps_per_episode=20)
+    
+    # Test 2: Profile with patches
+    profiler.profile_with_patches(num_episodes=2, steps_per_episode=20)
+    
+    # Test 3: Profile specific operations
+    profiler.profile_specific_operations()
+    
+    # Test 4: Run cProfile analysis
+    profiler.run_cprofile_analysis()
+    
+    # Test 5: Check import times
+    asyncio.run(check_import_time())
+    
+    # Print summary
+    profiler.print_summary()
+    
+    # Recommendations
+    logger.info("\n" + "=" * 60)
+    logger.info("RECOMMENDATIONS")
+    logger.info("=" * 60)
+    
+    mean_step = statistics.mean(profiler.timings["env_step"]) if profiler.timings["env_step"] else 0
+    mean_render = statistics.mean(profiler.timings["env_render"]) if profiler.timings["env_render"] else 0
+    
+    if mean_step > 0.1:
+        logger.info("⚠️  Step operations are slow (>100ms). Consider:")
+        logger.info("   - Check if debug mode is enabled")
+        logger.info("   - Profile the step() method specifically")
+        logger.info("   - Check for unnecessary computations")
+        
+    if mean_render > 0.05:
+        logger.info("⚠️  Render operations are slow (>50ms). Consider:")
+        logger.info("   - Caching rendered observations")
+        logger.info("   - Optimizing image generation")
+        
+    if profiler.timings["serialization"] and statistics.mean(profiler.timings["serialization"]) > 0.5:
+        logger.info("⚠️  Serialization is slow (>500ms). Consider:")
+        logger.info("   - Using more efficient serialization format")
+        logger.info("   - Reducing state size")
+
+
+if __name__ == "__main__":
+    main()