synth-ai 0.2.14__py3-none-any.whl → 0.4.1__py3-none-any.whl

synth_ai/environments/examples/crafter_classic/agent_demos/old/profile_env_slowness.py

@@ -1,456 +0,0 @@
-#!/usr/bin/env python3
-"""Deep profiling of environment service slowness."""
-
-import asyncio
-import time
-import httpx
-import json
-import cProfile
-import pstats
-import io
-from typing import Dict, List, Any, Tuple
-import statistics
-import logging
-
-# Configure logging
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(levelname)s - %(message)s'
-)
-logger = logging.getLogger(__name__)
-
-
-class EnvironmentProfiler:
-    """Profile different layers of the environment service stack."""
-    
-    def __init__(self, service_url: str = "http://localhost:8901"):
-        self.service_url = service_url
-        self.timings: Dict[str, List[float]] = {
-            # Network layer
-            "http_request_total": [],
-            "http_request_only": [],
-            "json_serialization": [],
-            "json_deserialization": [],
-            
-            # Service layer
-            "service_processing": [],
-            "env_initialization": [],
-            "env_step": [],
-            
-            # Data transfer
-            "request_size_bytes": [],
-            "response_size_bytes": [],
-            
-            # Connection
-            "connection_setup": [],
-            "dns_lookup": [],
-        }
-        
-    async def profile_single_step(self, client: httpx.AsyncClient, env_id: str) -> Dict[str, Any]:
-        """Profile a single environment step with detailed timing."""
-        
-        # Prepare request payload
-        payload = {
-            "env_id": env_id,
-            "action": {
-                "tool_calls": [{
-                    "tool": "interact",
-                    "args": {"action": 0}
-                }]
-            }
-        }
-        
-        # Time JSON serialization
-        json_start = time.time()
-        json_data = json.dumps(payload)
-        json_time = time.time() - json_start
-        self.timings["json_serialization"].append(json_time)
-        self.timings["request_size_bytes"].append(len(json_data))
-        
-        # Time the full HTTP request
-        total_start = time.time()
-        
-        # Make request with detailed timing
-        response = await client.post(
-            f"{self.service_url}/env/CrafterClassic/step",
-            content=json_data,
-            headers={"Content-Type": "application/json"},
-            timeout=30.0
-        )
-        
-        total_time = time.time() - total_start
-        self.timings["http_request_total"].append(total_time)
-        
-        # Time JSON deserialization
-        response_text = response.text
-        self.timings["response_size_bytes"].append(len(response_text))
-        
-        deser_start = time.time()
-        response_data = json.loads(response_text)
-        deser_time = time.time() - deser_start
-        self.timings["json_deserialization"].append(deser_time)
-        
-        # Calculate network-only time (excluding serialization)
-        network_only = total_time - json_time - deser_time
-        self.timings["http_request_only"].append(network_only)
-        
-        return {
-            "total_time": total_time,
-            "network_time": network_only,
-            "json_serialize": json_time,
-            "json_deserialize": deser_time,
-            "request_size": len(json_data),
-            "response_size": len(response_text),
-            "response_data": response_data
-        }
-        
-    async def profile_with_connection_reuse(self):
-        """Test performance with connection reuse vs new connections."""
-        logger.info("\n" + "="*60)
-        logger.info("TESTING CONNECTION REUSE IMPACT")
-        logger.info("="*60)
-        
-        # Test 1: Reusing connection
-        logger.info("\n1. With connection reuse:")
-        async with httpx.AsyncClient() as client:
-            # Initialize environment
-            init_response = await client.post(
-                f"{self.service_url}/env/CrafterClassic/initialize",
-                json={"initial_state": {}, "config": {"area": [64, 64], "length": 100}}
-            )
-            env_id = init_response.json()["env_id"]
-            
-            # Run 10 steps with same connection
-            reuse_times = []
-            for i in range(10):
-                result = await self.profile_single_step(client, env_id)
-                reuse_times.append(result["total_time"])
-                logger.debug(f"  Step {i+1}: {result['total_time']:.3f}s")
-                
-            # Cleanup
-            await client.post(f"{self.service_url}/env/CrafterClassic/terminate", json={"env_id": env_id})
-            
-        logger.info(f"  Mean time with reuse: {statistics.mean(reuse_times):.3f}s")
-        
-        # Test 2: New connection each time
-        logger.info("\n2. With new connection each time:")
-        
-        # Initialize environment first
-        async with httpx.AsyncClient() as client:
-            init_response = await client.post(
-                f"{self.service_url}/env/CrafterClassic/initialize",
-                json={"initial_state": {}, "config": {"area": [64, 64], "length": 100}}
-            )
-            env_id = init_response.json()["env_id"]
-        
-        new_conn_times = []
-        for i in range(10):
-            # New client each time
-            async with httpx.AsyncClient() as client:
-                result = await self.profile_single_step(client, env_id)
-                new_conn_times.append(result["total_time"])
-                logger.debug(f"  Step {i+1}: {result['total_time']:.3f}s")
-                
-        # Cleanup
-        async with httpx.AsyncClient() as client:
-            await client.post(f"{self.service_url}/env/CrafterClassic/terminate", json={"env_id": env_id})
-            
-        logger.info(f"  Mean time with new connections: {statistics.mean(new_conn_times):.3f}s")
-        logger.info(f"  Overhead from new connections: {statistics.mean(new_conn_times) - statistics.mean(reuse_times):.3f}s")
-        
-    async def profile_payload_size_impact(self):
-        """Test if large payloads are causing slowness."""
-        logger.info("\n" + "="*60)
-        logger.info("TESTING PAYLOAD SIZE IMPACT")
-        logger.info("="*60)
-        
-        async with httpx.AsyncClient() as client:
-            # Initialize environment
-            init_response = await client.post(
-                f"{self.service_url}/env/CrafterClassic/initialize",
-                json={"initial_state": {}, "config": {"area": [64, 64], "length": 100}}
-            )
-            env_data = init_response.json()
-            env_id = env_data["env_id"]
-            
-            # Check observation size
-            obs_json = json.dumps(env_data["observation"])
-            logger.info(f"\nObservation size: {len(obs_json)} bytes ({len(obs_json)/1024:.1f} KB)")
-            
-            # Analyze observation structure
-            obs = env_data["observation"]
-            field_sizes = {}
-            for key, value in obs.items():
-                if isinstance(value, (list, dict)):
-                    field_sizes[key] = len(json.dumps(value))
-                else:
-                    field_sizes[key] = len(str(value))
-                    
-            # Sort by size
-            sorted_fields = sorted(field_sizes.items(), key=lambda x: x[1], reverse=True)
-            logger.info("\nLargest observation fields:")
-            for field, size in sorted_fields[:5]:
-                logger.info(f"  {field}: {size} bytes ({size/1024:.1f} KB)")
-                
-            # Test step timing
-            step_times = []
-            for i in range(5):
-                result = await self.profile_single_step(client, env_id)
-                step_times.append(result)
-                
-            # Analyze
-            logger.info("\nTiming breakdown (average of 5 steps):")
-            logger.info(f"  Total time: {statistics.mean([r['total_time'] for r in step_times]):.3f}s")
-            logger.info(f"  Network only: {statistics.mean([r['network_time'] for r in step_times]):.3f}s")
-            logger.info(f"  JSON serialize: {statistics.mean([r['json_serialize'] for r in step_times]):.6f}s")
-            logger.info(f"  JSON deserialize: {statistics.mean([r['json_deserialize'] for r in step_times]):.6f}s")
-            logger.info(f"  Response size: {statistics.mean([r['response_size'] for r in step_times]):.0f} bytes")
-            
-            # Cleanup
-            await client.post(f"{self.service_url}/env/CrafterClassic/terminate", json={"env_id": env_id})
-            
-    async def test_concurrent_environments(self):
-        """Test if multiple environments interfere with each other."""
-        logger.info("\n" + "="*60)
-        logger.info("TESTING CONCURRENT ENVIRONMENT INTERFERENCE")
-        logger.info("="*60)
-        
-        async with httpx.AsyncClient() as client:
-            # Create 5 environments
-            env_ids = []
-            for i in range(5):
-                init_response = await client.post(
-                    f"{self.service_url}/env/CrafterClassic/initialize",
-                    json={"initial_state": {}, "config": {"area": [64, 64], "length": 100}}
-                )
-                env_ids.append(init_response.json()["env_id"])
-                
-            logger.info(f"Created {len(env_ids)} environments")
-            
-            # Test 1: Sequential steps across different environments
-            logger.info("\n1. Sequential steps across environments:")
-            seq_times = []
-            for i in range(10):
-                env_id = env_ids[i % len(env_ids)]
-                result = await self.profile_single_step(client, env_id)
-                seq_times.append(result["total_time"])
-                
-            logger.info(f"  Mean time: {statistics.mean(seq_times):.3f}s")
-            
-            # Test 2: Concurrent steps
-            logger.info("\n2. Concurrent steps:")
-            
-            async def concurrent_step(env_id: str) -> float:
-                result = await self.profile_single_step(client, env_id)
-                return result["total_time"]
-                
-            # Run 5 concurrent steps
-            start = time.time()
-            concurrent_results = await asyncio.gather(*[
-                concurrent_step(env_id) for env_id in env_ids
-            ])
-            concurrent_time = time.time() - start
-            
-            logger.info(f"  Total time for 5 concurrent steps: {concurrent_time:.3f}s")
-            logger.info(f"  Mean individual step time: {statistics.mean(concurrent_results):.3f}s")
-            
-            # Cleanup
-            for env_id in env_ids:
-                await client.post(f"{self.service_url}/env/CrafterClassic/terminate", json={"env_id": env_id})
-                
-    async def profile_service_internals(self):
-        """Try to understand what the service is doing internally."""
-        logger.info("\n" + "="*60)
-        logger.info("ANALYZING SERVICE BEHAVIOR")
-        logger.info("="*60)
-        
-        async with httpx.AsyncClient() as client:
-            # Test with minimal config
-            logger.info("\n1. Testing with minimal world size:")
-            
-            # Small world
-            small_response = await client.post(
-                f"{self.service_url}/env/CrafterClassic/initialize",
-                json={"initial_state": {}, "config": {"area": [32, 32], "length": 10}}
-            )
-            small_env_id = small_response.json()["env_id"]
-            
-            # Time steps
-            small_times = []
-            for i in range(5):
-                result = await self.profile_single_step(client, small_env_id)
-                small_times.append(result["total_time"])
-                
-            logger.info(f"  32x32 world mean step time: {statistics.mean(small_times):.3f}s")
-            
-            # Normal world
-            logger.info("\n2. Testing with normal world size:")
-            normal_response = await client.post(
-                f"{self.service_url}/env/CrafterClassic/initialize",
-                json={"initial_state": {}, "config": {"area": [64, 64], "length": 100}}
-            )
-            normal_env_id = normal_response.json()["env_id"]
-            
-            normal_times = []
-            for i in range(5):
-                result = await self.profile_single_step(client, normal_env_id)
-                normal_times.append(result["total_time"])
-                
-            logger.info(f"  64x64 world mean step time: {statistics.mean(normal_times):.3f}s")
-            
-            # Cleanup
-            await client.post(f"{self.service_url}/env/CrafterClassic/terminate", json={"env_id": small_env_id})
-            await client.post(f"{self.service_url}/env/CrafterClassic/terminate", json={"env_id": normal_env_id})
-            
-    def print_summary(self):
-        """Print timing summary."""
-        logger.info("\n" + "="*60)
-        logger.info("TIMING SUMMARY")
-        logger.info("="*60)
-        
-        for category, times in self.timings.items():
-            if times and category not in ["request_size_bytes", "response_size_bytes"]:
-                logger.info(f"\n{category}:")
-                logger.info(f"  Samples: {len(times)}")
-                logger.info(f"  Mean: {statistics.mean(times):.3f}s")
-                logger.info(f"  Median: {statistics.median(times):.3f}s")
-                logger.info(f"  Min: {min(times):.3f}s")
-                logger.info(f"  Max: {max(times):.3f}s")
-                
-        # Print size statistics
-        if self.timings["response_size_bytes"]:
-            logger.info(f"\nResponse sizes:")
-            logger.info(f"  Mean: {statistics.mean(self.timings['response_size_bytes'])/1024:.1f} KB")
-            logger.info(f"  Max: {max(self.timings['response_size_bytes'])/1024:.1f} KB")
-
-
-async def trace_single_request():
-    """Trace a single request in detail to see where time goes."""
-    logger.info("\n" + "="*60)
-    logger.info("TRACING SINGLE REQUEST IN DETAIL")
-    logger.info("="*60)
-    
-    # Use httpx events to trace request lifecycle
-    async def log_request_start(request):
-        logger.info(f"  [REQUEST START] {request.method} {request.url}")
-        
-    async def log_request_end(request):
-        logger.info(f"  [REQUEST END] {request.method} {request.url}")
-        
-    async def log_response_start(response):
-        logger.info(f"  [RESPONSE START] Status: {response.status_code}")
-        
-    async def log_response_end(response):
-        logger.info(f"  [RESPONSE END] Status: {response.status_code}")
-    
-    event_hooks = {
-        "request": [log_request_start, log_request_end],
-        "response": [log_response_start, log_response_end]
-    }
-    
-    async with httpx.AsyncClient(event_hooks=event_hooks) as client:
-        # Initialize
-        logger.info("\nInitializing environment...")
-        init_start = time.time()
-        
-        init_response = await client.post(
-            "http://localhost:8901/env/CrafterClassic/initialize",
-            json={"initial_state": {}, "config": {"area": [64, 64], "length": 100}},
-            timeout=30.0
-        )
-        
-        init_time = time.time() - init_start
-        env_id = init_response.json()["env_id"]
-        logger.info(f"  Initialization took: {init_time:.3f}s")
-        
-        # Single step with detailed timing
-        logger.info("\nExecuting single step...")
-        
-        payload = {
-            "env_id": env_id,
-            "action": {"tool_calls": [{"tool": "interact", "args": {"action": 0}}]}
-        }
-        
-        # Time each phase
-        phases = {}
-        
-        # Phase 1: Serialize
-        phase_start = time.time()
-        json_data = json.dumps(payload)
-        phases["serialize"] = time.time() - phase_start
-        
-        # Phase 2: Send request and wait for response
-        phase_start = time.time()
-        response = await client.post(
-            "http://localhost:8901/env/CrafterClassic/step",
-            content=json_data,
-            headers={"Content-Type": "application/json"},
-            timeout=30.0
-        )
-        phases["network"] = time.time() - phase_start
-        
-        # Phase 3: Read response
-        phase_start = time.time()
-        response_text = response.text
-        phases["read_response"] = time.time() - phase_start
-        
-        # Phase 4: Parse JSON
-        phase_start = time.time()
-        response_data = json.loads(response_text)
-        phases["deserialize"] = time.time() - phase_start
-        
-        # Print breakdown
-        total_time = sum(phases.values())
-        logger.info(f"\n  Total step time: {total_time:.3f}s")
-        logger.info("  Breakdown:")
-        for phase, duration in phases.items():
-            percentage = (duration / total_time) * 100
-            logger.info(f"    {phase}: {duration:.3f}s ({percentage:.1f}%)")
-            
-        # Cleanup
-        await client.post(
-            "http://localhost:8901/env/CrafterClassic/terminate",
-            json={"env_id": env_id}
-        )
-
-
-async def main():
-    """Run all profiling tests."""
-    profiler = EnvironmentProfiler()
-    
-    # First trace a single request
-    await trace_single_request()
-    
-    # Run profiling tests
-    await profiler.profile_with_connection_reuse()
-    await profiler.profile_payload_size_impact()
-    await profiler.test_concurrent_environments()
-    await profiler.profile_service_internals()
-    
-    # Print summary
-    profiler.print_summary()
-    
-    # Final analysis
-    logger.info("\n" + "="*60)
-    logger.info("ANALYSIS & RECOMMENDATIONS")
-    logger.info("="*60)
-    
-    if profiler.timings["http_request_only"]:
-        mean_network = statistics.mean(profiler.timings["http_request_only"])
-        if mean_network > 1.0:
-            logger.info("\n⚠️  Network latency is high (>1s). Possible causes:")
-            logger.info("   - Service is overloaded")
-            logger.info("   - Python GIL blocking with concurrent requests")
-            logger.info("   - Inefficient service implementation")
-            
-    if profiler.timings["response_size_bytes"]:
-        mean_size = statistics.mean(profiler.timings["response_size_bytes"])
-        if mean_size > 50000:  # 50KB
-            logger.info("\n⚠️  Large response payloads. Consider:")
-            logger.info("   - Compressing responses")
-            logger.info("   - Removing unnecessary fields from observations")
-            logger.info("   - Using binary protocols instead of JSON")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

synth_ai/environments/examples/crafter_classic/agent_demos/old/replicate_issue.py

@@ -1,166 +0,0 @@
-#!/usr/bin/env python3
-"""
-Script to replicate the issue with Qwen 7B model inference
-"""
-
-import httpx
-import asyncio
-import json
-from datetime import datetime
-
-# Configuration
-BASE_URL = "http://localhost:8000/api/v1/learning"
-API_KEY = "test-api-key"  # Replace with your actual API key
-MODEL = "Qwen/Qwen2.5-7B-Instruct"
-
-async def test_base_model_inference():
-    """Test base model inference through the local learning service."""
-    
-    print(f"🔍 Testing Base Model Inference")
-    print(f"Time: {datetime.now()}")
-    print(f"Endpoint: {BASE_URL}/v1/chat/completions")
-    print(f"Model: {MODEL}")
-    print("=" * 60)
-    
-    async with httpx.AsyncClient(timeout=30.0) as client:
-        headers = {
-            "Authorization": f"Bearer {API_KEY}",
-            "Content-Type": "application/json"
-        }
-        
-        payload = {
-            "model": MODEL,
-            "messages": [
-                {"role": "user", "content": "Hello, can you help me play Crafter?"}
-            ],
-            "temperature": 0.7,
-            "max_tokens": 100
-        }
-        
-        print("\n📤 Request:")
-        print(f"Headers: {json.dumps(headers, indent=2)}")
-        print(f"Payload: {json.dumps(payload, indent=2)}")
-        
-        try:
-            print("\n⏳ Sending request...")
-            response = await client.post(
-                f"{BASE_URL}/v1/chat/completions",
-                headers=headers,
-                json=payload
-            )
-            
-            print(f"\n📥 Response Status: {response.status_code}")
-            print(f"Response Headers: {dict(response.headers)}")
-            
-            if response.status_code == 200:
-                data = response.json()
-                print(f"\n✅ Success!")
-                print(f"Response: {json.dumps(data, indent=2)}")
-                
-                # Extract the assistant's message
-                if "choices" in data and data["choices"]:
-                    content = data["choices"][0]["message"]["content"]
-                    print(f"\n🤖 Assistant says: {content}")
-                    
-                    # Check if it's the "not implemented" message
-                    if "Base model inference not implemented" in content:
-                        print("\n⚠️  WARNING: The service returned a 'not implemented' message!")
-                        print("This means the service is responding but base models aren't supported yet.")
-            else:
-                print(f"\n❌ Error Response:")
-                print(f"Body: {response.text}")
-                
-        except httpx.ConnectError as e:
-            print(f"\n❌ Connection Error: Could not connect to {BASE_URL}")
-            print(f"Error: {e}")
-            print("\nPossible issues:")
-            print("1. The backend service is not running on port 8000")
-            print("2. The learning service is not properly configured")
-            print("3. The proxy route /api/v1/learning is not set up")
-            
-        except httpx.TimeoutException as e:
-            print(f"\n❌ Timeout Error: Request timed out after 30 seconds")
-            print(f"Error: {e}")
-            
-        except Exception as e:
-            print(f"\n❌ Unexpected Error: {type(e).__name__}")
-            print(f"Error: {e}")
-
-
-async def test_crafter_integration():
-    """Test the full Crafter + LLM integration."""
-    
-    print("\n\n🎮 Testing Crafter Integration")
-    print("=" * 60)
-    
-    # First check if Crafter service is running
-    crafter_url = "http://localhost:8901"
-    
-    async with httpx.AsyncClient(timeout=5.0) as client:
-        try:
-            print(f"\n1️⃣ Checking Crafter service at {crafter_url}...")
-            response = await client.get(f"{crafter_url}/health")
-            if response.status_code == 200:
-                print("✅ Crafter service is running!")
-            else:
-                print(f"⚠️  Crafter service returned status {response.status_code}")
-        except Exception as e:
-            print(f"❌ Crafter service is not running on port 8901")
-            print(f"   Error: {e}")
-            print("\n   To start Crafter service:")
-            print("   uv run python -m uvicorn synth_ai.environments.service.app:app --host 0.0.0.0 --port 8901")
-    
-    # Test creating an environment
-    if True:  # You can set to False to skip environment creation
-        print(f"\n2️⃣ Testing environment creation...")
-        async with httpx.AsyncClient(timeout=10.0) as client:
-            try:
-                response = await client.post(
-                    f"{crafter_url}/create_env",
-                    json={
-                        "instance_id": "test_instance",
-                        "render_mode": "rgb_array",
-                        "difficulty": "easy",
-                        "seed": 42
-                    }
-                )
-                if response.status_code == 200:
-                    print("✅ Environment created successfully!")
-                else:
-                    print(f"❌ Failed to create environment: {response.text}")
-            except Exception as e:
-                print(f"❌ Error creating environment: {e}")
-
-
-def main():
-    """Run all tests."""
-    print("🚀 Replicating Qwen 7B Model Inference Issue")
-    print("=" * 60)
-    
-    # Run async tests
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
-    
-    try:
-        # Test base model inference
-        loop.run_until_complete(test_base_model_inference())
-        
-        # Test Crafter integration
-        loop.run_until_complete(test_crafter_integration())
-        
-    finally:
-        loop.close()
-    
-    print("\n\n📋 Summary:")
-    print("=" * 60)
-    print("This script tested:")
-    print("1. Base model inference through the learning service proxy")
-    print("2. Crafter environment service availability")
-    print("\nTo fix the issues:")
-    print("1. Ensure the backend is running: cd backend && uvicorn app.main:app --reload")
-    print("2. Ensure the learning service is configured and running")
-    print("3. Start Crafter service: uv run python -m uvicorn synth_ai.environments.service.app:app --host 0.0.0.0 --port 8901")
-
-
-if __name__ == "__main__":
-    main()