qyro 2.0.0__py3-none-any.whl

qyro/common/rpc.py

@@ -0,0 +1,287 @@
+"""
+Nexus Cross-Language Function Calling System (Redis-Backed)
+Enables any language to call functions registered in any other language.
+
+Re-architected for Phase 0 Core Repair to use Redis Lists/PubSub.
+"""
+
+import json
+import time
+import uuid
+import threading
+from typing import Dict, Any, Callable, Optional, List
+from dataclasses import dataclass, field, asdict
+from enum import IntEnum
+from functools import wraps
+
+from .logging import get_logger
+
+logger = get_logger("nexus.rpc")
+
+class CallStatus(IntEnum):
+    PENDING = 0
+    PROCESSING = 1
+    COMPLETED = 2
+    FAILED = 3
+    TIMEOUT = 4
+
+@dataclass
+class FunctionInfo:
+    name: str
+    language: str
+    process_id: str
+    param_types: List[str] = field(default_factory=list)
+    return_type: str = "any"
+    
+    def to_dict(self): return asdict(self)
+    @classmethod
+    def from_dict(cls, d): return cls(**d)
+
+@dataclass
+class FunctionCall:
+    call_id: str
+    function_name: str
+    args: List[Any]
+    kwargs: Dict[str, Any]
+    caller_id: str
+    timestamp: float
+    status: CallStatus = CallStatus.PENDING
+    result: Any = None
+    error: Optional[str] = None
+    
+    def to_dict(self):
+        d = asdict(self)
+        d['status'] = int(self.status)
+        return d
+    
+    @classmethod
+    def from_dict(cls, d):
+        d['status'] = CallStatus(d.get('status', 0))
+        return cls(**d)
+
+class NexusRPC:
+    """
+    Redis-backed RPC System.
+    Uses Redis Lists for Task Queue and Pub/Sub for Results.
+    """
+
+    # Redis Keys
+    KEY_REGISTRY = "nexus:rpc:registry"
+    KEY_QUEUE = "nexus:rpc:queue"
+    KEY_RESULT_PREFIX = "nexus:rpc:result:"
+    KEY_BROKER_LOCK = "nexus:rpc:broker_lock"
+    
+    def __init__(self, memory, mode: str = "event"):
+        self._memory = memory  # Expects RedisNexusMemory
+        self._mode = mode
+        self._local_handlers: Dict[str, Callable] = {}
+        self._process_id = str(uuid.uuid4())[:8]
+        self._running = False
+        self._worker_thread: Optional[threading.Thread] = None
+        
+        # Ensure we have a Redis client
+        if not hasattr(memory, '_redis'):
+             logger.warning("rpc_compatibility_mode", msg="Falling back to legacy memory (not supported)")
+             self._redis = None
+        else:
+             self._redis = memory._redis
+
+    def register(self, name: str, handler: Callable, **kwargs):
+        """Register a function."""
+        self._local_handlers[name] = handler
+        
+        info = FunctionInfo(
+            name=name,
+            language="python",
+            process_id=self._process_id,
+            param_types=kwargs.get('param_types', []),
+            return_type=kwargs.get('return_type', 'any')
+        )
+        
+        if self._redis:
+            self._redis.hset(self.KEY_REGISTRY, name, json.dumps(info.to_dict()))
+        
+        logger.info("rpc_registered", name=name)
+
+    def unregister(self, name: str):
+        if name in self._local_handlers:
+            del self._local_handlers[name]
+        if self._redis:
+            self._redis.hdel(self.KEY_REGISTRY, name)
+
+    def export(self, name: str = None, **kwargs):
+        """Decorator to export function."""
+        def decorator(func: Callable):
+            fn_name = name or func.__name__
+            self.register(fn_name, func, **kwargs)
+            
+            @wraps(func)
+            def wrapper(*args, **kw):
+                return func(*args, **kw)
+            return wrapper
+        return decorator
+
+    def call(self, function_name: str, *args, timeout: float = 10.0, **kwargs) -> Any:
+        """Execute a remote function call."""
+        if not self._redis:
+            raise RuntimeError("RPC requires Redis connection")
+
+        # 1. Look up function owner
+        raw_info = self._redis.hget(self.KEY_REGISTRY, function_name)
+        if not raw_info:
+            raise ValueError(f"Function not found in registry: {function_name}")
+
+        info = json.loads(raw_info)
+        target_pid = info.get('process_id')
+
+        if not target_pid:
+             raise ValueError(f"Function {function_name} has no owner (僵尸 state?)")
+
+        call_id = str(uuid.uuid4())
+        call = FunctionCall(
+            call_id=call_id,
+            function_name=function_name,
+            args=list(args),
+            kwargs=kwargs,
+            caller_id=self._process_id,
+            timestamp=time.time()
+        )
+
+        # 2. Push to TARGET Queue (O(1) routing)
+        target_queue = f"{self.KEY_QUEUE}:{target_pid}"
+        self._redis.rpush(target_queue, json.dumps(call.to_dict()))
+
+        # 3. Use Pub/Sub for result instead of polling
+        result_channel = f"{self.KEY_RESULT_PREFIX}{call_id}"
+
+        # Create a pubsub connection to listen for the result
+        pubsub = self._redis.pubsub()
+        pubsub.subscribe(result_channel)
+
+        # 4. Wait for Result via Pub/Sub with proper timeout handling
+        start = time.time()
+        try:
+            for message in pubsub.listen():
+                if message['type'] == 'message':
+                    try:
+                        res = FunctionCall.from_dict(json.loads(message['data']))
+
+                        if res.status == CallStatus.FAILED:
+                            raise RuntimeError(res.error)
+                        return res.result
+                    except json.JSONDecodeError as e:
+                        logger.error("rpc_result_parse_error", call_id=call_id, error=str(e))
+                        continue
+                    except Exception as e:
+                        logger.error("rpc_result_processing_error", call_id=call_id, error=str(e))
+                        raise
+
+                # Check for timeout
+                if time.time() - start >= timeout:
+                    raise TimeoutError(f"RPC Timeout: {function_name} after {timeout}s")
+
+        except Exception as e:
+            # Clean up the result key if there's an error
+            try:
+                self._redis.delete(f"{self.KEY_RESULT_PREFIX}{call_id}")
+            except:
+                pass
+            raise
+        finally:
+            try:
+                pubsub.close()
+            except:
+                pass
+
+    def start_handler(self):
+        """Start the worker thread."""
+        if self._running: return
+        self._running = True
+        self._worker_thread = threading.Thread(target=self._worker_loop, daemon=True)
+        self._worker_thread.start()
+        logger.info("rpc_worker_started", pid=self._process_id)
+
+    def stop_handler(self):
+        self._running = False
+        if self._worker_thread:
+            self._worker_thread.join(timeout=1.0)
+
+    def _worker_loop(self):
+        """Continuously process calls from MY queue."""
+        my_queue = f"{self.KEY_QUEUE}:{self._process_id}"
+
+        while self._running:
+            try:
+                if not self._redis:
+                    time.sleep(1)
+                    continue
+
+                # BLPOP from MY dedicated queue with proper error handling
+                try:
+                    item = self._redis.blpop(my_queue, timeout=1)
+                    if not item:
+                        continue
+
+                    _, data = item
+                    call = FunctionCall.from_dict(json.loads(data))
+
+                    # Execute (No need to check ownership, if it's in my queue, it's for me)
+                    self._execute_call(call)
+
+                except Exception as blpop_error:
+                    logger.error("rpc_blpop_error", error=str(blpop_error))
+                    # Brief pause before continuing to avoid tight loop on connection errors
+                    time.sleep(0.1)
+                    continue
+
+            except Exception as e:
+                logger.error("rpc_worker_error", error=str(e))
+                time.sleep(1)
+
+    def _execute_call(self, call: FunctionCall):
+        """Execute a function call and publish the result."""
+        try:
+            if call.function_name in self._local_handlers:
+                handler = self._local_handlers[call.function_name]
+
+                # Update call status to processing
+                call.status = CallStatus.PROCESSING
+
+                # Execute the handler
+                result = handler(*call.args, **call.kwargs)
+
+                # Update call with result
+                call.status = CallStatus.COMPLETED
+                call.result = result
+            else:
+                # Function not found in this process
+                call.status = CallStatus.FAILED
+                call.error = f"Function '{call.function_name}' not found in this module"
+                logger.error("rpc_function_not_found", function_name=call.function_name)
+
+        except Exception as e:
+            call.status = CallStatus.FAILED
+            call.error = str(e)
+            logger.error("rpc_execution_failed", fn=call.function_name, error=str(e))
+
+        # Publish result via Pub/Sub instead of storing in key
+        result_channel = f"{self.KEY_RESULT_PREFIX}{call.call_id}"
+
+        # Use Redis pipeline for atomic operation
+        with self._redis.pipeline() as pipe:
+            pipe.publish(result_channel, json.dumps(call.to_dict()))
+            # Set expiration for the result channel to prevent accumulation
+            pipe.expire(result_channel, 300)  # 5 minutes
+            pipe.execute()
+
+        logger.debug("rpc_call_executed", call_id=call.call_id, status=call.status.name)
+
+
+
+# Dummy Registry class for backwards compatibility imports if needed
+class FunctionRegistry:
+    def __init__(self, *args): pass
+
+def nexus_export(name=None, **kwargs):
+    def dec(f): return f
+    return dec

qyro/common/sandbox.py

@@ -0,0 +1,191 @@
+"""
+Nexus Secure Sandbox System
+Provides containerized execution for untrusted code using Docker isolation.
+"""
+
+import docker
+import tempfile
+import os
+import json
+from pathlib import Path
+from typing import Dict, Any, Optional, List
+import time
+import signal
+import subprocess
+
+
+class NexusSandbox:
+    """
+    Secure execution environment for untrusted code using Docker containers.
+    
+    Features:
+    - Isolated container execution
+    - Resource limits (CPU, memory, disk)
+    - Network isolation
+    - File system restrictions
+    - Timeout enforcement
+    """
+    
+    def __init__(self):
+        try:
+            self.client = docker.from_env()
+            # Test connection
+            self.client.ping()
+        except Exception as e:
+            raise RuntimeError(f"Docker is required for sandboxing: {e}")
+    
+    def execute_untrusted_code(
+        self, 
+        code: str, 
+        language: str, 
+        timeout: int = 30,
+        memory_limit: str = "128m",
+        cpu_quota: int = 100000,  # 10% of 1 CPU
+        network_disabled: bool = True
+    ) -> Dict[str, Any]:
+        """
+        Execute untrusted code in a secure Docker container.
+        
+        Args:
+            code: Source code to execute
+            language: Programming language ('python', 'javascript', 'go', etc.)
+            timeout: Execution timeout in seconds
+            memory_limit: Memory limit (e.g., '128m', '1g')
+            cpu_quota: CPU quota in microseconds per period
+            network_disabled: Whether to disable network access
+            
+        Returns:
+            Dictionary with execution results
+        """
+        # Map language to Docker image
+        image_map = {
+            'python': 'python:3.11-alpine',
+            'js': 'node:18-alpine',
+            'javascript': 'node:18-alpine',
+            'go': 'golang:1.21-alpine',
+            'rust': 'rust:1.75-slim',
+            'java': 'openjdk:17-jdk-slim',
+            'c': 'gcc:11.4-bookworm',
+            'cpp': 'gcc:11.4-bookworm',
+            'ruby': 'ruby:3.2-alpine',
+            'php': 'php:8.2-cli-alpine',
+            'lua': 'lua:5.4-alpine'
+        }
+        
+        if language not in image_map:
+            raise ValueError(f"Unsupported language for sandbox: {language}")
+        
+        image = image_map[language]
+        
+        # Create temporary directory for code
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_path = Path(temp_dir)
+            
+            # Write code to temporary file based on language
+            file_extensions = {
+                'python': '.py',
+                'js': '.js',
+                'javascript': '.js',
+                'go': '.go',
+                'rust': '.rs',
+                'java': '.java',
+                'c': '.c',
+                'cpp': '.cpp',
+                'ruby': '.rb',
+                'php': '.php',
+                'lua': '.lua'
+            }
+            
+            ext = file_extensions[language]
+            code_file = temp_path / f"code{ext}"
+            code_file.write_text(code)
+            
+            # Determine execution command based on language
+            commands = {
+                'python': ['python', '/code/code.py'],
+                'js': ['node', '/code/code.js'],
+                'javascript': ['node', '/code/code.js'],
+                'go': ['sh', '-c', 'cd /code && go run *.go'],
+                'rust': ['sh', '-c', 'rustc /code/code.rs -o /tmp/program && /tmp/program'],
+                'java': ['sh', '-c', 'cd /code && javac *.java && java $(ls *.class | head -c -7)'],
+                'c': ['sh', '-c', 'cd /code && gcc code.c -o program && ./program'],
+                'cpp': ['sh', '-c', 'cd /code && g++ code.cpp -o program && ./program'],
+                'ruby': ['ruby', '/code/code.rb'],
+                'php': ['php', '/code/code.php'],
+                'lua': ['lua', '/code/code.lua']
+            }
+            
+            cmd = commands[language]
+            
+            try:
+                # Run container with security restrictions
+                container = self.client.containers.run(
+                    image=image,
+                    command=cmd,
+                    volumes={str(temp_path): {'bind': '/code', 'mode': 'ro'}},  # Read-only mount
+                    network_mode='none' if network_disabled else None,
+                    mem_limit=memory_limit,
+                    cpu_quota=cpu_quota,
+                    environment={
+                        'HOME': '/tmp',
+                        'PATH': '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin'
+                    },
+                    working_dir='/code',
+                    remove=True,  # Auto-remove when done
+                    stdout=True,
+                    stderr=True,
+                    detach=False,
+                    timeout=timeout
+                )
+                
+                # Parse output
+                stdout = container[0].decode('utf-8', errors='replace') if container[0] else ""
+                stderr = container[1].decode('utf-8', errors='replace') if container[1] else ""
+                
+                return {
+                    'success': True,
+                    'stdout': stdout,
+                    'stderr': stderr,
+                    'exit_code': 0  # Docker run returns combined output
+                }
+                
+            except docker.errors.ContainerError as e:
+                return {
+                    'success': False,
+                    'stdout': e.stdout.decode('utf-8', errors='replace') if e.stdout else "",
+                    'stderr': e.stderr.decode('utf-8', errors='replace') if e.stderr else "",
+                    'exit_code': e.exit_code,
+                    'error': f"Container execution failed: {e}"
+                }
+            except Exception as e:
+                return {
+                    'success': False,
+                    'stdout': '',
+                    'stderr': '',
+                    'exit_code': -1,
+                    'error': str(e)
+                }
+    
+    def is_available(self) -> bool:
+        """Check if Docker is available for sandboxing."""
+        try:
+            self.client.ping()
+            return True
+        except:
+            return False
+
+
+# Singleton instance
+_nexus_sandbox = None
+
+
+def get_sandbox() -> Optional[NexusSandbox]:
+    """Get the global sandbox instance."""
+    global _nexus_sandbox
+    if _nexus_sandbox is None:
+        try:
+            _nexus_sandbox = NexusSandbox()
+        except RuntimeError:
+            # Docker not available
+            return None
+    return _nexus_sandbox

qyro/common/schema_loader.py

@@ -0,0 +1,33 @@
+import json
+from .type_generator import QyroTypeGenerator
+import os
+
+class QyroSchemaLoader:
+    def __init__(self):
+        pass
+
+    def process_schema(self, schema_block: str, output_dir="nexus_generated"):
+        try:
+            schema = json.loads(schema_block)
+            generator = QyroTypeGenerator(schema)
+            
+            os.makedirs(output_dir, exist_ok=True)
+            
+            with open(os.path.join(output_dir, "nexus_types.h"), "w") as f:
+                f.write(generator.generate_c_structs())
+                
+            with open(os.path.join(output_dir, "nexus_types.rs"), "w") as f:
+                f.write(generator.generate_rust_structs())
+                
+            with open(os.path.join(output_dir, "GlobalState.java"), "w") as f:
+                f.write(generator.generate_java_class())
+
+            with open(os.path.join(output_dir, "types.ts"), "w") as f:
+                f.write(generator.generate_ts_interface())
+                
+            print(f"[NEXUS] Schema processed. Types generated in {output_dir}/")
+            return schema
+            
+        except json.JSONDecodeError as e:
+            print(f"[NEXUS] Error parsing schema block: {e}")
+            return None