zexus 1.6.2

package/src/zexus/persistence.py

@@ -0,0 +1,391 @@
+# src/zexus/persistence.py
+"""
+Persistent Memory Management for Zexus
+Extends persistent storage beyond contracts to all storage keywords (LET, CONST, ENTITY, etc.)
+"""
+
+import os
+import json
+import sqlite3
+import weakref
+from threading import Lock
+from typing import Dict, Any, Optional, Set
+from .object import (
+    Object, Integer, Float, String, Boolean as BooleanObj, 
+    Null, NULL, List, Map, EntityInstance
+)
+
+# Storage directory for persistent data
+PERSISTENCE_DIR = os.path.expanduser("~/.zexus/persistence")
+os.makedirs(PERSISTENCE_DIR, exist_ok=True)
+
+
+# ===============================================
+# MEMORY LEAK TRACKING
+# ===============================================
+
+class MemoryTracker:
+    """Track object allocations and detect potential memory leaks"""
+    
+    def __init__(self):
+        self.allocations: Dict[int, Dict[str, Any]] = {}
+        self.weak_refs: Dict[int, weakref.ref] = {}
+        self.lock = Lock()
+        self.enabled = True
+        self.allocation_count = 0
+        self.max_allocations = 100000  # Alert threshold
+        
+    def track(self, obj: Object, context: str = "unknown"):
+        """Track an object allocation"""
+        if not self.enabled:
+            return
+            
+        with self.lock:
+            obj_id = id(obj)
+            self.allocation_count += 1
+            
+            # Create weak reference to detect when object is garbage collected
+            def cleanup(ref):
+                with self.lock:
+                    if obj_id in self.allocations:
+                        del self.allocations[obj_id]
+                    if obj_id in self.weak_refs:
+                        del self.weak_refs[obj_id]
+            
+            self.weak_refs[obj_id] = weakref.ref(obj, cleanup)
+            self.allocations[obj_id] = {
+                'type': obj.type() if hasattr(obj, 'type') else type(obj).__name__,
+                'context': context,
+                'allocation_number': self.allocation_count
+            }
+            
+            # Check for potential memory leak
+            if len(self.allocations) > self.max_allocations:
+                self._report_potential_leak()
+    
+    def untrack(self, obj: Object):
+        """Manually untrack an object"""
+        with self.lock:
+            obj_id = id(obj)
+            if obj_id in self.allocations:
+                del self.allocations[obj_id]
+            if obj_id in self.weak_refs:
+                del self.weak_refs[obj_id]
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """Get memory usage statistics"""
+        with self.lock:
+            type_counts = {}
+            for alloc in self.allocations.values():
+                obj_type = alloc['type']
+                type_counts[obj_type] = type_counts.get(obj_type, 0) + 1
+            
+            return {
+                'total_tracked': len(self.allocations),
+                'total_allocated': self.allocation_count,
+                'by_type': type_counts
+            }
+    
+    def _report_potential_leak(self):
+        """Report potential memory leak"""
+        stats = self.get_stats()
+        print(f"⚠️  MEMORY WARNING: {stats['total_tracked']} objects tracked (threshold: {self.max_allocations})")
+        print(f"   Breakdown: {stats['by_type']}")
+    
+    def clear(self):
+        """Clear all tracking data"""
+        with self.lock:
+            self.allocations.clear()
+            self.weak_refs.clear()
+            self.allocation_count = 0
+
+
+# Global memory tracker instance
+_memory_tracker = MemoryTracker()
+
+
+def track_allocation(obj: Object, context: str = "unknown"):
+    """Track an object allocation"""
+    _memory_tracker.track(obj, context)
+
+
+def get_memory_stats() -> Dict[str, Any]:
+    """Get current memory statistics"""
+    return _memory_tracker.get_stats()
+
+
+def enable_memory_tracking():
+    """Enable memory tracking"""
+    _memory_tracker.enabled = True
+
+
+def disable_memory_tracking():
+    """Disable memory tracking"""
+    _memory_tracker.enabled = False
+
+
+# ===============================================
+# PERSISTENT STORAGE BACKEND
+# ===============================================
+
+class PersistentStorage:
+    """Persistent storage for variables using SQLite"""
+    
+    def __init__(self, scope_id: str, storage_dir: str = PERSISTENCE_DIR):
+        self.scope_id = scope_id
+        self.db_path = os.path.join(storage_dir, f"{scope_id}.sqlite")
+        self.conn = None
+        self.lock = Lock()
+        self._init_db()
+    
+    def _init_db(self):
+        """Initialize SQLite database"""
+        self.conn = sqlite3.connect(self.db_path, check_same_thread=False)
+        cursor = self.conn.cursor()
+        cursor.execute('''
+            CREATE TABLE IF NOT EXISTS variables (
+                name TEXT PRIMARY KEY,
+                type TEXT NOT NULL,
+                value TEXT NOT NULL,
+                is_const INTEGER DEFAULT 0,
+                created_at REAL NOT NULL,
+                updated_at REAL NOT NULL
+            )
+        ''')
+        cursor.execute('''
+            CREATE INDEX IF NOT EXISTS idx_name ON variables(name)
+        ''')
+        self.conn.commit()
+    
+    def set(self, name: str, value: Object, is_const: bool = False):
+        """Persist a variable"""
+        with self.lock:
+            serialized = self._serialize(value)
+            cursor = self.conn.cursor()
+            
+            import time
+            timestamp = time.time()
+            
+            cursor.execute('''
+                INSERT OR REPLACE INTO variables (name, type, value, is_const, created_at, updated_at)
+                VALUES (?, ?, ?, ?, 
+                    COALESCE((SELECT created_at FROM variables WHERE name = ?), ?),
+                    ?)
+            ''', (name, serialized['type'], serialized['value'], 1 if is_const else 0, 
+                  name, timestamp, timestamp))
+            
+            self.conn.commit()
+    
+    def get(self, name: str) -> Optional[Object]:
+        """Retrieve a persisted variable"""
+        with self.lock:
+            cursor = self.conn.cursor()
+            cursor.execute('SELECT type, value FROM variables WHERE name = ?', (name,))
+            row = cursor.fetchone()
+            
+            if row is None:
+                return None
+            
+            return self._deserialize({'type': row[0], 'value': row[1]})
+    
+    def delete(self, name: str):
+        """Delete a persisted variable"""
+        with self.lock:
+            cursor = self.conn.cursor()
+            cursor.execute('DELETE FROM variables WHERE name = ?', (name,))
+            self.conn.commit()
+    
+    def is_const(self, name: str) -> bool:
+        """Check if a variable is const"""
+        with self.lock:
+            cursor = self.conn.cursor()
+            cursor.execute('SELECT is_const FROM variables WHERE name = ?', (name,))
+            row = cursor.fetchone()
+            return bool(row[0]) if row else False
+    
+    def list_variables(self) -> list:
+        """List all persisted variables"""
+        with self.lock:
+            cursor = self.conn.cursor()
+            cursor.execute('SELECT name, type, is_const FROM variables')
+            return [{'name': row[0], 'type': row[1], 'const': bool(row[2])} for row in cursor.fetchall()]
+    
+    def clear(self):
+        """Clear all persisted variables"""
+        with self.lock:
+            cursor = self.conn.cursor()
+            cursor.execute('DELETE FROM variables')
+            self.conn.commit()
+    
+    def _serialize(self, obj: Object) -> Dict[str, str]:
+        """Serialize Zexus object to JSON"""
+        if isinstance(obj, String):
+            return {'type': 'string', 'value': json.dumps(obj.value)}
+        elif isinstance(obj, Integer):
+            return {'type': 'integer', 'value': json.dumps(obj.value)}
+        elif isinstance(obj, Float):
+            return {'type': 'float', 'value': json.dumps(obj.value)}
+        elif isinstance(obj, BooleanObj):
+            return {'type': 'boolean', 'value': json.dumps(obj.value)}
+        elif isinstance(obj, List):
+            serialized = [self._serialize(e) for e in obj.elements]
+            return {'type': 'list', 'value': json.dumps(serialized)}
+        elif isinstance(obj, Map):
+            serialized = {k: self._serialize(v) for k, v in obj.pairs.items()}
+            return {'type': 'map', 'value': json.dumps(serialized)}
+        elif obj is Null or obj is NULL:
+            return {'type': 'null', 'value': json.dumps(None)}
+        elif isinstance(obj, EntityInstance):
+            serialized_values = {k: self._serialize(v) for k, v in obj.values.items()}
+            return {
+                'type': 'entity_instance',
+                'value': json.dumps({
+                    'entity_name': obj.entity_def.name,
+                    'values': serialized_values
+                })
+            }
+        else:
+            # Fallback: convert to string
+            return {'type': 'string', 'value': json.dumps(str(obj.inspect() if hasattr(obj, 'inspect') else obj))}
+    
+    def _deserialize(self, data: Dict[str, str]) -> Object:
+        """Deserialize JSON to Zexus object"""
+        obj_type = data['type']
+        value = json.loads(data['value'])
+        
+        if obj_type == 'string':
+            return String(value)
+        elif obj_type == 'integer':
+            return Integer(value)
+        elif obj_type == 'float':
+            return Float(value)
+        elif obj_type == 'boolean':
+            return BooleanObj(value)
+        elif obj_type == 'null':
+            return NULL
+        elif obj_type == 'list':
+            elements = [self._deserialize(e) for e in value]
+            return List(elements)
+        elif obj_type == 'map':
+            pairs = {k: self._deserialize(v) for k, v in value.items()}
+            return Map(pairs)
+        elif obj_type == 'entity_instance':
+            # Note: This creates a basic EntityInstance without full EntityDefinition
+            # For production, you'd need to store/restore the entity definition
+            from .object import EntityDefinition
+            entity_name = value['entity_name']
+            serialized_values = value['values']
+            deserialized_values = {k: self._deserialize(v) for k, v in serialized_values.items()}
+            
+            # Create minimal entity definition
+            entity_def = EntityDefinition(entity_name, [])
+            return EntityInstance(entity_def, deserialized_values)
+        else:
+            return String(str(value))
+    
+    def close(self):
+        """Close database connection"""
+        if self.conn:
+            self.conn.close()
+
+
+# ===============================================
+# PERSISTENT ENVIRONMENT MIXIN
+# ===============================================
+
+class PersistentEnvironmentMixin:
+    """Mixin to add persistence to Environment class"""
+    
+    def __init__(self, *args, persistence_scope: Optional[str] = None, enable_persistence: bool = False, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.persistence_enabled = enable_persistence
+        self.persistence_scope = persistence_scope
+        self.persistent_storage = None
+        self.persisted_vars: Set[str] = set()  # Track which vars are persisted
+        
+        if enable_persistence and persistence_scope:
+            self.persistent_storage = PersistentStorage(persistence_scope)
+            self._load_persisted_vars()
+    
+    def _load_persisted_vars(self):
+        """Load persisted variables into environment"""
+        if not self.persistent_storage:
+            return
+        
+        for var_info in self.persistent_storage.list_variables():
+            name = var_info['name']
+            value = self.persistent_storage.get(name)
+            if value is not None:
+                self.store[name] = value
+                self.persisted_vars.add(name)
+                if var_info['const']:
+                    self.const_vars.add(name)
+    
+    def set_persistent(self, name: str, val: Object, is_const: bool = False):
+        """Set a variable with persistence"""
+        # Set in memory
+        if is_const:
+            self.set_const(name, val)
+        else:
+            self.set(name, val)
+        
+        # Persist to storage
+        if self.persistence_enabled and self.persistent_storage:
+            self.persistent_storage.set(name, val, is_const)
+            self.persisted_vars.add(name)
+            track_allocation(val, f"persistent:{name}")
+        
+        return val
+    
+    def get_persistent(self, name: str) -> Optional[Object]:
+        """Get a variable, checking persistence if not in memory"""
+        # Check memory first
+        val = self.get(name)
+        if val is not None:
+            return val
+        
+        # Check persistent storage
+        if self.persistence_enabled and self.persistent_storage:
+            val = self.persistent_storage.get(name)
+            if val is not None:
+                self.store[name] = val
+                self.persisted_vars.add(name)
+                return val
+        
+        return None
+    
+    def clear_persistence(self):
+        """Clear all persisted variables"""
+        if self.persistent_storage:
+            self.persistent_storage.clear()
+            self.persisted_vars.clear()
+
+
+# ===============================================
+# UTILITY FUNCTIONS
+# ===============================================
+
+def create_persistent_scope(scope_name: str) -> PersistentStorage:
+    """Create a new persistent storage scope"""
+    return PersistentStorage(scope_name)
+
+
+def list_persistent_scopes() -> list:
+    """List all persistent storage scopes"""
+    if not os.path.exists(PERSISTENCE_DIR):
+        return []
+    
+    scopes = []
+    for filename in os.listdir(PERSISTENCE_DIR):
+        if filename.endswith('.sqlite'):
+            scope_name = filename[:-7]  # Remove .sqlite extension
+            scopes.append(scope_name)
+    
+    return scopes
+
+
+def delete_persistent_scope(scope_name: str):
+    """Delete a persistent storage scope"""
+    db_path = os.path.join(PERSISTENCE_DIR, f"{scope_name}.sqlite")
+    if os.path.exists(db_path):
+        os.remove(db_path)

package/src/zexus/plugin_system.py

@@ -0,0 +1,290 @@
+"""
+Plugin system for Zexus interpreter.
+
+Enables third-party extensions through hooks and capability declarations.
+Plugins are self-contained modules that extend the language without
+modifying core functionality.
+"""
+
+from typing import Dict, List, Callable, Any, Optional, Set
+from dataclasses import dataclass, field
+from collections import defaultdict
+import os
+import sys
+
+
+@dataclass
+class PluginMetadata:
+    """Metadata for a plugin."""
+    name: str
+    version: str
+    author: str = ""
+    description: str = ""
+    requires: List[str] = field(default_factory=list)  # Capabilities required
+    provides: List[str] = field(default_factory=list)  # Capabilities provided
+    hooks: List[str] = field(default_factory=list)  # Hook names registered
+    config: Dict[str, Any] = field(default_factory=dict)  # Configuration schema
+
+    @classmethod
+    def from_dict(cls, data: dict) -> 'PluginMetadata':
+        """Create metadata from dictionary."""
+        return cls(
+            name=data.get('name', ''),
+            version=data.get('version', '1.0.0'),
+            author=data.get('author', ''),
+            description=data.get('description', ''),
+            requires=data.get('requires', []),
+            provides=data.get('provides', []),
+            hooks=data.get('hooks', []),
+            config=data.get('config', {})
+        )
+
+
+@dataclass
+class Hook:
+    """A registered hook handler."""
+    name: str
+    handler: Callable
+    plugin_name: str
+    priority: int = 0  # Higher priority executes first
+
+
+class PluginManager:
+    """
+    Manages plugin loading, registration, and execution.
+    
+    Handles:
+    - Plugin discovery and loading
+    - Hook registration and execution
+    - Capability tracking and validation
+    - Dependency resolution
+    - Sandbox enforcement
+    """
+    
+    def __init__(self):
+        """Initialize the plugin manager."""
+        self.loaded_plugins: Dict[str, PluginMetadata] = {}
+        self.hooks: Dict[str, List[Hook]] = defaultdict(list)
+        self.capabilities: Set[str] = set()
+        self.plugin_modules: Dict[str, Any] = {}  # Loaded plugin modules
+        self.config: Dict[str, Dict[str, Any]] = {}  # Per-plugin config
+        
+        # Builtin capabilities always available
+        self.capabilities.add("core")
+        
+    def load_plugin(self, module_path: str, config: Optional[Dict[str, Any]] = None) -> PluginMetadata:
+        """
+        Load a plugin from a module path.
+        
+        Args:
+            module_path: Path to plugin module (.zx file or directory)
+            config: Configuration dictionary for the plugin
+            
+        Returns:
+            PluginMetadata object
+            
+        Raises:
+            FileNotFoundError: If plugin module not found
+            ValueError: If plugin metadata invalid or dependencies unmet
+        """
+        if not os.path.exists(module_path):
+            raise FileNotFoundError(f"Plugin module not found: {module_path}")
+        
+        # Note: Actual implementation will parse .zx module
+        # For now, this is the interface
+        metadata = PluginMetadata(
+            name="placeholder",
+            version="1.0.0"
+        )
+        
+        # Validate dependencies
+        for required_cap in metadata.requires:
+            if required_cap not in self.capabilities:
+                raise ValueError(f"Plugin {metadata.name} requires unavailable capability: {required_cap}")
+        
+        # Store metadata and config
+        self.loaded_plugins[metadata.name] = metadata
+        if config:
+            self.config[metadata.name] = config
+        
+        # Add provided capabilities
+        for cap in metadata.provides:
+            self.capabilities.add(cap)
+        
+        return metadata
+    
+    def register_hook(self, hook_name: str, handler: Callable, 
+                     plugin_name: str, priority: int = 0) -> None:
+        """
+        Register a hook handler.
+        
+        Args:
+            hook_name: Name of the hook (e.g., "pre_eval", "import_resolver")
+            handler: Callable that handles the hook
+            plugin_name: Name of plugin registering the hook
+            priority: Execution priority (higher = earlier)
+        """
+        hook = Hook(name=hook_name, handler=handler, plugin_name=plugin_name, priority=priority)
+        self.hooks[hook_name].append(hook)
+        
+        # Sort by priority (descending)
+        self.hooks[hook_name].sort(key=lambda h: h.priority, reverse=True)
+    
+    def call_hooks(self, hook_name: str, *args, **kwargs) -> Any:
+        """
+        Call all registered handlers for a hook.
+        
+        Handlers are called in priority order (highest first).
+        If a handler returns non-None, stop and return that value.
+        Otherwise, return the first argument (usually the transformed input).
+        
+        Args:
+            hook_name: Name of the hook to call
+            *args: Arguments to pass to handlers
+            **kwargs: Keyword arguments to pass to handlers
+            
+        Returns:
+            Result from first non-None handler return, or first argument
+        """
+        if hook_name not in self.hooks:
+            # No handlers registered, return first arg unchanged
+            return args[0] if args else None
+        
+        result = args[0] if args else None
+        
+        for hook in self.hooks[hook_name]:
+            try:
+                # Call handler with arguments
+                handler_result = hook.handler(*args, **kwargs)
+                if handler_result is not None:
+                    # Handler returned a value, update result
+                    result = handler_result
+                    # Update args[0] for next handler
+                    if args:
+                        args = (result,) + args[1:]
+            except Exception as e:
+                # Log hook error but continue
+                print(f"Error in hook {hook_name} from {hook.plugin_name}: {e}", 
+                      file=sys.stderr)
+        
+        return result
+    
+    def check_capability(self, capability: str) -> bool:
+        """
+        Check if a capability is available.
+        
+        Args:
+            capability: Capability name to check
+            
+        Returns:
+            True if capability is available, False otherwise
+        """
+        return capability in self.capabilities
+    
+    def grant_capability(self, capability: str) -> None:
+        """
+        Manually grant a capability.
+        
+        Args:
+            capability: Capability name to grant
+        """
+        self.capabilities.add(capability)
+    
+    def has_plugin(self, plugin_name: str) -> bool:
+        """Check if a plugin is loaded."""
+        return plugin_name in self.loaded_plugins
+    
+    def get_plugin_metadata(self, plugin_name: str) -> Optional[PluginMetadata]:
+        """Get metadata for a loaded plugin."""
+        return self.loaded_plugins.get(plugin_name)
+    
+    def get_capabilities(self) -> List[str]:
+        """Get list of available capabilities."""
+        return sorted(list(self.capabilities))
+    
+    def get_loaded_plugins(self) -> List[str]:
+        """Get list of loaded plugin names."""
+        return sorted(list(self.loaded_plugins.keys()))
+    
+    def get_hooks(self, hook_name: Optional[str] = None) -> Dict[str, List[str]]:
+        """
+        Get registered hooks.
+        
+        Args:
+            hook_name: If specified, return handlers for specific hook
+            
+        Returns:
+            Dict of hook_name -> [plugin_names] or filtered results
+        """
+        if hook_name:
+            return {hook_name: [h.plugin_name for h in self.hooks.get(hook_name, [])]}
+        
+        result = {}
+        for name, hooks_list in self.hooks.items():
+            result[name] = [h.plugin_name for h in hooks_list]
+        return result
+
+
+class PluginGlobalObject:
+    """
+    The `plugin` global object exposed to plugin code.
+    
+    Provides interface for:
+    - Registering hooks
+    - Declaring capabilities
+    - Accessing metadata
+    - Introspection
+    """
+    
+    def __init__(self, manager: PluginManager, current_plugin: str):
+        """
+        Initialize the plugin global.
+        
+        Args:
+            manager: PluginManager instance
+            current_plugin: Name of the currently-executing plugin
+        """
+        self.manager = manager
+        self.current_plugin = current_plugin
+    
+    def register_hook(self, hook_name: str, handler: Callable) -> None:
+        """Register a hook handler for the current plugin."""
+        self.manager.register_hook(hook_name, handler, self.current_plugin)
+    
+    def grant_capability(self, capability: str) -> None:
+        """Declare that this plugin grants a capability."""
+        self.manager.grant_capability(capability)
+    
+    def has_capability(self, capability: str) -> bool:
+        """Check if a capability is available."""
+        return self.manager.check_capability(capability)
+    
+    def get_hooks(self) -> Dict[str, List[str]]:
+        """Get all registered hooks."""
+        return self.manager.get_hooks()
+    
+    def get_capabilities(self) -> List[str]:
+        """Get all available capabilities."""
+        return self.manager.get_capabilities()
+    
+    def get_loaded_plugins(self) -> List[str]:
+        """Get list of loaded plugins."""
+        return self.manager.get_loaded_plugins()
+    
+    def metadata(self) -> PluginMetadata:
+        """Get metadata for current plugin."""
+        return self.manager.get_plugin_metadata(self.current_plugin)
+    
+    def load(self, plugin_name: str, config: Optional[Dict[str, Any]] = None) -> bool:
+        """
+        Load another plugin by name.
+        
+        Returns:
+            True if loaded, False if already loaded
+        """
+        if self.manager.has_plugin(plugin_name):
+            return False
+        
+        # Would resolve plugin path and load
+        # Simplified for now
+        return True