zexus 1.7.1 → 1.7.2

package/src/zexus/evaluator/bytecode_compiler.py

@@ -3,18 +3,54 @@ Bytecode Compiler for Evaluator
 
 This module allows the evaluator to compile AST nodes to bytecode
 for VM execution when performance is critical.
+
+Features:
+- AST to bytecode compilation
+- File-based persistent caching (faster repeat runs)
+- Pattern-based caching (similar code reuse)
+- Multi-pass optimization (peephole, constant folding)
+- Seamless integration with UnifiedExecutor
 """
 import os
 from typing import Dict, List, Optional
 from .. import zexus_ast
 from ..vm.bytecode import Bytecode, BytecodeBuilder
 
-# Try to import cache (optional dependency)
+# Import cache (always available now)
+from ..vm.cache import BytecodeCache
+
+# Import optimizers (standard, not optional)
+try:
+    from ..vm.optimizer import BytecodeOptimizer
+    OPTIMIZER_AVAILABLE = True
+except ImportError:
+    BytecodeOptimizer = None
+    OPTIMIZER_AVAILABLE = False
+
 try:
-    from ..vm.cache import BytecodeCache
-    CACHE_AVAILABLE = True
+    from ..vm.peephole_optimizer import PeepholeOptimizer, OptimizationLevel
+    PEEPHOLE_AVAILABLE = True
 except ImportError:
-    CACHE_AVAILABLE = False
+    PeepholeOptimizer = None
+    OptimizationLevel = None
+    PEEPHOLE_AVAILABLE = False
+
+
+# Global shared cache for cross-compiler persistence
+_SHARED_CACHE: Optional[BytecodeCache] = None
+
+
+def get_shared_cache() -> BytecodeCache:
+    """Get the global shared bytecode cache (persistent across runs)"""
+    global _SHARED_CACHE
+    if _SHARED_CACHE is None:
+        _SHARED_CACHE = BytecodeCache(
+            max_size=2000,
+            max_memory_mb=100,
+            persistent=True,  # Enable disk persistence
+            debug=False
+        )
+    return _SHARED_CACHE
 
 
 class EvaluatorBytecodeCompiler:
@@ -24,8 +60,9 @@ class EvaluatorBytecodeCompiler:
     
     Features:
     - AST to bytecode compilation
-    - Bytecode caching for repeated code
-    - Optimization support
+    - File-based caching for repeat runs
+    - Pattern caching for similar code
+    - Multi-pass optimization
     - Error tracking
     """
     
@@ -35,31 +72,81 @@ class EvaluatorBytecodeCompiler:
         
         Args:
             use_cache: Enable bytecode caching
-            cache_size: Maximum cache entries
+            cache_size: Maximum cache entries (ignored if using shared cache)
         """
         self.builder: Optional[BytecodeBuilder] = None
         self.errors: List[str] = []
         self._loop_stack: List[Dict[str, str]] = []  # Stack of loop labels
         
-        # Bytecode cache
+        # Use shared persistent cache
         self.cache: Optional[BytecodeCache] = None
-        if use_cache and CACHE_AVAILABLE:
-            self.cache = BytecodeCache(
-                max_size=cache_size,
-                max_memory_mb=50,
-                persistent=False,
-                debug=False
+        if use_cache:
+            self.cache = get_shared_cache()
+            self._cache_stats_baseline = self._snapshot_cache_stats()
+        
+        # Initialize optimizers (standard, not optional)
+        self.bytecode_optimizer = None
+        self.peephole_optimizer = None
+        
+        if OPTIMIZER_AVAILABLE:
+            self.bytecode_optimizer = BytecodeOptimizer(level=2, debug=False)
+        
+        if PEEPHOLE_AVAILABLE:
+            self.peephole_optimizer = PeepholeOptimizer(
+                level=OptimizationLevel.MODERATE
             )
+        if not use_cache or self.cache is None:
+            self._cache_stats_baseline = {"hits": 0, "misses": 0, "pattern_hits": 0}
+    
+    def compile_file(self, file_path: str, ast_root, optimize: bool = True) -> Optional[List[Bytecode]]:
+        """
+        Compile an entire file's AST with file-based caching.
+        
+        This enables faster repeat runs - if the file hasn't changed,
+        we skip compilation entirely and return cached bytecode.
+        
+        Args:
+            file_path: Path to source file
+            ast_root: Parsed AST root
+            optimize: Whether to apply optimizations
+            
+        Returns:
+            List of bytecode objects or None if cached invalid
+        """
+        if self.cache:
+            # Check file-based cache first
+            cached = self.cache.get_by_file(file_path)
+            if cached:
+                return cached
+        
+        # Cache miss - compile
+        bytecodes = []
+        if hasattr(ast_root, 'statements'):
+            for stmt in ast_root.statements:
+                bc = self.compile(stmt, optimize=optimize, use_cache=False)
+                if bc:
+                    bytecodes.append(bc)
+        else:
+            bc = self.compile(ast_root, optimize=optimize, use_cache=False)
+            if bc:
+                bytecodes.append(bc)
+        
+        # Store in file cache
+        if self.cache and bytecodes:
+            self.cache.put_by_file(file_path, bytecodes)
+        
+        return bytecodes if bytecodes else None
     
     def compile(self, node, optimize: bool = True, use_cache: bool = True) -> Optional[Bytecode]:
         """
         Compile an AST node to bytecode.
         
         Process:
-        1. Check cache for existing bytecode
-        2. If cache miss, compile AST to bytecode
-        3. Apply optimizations if requested
-        4. Store in cache for future use
+        1. Check AST cache for existing bytecode
+        2. Check pattern cache for similar code
+        3. If cache miss, compile AST to bytecode
+        4. Apply multi-pass optimizations
+        5. Store in caches for future use
         
         Args:
             node: AST node to compile
@@ -69,11 +156,16 @@ class EvaluatorBytecodeCompiler:
         Returns:
             Bytecode object or None if compilation failed
         """
-        # Check cache first
+        # Check AST cache first
         if use_cache and self.cache:
             cached = self.cache.get(node)
             if cached:
                 return cached
+            
+            # Check pattern cache (similar code structure)
+            pattern_cached = self.cache.get_by_pattern(node)
+            if pattern_cached:
+                return pattern_cached
         
         # Cache miss - compile
         self.builder = BytecodeBuilder()
@@ -92,9 +184,10 @@ class EvaluatorBytecodeCompiler:
             
             bytecode.set_metadata('created_by', 'evaluator')
             
-            # Store in cache
+            # Store in caches
             if use_cache and self.cache:
                 self.cache.put(node, bytecode)
+                self.cache.put_by_pattern(node, bytecode)
             
             return bytecode
             
@@ -124,6 +217,31 @@ class EvaluatorBytecodeCompiler:
             error_msg = f"Unsupported node type for bytecode: {node_type}"
             self.errors.append(error_msg)
             self.builder.emit_constant(None)
+
+    def _coerce_string(self, candidate) -> str:
+        """Convert literal or identifier nodes into raw string values."""
+        if candidate is None:
+            return ""
+        if hasattr(candidate, 'value'):
+            return str(candidate.value)
+        return str(candidate)
+
+    def _coerce_identifier(self, candidate) -> str:
+        """Extract identifier name as string, handling AST nodes and raw strings."""
+        if candidate is None:
+            return ""
+        if hasattr(candidate, 'value'):
+            return str(candidate.value)
+        return str(candidate)
+
+    def _emit_vm_builtin_call(self, builtin_name: str, payload, discard_result: bool = True):
+        """Helper to push a payload and invoke a VM builtin handler."""
+        if self.builder is None:
+            return
+        self.builder.emit_constant(payload)
+        self.builder.emit_call(builtin_name, 1)
+        if discard_result:
+            self.builder.emit("POP")
     
     # === Statement Compilation ===
     
@@ -139,7 +257,7 @@ class EvaluatorBytecodeCompiler:
         """Compile expression statement"""
         self._compile_node(node.expression)
         # Pop result unless it's the last statement
-        # self.builder.emit("POP")
+        self.builder.emit("POP")
     
     def _compile_LetStatement(self, node: zexus_ast.LetStatement):
         """Compile let statement"""
@@ -169,6 +287,19 @@ class EvaluatorBytecodeCompiler:
         # Emit a special instruction or constant to signal continue mode
         # For now, emit a CONTINUE instruction that the VM can handle
         self.builder.emit("CONTINUE")
+
+    def _compile_BreakStatement(self, node: zexus_ast.BreakStatement):
+        """Compile break statement by jumping to loop end label"""
+        if not self._loop_stack:
+            self.errors.append("Break statement outside loop")
+            self.builder.emit_constant(None)
+            return
+        end_label = self._loop_stack[-1].get('end')
+        if not end_label:
+            self.errors.append("Break statement missing loop end label")
+            self.builder.emit_constant(None)
+            return
+        self.builder.emit_jump(end_label)
     
     def _compile_IfStatement(self, node: zexus_ast.IfStatement):
         """Compile if statement with elif and else branches"""
@@ -336,6 +467,86 @@ class EvaluatorBytecodeCompiler:
         
         # Emit PRINT opcode
         self.builder.emit("PRINT")
+
+    def _compile_UseStatement(self, node: zexus_ast.UseStatement):
+        """Compile module use/import statement for VM execution."""
+        module_path = self._coerce_string(getattr(node, 'file_path', None))
+        alias_name = self._coerce_identifier(getattr(node, 'alias', None)) if getattr(node, 'alias', None) else ""
+
+        names_list = []
+        if getattr(node, 'names', None):
+            for entry in node.names:
+                names_list.append(self._coerce_identifier(entry))
+
+        spec = {
+            "file": module_path,
+            "alias": alias_name,
+            "names": names_list,
+            "is_named": bool(getattr(node, 'is_named_import', False)) and len(names_list) > 0,
+        }
+
+        spec_idx = self.builder.bytecode.add_constant(spec)
+        self.builder.emit("LOAD_CONST", spec_idx)
+        name_idx = self.builder.bytecode.add_constant("__vm_use_module__")
+        self.builder.emit("CALL_NAME", (name_idx, 1))
+        self.builder.emit("POP")
+
+    def _compile_FromStatement(self, node: zexus_ast.FromStatement):
+        """Compile from-import statements into VM-friendly builtin calls."""
+        module_path = self._coerce_string(getattr(node, 'file_path', None))
+        entries = []
+
+        for entry in getattr(node, 'imports', []) or []:
+            if isinstance(entry, (list, tuple)):
+                base = entry[0] if len(entry) > 0 else None
+                alias = entry[1] if len(entry) > 1 else None
+            else:
+                base = entry
+                alias = None
+
+            entries.append({
+                "name": self._coerce_identifier(base),
+                "alias": self._coerce_identifier(alias) if alias else "",
+            })
+
+        spec = {
+            "file": module_path,
+            "imports": entries,
+        }
+
+        spec_idx = self.builder.bytecode.add_constant(spec)
+        self.builder.emit("LOAD_CONST", spec_idx)
+        name_idx = self.builder.bytecode.add_constant("__vm_from_module__")
+        self.builder.emit("CALL_NAME", (name_idx, 1))
+        self.builder.emit("POP")
+
+    def _compile_NativeStatement(self, node: zexus_ast.NativeStatement):
+        """Compile native performance statements via VM builtins."""
+        self._emit_vm_builtin_call("__vm_native_statement__", node, discard_result=False)
+
+    def _compile_GCStatement(self, node: zexus_ast.GCStatement):
+        """Compile garbage collection directives into VM builtins."""
+        self._emit_vm_builtin_call("__vm_gc_statement__", node, discard_result=False)
+
+    def _compile_InlineStatement(self, node: zexus_ast.InlineStatement):
+        """Compile inline optimization directives into VM builtins."""
+        self._emit_vm_builtin_call("__vm_inline_statement__", node, discard_result=False)
+
+    def _compile_BufferStatement(self, node: zexus_ast.BufferStatement):
+        """Compile buffer directives to VM builtins."""
+        self._emit_vm_builtin_call("__vm_buffer_statement__", node, discard_result=False)
+
+    def _compile_SIMDStatement(self, node: zexus_ast.SIMDStatement):
+        """Compile SIMD directives to VM builtins."""
+        self._emit_vm_builtin_call("__vm_simd_statement__", node, discard_result=False)
+
+    def _compile_DeferStatement(self, node: zexus_ast.DeferStatement):
+        """Compile defer cleanup directives to VM builtins."""
+        self._emit_vm_builtin_call("__vm_defer_statement__", node, discard_result=False)
+
+    def _compile_PatternStatement(self, node: zexus_ast.PatternStatement):
+        """Compile pattern matching directives to VM builtins."""
+        self._emit_vm_builtin_call("__vm_pattern_statement__", node, discard_result=False)
     
     # === Blockchain Statement Compilation ===
     
@@ -485,7 +696,36 @@ class EvaluatorBytecodeCompiler:
             self._compile_node(value_expr)
         # Build map from stack
         self.builder.emit("BUILD_MAP", len(node.pairs))
-    
+
+    def _compile_PropertyAccessExpression(self, node: zexus_ast.PropertyAccessExpression):
+        """Compile property access"""
+        if self.builder is None:
+            return
+        
+        # Compile object
+        self._compile_node(node.object)
+        
+        if node.computed:
+            # Bracket notation: obj[expr]
+            self._compile_node(node.property)
+            self._builder_emit("INDEX")
+        else:
+            # Dot notation: obj.prop
+            if hasattr(node.property, 'value'):
+                name = node.property.value
+            else:
+                name = str(node.property)
+            
+            # Push property name as constant
+            self.builder.emit_constant(name)
+            self._builder_emit("GET_ATTR")
+            
+    def _builder_emit(self, opcode):
+        """Helper to emit opcode string or enum"""
+        # If builder has emit, use it. Some builders might use emit(opcode, operand).
+        # Wrapper to handle potential string usage if enum not fully refreshed
+        self.builder.emit(opcode)
+
     def _compile_InfixExpression(self, node: zexus_ast.InfixExpression):
         """Compile infix expression"""
         if self.builder is None:
@@ -523,6 +763,26 @@ class EvaluatorBytecodeCompiler:
     
     def _compile_CallExpression(self, node: zexus_ast.CallExpression):
         """Compile function call"""
+        # Method call on property access (obj.method(...))
+        if isinstance(node.function, zexus_ast.PropertyAccessExpression):
+            # Only handle non-computed property names here
+            if hasattr(node.function, 'computed') and node.function.computed:
+                pass
+            else:
+                method_name = None
+                if hasattr(node.function.property, 'value'):
+                    method_name = node.function.property.value
+                elif isinstance(node.function.property, str):
+                    method_name = node.function.property
+
+                if method_name is not None:
+                    # Load object first, then args, then CALL_METHOD
+                    self._compile_node(node.function.object)
+                    for arg in node.arguments:
+                        self._compile_node(arg)
+                    self.builder.emit_call_method(method_name, len(node.arguments))
+                    return
+
         # Check for blockchain-specific function calls that have dedicated opcodes
         if isinstance(node.function, zexus_ast.Identifier):
             func_name = node.function.value
@@ -594,16 +854,57 @@ class EvaluatorBytecodeCompiler:
     
     def _compile_AssignmentExpression(self, node: zexus_ast.AssignmentExpression):
         """Compile assignment expression"""
-        # Compile the value
-        self._compile_node(node.value)
         # Store to name
         if isinstance(node.name, zexus_ast.Identifier):
+            # Compile the value
+            self._compile_node(node.value)
             self.builder.emit("DUP")  # Keep value on stack
             name = str(node.name.value).strip()
             self.builder.emit_store(name)
-        else:
-            self.errors.append(
-                "Complex assignment targets not yet supported in bytecode")
+            return
+
+        # Property/index assignment (obj.prop = value / obj[expr] = value)
+        if isinstance(node.name, zexus_ast.PropertyAccessExpression):
+            tmp_name = "__assign_tmp"
+            # Evaluate value first and stash it
+            self._compile_node(node.value)
+            self.builder.emit_store(tmp_name)
+
+            # Compile object
+            self._compile_node(node.name.object)
+
+            # Compile property key
+            if hasattr(node.name, 'computed') and node.name.computed:
+                self._compile_node(node.name.property)
+            else:
+                if hasattr(node.name.property, 'value'):
+                    prop_name = node.name.property.value
+                    self.builder.emit_constant(prop_name)
+                else:
+                    self._compile_node(node.name.property)
+
+            # Push value and call set
+            self.builder.emit_load(tmp_name)
+            self.builder.emit_call_method("set", 2)
+
+            # Return assigned value
+            self.builder.emit_load(tmp_name)
+            return
+
+        if isinstance(node.name, zexus_ast.IndexExpression):
+            tmp_name = "__assign_tmp"
+            self._compile_node(node.value)
+            self.builder.emit_store(tmp_name)
+
+            self._compile_node(node.name.left)
+            self._compile_node(node.name.index)
+            self.builder.emit_load(tmp_name)
+            self.builder.emit_call_method("set", 2)
+            self.builder.emit_load(tmp_name)
+            return
+
+        self.errors.append(
+            "Complex assignment targets not yet supported in bytecode")
     
     def _compile_IndexExpression(self, node):
         """Compile index expression"""
@@ -613,6 +914,42 @@ class EvaluatorBytecodeCompiler:
         self._compile_node(node.index)
         # Emit index operation
         self.builder.emit("INDEX")
+
+    def _compile_SliceExpression(self, node):
+        """Compile slice expression"""
+        self._compile_node(node.object)
+
+        if node.start is not None:
+            self._compile_node(node.start)
+        else:
+            self.builder.emit_constant(None)
+
+        if node.end is not None:
+            self._compile_node(node.end)
+        else:
+            self.builder.emit_constant(None)
+
+        self.builder.emit("SLICE")
+
+    def _compile_ThisExpression(self, node):
+        """Compile 'this' expression"""
+        self.builder.emit_load("this")
+
+    def _compile_TernaryExpression(self, node: zexus_ast.TernaryExpression):
+        """Compile ternary expression: condition ? true_value : false_value"""
+        else_label = f"ternary_else_{id(node)}"
+        end_label = f"ternary_end_{id(node)}"
+
+        self._compile_node(node.condition)
+        self.builder.emit_jump_if_false(else_label)
+
+        self._compile_node(node.true_value)
+        self.builder.emit_jump(end_label)
+
+        self.builder.mark_label(else_label)
+        self._compile_node(node.false_value)
+
+        self.builder.mark_label(end_label)
     
     def _compile_PropertyAccessExpression(self, node):
         """Compile property access (obj.property or obj[property])"""
@@ -680,13 +1017,19 @@ class EvaluatorBytecodeCompiler:
     
     def _optimize(self, bytecode: Bytecode) -> Bytecode:
         """
-        Apply peephole optimizations to bytecode.
+        Apply multi-pass optimizations to bytecode.
+        
+        Optimization pipeline:
+        1. BytecodeOptimizer (if available) - advanced passes
+        2. PeepholeOptimizer (if available) - local patterns
+        3. Built-in optimizations - fallback
         
-        Optimizations:
-        - Remove unnecessary POP instructions
+        Optimizations include:
         - Constant folding (compile-time evaluation)
-        - Dead code elimination  
-        - Algebraic simplifications
+        - Dead code elimination
+        - Strength reduction
+        - Instruction fusion
+        - Jump threading
         - Redundant operation removal
         """
         from zexus.vm.bytecode import Bytecode, Opcode
@@ -695,8 +1038,33 @@ class EvaluatorBytecodeCompiler:
         if not instructions:
             return bytecode
         
-        # Multiple optimization passes for better results
         optimized = instructions
+        
+        # Pass 1: BytecodeOptimizer (advanced optimizations)
+        if self.bytecode_optimizer:
+            try:
+                optimized = self.bytecode_optimizer.optimize(
+                    optimized, 
+                    bytecode.constants
+                )
+            except Exception:
+                pass  # Fall through to other optimizers
+        
+        # Pass 2: PeepholeOptimizer (local patterns)
+        if self.peephole_optimizer:
+            try:
+                # Convert to format peephole expects
+                from ..vm.peephole_optimizer import Instruction
+                peep_insts = [
+                    Instruction(opcode=inst[0], arg=inst[1] if len(inst) > 1 else None)
+                    for inst in optimized
+                ]
+                peep_result = self.peephole_optimizer.optimize(peep_insts)
+                optimized = [(inst.opcode, inst.arg) for inst in peep_result]
+            except Exception:
+                pass  # Fall through to built-in
+        
+        # Pass 3: Built-in optimizations (always available)
         optimized = self._constant_folding(optimized, bytecode.constants)
         optimized = self._dead_code_elimination(optimized)
         optimized = self._peephole_patterns(optimized)
@@ -916,8 +1284,12 @@ class EvaluatorBytecodeCompiler:
             'StringLiteral', 'Boolean', 'ListLiteral', 'MapLiteral', 'NullLiteral',
             'InfixExpression', 'PrefixExpression', 'CallExpression',
             'AwaitExpression', 'SpawnExpression', 'AssignmentExpression', 'IndexExpression',
-            'PropertyAccessExpression', 'LambdaExpression',
+            'PropertyAccessExpression', 'SliceExpression', 'LambdaExpression',
+            'ThisExpression', 'TernaryExpression',
             'FindExpression', 'LoadExpression',
+            'UseStatement', 'FromStatement',
+            'NativeStatement', 'GCStatement', 'InlineStatement',
+            'BufferStatement', 'SIMDStatement', 'DeferStatement', 'PatternStatement',
             # Blockchain nodes
             'TxStatement', 'RevertStatement', 'RequireStatement',
             'StateAccessExpression', 'LedgerAppendStatement', 'GasChargeStatement'
@@ -927,6 +1299,17 @@ class EvaluatorBytecodeCompiler:
     
     # ==================== Cache Management ====================
     
+    def _snapshot_cache_stats(self) -> Dict[str, int]:
+        """Capture current cache stats to use as a baseline."""
+        if not self.cache:
+            return {"hits": 0, "misses": 0, "pattern_hits": 0}
+        stats = self.cache.get_stats() or {}
+        return {
+            "hits": int(stats.get("hits", 0)),
+            "misses": int(stats.get("misses", 0)),
+            "pattern_hits": int(stats.get("pattern_hits", 0)),
+        }
+
     def get_cache_stats(self) -> Optional[Dict]:
         """
         Get cache statistics
@@ -934,14 +1317,34 @@ class EvaluatorBytecodeCompiler:
         Returns:
             Dictionary with cache stats or None if cache disabled
         """
-        if self.cache:
-            return self.cache.get_stats()
-        return None
+        if not self.cache:
+            return None
+
+        stats = self.cache.get_stats() or {}
+        baseline = getattr(self, "_cache_stats_baseline", {"hits": 0, "misses": 0, "pattern_hits": 0})
+
+        hits = max(0, int(stats.get("hits", 0)) - baseline.get("hits", 0))
+        misses = max(0, int(stats.get("misses", 0)) - baseline.get("misses", 0))
+        pattern_hits = max(0, int(stats.get("pattern_hits", 0)) - baseline.get("pattern_hits", 0))
+
+        total = hits + misses
+        hit_rate = (hits / total * 100.0) if total else 0.0
+
+        adjusted_stats = dict(stats)
+        adjusted_stats.update({
+            "hits": hits,
+            "misses": misses,
+            "pattern_hits": pattern_hits,
+            "total": total,
+            "hit_rate": hit_rate,
+        })
+        return adjusted_stats
     
     def clear_cache(self):
         """Clear bytecode cache"""
         if self.cache:
             self.cache.clear()
+            self._cache_stats_baseline = {"hits": 0, "misses": 0, "pattern_hits": 0}
     
     def invalidate_cache(self, node):
         """Invalidate cached bytecode for a node"""
@@ -952,6 +1355,7 @@ class EvaluatorBytecodeCompiler:
         """Reset cache statistics"""
         if self.cache:
             self.cache.reset_stats()
+            self._cache_stats_baseline = self._snapshot_cache_stats()
     
     def cache_size(self) -> int:
         """Get current cache size"""