zexus 1.6.8 → 1.7.2

package/src/zexus/module_cache.py

@@ -6,22 +6,72 @@ re-parsing and re-evaluating modules that have already been loaded.
 """
 
 import os
+import hashlib
 import threading
-from typing import Dict, Optional
+from typing import Dict, Optional, Any, Tuple, Set
 from .object import Environment
 
-_MODULE_CACHE: Dict[str, Environment] = {}
+# Module cache stores: (environment, bytecode, ast)
+_MODULE_CACHE: Dict[str, Tuple[Environment, Any, Any]] = {}
 _MODULE_CACHE_LOCK = threading.Lock()
 
-def get_cached_module(module_path: str) -> Optional[Environment]:
-    """Get a cached module environment if available"""
+# Circular-import detection: set of normalized paths currently being loaded
+_LOADING_SET: Set[str] = set()
+_LOADING_SET_LOCK = threading.Lock()
+
+# Contract AST cache by source hash
+_CONTRACT_AST_CACHE: Dict[str, Any] = {}
+_CONTRACT_AST_LOCK = threading.Lock()
+
+
+class CircularImportError(Exception):
+    """Raised when a circular import dependency is detected."""
+    def __init__(self, path: str, chain: Optional[list] = None):
+        self.path = path
+        self.chain = chain or []
+        if chain:
+            cycle = " -> ".join(chain + [path])
+            msg = f"Circular import detected: {cycle}"
+        else:
+            msg = f"Circular import detected while loading: {path}"
+        super().__init__(msg)
+
+
+def begin_loading(module_path: str) -> None:
+    """Mark *module_path* as currently being loaded.
+    
+    Raises ``CircularImportError`` if the module is already in the loading set
+    (i.e. a circular dependency has been encountered).
+    """
+    norm = normalize_path(module_path)
+    with _LOADING_SET_LOCK:
+        if norm in _LOADING_SET:
+            raise CircularImportError(norm, list(_LOADING_SET))
+        _LOADING_SET.add(norm)
+
+
+def end_loading(module_path: str) -> None:
+    """Remove *module_path* from the loading set after it has finished loading."""
+    norm = normalize_path(module_path)
+    with _LOADING_SET_LOCK:
+        _LOADING_SET.discard(norm)
+
+
+def is_loading(module_path: str) -> bool:
+    """Check whether *module_path* is currently being loaded."""
+    norm = normalize_path(module_path)
+    with _LOADING_SET_LOCK:
+        return norm in _LOADING_SET
+
+def get_cached_module(module_path: str) -> Optional[Tuple[Environment, Any, Any]]:
+    """Get a cached module (environment, bytecode, ast) if available"""
     with _MODULE_CACHE_LOCK:
         return _MODULE_CACHE.get(module_path)
 
-def cache_module(module_path: str, module_env: Environment) -> None:
-    """Cache a loaded module environment"""
+def cache_module(module_path: str, module_env: Environment, bytecode: Any = None, ast: Any = None) -> None:
+    """Cache a loaded module environment with optional bytecode and AST"""
     with _MODULE_CACHE_LOCK:
-        _MODULE_CACHE[module_path] = module_env
+        _MODULE_CACHE[module_path] = (module_env, bytecode, ast)
 
 def clear_module_cache() -> None:
     """Clear the entire module cache"""
@@ -40,6 +90,16 @@ def list_cached_modules() -> list[str]:
     with _MODULE_CACHE_LOCK:
         return list(_MODULE_CACHE.keys())
 
+def get_cached_contract_ast(source_hash: str) -> Optional[Any]:
+    """Get a cached contract AST by source hash"""
+    with _CONTRACT_AST_LOCK:
+        return _CONTRACT_AST_CACHE.get(source_hash)
+
+def cache_contract_ast(source_hash: str, ast: Any) -> None:
+    """Cache a parsed contract AST"""
+    with _CONTRACT_AST_LOCK:
+        _CONTRACT_AST_CACHE[source_hash] = ast
+
 def get_module_candidates(file_path: str, importer_file: str = None) -> list[str]:
     """Get candidate paths for a module, checking zpm_modules etc.
     
@@ -62,13 +122,19 @@ def get_module_candidates(file_path: str, importer_file: str = None) -> list[str
             importer_dir = os.path.dirname(importer_file)
             resolved_path = os.path.join(importer_dir, file_path[2:])  # Remove './'
             candidates.append(resolved_path)
+            # Also consider project-root relative paths like "./tests/..."
+            candidates.append(os.path.join(os.getcwd(), file_path[2:]))
         elif importer_file and file_path.startswith('../'):
             # Parent directory relative to importing file
             importer_dir = os.path.dirname(importer_file)
             resolved_path = os.path.join(importer_dir, file_path)
             candidates.append(resolved_path)
         else:
-            # Relative to current working directory
+            # For bare imports (no ./ or ../ prefix), check relative to importer first
+            if importer_file:
+                importer_dir = os.path.dirname(importer_file)
+                candidates.append(os.path.join(importer_dir, file_path))
+            # Then relative to current working directory
             candidates.append(os.path.join(os.getcwd(), file_path))
         
         # Also check zpm_modules directory
@@ -86,4 +152,168 @@ def get_module_candidates(file_path: str, importer_file: str = None) -> list[str
 
 def normalize_path(path: str) -> str:
     """Normalize a path for consistent cache keys"""
-    return os.path.abspath(os.path.expanduser(path))
+    return os.path.abspath(os.path.expanduser(path))
+
+
+# ---------------------------------------------------------------------------
+# Module Pre-compilation
+# ---------------------------------------------------------------------------
+
+def _extract_import_paths(node) -> list:
+    """Recursively walk an AST and return all import file paths."""
+    from . import zexus_ast
+    paths = []
+    if isinstance(node, zexus_ast.Program):
+        for stmt in getattr(node, 'statements', []):
+            paths.extend(_extract_import_paths(stmt))
+    elif isinstance(node, zexus_ast.UseStatement):
+        fp = node.file_path
+        if hasattr(fp, 'value'):
+            fp = fp.value
+        if isinstance(fp, str):
+            paths.append(fp)
+    elif isinstance(node, zexus_ast.FromStatement):
+        fp = node.file_path
+        if hasattr(fp, 'value'):
+            fp = fp.value
+        if isinstance(fp, str):
+            paths.append(fp)
+    elif isinstance(node, (zexus_ast.BlockStatement,)):
+        for stmt in getattr(node, 'statements', []):
+            paths.extend(_extract_import_paths(stmt))
+    elif isinstance(node, zexus_ast.IfStatement):
+        paths.extend(_extract_import_paths(node.consequence))
+        if node.alternative:
+            paths.extend(_extract_import_paths(node.alternative))
+        for cond, body in getattr(node, 'elif_parts', []):
+            paths.extend(_extract_import_paths(body))
+    elif isinstance(node, (zexus_ast.WhileStatement, zexus_ast.ForEachStatement)):
+        paths.extend(_extract_import_paths(getattr(node, 'body', None) or getattr(node, 'block', None)))
+    elif isinstance(node, zexus_ast.FunctionStatement):
+        paths.extend(_extract_import_paths(getattr(node, 'body', None)))
+    elif isinstance(node, zexus_ast.ActionStatement):
+        paths.extend(_extract_import_paths(getattr(node, 'body', None)))
+    elif isinstance(node, zexus_ast.TryCatchStatement):
+        paths.extend(_extract_import_paths(getattr(node, 'try_body', None)))
+        paths.extend(_extract_import_paths(getattr(node, 'catch_body', None)))
+        if getattr(node, 'finally_body', None):
+            paths.extend(_extract_import_paths(node.finally_body))
+    return [p for p in paths if p]
+
+
+def precompile_modules(program, main_file: str, compile_bytecode: bool = True) -> dict:
+    """Pre-compile all imported modules before execution.
+    
+    Walks the AST of *program*, resolves every ``use`` / ``from`` import,
+    lexes + parses each module file, optionally compiles to bytecode, and
+    stores everything in the global ``_MODULE_CACHE``.  Processes modules
+    recursively so transitive dependencies are also cached.
+    
+    Args:
+        program: The parsed AST (Program node) of the main file.
+        main_file: Absolute path of the main source file.
+        compile_bytecode: If True, also compile each module to VM bytecode.
+    
+    Returns:
+        dict mapping normalised module path → (ast, bytecode_or_None).
+    """
+    from .lexer import Lexer
+    from .parser import Parser
+    from .stdlib_integration import is_stdlib_module
+    from .builtin_modules import is_builtin_module
+
+    compiler_mod = None
+    if compile_bytecode:
+        try:
+            from .vm.compiler import compile_ast_to_bytecode
+            compiler_mod = compile_ast_to_bytecode
+        except Exception:
+            compiler_mod = None
+
+    results: Dict[str, Any] = {}
+    visited: Set[str] = set()
+
+    def _resolve_and_cache(import_path: str, importer_file: str):
+        """Resolve a single import and cache it, then recurse."""
+        # Skip stdlib / builtin modules — they aren't file-based
+        if is_stdlib_module(import_path) or is_builtin_module(import_path):
+            return
+
+        candidates = get_module_candidates(import_path, importer_file)
+        resolved = None
+        for cand in candidates:
+            norm = normalize_path(cand)
+            if norm in visited:
+                import warnings
+                warnings.warn(
+                    f"Circular import detected during pre-compilation: {import_path} "
+                    f"(resolved to {norm})",
+                    stacklevel=2,
+                )
+                return  # Already processed (or in progress — avoids cycles)
+            if os.path.isfile(cand):
+                resolved = cand
+                break
+
+        if resolved is None:
+            return  # Unresolvable — runtime will report the error
+
+        norm = normalize_path(resolved)
+        if norm in visited:
+            return
+        visited.add(norm)
+
+        # Already cached from a previous run?
+        cached = get_cached_module(norm)
+        if cached is not None:
+            results[norm] = (cached[2], cached[1])  # (ast, bytecode)
+            # Still recurse into dependencies of the cached module
+            if cached[2] is not None:
+                sub_paths = _extract_import_paths(cached[2])
+                for sp in sub_paths:
+                    _resolve_and_cache(sp, resolved)
+            return
+
+        # Read, lex, parse
+        try:
+            with open(resolved, 'r', encoding='utf-8') as f:
+                source = f.read()
+        except (OSError, IOError):
+            return
+
+        try:
+            lexer = Lexer(source, filename=resolved)
+            parser = Parser(lexer, 'universal', enable_advanced_strategies=True)
+            mod_ast = parser.parse_program()
+        except Exception:
+            return  # Parse failure — runtime will handle
+
+        # Optionally compile to bytecode
+        mod_bytecode = None
+        if compiler_mod is not None:
+            try:
+                mod_bytecode = compiler_mod(mod_ast, optimize=True)
+            except Exception:
+                pass  # Compilation failure is non-fatal; interpreter fallback
+
+        # Cache a placeholder env + ast + bytecode so runtime finds them.
+        # Mark the env as pre-compiled but not yet evaluated — the runtime
+        # will execute the bytecode/AST to populate it on first use.
+        mod_env = Environment()
+        mod_env.set("__file__", resolved)
+        mod_env.set("__MODULE__", os.path.splitext(os.path.basename(resolved))[0])
+        mod_env._precompiled = True  # Marker for eval_use_statement
+        cache_module(norm, mod_env, mod_bytecode, mod_ast)
+        results[norm] = (mod_ast, mod_bytecode)
+
+        # Recurse into sub-imports
+        sub_paths = _extract_import_paths(mod_ast)
+        for sp in sub_paths:
+            _resolve_and_cache(sp, resolved)
+
+    # Kick off from the main program's imports
+    import_paths = _extract_import_paths(program)
+    for ip in import_paths:
+        _resolve_and_cache(ip, main_file)
+
+    return results

package/src/zexus/module_manager.py

@@ -13,6 +13,7 @@ class ModuleManager:
             self.base_path / "lib"
         ]
         self._debug = False
+        self._max_find_results = 50
 
     def normalize_path(self, path):
         """Normalize a module path"""
@@ -87,6 +88,129 @@ class ModuleManager:
         """Disable debug logging"""
         self._debug = False
 
+    def find_files(self, pattern, current_dir=None, scope=None, max_results=None):
+        """Search for files matching pattern within known search paths.
+
+        Args:
+            pattern: File name or relative pattern to search for.
+            current_dir: Directory of the importing file (for relative priority).
+            scope: Optional directory hint (absolute or relative) to limit search.
+            max_results: Maximum matches to return (defaults to manager limit).
+
+        Returns:
+            List of normalized absolute paths matching the pattern.
+        """
+        max_results = max_results or self._max_find_results
+
+        if not pattern:
+            return []
+
+        # Absolute pattern short-circuit
+        try:
+            pattern_path = Path(pattern)
+        except TypeError:
+            return []
+
+        if pattern_path.is_absolute():
+            if pattern_path.exists():
+                return [self.normalize_path(pattern_path)]
+            # Try with known extensions
+            for ext in (".zx", ".zexus"):
+                candidate = pattern_path.with_suffix(ext)
+                if candidate.exists():
+                    return [self.normalize_path(candidate)]
+            return []
+
+        roots = []
+
+        def _append_root(path_candidate):
+            if not path_candidate:
+                return
+            try:
+                resolved = Path(path_candidate).resolve()
+            except (OSError, RuntimeError):
+                return
+            if resolved not in roots and resolved.exists():
+                roots.append(resolved)
+
+        # Scope hint first (if provided)
+        if scope:
+            scope_path = Path(scope)
+            if not scope_path.is_absolute() and current_dir:
+                _append_root(Path(current_dir) / scope_path)
+            _append_root(self.base_path / scope_path)
+            if scope_path.is_absolute():
+                _append_root(scope_path)
+
+        # Current file directory has highest priority after scope
+        if current_dir:
+            _append_root(current_dir)
+
+        # Standard search paths (project root, modules, lib, etc.)
+        for search_path in self.search_paths:
+            _append_root(search_path)
+
+        # Deduplicate while preserving order
+        seen = set()
+        ordered_roots = []
+        for root in roots:
+            if root in seen:
+                continue
+            seen.add(root)
+            ordered_roots.append(root)
+
+        matches = []
+        match_set = set()
+
+        def _record(path_obj):
+            normalized = self.normalize_path(path_obj)
+            if normalized not in match_set:
+                match_set.add(normalized)
+                matches.append(normalized)
+
+        # Exact relative matches via direct join first
+        for root in ordered_roots:
+            candidate = root / pattern_path
+            if candidate.exists() and candidate.is_file():
+                _record(candidate)
+                if len(matches) >= max_results:
+                    return matches
+            else:
+                for ext in (".zx", ".zexus"):
+                    candidate_ext = candidate.with_suffix(ext)
+                    if candidate_ext.exists() and candidate_ext.is_file():
+                        _record(candidate_ext)
+                        if len(matches) >= max_results:
+                            return matches
+
+        # Fallback to glob search across roots
+        if pattern_path.name:
+            search_pattern = str(pattern_path)
+            basename_pattern = pattern_path.name
+        else:
+            search_pattern = pattern
+            basename_pattern = pattern
+
+        for root in ordered_roots:
+            if len(matches) >= max_results:
+                break
+            try:
+                iterator = root.rglob(search_pattern)
+            except (ValueError, OSError):
+                try:
+                    iterator = root.rglob(basename_pattern)
+                except (ValueError, OSError):
+                    continue
+
+            for found in iterator:
+                if not found.is_file():
+                    continue
+                _record(found)
+                if len(matches) >= max_results:
+                    break
+
+        return matches
+
 # Create a default instance for backwards compatibility
 _default_manager = ModuleManager()
 
@@ -104,4 +228,8 @@ def cache_module(path, module_env):
     return _default_manager.cache_module(path, module_env)
 
 def clear_cache():
-    return _default_manager.clear_cache()
+    return _default_manager.clear_cache()
+
+
+def find_files(pattern, current_dir=None, scope=None, max_results=None):
+    return _default_manager.find_files(pattern, current_dir=current_dir, scope=scope, max_results=max_results)

package/src/zexus/object.py

@@ -98,6 +98,20 @@ class List(Object):
         except Exception:
             return NULL
     
+    def set(self, index, value):
+        """Set element at index, mirroring array-style assignment."""
+        try:
+            idx = index.value if hasattr(index, 'value') else index
+            idx = int(idx)
+        except Exception as exc:
+            raise EvaluationError(f"Invalid index for assignment: {index}") from exc
+
+        if idx < 0 or idx >= len(self.elements):
+            raise EvaluationError(f"Index out of range for List assignment: {idx}")
+
+        self.elements[idx] = value
+        return value
+
     def append(self, item):
         """Append item to list in-place (mutating operation)"""
         self.elements.append(item)
@@ -122,16 +136,35 @@ class Map(Object):
             pairs.append(f"{key_str}: {value_str}")
         return "{" + ", ".join(pairs) + "}"
 
+    def _normalize_key(self, key):
+        if hasattr(key, 'inspect'):
+            return key.inspect()
+        return str(key)
+
     def get(self, key):
-        """Get value by key (compatible with string keys)"""
-        return self.pairs.get(key)
+        """Get value by key (compatible with both string and String keys)"""
+        # Try direct lookup first (works for String-keyed and plain-keyed maps)
+        val = self.pairs.get(key)
+        if val is not None:
+            return val
+        # Try with String object key (for maps created by _python_to_zexus)
+        if isinstance(key, str):
+            str_key = String(key)
+            val = self.pairs.get(str_key)
+            if val is not None:
+                return val
+        # Try with normalized plain string key (for maps with plain string keys)
+        norm_key = self._normalize_key(key)
+        return self.pairs.get(norm_key)
 
     def set(self, key, value):
         """Set value for key, blocking modification if key is sealed."""
-        existing = self.pairs.get(key)
+        norm_key = self._normalize_key(key)
+        existing = self.pairs.get(norm_key)
         if existing is not None and existing.__class__.__name__ == 'SealedObject':
             raise EvaluationError(f"Cannot modify sealed map key: {key}")
-        self.pairs[key] = value
+        self.pairs[norm_key] = value
+        return value
 
     def keys(self):
         """Return array of map keys"""
@@ -860,6 +893,18 @@ class Environment:
         self.store[name] = val
         self.notify_watchers(name, val)
         return val
+
+    def clone_for_closure(self):
+        """Create a shallow copy of the environment for closure capture."""
+        cloned = Environment(outer=self.outer, persistence_scope=self.persistence_scope)
+        cloned.store = dict(self.store)
+        cloned.const_vars = set(self.const_vars)
+        cloned.exports = dict(self.exports)
+        cloned.watchers = {k: list(v) for k, v in self.watchers.items()}
+        cloned.debug_mode = self.debug_mode
+        cloned._persistent_storage = self._persistent_storage
+        cloned._memory_tracker = self._memory_tracker
+        return cloned
     
     def assign(self, name, val):
         """Assign to an existing variable in the scope chain, or create in current if not found."""
@@ -1101,14 +1146,39 @@ class EvaluationError(Object):
             
             # Add stack trace if available
             if self.stack_trace:
-                trace = "\n".join(self.stack_trace[-5:])
+                formatted_trace = []
+                for frame in self.stack_trace[-5:]:
+                    if isinstance(frame, str):
+                        formatted_trace.append(frame)
+                    elif isinstance(frame, tuple) and len(frame) == 2:
+                        node_type, line = frame
+                        item = f"  at {node_type.__name__}"
+                        if line:
+                            item += f" (line {line})"
+                        formatted_trace.append(item)
+                
+                trace = "\n".join(formatted_trace)
                 temp_error.message += f"\n\nStack trace:\n{trace}"
             
             return temp_error.format_error()
         except Exception:
             # Fallback to simple format if error reporter not available
             location = f"Line {self.line}:{self.column}" if self.line and self.column else "Unknown location"
-            trace = "\n".join(self.stack_trace[-3:]) if self.stack_trace else ""
+            
+            # Format simple trace
+            formatted_trace = []
+            if self.stack_trace:
+                for frame in self.stack_trace[-3:]:
+                    if isinstance(frame, str):
+                        formatted_trace.append(frame)
+                    elif isinstance(frame, tuple) and len(frame) == 2:
+                        node_type, line = frame
+                        item = f"  at {node_type.__name__}"
+                        if line:
+                            item += f" (line {line})"
+                        formatted_trace.append(item)
+            
+            trace = "\n".join(formatted_trace)
             trace_section = f"\n   Stack:\n{trace}" if trace else ""
             suggestion_section = f"\n   💡 Suggestion: {self.suggestion}" if self.suggestion else ""
             return f"❌ Runtime Error at {location}\n   {self.message}{suggestion_section}{trace_section}"