zexus 1.6.2

package/src/zexus/zexus_token.py

@@ -0,0 +1,253 @@
+# zexus_token.py (ENHANCED WITH PHASE 1 TOKENS)
+
+# Special Tokens
+ILLEGAL = "ILLEGAL"
+EOF = "EOF"
+
+# Identifiers + Literals
+IDENT = "IDENT"
+INT = "INT"
+STRING = "STRING"
+FLOAT = "FLOAT"
+
+# Operators
+ASSIGN = "="
+PLUS = "+"
+MINUS = "-"
+SLASH = "/"
+STAR = "*"
+BANG = "!"
+LT = "<"
+GT = ">"
+EQ = "=="
+NOT_EQ = "!="
+MOD = "%"
+DOT = "."
+LTE = "<="
+GTE = ">="
+APPEND = ">>"           # Append operator for LOG: log >> file
+IMPORT_OP = "<<"        # Import operator: << file or let x << file
+AND = "&&"
+OR = "||"
+QUESTION = "?"          # Ternary operator: condition ? true_val : false_val
+NULLISH = "??"          # Nullish coalescing: value ?? default
+
+# Delimiters
+COMMA = ","
+SEMICOLON = ";"
+COLON = ":"
+LPAREN = "("
+RPAREN = ")"
+LBRACE = "{"
+RBRACE = "}"
+LBRACKET = "["
+RBRACKET = "]"
+AT = "@"                # Decorator symbol: @logged, @cached, etc.
+
+# Backwards-compatible alias: some parts of the codebase expect the name
+# ASTERISK for the multiplication token. Provide a stable alias to avoid
+# NameError when older modules reference ASTERISK.
+ASTERISK = STAR
+
+# Keywords
+LET = "LET"
+CONST = "CONST"                # NEW: Const immutable variables
+DATA = "DATA"                  # NEW: Dataclass definitions
+PRINT = "PRINT"
+IF = "IF"
+THEN = "THEN"                  # NEW: Then for if-then-else expressions
+ELIF = "ELIF"                  # NEW: Elif else-if conditionals
+ELSE = "ELSE"
+RETURN = "RETURN"
+TRUE = "TRUE"
+FALSE = "FALSE"
+NULL = "NULL"
+FOR = "FOR"
+EACH = "EACH"
+IN = "IN"
+ACTION = "ACTION"
+FUNCTION = "FUNCTION"
+SCREEN = "SCREEN"
+COMPONENT = "COMPONENT"
+THEME = "THEME"
+COLOR = "COLOR"
+GRAPHICS = "GRAPHICS"
+CANVAS = "CANVAS"
+ANIMATION = "ANIMATION"
+CLOCK = "CLOCK"
+MAP = "MAP"
+WHILE = "WHILE"
+USE = "USE"
+EXACTLY = "EXACTLY"
+EMBEDDED = "EMBEDDED"
+EXPORT = "EXPORT"
+LAMBDA = "LAMBDA"
+DEBUG = "DEBUG"      # NEW: Debug token
+TRY = "TRY"          # NEW: Try token
+CATCH = "CATCH"      # NEW: Catch token
+CONTINUE = "CONTINUE" # NEW: Continue on error token
+BREAK = "BREAK"      # NEW: Break loop statement
+THROW = "THROW"      # NEW: Throw error statement
+EXTERNAL = "EXTERNAL" # NEW: From token
+FROM = "FROM"        # NEW: From token
+
+# ASYNC / AWAIT / MODULE / EVENT / ENUM / PROTOCOL tokens
+ASYNC = "ASYNC"
+AWAIT = "AWAIT"
+EVENT = "EVENT"
+EMIT = "EMIT"
+ENUM = "ENUM"
+PROTOCOL = "PROTOCOL"
+IMPORT = "IMPORT"
+
+# SECURITY & ADVANCED FEATURES
+ENTITY = "ENTITY"              # Entity declaration: entity User { ... }
+VERIFY = "VERIFY"              # Verify checks: verify(action_name, ...conditions)
+CONTRACT = "CONTRACT"          # Smart contracts: contract Token { ... }
+PROTECT = "PROTECT"            # Protection guardrails: protect(action, rules)
+SEAL = "SEAL"                  # Seal objects: seal identifier
+AUDIT = "AUDIT"                # Audit log: audit data_name, action_type, timestamp
+RESTRICT = "RESTRICT"          # Field-level access control: restrict obj.field = "rule"
+SANDBOX = "SANDBOX"            # Isolated execution environment: sandbox { code }
+TRAIL = "TRAIL"                # Real-time audit/debug/print trail: trail audit_or_print_or_debug
+
+# CAPABILITY-BASED SECURITY (NEW)
+CAPABILITY = "CAPABILITY"      # Define capabilities: capability(name, description, scope)
+GRANT = "GRANT"                # Grant capabilities: grant entity_name capability(name)
+REVOKE = "REVOKE"              # Revoke capabilities: revoke entity_name capability(name)
+IMMUTABLE = "IMMUTABLE"        # Enforce immutability: immutable identifier
+
+# DATA VALIDATION & SANITIZATION (NEW)
+VALIDATE = "VALIDATE"          # Validate data: validate(value, schema, options)
+SANITIZE = "SANITIZE"          # Sanitize input: sanitize(value, rules, encoding)
+
+# COMPLEXITY & LARGE PROJECT MANAGEMENT (NEW)
+INTERFACE = "INTERFACE"        # Define formal interface: interface Drawable { ... }
+TYPE_ALIAS = "TYPE_ALIAS"      # Type alias: type_alias UserID = integer;
+MODULE = "MODULE"              # Module definition: module database { ... }
+USING = "USING"                # Resource management: using(file) { ... }
+PACKAGE = "PACKAGE"            # Package/namespace: package myapp.utils { ... }
+
+# CONCURRENCY & PERFORMANCE (NEW)
+CHANNEL = "CHANNEL"            # Channel definition: channel<integer> data;
+SEND = "SEND"                  # Send to channel: send(channel, value);
+RECEIVE = "RECEIVE"            # Receive from channel: receive(channel);
+ATOMIC = "ATOMIC"              # Atomic operations: atomic(counter++);
+
+# RENDERER OPERATIONS (ADD THESE)
+MIX = "MIX"                    # Color mixing: mix("blue", "red", 0.5)
+RENDER = "RENDER"              # Render screen: render_screen("login")
+ADD_TO = "ADD_TO"              # Add component: add_to_screen("login", "button")
+SET_THEME = "SET_THEME"        # Set theme: set_theme("dark")
+CREATE_CANVAS = "CREATE_CANVAS" # Create canvas: create_canvas(80, 25)
+DRAW = "DRAW"                  # Draw operation: draw_line(x1, y1, x2, y2)
+
+# PROPERTY TOKENS (ADD THESE)
+WIDTH = "WIDTH"
+HEIGHT = "HEIGHT"
+X = "X"
+Y = "Y"
+TEXT = "TEXT"
+BACKGROUND = "BACKGROUND"
+BORDER = "BORDER"
+STYLE = "STYLE"
+RADIUS = "RADIUS"
+FILL = "FILL"
+
+# ADVANCED FEATURE TOKENS
+MIDDLEWARE = "MIDDLEWARE"
+AUTH = "AUTH"
+THROTTLE = "THROTTLE"
+CACHE = "CACHE"
+INJECT = "INJECT"  # Dependency injection: inject DatabaseAPI
+
+# BLOCKCHAIN & SMART CONTRACT TOKENS
+LEDGER = "LEDGER"              # Immutable state ledger: ledger balances;
+STATE = "STATE"                # State management: state counter = 0;
+TX = "TX"                      # Transaction context: TX.caller, TX.timestamp
+REVERT = "REVERT"              # Revert transaction: revert("reason");
+REQUIRE = "REQUIRE"            # Require condition: require(condition, "error");
+HASH = "HASH"                  # Hash function: hash(data, "SHA256");
+SIGNATURE = "SIGNATURE"        # Create signature: signature(data, private_key);
+VERIFY_SIG = "VERIFY_SIG"      # Verify signature: verify_sig(data, sig, public_key);
+LIMIT = "LIMIT"                # Gas/resource limit: action transfer() limit 1000 { ... };
+GAS = "GAS"                    # Gas tracking: gas_used(), gas_remaining()
+PERSISTENT = "PERSISTENT"
+STORAGE = "STORAGE"
+REQUIRE = "REQUIRE"
+
+# PERFORMANCE OPTIMIZATION TOKENS
+NATIVE = "NATIVE"            # Call C/C++ code: native { "func_name", arg1, arg2 }
+GC = "GC"                    # Control garbage collection: gc "collect" or gc "pause"
+INLINE = "INLINE"            # Function inlining: inline func_name
+BUFFER = "BUFFER"            # Direct memory access: buffer(ptr, size)
+SIMD = "SIMD"                # Vector operations: simd(operation, vector1, vector2)
+
+# CONVENIENCE FEATURES TOKENS
+DEFER = "DEFER"              # Cleanup code execution: defer cleanup_code;
+PATTERN = "PATTERN"          # Pattern matching: pattern value { case x => ...; }
+MATCH = "MATCH"              # Match expression: match value { Point(x, y) => ... }
+
+# ADVANCED FEATURES TOKENS
+ENUM = "ENUM"                # Type-safe enumerations: enum Color { Red, Green, Blue }
+STREAM = "STREAM"            # Event streaming: stream name as event => handler;
+WATCH = "WATCH"              # Reactive state management: watch variable => reaction;
+LOG = "LOG"                  # Output logging: log > filename.txt
+
+# Modifiers (single keyword to extend declarations)
+PUBLIC = "PUBLIC"
+PRIVATE = "PRIVATE"
+SEALED = "SEALED"
+ASYNC = "ASYNC"
+NATIVE = "NATIVE"
+INLINE = "INLINE"
+SECURE = "SECURE"
+PURE = "PURE"
+VIEW = "VIEW"                  # View function (alias for pure, read-only)
+PAYABLE = "PAYABLE"            # Payable function (can receive tokens)
+MODIFIER = "MODIFIER"          # Function modifier (like onlyOwner)
+
+# Contract & Protocol
+IMPLEMENTS = "IMPLEMENTS"      # Protocol implementation: contract X implements Y
+THIS = "THIS"                  # Reference to current contract instance
+
+class Token:
+    def __init__(self, token_type, literal, line=None, column=None):
+        self.type = token_type
+        self.literal = literal
+        self.line = line  # ✅ ADD line tracking
+        self.column = column  # ✅ ADD column tracking
+        
+        # For backward compatibility with code expecting dict-like tokens
+        self.value = literal  # Alias for literal
+
+    def __repr__(self):
+        if self.line and self.column:
+            return f"Token({self.type}, '{self.literal}', line={self.line}, col={self.column})"
+        return f"Token({self.type}, '{self.literal}')"
+    
+    def get(self, key, default=None):
+        """Dict-like get method for backward compatibility"""
+        if hasattr(self, key):
+            return getattr(self, key)
+        return default
+    
+    def __getitem__(self, key):
+        """Allow dict-like access for compatibility"""
+        if hasattr(self, key):
+            return getattr(self, key)
+        raise KeyError(f"Token has no attribute '{key}'")
+    
+    def __contains__(self, key):
+        """Check if token has attribute"""
+        return hasattr(self, key)
+    
+    def to_dict(self):
+        """Convert token to dictionary for compatibility"""
+        return {
+            'type': self.type,
+            'literal': self.literal,
+            'value': self.literal,
+            'line': self.line,
+            'column': self.column
+        }

package/src/zexus/zpm/__init__.py

@@ -0,0 +1,15 @@
+"""
+ZPM - Zexus Package Manager
+"""
+
+from .package_manager import PackageManager
+from .registry import PackageRegistry
+from .installer import PackageInstaller
+from .publisher import PackagePublisher
+
+__all__ = [
+    'PackageManager',
+    'PackageRegistry',
+    'PackageInstaller',
+    'PackagePublisher'
+]

package/src/zexus/zpm/installer.py

@@ -0,0 +1,116 @@
+"""
+Package Installer - Handles package installation and dependencies
+"""
+import os
+import json
+import shutil
+from pathlib import Path
+from typing import Dict, Optional
+
+
+class PackageInstaller:
+    """Handles package installation"""
+    
+    def __init__(self, install_dir: Path):
+        self.install_dir = Path(install_dir)
+        self.install_dir.mkdir(parents=True, exist_ok=True)
+    
+    def install(self, package_info: Dict) -> bool:
+        """Install a package"""
+        name = package_info["name"]
+        version = package_info["version"]
+        pkg_type = package_info.get("type", "normal")
+        
+        target_dir = self.install_dir / name
+        
+        # Check if already installed
+        if target_dir.exists():
+            existing_pkg = target_dir / "zexus.json"
+            if existing_pkg.exists():
+                with open(existing_pkg) as f:
+                    existing_info = json.load(f)
+                    if existing_info.get("version") == version:
+                        print(f"ℹ️  {name}@{version} already installed")
+                        return True
+        
+        # Create package directory
+        target_dir.mkdir(parents=True, exist_ok=True)
+        
+        # For built-in packages, create stub
+        if pkg_type == "builtin":
+            self._install_builtin(name, version, target_dir)
+        else:
+            # TODO: Download and extract package
+            self._install_from_source(package_info, target_dir)
+        
+        return True
+    
+    def _install_builtin(self, name: str, version: str, target_dir: Path):
+        """Install a built-in package"""
+        # Create package.json
+        pkg_json = {
+            "name": name,
+            "version": version,
+            "type": "builtin",
+            "main": "index.zx"
+        }
+        
+        with open(target_dir / "zexus.json", "w") as f:
+            json.dump(pkg_json, f, indent=2)
+        
+        # Create stub main file
+        main_file = target_dir / "index.zx"
+        main_file.write_text(f"""// {name} - Built-in Zexus package
+// Version: {version}
+
+// This is a built-in package provided by Zexus
+// Functions are available globally when imported
+
+export {{
+    // Package exports will be defined here
+}}
+""")
+    
+    def _install_from_source(self, package_info: Dict, target_dir: Path):
+        """Install package from source/tarball"""
+        # TODO: Implement actual download and extraction
+        
+        # For now, create placeholder
+        pkg_json = {
+            "name": package_info["name"],
+            "version": package_info["version"],
+            "description": package_info.get("description", ""),
+        }
+        
+        with open(target_dir / "zexus.json", "w") as f:
+            json.dump(pkg_json, f, indent=2)
+        
+        main_file = target_dir / "index.zx"
+        main_file.write_text(f"""// {package_info['name']}
+// Placeholder - package installation from remote sources not yet implemented
+""")
+    
+    def uninstall(self, package: str) -> bool:
+        """Uninstall a package"""
+        target_dir = self.install_dir / package
+        
+        if not target_dir.exists():
+            print(f"⚠️  Package {package} not installed")
+            return False
+        
+        shutil.rmtree(target_dir)
+        return True
+    
+    def is_installed(self, package: str) -> bool:
+        """Check if a package is installed"""
+        return (self.install_dir / package).exists()
+    
+    def get_installed_version(self, package: str) -> Optional[str]:
+        """Get installed version of a package"""
+        pkg_json = self.install_dir / package / "zexus.json"
+        if not pkg_json.exists():
+            return None
+        
+        with open(pkg_json) as f:
+            info = json.load(f)
+            return info.get("version")

package/src/zexus/zpm/package_manager.py

@@ -0,0 +1,208 @@
+"""
+Package Manager Core - Main ZPM interface
+"""
+import os
+import json
+from pathlib import Path
+from typing import Dict, List, Optional
+from .registry import PackageRegistry
+from .installer import PackageInstaller
+from .publisher import PackagePublisher
+
+
+class PackageManager:
+    """Main package manager interface"""
+    
+    def __init__(self, project_root: str = "."):
+        self.project_root = Path(project_root).resolve()
+        self.zpm_dir = self.project_root / "zpm_modules"
+        self.config_file = self.project_root / "zexus.json"
+        self.lock_file = self.project_root / "zexus-lock.json"
+        
+        self.registry = PackageRegistry()
+        self.installer = PackageInstaller(self.zpm_dir)
+        self.publisher = PackagePublisher(self.registry)
+        
+    def init(self, name: str = None, version: str = "1.6.2") -> Dict:
+        """Initialize a new Zexus project with package.json"""
+        if self.config_file.exists():
+            print(f"⚠️  {self.config_file} already exists")
+            return self.load_config()
+        
+        if not name:
+            name = self.project_root.name
+        
+        config = {
+            "name": name,
+            "version": version,
+            "description": "",
+            "main": "main.zx",
+            "dependencies": {},
+            "devDependencies": {},
+            "scripts": {
+                "test": "zx run tests/test_*.zx",
+                "build": "zx compile main.zx"
+            },
+            "author": "",
+            "license": "MIT"
+        }
+        
+        self.save_config(config)
+        print(f"✅ Created {self.config_file}")
+        print(f"📝 Edit {self.config_file} to customize your project")
+        
+        return config
+    
+    def install(self, package: str = None, dev: bool = False) -> bool:
+        """Install a package or all packages from zexus.json"""
+        if not package:
+            # Install all dependencies
+            return self.install_all()
+        
+        # Install specific package
+        print(f"📦 Installing {package}...")
+        
+        # Parse package@version
+        if "@" in package:
+            name, version = package.split("@", 1)
+        else:
+            name, version = package, "latest"
+        
+        # Get package info from registry
+        pkg_info = self.registry.get_package(name, version)
+        if not pkg_info:
+            print(f"❌ Package {name}@{version} not found")
+            return False
+        
+        # Install package
+        success = self.installer.install(pkg_info)
+        if not success:
+            return False
+        
+        # Update config
+        config = self.load_config()
+        dep_key = "devDependencies" if dev else "dependencies"
+        if dep_key not in config:
+            config[dep_key] = {}
+        
+        config[dep_key][name] = version
+        self.save_config(config)
+        
+        # Update lock file
+        self.update_lock_file()
+        
+        print(f"✅ Installed {name}@{version}")
+        return True
+    
+    def install_all(self) -> bool:
+        """Install all dependencies from zexus.json"""
+        config = self.load_config()
+        if not config:
+            print("❌ No zexus.json found. Run 'zpm init' first.")
+            return False
+        
+        dependencies = config.get("dependencies", {})
+        dev_dependencies = config.get("devDependencies", {})
+        all_deps = {**dependencies, **dev_dependencies}
+        
+        if not all_deps:
+            print("✅ No dependencies to install")
+            return True
+        
+        print(f"📦 Installing {len(all_deps)} package(s)...")
+        
+        success_count = 0
+        for name, version in all_deps.items():
+            pkg_info = self.registry.get_package(name, version)
+            if pkg_info and self.installer.install(pkg_info):
+                success_count += 1
+            else:
+                print(f"⚠️  Failed to install {name}@{version}")
+        
+        self.update_lock_file()
+        print(f"✅ Installed {success_count}/{len(all_deps)} package(s)")
+        return success_count == len(all_deps)
+    
+    def uninstall(self, package: str) -> bool:
+        """Uninstall a package"""
+        print(f"🗑️  Uninstalling {package}...")
+        
+        success = self.installer.uninstall(package)
+        if not success:
+            return False
+        
+        # Update config
+        config = self.load_config()
+        if package in config.get("dependencies", {}):
+            del config["dependencies"][package]
+        if package in config.get("devDependencies", {}):
+            del config["devDependencies"][package]
+        
+        self.save_config(config)
+        self.update_lock_file()
+        
+        print(f"✅ Uninstalled {package}")
+        return True
+    
+    def list(self) -> List[Dict]:
+        """List installed packages"""
+        if not self.zpm_dir.exists():
+            return []
+        
+        packages = []
+        for item in self.zpm_dir.iterdir():
+            if item.is_dir():
+                pkg_json = item / "zexus.json"
+                if pkg_json.exists():
+                    with open(pkg_json) as f:
+                        pkg_info = json.load(f)
+                        packages.append({
+                            "name": pkg_info.get("name", item.name),
+                            "version": pkg_info.get("version", "unknown"),
+                            "path": str(item)
+                        })
+        
+        return packages
+    
+    def search(self, query: str) -> List[Dict]:
+        """Search for packages in registry"""
+        return self.registry.search(query)
+    
+    def publish(self) -> bool:
+        """Publish current package to registry"""
+        config = self.load_config()
+        if not config:
+            print("❌ No zexus.json found")
+            return False
+        
+        return self.publisher.publish(self.project_root, config)
+    
+    def load_config(self) -> Optional[Dict]:
+        """Load zexus.json"""
+        if not self.config_file.exists():
+            return None
+        
+        with open(self.config_file) as f:
+            return json.load(f)
+    
+    def save_config(self, config: Dict):
+        """Save zexus.json"""
+        with open(self.config_file, "w") as f:
+            json.dump(config, f, indent=2)
+    
+    def update_lock_file(self):
+        """Update zexus-lock.json with installed packages"""
+        packages = self.list()
+        lock_data = {
+            "lockfileVersion": 1,
+            "packages": {
+                pkg["name"]: {
+                    "version": pkg["version"],
+                    "path": pkg["path"]
+                }
+                for pkg in packages
+            }
+        }
+        
+        with open(self.lock_file, "w") as f:
+            json.dump(lock_data, f, indent=2)

package/src/zexus/zpm/publisher.py

@@ -0,0 +1,98 @@
+"""
+Package Publisher - Handles package publishing to registry
+"""
+import os
+import json
+import tarfile
+from pathlib import Path
+from typing import Dict
+
+
+class PackagePublisher:
+    """Handles package publishing"""
+    
+    def __init__(self, registry):
+        self.registry = registry
+    
+    def publish(self, project_root: Path, config: Dict) -> bool:
+        """Publish a package"""
+        name = config.get("name")
+        version = config.get("version")
+        
+        if not name or not version:
+            print("❌ Package name and version required")
+            return False
+        
+        print(f"📤 Publishing {name}@{version}...")
+        
+        # Collect files
+        files = self._collect_files(project_root, config)
+        
+        if not files:
+            print("❌ No files to publish")
+            return False
+        
+        # Create tarball
+        tarball_path = self._create_tarball(project_root, name, version, files)
+        
+        # Publish to registry
+        package_data = {
+            **config,
+            "tarball": str(tarball_path),
+            "files": files
+        }
+        
+        success = self.registry.publish_package(package_data, files)
+        
+        if success:
+            print(f"✅ Published {name}@{version}")
+            print(f"📦 Tarball: {tarball_path}")
+        else:
+            print(f"❌ Failed to publish {name}@{version}")
+        
+        return success
+    
+    def _collect_files(self, project_root: Path, config: Dict) -> list:
+        """Collect files to include in package"""
+        files = []
+        
+        # Include main file
+        main_file = config.get("main", "main.zx")
+        if (project_root / main_file).exists():
+            files.append(main_file)
+        
+        # Include all .zx files (excluding node_modules equivalent)
+        for zx_file in project_root.rglob("*.zx"):
+            relative = zx_file.relative_to(project_root)
+            # Exclude zpm_modules, tests, etc.
+            if not any(part.startswith(".") or part in ["zpm_modules", "node_modules"] 
+                      for part in relative.parts):
+                files.append(str(relative))
+        
+        # Include package.json
+        files.append("zexus.json")
+        
+        # Include README if exists
+        for readme in ["README.md", "README.txt", "README"]:
+            if (project_root / readme).exists():
+                files.append(readme)
+                break
+        
+        # Include LICENSE if exists
+        if (project_root / "LICENSE").exists():
+            files.append("LICENSE")
+        
+        return files
+    
+    def _create_tarball(self, project_root: Path, name: str, version: str, files: list) -> Path:
+        """Create a tarball of the package"""
+        tarball_name = f"{name}-{version}.tar.gz"
+        tarball_path = project_root / tarball_name
+        
+        with tarfile.open(tarball_path, "w:gz") as tar:
+            for file in files:
+                file_path = project_root / file
+                if file_path.exists():
+                    tar.add(file_path, arcname=f"{name}/{file}")
+        
+        return tarball_path