exonware-xwlazy 0.1.0.10__py3-none-any.whl → 0.1.0.19__py3-none-any.whl

exonware/xwlazy/defs.py

@@ -0,0 +1,397 @@
+"""
+#exonware/xwlazy/src/exonware/xwlazy/defs.py
+
+Company: eXonware.com
+Author: Eng. Muhammad AlShehri
+Email: connect@exonware.com
+Version: 0.1.0.19
+Generation Date: 10-Oct-2025
+
+Type Definitions and Constants for Lazy Loading System
+
+This module defines type definitions, constants, and TypedDict structures
+for the lazy loading system following GUIDE_ARCH.md structure.
+"""
+
+from enum import Enum
+from typing import TypedDict, Dict, List, Optional, Any, Tuple
+from dataclasses import dataclass, field
+from types import ModuleType
+
+
+# =============================================================================
+# ENUMS
+# =============================================================================
+
+class LazyLoadMode(Enum):
+    """Controls lazy module loading behavior."""
+    NONE = "none"           # Standard imports (no lazy loading)
+    AUTO = "auto"           # Lazy loading enabled (deferred module loading)
+    PRELOAD = "preload"     # Preload all modules on start
+    BACKGROUND = "background"  # Load modules in background threads
+    CACHED = "cached"       # Cache loaded modules but allow unloading
+    # Superior performance modes
+    TURBO = "turbo"         # Multi-tier cache + parallel preloading + bytecode caching
+    ADAPTIVE = "adaptive"   # Self-optimizing with pattern learning
+    HYPERPARALLEL = "hyperparallel"  # Maximum parallelism with multi-threading
+    STREAMING = "streaming" # Asynchronous background loading with streaming
+    ULTRA = "ultra"         # Aggressive optimizations (pre-compiled bytecode, mmap, etc.)
+    INTELLIGENT = "intelligent"  # Automatically switches to fastest mode based on load level
+
+
+class LazyInstallMode(Enum):
+    """Lazy installation modes."""
+    # Core modes
+    NONE = "none"           # No auto-installation
+    SMART = "smart"         # Install on first usage (on-demand) - replaces AUTO
+    FULL = "full"           # Install all dependencies on start
+    CLEAN = "clean"         # Install on usage + uninstall after completion
+    TEMPORARY = "temporary"  # Always uninstall after use (more aggressive than CLEAN)
+    SIZE_AWARE = "size_aware"  # Install small packages, skip large ones
+    
+    # Special purpose modes (kept for specific use cases)
+    INTERACTIVE = "interactive"  # Ask user before installing
+    WARN = "warn"           # Log warning but don't install (for monitoring)
+    DISABLED = "disabled"   # Don't install anything (alias for NONE, more explicit)
+    DRY_RUN = "dry_run"    # Show what would be installed but don't install
+
+
+class PathType(Enum):
+    """Path types for validation."""
+    FILE = "file"
+    DIRECTORY = "directory"
+    UNKNOWN = "unknown"
+
+
+class InstallStatus(Enum):
+    """Installation status."""
+    PENDING = "pending"
+    RUNNING = "running"
+    SUCCESS = "success"
+    FAILED = "failed"
+    SKIPPED = "skipped"
+
+
+class LoadLevel(Enum):
+    """Load level categories."""
+    LIGHT = "light_load"
+    MEDIUM = "medium_load"
+    HEAVY = "heavy_load"
+    ENTERPRISE = "enterprise_load"
+
+
+# =============================================================================
+# DATACLASSES
+# =============================================================================
+
+@dataclass
+class DependencyInfo:
+    """Information about a discovered dependency."""
+    import_name: str
+    package_name: str
+    version: Optional[str] = None
+    source: str = "unknown"
+    category: str = "general"
+
+
+@dataclass
+class LazyModeConfig:
+    """Two-dimensional lazy mode configuration combining load and install modes."""
+    load_mode: LazyLoadMode = LazyLoadMode.NONE
+    install_mode: LazyInstallMode = LazyInstallMode.NONE
+    
+    # Additional configuration options
+    auto_uninstall_large: bool = False  # For AUTO_MODE behavior
+    large_package_threshold_mb: float = 50.0  # Size threshold for SIZE_AWARE mode
+    preload_priority: List[str] = field(default_factory=list)  # Priority modules for PRELOAD
+    background_workers: int = 2  # Workers for BACKGROUND mode
+    
+    def __post_init__(self):
+        """Normalize enum values."""
+        if isinstance(self.load_mode, str):
+            self.load_mode = LazyLoadMode(self.load_mode)
+        if isinstance(self.install_mode, str):
+            self.install_mode = LazyInstallMode(self.install_mode)
+
+
+@dataclass
+class InstallResult:
+    """Result of an installation operation."""
+    package_name: str
+    success: bool
+    status: InstallStatus
+    error: Optional[str] = None
+    version: Optional[str] = None
+    source: Optional[str] = None  # "cache", "pip", "wheel", etc.
+
+
+@dataclass
+class LazyConfig:
+    """Bridge configuration settings with the lazy package implementation."""
+    packages: Tuple[str, ...] = field(
+        default_factory=lambda: ("default",)
+    )
+    
+    def __post_init__(self) -> None:
+        self.packages = tuple(package.lower() for package in self.packages)
+
+
+@dataclass(frozen=True)
+class PackageManifest:
+    """Resolved manifest data for a single package."""
+    package: str
+    dependencies: Dict[str, str] = field(default_factory=dict)
+    watched_prefixes: Tuple[str, ...] = ()
+    async_installs: bool = False
+    async_workers: int = 1
+    class_wrap_prefixes: Tuple[str, ...] = ()
+    metadata: Dict[str, Any] = field(default_factory=dict)
+    
+    def get_dependency(self, import_name: str) -> Optional[str]:
+        """Return the declared package for the given import name."""
+        if not import_name:
+            return None
+        direct = self.dependencies.get(import_name)
+        if direct is not None:
+            return direct
+        # Case-insensitive fallback for convenience
+        return self.dependencies.get(import_name.lower())
+
+
+@dataclass(frozen=True)
+class PackageData:
+    """
+    Immutable package data - same across all strategies.
+    
+    This data structure is used by all package caching, helper, and manager strategies.
+    """
+    name: str
+    version: Optional[str] = None
+    installed: bool = False
+    install_time: Optional[float] = None
+    access_count: int = 0
+    install_mode: Optional['LazyInstallMode'] = None
+    error: Optional[Exception] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class ModuleData:
+    """
+    Immutable module data - same across all strategies.
+    
+    This data structure is used by all module caching, helper, and manager strategies.
+    """
+    path: str
+    loaded_module: Optional['ModuleType'] = None
+    loading: bool = False
+    load_time: Optional[float] = None
+    access_count: int = 0
+    error: Optional[Exception] = None
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+# =============================================================================
+# TYPE DEFINITIONS
+# =============================================================================
+
+class DependencyMapping(TypedDict, total=False):
+    """Type definition for dependency mapping structure."""
+    import_name: str
+    package_name: str
+    version: Optional[str]
+    source: str
+    category: str
+
+
+class PackageStats(TypedDict, total=False):
+    """Type definition for package statistics."""
+    enabled: bool
+    mode: str
+    package_name: str
+    installed_packages: List[str]
+    failed_packages: List[str]
+    total_installed: int
+    total_failed: int
+
+
+class LazyStatus(TypedDict, total=False):
+    """Type definition for lazy mode status."""
+    enabled: bool
+    hook_installed: bool
+    lazy_install_enabled: bool
+    active: bool
+    error: Optional[str]
+
+
+# =============================================================================
+# CONSTANTS
+# =============================================================================
+
+# Default configuration values
+DEFAULT_LARGE_PACKAGE_THRESHOLD_MB: float = 50.0
+DEFAULT_BACKGROUND_WORKERS: int = 2
+DEFAULT_PRELOAD_PRIORITY: List[str] = []
+
+# Common import -> package mappings (will be populated from discovery)
+COMMON_IMPORT_MAPPINGS: Dict[str, str] = {
+    # Common mappings that are frequently used
+    'cv2': 'opencv-python',
+    'PIL': 'Pillow',
+    'yaml': 'PyYAML',
+    'toml': 'toml',
+    'pandas': 'pandas',
+    'numpy': 'numpy',
+    'scipy': 'scipy',
+    'sklearn': 'scikit-learn',
+    'matplotlib': 'matplotlib',
+    'seaborn': 'seaborn',
+    'requests': 'requests',
+    'urllib3': 'urllib3',
+    'bs4': 'beautifulsoup4',
+    'lxml': 'lxml',
+    'jinja2': 'Jinja2',
+    'flask': 'Flask',
+    'django': 'Django',
+    'fastapi': 'fastapi',
+    'pydantic': 'pydantic',
+    'sqlalchemy': 'SQLAlchemy',
+    'psycopg2': 'psycopg2-binary',
+    'pymongo': 'pymongo',
+    'redis': 'redis',
+    'celery': 'celery',
+    'boto3': 'boto3',
+    'azure': 'azure-storage-blob',
+    'google': 'google-cloud-storage',
+    'tensorflow': 'tensorflow',
+    'torch': 'torch',
+    'transformers': 'transformers',
+    'openai': 'openai',
+    'anthropic': 'anthropic',
+}
+
+# Package discovery source names
+DISCOVERY_SOURCE_PYPROJECT = "pyproject.toml"
+DISCOVERY_SOURCE_REQUIREMENTS = "requirements.txt"
+DISCOVERY_SOURCE_SETUP = "setup.py"
+DISCOVERY_SOURCE_POETRY = "poetry.lock"
+DISCOVERY_SOURCE_PIPFILE = "Pipfile"
+DISCOVERY_SOURCE_MANIFEST = "manifest.json"
+
+# Installation mode aliases
+INSTALL_MODE_ALIASES: Dict[str, str] = {
+    'auto': 'smart',
+    'on_demand': 'smart',
+    'on-demand': 'smart',
+    'lazy': 'smart',
+}
+
+# Cache keys
+CACHE_KEY_DEPENDENCIES = "dependencies"
+CACHE_KEY_PACKAGE_INFO = "package_info"
+CACHE_KEY_INSTALL_STATUS = "install_status"
+CACHE_KEY_DISCOVERY_SOURCES = "discovery_sources"
+
+# File patterns for discovery
+PYPROJECT_PATTERN = "pyproject.toml"
+REQUIREMENTS_PATTERN = "requirements*.txt"
+SETUP_PATTERN = "setup.py"
+POETRY_PATTERN = "poetry.lock"
+PIPFILE_PATTERN = "Pipfile"
+
+# =============================================================================
+# PRESET MODE MAPPINGS
+# =============================================================================
+
+# Preset mode combinations for convenience
+PRESET_MODES: Dict[str, LazyModeConfig] = {
+    "none": LazyModeConfig(
+        load_mode=LazyLoadMode.NONE,
+        install_mode=LazyInstallMode.NONE
+    ),
+    "lite": LazyModeConfig(
+        load_mode=LazyLoadMode.AUTO,
+        install_mode=LazyInstallMode.NONE
+    ),
+    "smart": LazyModeConfig(
+        load_mode=LazyLoadMode.AUTO,
+        install_mode=LazyInstallMode.SMART
+    ),
+    "full": LazyModeConfig(
+        load_mode=LazyLoadMode.AUTO,
+        install_mode=LazyInstallMode.FULL
+    ),
+    "clean": LazyModeConfig(
+        load_mode=LazyLoadMode.AUTO,
+        install_mode=LazyInstallMode.CLEAN
+    ),
+    "temporary": LazyModeConfig(
+        load_mode=LazyLoadMode.AUTO,
+        install_mode=LazyInstallMode.TEMPORARY
+    ),
+    "size_aware": LazyModeConfig(
+        load_mode=LazyLoadMode.AUTO,
+        install_mode=LazyInstallMode.SIZE_AWARE
+    ),
+    "auto": LazyModeConfig(
+        load_mode=LazyLoadMode.AUTO,
+        install_mode=LazyInstallMode.SMART,
+        auto_uninstall_large=True
+    ),
+}
+
+
+def get_preset_mode(preset_name: str) -> Optional[LazyModeConfig]:
+    """Get preset mode configuration by name."""
+    return PRESET_MODES.get(preset_name.lower())
+
+
+# =============================================================================
+# EXPORT ALL
+# =============================================================================
+
+__all__ = [
+    # Enums
+    'LazyLoadMode',
+    'LazyInstallMode',
+    'PathType',
+    'InstallStatus',
+    'LoadLevel',
+    # Dataclasses
+    'DependencyInfo',
+    'LazyModeConfig',
+    'InstallResult',
+    'LazyConfig',
+    'PackageManifest',
+    'PackageData',
+    'ModuleData',
+    # Type definitions
+    'DependencyMapping',
+    'PackageStats',
+    'LazyStatus',
+    # Constants
+    'DEFAULT_LARGE_PACKAGE_THRESHOLD_MB',
+    'DEFAULT_BACKGROUND_WORKERS',
+    'DEFAULT_PRELOAD_PRIORITY',
+    'COMMON_IMPORT_MAPPINGS',
+    'DISCOVERY_SOURCE_PYPROJECT',
+    'DISCOVERY_SOURCE_REQUIREMENTS',
+    'DISCOVERY_SOURCE_SETUP',
+    'DISCOVERY_SOURCE_POETRY',
+    'DISCOVERY_SOURCE_PIPFILE',
+    'DISCOVERY_SOURCE_MANIFEST',
+    'INSTALL_MODE_ALIASES',
+    'CACHE_KEY_DEPENDENCIES',
+    'CACHE_KEY_PACKAGE_INFO',
+    'CACHE_KEY_INSTALL_STATUS',
+    'CACHE_KEY_DISCOVERY_SOURCES',
+    'PYPROJECT_PATTERN',
+    'REQUIREMENTS_PATTERN',
+    'SETUP_PATTERN',
+    'POETRY_PATTERN',
+    'PIPFILE_PATTERN',
+    # Preset modes
+    'PRESET_MODES',
+    'get_preset_mode',
+]
+

exonware/xwlazy/errors.py

@@ -0,0 +1,284 @@
+"""
+#exonware/xwlazy/src/exonware/xwlazy/errors.py
+
+Company: eXonware.com
+Author: Eng. Muhammad AlShehri
+Email: connect@exonware.com
+Version: 0.1.0.19
+Generation Date: 10-Oct-2025
+
+Errors for Lazy Loading System
+
+This module defines all exception classes for the lazy loading system
+following GUIDE_ARCH.md structure.
+"""
+
+from typing import Optional, Any
+
+
+# =============================================================================
+# BASE EXCEPTION
+# =============================================================================
+
+class LazySystemError(Exception):
+    """
+    Base exception for all lazy system errors.
+    
+    All lazy system exceptions inherit from this for easy error handling.
+    """
+    
+    def __init__(self, message: str, package_name: Optional[str] = None):
+        """
+        Initialize lazy system error.
+        
+        Args:
+            message: Error message
+            package_name: Optional package name for scoped errors
+        """
+        self.package_name = package_name
+        if package_name:
+            message = f"[{package_name}] {message}"
+        super().__init__(message)
+
+
+# =============================================================================
+# SPECIFIC EXCEPTIONS
+# =============================================================================
+
+class LazyInstallError(LazySystemError):
+    """
+    Raised when package installation fails.
+    
+    Examples:
+        - pip install command fails
+        - Package not found in PyPI
+        - Network error during installation
+    """
+    pass
+
+
+class LazyDiscoveryError(LazySystemError):
+    """
+    Raised when dependency discovery fails.
+    
+    Examples:
+        - Cannot read pyproject.toml
+        - Invalid TOML syntax
+        - Missing dependency configuration
+    """
+    pass
+
+
+class LazyHookError(LazySystemError):
+    """
+    Raised when import hook operation fails.
+    
+    Examples:
+        - Cannot install hook in sys.meta_path
+        - Hook is already installed
+        - Hook interception fails
+    """
+    pass
+
+
+class LazySecurityError(LazySystemError):
+    """
+    Raised when security policy is violated.
+    
+    Examples:
+        - Package not in allow list
+        - Package in deny list
+        - Untrusted package source
+    """
+    pass
+
+
+class ExternallyManagedError(LazyInstallError):
+    """
+    Raised when environment is externally managed (PEP 668).
+    
+    This happens when the Python environment has an EXTERNALLY-MANAGED
+    marker file, preventing pip installations. Common in system Python
+    installations on Linux distributions.
+    
+    Solutions:
+        1. Use a virtual environment
+        2. Use pipx for isolated installations
+        3. Override with --break-system-packages (not recommended)
+    """
+    
+    def __init__(self, package_name: str):
+        """
+        Initialize externally managed error.
+        
+        Args:
+            package_name: Package that cannot be installed
+        """
+        message = (
+            f"Cannot install '{package_name}': Environment is externally managed (PEP 668). "
+            f"Please use a virtual environment or pipx."
+        )
+        super().__init__(message, package_name=None)
+
+
+class DeferredImportError(Exception):
+    """
+    Placeholder for a failed import that will be retried when accessed.
+    
+    This enables two-stage lazy loading:
+    - Stage 1: Import fails → Return DeferredImportError placeholder
+    - Stage 2: On first use → Install missing package and replace with real module
+    
+    Performance optimized:
+    - Zero overhead until user actually accesses the deferred import
+    - Only installs dependencies when truly needed
+    - Caches successful imports to avoid repeated installs
+    
+    Note: This is both an error class and a proxy object. It stays in
+    errors.py because it represents an error state, but acts as
+    a proxy until resolved.
+    """
+    
+    __slots__ = (
+        '_import_name',
+        '_original_error',
+        '_installer_package',
+        '_retry_attempted',
+        '_real_module',
+        '_async_handle',
+    )
+    
+    def __init__(
+        self,
+        import_name: str,
+        original_error: Exception,
+        installer_package: str,
+        async_handle: Optional[Any] = None,
+    ):
+        """
+        Initialize deferred import placeholder.
+        
+        Args:
+            import_name: Name of the module that failed to import (e.g., 'fastavro')
+            original_error: The original ImportError that was caught
+            installer_package: Package name to use for lazy installation (e.g., 'xwsystem')
+        """
+        self._import_name = import_name
+        self._original_error = original_error
+        self._installer_package = installer_package
+        self._retry_attempted = False
+        self._real_module = None
+        self._async_handle = async_handle
+        super().__init__(f"Deferred import: {import_name}")
+    
+    def _try_install_and_import(self):
+        """
+        Attempt to install missing package and import it.
+        
+        Returns:
+            The real module if installation succeeds
+            
+        Raises:
+            Original ImportError if installation fails or is disabled
+        """
+        # Import from facade and new structure
+        from .facade import lazy_import_with_install, is_lazy_install_enabled
+        from .common.logger import get_logger
+        
+        logger = get_logger("xwlazy")
+        logger.info(f"[STAGE 2] _try_install_and_import called for '{self._import_name}'")
+        
+        # Return cached module if already installed
+        if self._real_module is not None:
+            logger.info(f"[STAGE 2] Using cached module for '{self._import_name}'")
+            return self._real_module
+        
+        # Only try once to avoid repeated failures
+        if self._retry_attempted:
+            logger.warning(f"[STAGE 2] Already attempted installation for '{self._import_name}', raising original error")
+            raise self._original_error
+        
+        self._retry_attempted = True
+        
+        if self._async_handle is not None:
+            logger.info(f"[STAGE 2] Waiting for async install of '{self._import_name}' to finish")
+            self._async_handle.wait()
+            
+            # After async install, try simple import first (cache invalidation needed)
+            try:
+                import importlib
+                import sys
+                importlib.invalidate_caches()
+                sys.path_importer_cache.clear()
+                module = importlib.import_module(self._import_name)
+                self._real_module = module
+                logger.info(f"✅ [STAGE 2] Successfully loaded '{self._import_name}' after async install")
+                return module
+            except ImportError:
+                pass  # Fall through to lazy_import_with_install
+        
+        if not is_lazy_install_enabled(self._installer_package):
+            logger.warning(f"[STAGE 2] Lazy install disabled for {self._installer_package}, cannot load {self._import_name}")
+            raise self._original_error
+        
+        logger.info(f"⏳ [STAGE 2] Installing '{self._import_name}' on first use...")
+        
+        # Try to install and import
+        module, success = lazy_import_with_install(
+            self._import_name,
+            installer_package=self._installer_package
+        )
+        
+        if success and module:
+            self._real_module = module
+            logger.info(f"✅ [STAGE 2] Successfully installed and loaded '{self._import_name}'")
+            return module
+        else:
+            logger.error(f"❌ [STAGE 2] Failed to install '{self._import_name}'")
+            raise self._original_error
+    
+    def __call__(self, *args, **kwargs):
+        """
+        When user tries to instantiate, install dependency first.
+        
+        This enables: serializer = AvroSerializer() → installs fastavro → creates instance
+        """
+        module = self._try_install_and_import()
+        # If module is callable (a class), instantiate it
+        if callable(module):
+            return module(*args, **kwargs)
+        return module
+    
+    def __getattr__(self, name):
+        """
+        When user accesses attributes, install dependency first.
+        
+        This enables: from fastavro import reader → installs fastavro → returns reader
+        """
+        module = self._try_install_and_import()
+        return getattr(module, name)
+    
+    def __repr__(self):
+        """Show helpful message about deferred import."""
+        if self._real_module is not None:
+            return f"<DeferredImport: {self._import_name} (loaded)>"
+        return f"<DeferredImport: {self._import_name} (will install on first use)>"
+
+
+# =============================================================================
+# EXPORT ALL
+# =============================================================================
+
+__all__ = [
+    # Base exception
+    'LazySystemError',
+    # Specific exceptions
+    'LazyInstallError',
+    'LazyDiscoveryError',
+    'LazyHookError',
+    'LazySecurityError',
+    'ExternallyManagedError',
+    # Two-stage loading
+    'DeferredImportError',
+]
+