exonware-xwlazy 0.1.0.23__py3-none-any.whl → 1.0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- exonware/__init__.py +85 -34
- exonware/xwlazy/version.py +5 -5
- exonware/xwlazy.py +2546 -0
- exonware/xwlazy_external_libs.toml +716 -0
- {exonware_xwlazy-0.1.0.23.dist-info → exonware_xwlazy-1.0.1.2.dist-info}/METADATA +5 -1
- exonware_xwlazy-1.0.1.2.dist-info/RECORD +8 -0
- exonware/xwlazy/__init__.py +0 -379
- exonware/xwlazy/common/__init__.py +0 -55
- exonware/xwlazy/common/base.py +0 -65
- exonware/xwlazy/common/cache.py +0 -504
- exonware/xwlazy/common/logger.py +0 -257
- exonware/xwlazy/common/services/__init__.py +0 -72
- exonware/xwlazy/common/services/dependency_mapper.py +0 -250
- exonware/xwlazy/common/services/install_async_utils.py +0 -170
- exonware/xwlazy/common/services/install_cache_utils.py +0 -245
- exonware/xwlazy/common/services/keyword_detection.py +0 -283
- exonware/xwlazy/common/services/spec_cache.py +0 -165
- exonware/xwlazy/common/services/state_manager.py +0 -84
- exonware/xwlazy/common/strategies/__init__.py +0 -28
- exonware/xwlazy/common/strategies/caching_dict.py +0 -44
- exonware/xwlazy/common/strategies/caching_installation.py +0 -88
- exonware/xwlazy/common/strategies/caching_lfu.py +0 -66
- exonware/xwlazy/common/strategies/caching_lru.py +0 -63
- exonware/xwlazy/common/strategies/caching_multitier.py +0 -59
- exonware/xwlazy/common/strategies/caching_ttl.py +0 -59
- exonware/xwlazy/common/utils.py +0 -142
- exonware/xwlazy/config.py +0 -193
- exonware/xwlazy/contracts.py +0 -1533
- exonware/xwlazy/defs.py +0 -378
- exonware/xwlazy/errors.py +0 -276
- exonware/xwlazy/facade.py +0 -1137
- exonware/xwlazy/host/__init__.py +0 -8
- exonware/xwlazy/host/conf.py +0 -16
- exonware/xwlazy/module/__init__.py +0 -18
- exonware/xwlazy/module/base.py +0 -622
- exonware/xwlazy/module/data.py +0 -17
- exonware/xwlazy/module/facade.py +0 -246
- exonware/xwlazy/module/importer_engine.py +0 -2964
- exonware/xwlazy/module/partial_module_detector.py +0 -275
- exonware/xwlazy/module/strategies/__init__.py +0 -22
- exonware/xwlazy/module/strategies/module_helper_lazy.py +0 -93
- exonware/xwlazy/module/strategies/module_helper_simple.py +0 -65
- exonware/xwlazy/module/strategies/module_manager_advanced.py +0 -111
- exonware/xwlazy/module/strategies/module_manager_simple.py +0 -95
- exonware/xwlazy/package/__init__.py +0 -18
- exonware/xwlazy/package/base.py +0 -863
- exonware/xwlazy/package/conf.py +0 -324
- exonware/xwlazy/package/data.py +0 -17
- exonware/xwlazy/package/facade.py +0 -480
- exonware/xwlazy/package/services/__init__.py +0 -84
- exonware/xwlazy/package/services/async_install_handle.py +0 -87
- exonware/xwlazy/package/services/config_manager.py +0 -249
- exonware/xwlazy/package/services/discovery.py +0 -435
- exonware/xwlazy/package/services/host_packages.py +0 -180
- exonware/xwlazy/package/services/install_async.py +0 -291
- exonware/xwlazy/package/services/install_cache.py +0 -145
- exonware/xwlazy/package/services/install_interactive.py +0 -59
- exonware/xwlazy/package/services/install_policy.py +0 -156
- exonware/xwlazy/package/services/install_registry.py +0 -54
- exonware/xwlazy/package/services/install_result.py +0 -17
- exonware/xwlazy/package/services/install_sbom.py +0 -153
- exonware/xwlazy/package/services/install_utils.py +0 -79
- exonware/xwlazy/package/services/installer_engine.py +0 -406
- exonware/xwlazy/package/services/lazy_installer.py +0 -803
- exonware/xwlazy/package/services/manifest.py +0 -503
- exonware/xwlazy/package/services/strategy_registry.py +0 -324
- exonware/xwlazy/package/strategies/__init__.py +0 -57
- exonware/xwlazy/package/strategies/package_discovery_file.py +0 -129
- exonware/xwlazy/package/strategies/package_discovery_hybrid.py +0 -84
- exonware/xwlazy/package/strategies/package_discovery_manifest.py +0 -101
- exonware/xwlazy/package/strategies/package_execution_async.py +0 -113
- exonware/xwlazy/package/strategies/package_execution_cached.py +0 -90
- exonware/xwlazy/package/strategies/package_execution_pip.py +0 -99
- exonware/xwlazy/package/strategies/package_execution_wheel.py +0 -106
- exonware/xwlazy/package/strategies/package_mapping_discovery_first.py +0 -100
- exonware/xwlazy/package/strategies/package_mapping_hybrid.py +0 -105
- exonware/xwlazy/package/strategies/package_mapping_manifest_first.py +0 -100
- exonware/xwlazy/package/strategies/package_policy_allow_list.py +0 -57
- exonware/xwlazy/package/strategies/package_policy_deny_list.py +0 -57
- exonware/xwlazy/package/strategies/package_policy_permissive.py +0 -46
- exonware/xwlazy/package/strategies/package_timing_clean.py +0 -67
- exonware/xwlazy/package/strategies/package_timing_full.py +0 -66
- exonware/xwlazy/package/strategies/package_timing_smart.py +0 -68
- exonware/xwlazy/package/strategies/package_timing_temporary.py +0 -66
- exonware/xwlazy/runtime/__init__.py +0 -18
- exonware/xwlazy/runtime/adaptive_learner.py +0 -129
- exonware/xwlazy/runtime/base.py +0 -274
- exonware/xwlazy/runtime/facade.py +0 -94
- exonware/xwlazy/runtime/intelligent_selector.py +0 -170
- exonware/xwlazy/runtime/metrics.py +0 -60
- exonware/xwlazy/runtime/performance.py +0 -37
- exonware_xwlazy-0.1.0.23.dist-info/RECORD +0 -93
- xwlazy/__init__.py +0 -14
- xwlazy/lazy.py +0 -30
- {exonware_xwlazy-0.1.0.23.dist-info → exonware_xwlazy-1.0.1.2.dist-info}/WHEEL +0 -0
- {exonware_xwlazy-0.1.0.23.dist-info → exonware_xwlazy-1.0.1.2.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,2964 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
#exonware/xwlazy/src/exonware/xwlazy/module/importer_engine.py
|
|
3
|
-
|
|
4
|
-
Import Engine - Unified engine for all import-related operations.
|
|
5
|
-
|
|
6
|
-
Company: eXonware.com
|
|
7
|
-
Author: Eng. Muhammad AlShehri
|
|
8
|
-
Email: connect@exonware.com
|
|
9
|
-
|
|
10
|
-
Generation Date: 15-Nov-2025
|
|
11
|
-
|
|
12
|
-
This module provides unified import engine for all import-related functionality.
|
|
13
|
-
All import-related functionality is centralized here.
|
|
14
|
-
|
|
15
|
-
Merged from:
|
|
16
|
-
- logging_utils.py (Logging utilities)
|
|
17
|
-
- import_tracking.py (Import tracking)
|
|
18
|
-
- prefix_trie.py (Prefix trie data structure)
|
|
19
|
-
- watched_registry.py (Watched prefix registry)
|
|
20
|
-
- deferred_loader.py (Deferred module loader)
|
|
21
|
-
- cache_utils.py (Multi-tier cache and bytecode cache)
|
|
22
|
-
- parallel_utils.py (Parallel loading utilities)
|
|
23
|
-
- module_patching.py (Module patching utilities)
|
|
24
|
-
- archive_imports.py (Archive import utilities)
|
|
25
|
-
- bootstrap.py (Bootstrap utilities)
|
|
26
|
-
- loader.py (Lazy loader)
|
|
27
|
-
- registry.py (Lazy module registry)
|
|
28
|
-
- importer.py (Lazy importer)
|
|
29
|
-
- import_hook.py (Import hook)
|
|
30
|
-
- meta_path_finder.py (Meta path finder)
|
|
31
|
-
|
|
32
|
-
Features:
|
|
33
|
-
- Unified import engine for all import operations
|
|
34
|
-
- Multi-tier caching (L1/L2/L3)
|
|
35
|
-
- Parallel loading support
|
|
36
|
-
- Import tracking and circular import prevention
|
|
37
|
-
- Watched prefix registry
|
|
38
|
-
- Meta path finder for intercepting imports
|
|
39
|
-
"""
|
|
40
|
-
|
|
41
|
-
# =============================================================================
|
|
42
|
-
# IMPORTS
|
|
43
|
-
# =============================================================================
|
|
44
|
-
|
|
45
|
-
from __future__ import annotations
|
|
46
|
-
|
|
47
|
-
import os
|
|
48
|
-
import sys
|
|
49
|
-
import json
|
|
50
|
-
import time
|
|
51
|
-
import asyncio
|
|
52
|
-
import pickle
|
|
53
|
-
import struct
|
|
54
|
-
import builtins
|
|
55
|
-
import atexit
|
|
56
|
-
import logging
|
|
57
|
-
import importlib
|
|
58
|
-
import importlib.util
|
|
59
|
-
import importlib.machinery
|
|
60
|
-
import importlib.abc
|
|
61
|
-
import threading
|
|
62
|
-
import subprocess
|
|
63
|
-
import concurrent.futures
|
|
64
|
-
from pathlib import Path
|
|
65
|
-
from types import ModuleType
|
|
66
|
-
from typing import Optional, Any, Iterable, Callable
|
|
67
|
-
from collections import OrderedDict, defaultdict, Counter, deque
|
|
68
|
-
from queue import Queue
|
|
69
|
-
from datetime import datetime
|
|
70
|
-
from enum import Enum
|
|
71
|
-
|
|
72
|
-
from ..defs import LazyLoadMode, LazyInstallMode
|
|
73
|
-
from ..common.services.dependency_mapper import DependencyMapper
|
|
74
|
-
from ..common.services.spec_cache import _spec_cache_get, _spec_cache_put
|
|
75
|
-
from ..package.services import LazyInstallerRegistry, LazyInstaller
|
|
76
|
-
from ..package.services.config_manager import LazyInstallConfig
|
|
77
|
-
from ..package.services.manifest import _normalize_prefix
|
|
78
|
-
from ..errors import DeferredImportError
|
|
79
|
-
from .base import AModuleHelper
|
|
80
|
-
|
|
81
|
-
# Import from common (logger and cache)
|
|
82
|
-
from ..common.logger import get_logger, log_event
|
|
83
|
-
from ..common.cache import MultiTierCache, BytecodeCache
|
|
84
|
-
|
|
85
|
-
# Import from runtime folder (moved from module folder)
|
|
86
|
-
from ..runtime.adaptive_learner import AdaptiveLearner
|
|
87
|
-
from ..runtime.intelligent_selector import IntelligentModeSelector, LoadLevel
|
|
88
|
-
|
|
89
|
-
# =============================================================================
|
|
90
|
-
# LOGGER (from common.logger)
|
|
91
|
-
# =============================================================================
|
|
92
|
-
|
|
93
|
-
logger = get_logger("xwlazy.importer_engine")
|
|
94
|
-
|
|
95
|
-
# =============================================================================
|
|
96
|
-
# IMPORT TRACKING (from import_tracking.py)
|
|
97
|
-
# =============================================================================
|
|
98
|
-
|
|
99
|
-
_thread_local = threading.local()
|
|
100
|
-
_importing = threading.local()
|
|
101
|
-
_installing = threading.local()
|
|
102
|
-
|
|
103
|
-
_installation_depth = 0
|
|
104
|
-
_installation_depth_lock = threading.Lock()
|
|
105
|
-
|
|
106
|
-
# Thread-local flag to prevent recursion during installation checks
|
|
107
|
-
_checking_installation = threading.local()
|
|
108
|
-
|
|
109
|
-
def _get_thread_imports() -> set[str]:
|
|
110
|
-
"""Get thread-local import set (creates if needed)."""
|
|
111
|
-
if not hasattr(_thread_local, 'imports'):
|
|
112
|
-
_thread_local.imports = set()
|
|
113
|
-
return _thread_local.imports
|
|
114
|
-
|
|
115
|
-
def _is_checking_installation() -> bool:
|
|
116
|
-
"""Check if we're currently checking installation status (to prevent recursion)."""
|
|
117
|
-
return getattr(_checking_installation, 'active', False)
|
|
118
|
-
|
|
119
|
-
def _set_checking_installation(value: bool) -> None:
|
|
120
|
-
"""Set the installation check flag."""
|
|
121
|
-
_checking_installation.active = value
|
|
122
|
-
|
|
123
|
-
def _is_import_in_progress(module_name: str) -> bool:
|
|
124
|
-
"""Check if a module import is currently in progress for this thread."""
|
|
125
|
-
return module_name in _get_thread_imports()
|
|
126
|
-
|
|
127
|
-
def _mark_import_started(module_name: str) -> None:
|
|
128
|
-
"""Mark a module import as started for this thread."""
|
|
129
|
-
_get_thread_imports().add(module_name)
|
|
130
|
-
|
|
131
|
-
def _mark_import_finished(module_name: str) -> None:
|
|
132
|
-
"""Mark a module import as finished for this thread."""
|
|
133
|
-
_get_thread_imports().discard(module_name)
|
|
134
|
-
|
|
135
|
-
def get_importing_state() -> threading.local:
|
|
136
|
-
"""Get thread-local importing state."""
|
|
137
|
-
return _importing
|
|
138
|
-
|
|
139
|
-
def get_installing_state() -> threading.local:
|
|
140
|
-
"""Get thread-local installing state."""
|
|
141
|
-
return _installing
|
|
142
|
-
|
|
143
|
-
# Thread-local storage for installation state
|
|
144
|
-
_installing_state = get_installing_state()
|
|
145
|
-
_importing_state = get_importing_state()
|
|
146
|
-
|
|
147
|
-
# Global recursion depth counter to prevent infinite recursion
|
|
148
|
-
_installation_depth = 0
|
|
149
|
-
_installation_depth_lock = threading.Lock()
|
|
150
|
-
|
|
151
|
-
# =============================================================================
|
|
152
|
-
# GLOBAL __import__ HOOK (Critical for Module-Level Imports)
|
|
153
|
-
# =============================================================================
|
|
154
|
-
|
|
155
|
-
# Global state for builtins.__import__ hook
|
|
156
|
-
_original_builtins_import: Optional[Callable] = None
|
|
157
|
-
_global_import_hook_installed: bool = False
|
|
158
|
-
_global_import_hook_lock = threading.RLock()
|
|
159
|
-
|
|
160
|
-
# Fast-path caches for O(1) lookup
|
|
161
|
-
_installed_cache: set[str] = set()
|
|
162
|
-
_failed_cache: set[str] = set()
|
|
163
|
-
|
|
164
|
-
# Registry of packages that should auto-install
|
|
165
|
-
_lazy_packages: dict[str, Any] = {}
|
|
166
|
-
|
|
167
|
-
def register_lazy_package(package_name: str, config: Optional[Any] = None) -> None:
|
|
168
|
-
"""
|
|
169
|
-
Register a package for lazy loading/installation.
|
|
170
|
-
|
|
171
|
-
Args:
|
|
172
|
-
package_name: Package name to register
|
|
173
|
-
config: Optional configuration object
|
|
174
|
-
"""
|
|
175
|
-
with _global_import_hook_lock:
|
|
176
|
-
_lazy_packages[package_name] = config or {}
|
|
177
|
-
logger.debug(f"Registered lazy package: {package_name}")
|
|
178
|
-
|
|
179
|
-
def _should_auto_install(module_name: str) -> bool:
|
|
180
|
-
"""
|
|
181
|
-
Check if module should be auto-installed.
|
|
182
|
-
|
|
183
|
-
Checks if:
|
|
184
|
-
1. Module's root package is registered, OR
|
|
185
|
-
2. Module maps to a known package (via DependencyMapper) AND packages are registered
|
|
186
|
-
|
|
187
|
-
Args:
|
|
188
|
-
module_name: Module name to check
|
|
189
|
-
|
|
190
|
-
Returns:
|
|
191
|
-
True if should auto-install, False otherwise
|
|
192
|
-
"""
|
|
193
|
-
root_package = module_name.split('.')[0]
|
|
194
|
-
|
|
195
|
-
# Fast path: root package is registered
|
|
196
|
-
if root_package in _lazy_packages:
|
|
197
|
-
return True
|
|
198
|
-
|
|
199
|
-
# If no packages are registered, don't auto-install
|
|
200
|
-
if not _lazy_packages:
|
|
201
|
-
return False
|
|
202
|
-
|
|
203
|
-
# Check if module maps to a known package via DependencyMapper
|
|
204
|
-
# This handles cases like:
|
|
205
|
-
# - 'yaml' -> 'PyYAML' (different name)
|
|
206
|
-
# - 'msgpack' -> 'msgpack' (same name, but still a known dependency)
|
|
207
|
-
# - 'bson' -> 'pymongo' (different name)
|
|
208
|
-
try:
|
|
209
|
-
# Use the first registered package to get the mapper context
|
|
210
|
-
# All registered packages should have the same dependency mappings
|
|
211
|
-
registered_package = next(iter(_lazy_packages.keys()), None)
|
|
212
|
-
if registered_package:
|
|
213
|
-
mapper = DependencyMapper(package_name=registered_package)
|
|
214
|
-
else:
|
|
215
|
-
mapper = DependencyMapper()
|
|
216
|
-
|
|
217
|
-
package_name = mapper.get_package_name(module_name)
|
|
218
|
-
|
|
219
|
-
# If DependencyMapper found a package name (not None), it's a known dependency
|
|
220
|
-
# Allow installation attempt - the installer will verify if it's actually needed
|
|
221
|
-
if package_name:
|
|
222
|
-
logger.debug(f"[AUTO-INSTALL] Module '{module_name}' maps to package '{package_name}', allowing auto-install")
|
|
223
|
-
return True
|
|
224
|
-
else:
|
|
225
|
-
logger.debug(f"[AUTO-INSTALL] Module '{module_name}' has no package mapping, skipping auto-install")
|
|
226
|
-
|
|
227
|
-
except Exception as e:
|
|
228
|
-
# If mapper fails, log and be conservative
|
|
229
|
-
logger.debug(f"[AUTO-INSTALL] DependencyMapper failed for '{module_name}': {e}")
|
|
230
|
-
pass
|
|
231
|
-
|
|
232
|
-
return False
|
|
233
|
-
|
|
234
|
-
def _try_install_package(module_name: str) -> bool:
|
|
235
|
-
"""
|
|
236
|
-
Try to install package for missing module.
|
|
237
|
-
|
|
238
|
-
Tries all registered packages until one succeeds.
|
|
239
|
-
|
|
240
|
-
Args:
|
|
241
|
-
module_name: Module name that failed to import
|
|
242
|
-
|
|
243
|
-
Returns:
|
|
244
|
-
True if installation successful, False otherwise
|
|
245
|
-
"""
|
|
246
|
-
# Try root package first
|
|
247
|
-
root_package = module_name.split('.')[0]
|
|
248
|
-
if root_package in _lazy_packages:
|
|
249
|
-
try:
|
|
250
|
-
logger.info(f"[AUTO-INSTALL] Trying root package {root_package} for module {module_name}")
|
|
251
|
-
installer = LazyInstallerRegistry.get_instance(root_package)
|
|
252
|
-
logger.info(f"[AUTO-INSTALL] Installer for {root_package}: {installer is not None}, enabled={installer.is_enabled() if installer else False}")
|
|
253
|
-
|
|
254
|
-
if installer is None:
|
|
255
|
-
logger.warning(f"[AUTO-INSTALL] Installer for {root_package} is None")
|
|
256
|
-
elif not installer.is_enabled():
|
|
257
|
-
logger.warning(f"[AUTO-INSTALL] Installer for {root_package} is disabled")
|
|
258
|
-
else:
|
|
259
|
-
logger.info(f"Auto-installing missing package for module: {module_name} (via {root_package})")
|
|
260
|
-
logger.info(f"[AUTO-INSTALL] Calling install_and_import('{module_name}')")
|
|
261
|
-
try:
|
|
262
|
-
module, success = installer.install_and_import(module_name)
|
|
263
|
-
logger.info(f"[AUTO-INSTALL] install_and_import returned: module={module is not None}, success={success}")
|
|
264
|
-
if success and module:
|
|
265
|
-
_installed_cache.add(module_name)
|
|
266
|
-
logger.info(f"[AUTO-INSTALL] Successfully installed and imported '{module_name}'")
|
|
267
|
-
return True
|
|
268
|
-
else:
|
|
269
|
-
logger.warning(f"[AUTO-INSTALL] install_and_import failed for '{module_name}': success={success}, module={module is not None}")
|
|
270
|
-
except Exception as install_exc:
|
|
271
|
-
logger.error(f"[AUTO-INSTALL] install_and_import raised exception: {install_exc}", exc_info=True)
|
|
272
|
-
raise
|
|
273
|
-
except Exception as e:
|
|
274
|
-
logger.error(f"[AUTO-INSTALL] Exception installing via root package {root_package}: {e}", exc_info=True)
|
|
275
|
-
|
|
276
|
-
# Try all registered packages (for dependencies like 'yaml' when 'xwsystem' is registered)
|
|
277
|
-
for registered_package in _lazy_packages.keys():
|
|
278
|
-
if registered_package == root_package:
|
|
279
|
-
continue # Already tried
|
|
280
|
-
|
|
281
|
-
try:
|
|
282
|
-
logger.debug(f"[AUTO-INSTALL] Trying to get installer for {registered_package}")
|
|
283
|
-
installer = LazyInstallerRegistry.get_instance(registered_package)
|
|
284
|
-
logger.debug(f"[AUTO-INSTALL] Installer retrieved: {installer is not None}")
|
|
285
|
-
|
|
286
|
-
if installer is None:
|
|
287
|
-
logger.warning(f"[AUTO-INSTALL] Installer for {registered_package} is None")
|
|
288
|
-
continue
|
|
289
|
-
|
|
290
|
-
if not installer.is_enabled():
|
|
291
|
-
logger.warning(f"[AUTO-INSTALL] Installer for {registered_package} is disabled")
|
|
292
|
-
continue
|
|
293
|
-
|
|
294
|
-
logger.info(f"Auto-installing missing package for module: {module_name} (via {registered_package})")
|
|
295
|
-
logger.info(f"[AUTO-INSTALL] Calling install_and_import('{module_name}')")
|
|
296
|
-
try:
|
|
297
|
-
module, success = installer.install_and_import(module_name)
|
|
298
|
-
logger.info(f"[AUTO-INSTALL] install_and_import returned: module={module is not None}, success={success}")
|
|
299
|
-
except Exception as install_exc:
|
|
300
|
-
logger.error(f"[AUTO-INSTALL] install_and_import raised exception: {install_exc}", exc_info=True)
|
|
301
|
-
raise
|
|
302
|
-
|
|
303
|
-
if success and module:
|
|
304
|
-
_installed_cache.add(module_name)
|
|
305
|
-
logger.info(f"[AUTO-INSTALL] Successfully installed and imported '{module_name}'")
|
|
306
|
-
return True
|
|
307
|
-
else:
|
|
308
|
-
logger.warning(f"[AUTO-INSTALL] install_and_import failed for '{module_name}': success={success}, module={module is not None}")
|
|
309
|
-
except Exception as e:
|
|
310
|
-
logger.error(f"[AUTO-INSTALL] Exception installing via {registered_package}: {e}", exc_info=True)
|
|
311
|
-
continue
|
|
312
|
-
|
|
313
|
-
logger.warning(f"[AUTO-INSTALL] Failed to install package for module '{module_name}' via all registered packages")
|
|
314
|
-
return False
|
|
315
|
-
|
|
316
|
-
def _intercepting_import(name: str, globals=None, locals=None, fromlist=(), level=0):
|
|
317
|
-
"""
|
|
318
|
-
Intercept ALL imports including module-level ones.
|
|
319
|
-
|
|
320
|
-
This is the global builtins.__import__ replacement that catches
|
|
321
|
-
ALL imports, including those at module level during package initialization.
|
|
322
|
-
|
|
323
|
-
CRITICAL: Skip relative imports (level > 0) - they must use normal import path.
|
|
324
|
-
Relative imports are package/module agnostic and should not be intercepted.
|
|
325
|
-
|
|
326
|
-
CRITICAL: When fromlist is non-empty (e.g., "from module import Class"),
|
|
327
|
-
Python's import machinery needs direct access to module attributes via getattr().
|
|
328
|
-
We MUST return the actual module object without any wrapping.
|
|
329
|
-
"""
|
|
330
|
-
# CRITICAL FIX: Handle relative imports (level > 0)
|
|
331
|
-
# Relative imports like "from .common import" have level=1
|
|
332
|
-
# We must use normal import BUT still enhance the module after import (package/module agnostic)
|
|
333
|
-
if level > 0:
|
|
334
|
-
result = _original_builtins_import(name, globals, locals, fromlist, level)
|
|
335
|
-
# CRITICAL: Enhance modules imported via relative imports (package/module agnostic)
|
|
336
|
-
# This ensures instance methods work on classes for ANY package/module structure
|
|
337
|
-
if result and isinstance(result, ModuleType) and fromlist:
|
|
338
|
-
try:
|
|
339
|
-
for pkg_name in _lazy_packages.keys():
|
|
340
|
-
if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
|
|
341
|
-
finder = _installed_hooks.get(pkg_name)
|
|
342
|
-
if finder:
|
|
343
|
-
finder._enhance_classes_with_class_methods(result)
|
|
344
|
-
break
|
|
345
|
-
except Exception:
|
|
346
|
-
pass
|
|
347
|
-
return result
|
|
348
|
-
|
|
349
|
-
# CRITICAL FIX: Handle fromlist imports specially
|
|
350
|
-
# When fromlist is present (e.g., "from module import Class"),
|
|
351
|
-
# Python needs direct access to module attributes via getattr(module, 'Class')
|
|
352
|
-
# We MUST return the actual module without any wrapping or modification
|
|
353
|
-
if fromlist:
|
|
354
|
-
# Import partial module detector (lazy import to avoid circular dependency)
|
|
355
|
-
try:
|
|
356
|
-
from .partial_module_detector import (
|
|
357
|
-
is_partially_initialized as _default_is_partially_initialized,
|
|
358
|
-
mark_module_importing,
|
|
359
|
-
unmark_module_importing,
|
|
360
|
-
DetectionStrategy,
|
|
361
|
-
PartialModuleDetector
|
|
362
|
-
)
|
|
363
|
-
# Allow strategy override via environment variable for testing
|
|
364
|
-
strategy_name = os.environ.get('XWLAZY_PARTIAL_DETECTION_STRATEGY', 'hybrid')
|
|
365
|
-
try:
|
|
366
|
-
strategy = DetectionStrategy(strategy_name)
|
|
367
|
-
detector = PartialModuleDetector(strategy)
|
|
368
|
-
is_partially_initialized = lambda n, m: detector.is_partially_initialized(n, m)
|
|
369
|
-
except (ValueError, AttributeError):
|
|
370
|
-
# Use default detector (HYBRID strategy)
|
|
371
|
-
is_partially_initialized = _default_is_partially_initialized
|
|
372
|
-
except ImportError:
|
|
373
|
-
# Fallback if detector not available
|
|
374
|
-
def is_partially_initialized(name, mod):
|
|
375
|
-
return False
|
|
376
|
-
def mark_module_importing(name):
|
|
377
|
-
pass
|
|
378
|
-
def unmark_module_importing(name):
|
|
379
|
-
pass
|
|
380
|
-
|
|
381
|
-
# Mark this module as being imported (for tracking)
|
|
382
|
-
mark_module_importing(name)
|
|
383
|
-
|
|
384
|
-
try:
|
|
385
|
-
# Fast path: already imported and in sys.modules
|
|
386
|
-
# CRITICAL: For fromlist imports, we must be VERY conservative
|
|
387
|
-
# Only return early if we're CERTAIN the module is fully initialized
|
|
388
|
-
# and NOT currently being imported
|
|
389
|
-
if name in sys.modules:
|
|
390
|
-
module = sys.modules[name]
|
|
391
|
-
# Ensure module is fully loaded (not a placeholder or lazy wrapper)
|
|
392
|
-
# Check if it's a real ModuleType (not a proxy/wrapper)
|
|
393
|
-
if isinstance(module, ModuleType):
|
|
394
|
-
# CRITICAL: Check if module is partially initialized
|
|
395
|
-
# We must NOT return a partially initialized module
|
|
396
|
-
# For fromlist imports, be extra conservative - only return if
|
|
397
|
-
# module is definitely fully loaded AND not currently importing
|
|
398
|
-
if not is_partially_initialized(name, module):
|
|
399
|
-
# Additional check: verify module has meaningful content
|
|
400
|
-
# (not just metadata attributes)
|
|
401
|
-
module_dict = getattr(module, '__dict__', {})
|
|
402
|
-
metadata_attrs = {'__name__', '__loader__', '__spec__', '__package__', '__file__', '__path__', '__cached__'}
|
|
403
|
-
content_attrs = set(module_dict.keys()) - metadata_attrs
|
|
404
|
-
|
|
405
|
-
# Only return if module has actual content (classes, functions, etc.)
|
|
406
|
-
# OR if it's a namespace package (has __path__)
|
|
407
|
-
has_content = len(content_attrs) > 0
|
|
408
|
-
is_namespace = hasattr(module, '__path__')
|
|
409
|
-
|
|
410
|
-
if has_content or is_namespace:
|
|
411
|
-
# Check if it's a placeholder by looking for common placeholder patterns
|
|
412
|
-
is_placeholder = (
|
|
413
|
-
hasattr(module, '__getattr__') and
|
|
414
|
-
not hasattr(module, '__file__') and
|
|
415
|
-
not hasattr(module, '__path__') # Namespace packages don't have __file__
|
|
416
|
-
)
|
|
417
|
-
if not is_placeholder:
|
|
418
|
-
# Module is fully loaded with content, return it directly
|
|
419
|
-
# CRITICAL FIX: Enhance classes before returning
|
|
420
|
-
try:
|
|
421
|
-
# Find which package this module belongs to
|
|
422
|
-
for pkg_name in _lazy_packages.keys():
|
|
423
|
-
# Check if module name starts with package name or "exonware." + package name
|
|
424
|
-
if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
|
|
425
|
-
finder = _installed_hooks.get(pkg_name)
|
|
426
|
-
if finder:
|
|
427
|
-
finder._enhance_classes_with_class_methods(module)
|
|
428
|
-
break
|
|
429
|
-
except Exception:
|
|
430
|
-
pass
|
|
431
|
-
unmark_module_importing(name)
|
|
432
|
-
return module
|
|
433
|
-
# Module is partially initialized or has no content - fall through to normal import
|
|
434
|
-
# If it's a placeholder, partially initialized, or has no content, fall through to normal import
|
|
435
|
-
finally:
|
|
436
|
-
# Always unmark when done (even if exception occurs)
|
|
437
|
-
unmark_module_importing(name)
|
|
438
|
-
|
|
439
|
-
# For fromlist imports, use normal import path to ensure classes/functions
|
|
440
|
-
# are accessible via getattr() - Python's import machinery handles extraction
|
|
441
|
-
try:
|
|
442
|
-
# Mark as importing before calling original import
|
|
443
|
-
try:
|
|
444
|
-
from .partial_module_detector import mark_module_importing, unmark_module_importing
|
|
445
|
-
mark_module_importing(name)
|
|
446
|
-
except ImportError:
|
|
447
|
-
pass
|
|
448
|
-
|
|
449
|
-
try:
|
|
450
|
-
result = _original_builtins_import(name, globals, locals, fromlist, level)
|
|
451
|
-
# Cache success but return actual module (no wrapping)
|
|
452
|
-
_installed_cache.add(name)
|
|
453
|
-
# CRITICAL FIX: Enhance classes in the module for class-level method access
|
|
454
|
-
# This makes instance methods callable on classes (e.g., BsonSerializer.encode(data))
|
|
455
|
-
if result and isinstance(result, ModuleType):
|
|
456
|
-
try:
|
|
457
|
-
# Find which package this module belongs to
|
|
458
|
-
for pkg_name in _lazy_packages.keys():
|
|
459
|
-
# Check if module name starts with package name or "exonware." + package name
|
|
460
|
-
if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
|
|
461
|
-
finder = _installed_hooks.get(pkg_name)
|
|
462
|
-
if finder:
|
|
463
|
-
finder._enhance_classes_with_class_methods(result)
|
|
464
|
-
break
|
|
465
|
-
except Exception as e:
|
|
466
|
-
# Enhancement failed - don't break the import, but log for debugging
|
|
467
|
-
logger.debug(f"Enhancement failed for {name}: {e}", exc_info=True)
|
|
468
|
-
pass
|
|
469
|
-
return result
|
|
470
|
-
finally:
|
|
471
|
-
# Always unmark when done
|
|
472
|
-
try:
|
|
473
|
-
from .partial_module_detector import unmark_module_importing
|
|
474
|
-
unmark_module_importing(name)
|
|
475
|
-
except ImportError:
|
|
476
|
-
pass
|
|
477
|
-
except ImportError as e:
|
|
478
|
-
# Only try auto-install if needed
|
|
479
|
-
if _should_auto_install(name):
|
|
480
|
-
try:
|
|
481
|
-
if _try_install_package(name):
|
|
482
|
-
# Retry import after installation - return actual module
|
|
483
|
-
result = _original_builtins_import(name, globals, locals, fromlist, level)
|
|
484
|
-
_installed_cache.add(name)
|
|
485
|
-
# CRITICAL FIX: Enhance classes in the module for class-level method access
|
|
486
|
-
if result and isinstance(result, ModuleType):
|
|
487
|
-
try:
|
|
488
|
-
# Find which package this module belongs to
|
|
489
|
-
for pkg_name in _lazy_packages.keys():
|
|
490
|
-
# Check if module name starts with package name or "exonware." + package name
|
|
491
|
-
if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
|
|
492
|
-
finder = _installed_hooks.get(pkg_name)
|
|
493
|
-
if finder:
|
|
494
|
-
finder._enhance_classes_with_class_methods(result)
|
|
495
|
-
break
|
|
496
|
-
except Exception:
|
|
497
|
-
pass
|
|
498
|
-
return result
|
|
499
|
-
except Exception:
|
|
500
|
-
# If installation fails, don't crash - just raise the original ImportError
|
|
501
|
-
pass
|
|
502
|
-
|
|
503
|
-
# Installation failed or not applicable - cache failure (but limit cache size)
|
|
504
|
-
if len(_failed_cache) < 1000: # Prevent unbounded growth
|
|
505
|
-
_failed_cache.add(name)
|
|
506
|
-
raise
|
|
507
|
-
|
|
508
|
-
# Fast path: cached as installed (but still enhance for fromlist)
|
|
509
|
-
if name in _installed_cache:
|
|
510
|
-
result = _original_builtins_import(name, globals, locals, fromlist, level)
|
|
511
|
-
# CRITICAL: Enhance even cached modules for fromlist imports (package/module agnostic)
|
|
512
|
-
if fromlist and result and isinstance(result, ModuleType):
|
|
513
|
-
try:
|
|
514
|
-
for pkg_name in _lazy_packages.keys():
|
|
515
|
-
if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
|
|
516
|
-
finder = _installed_hooks.get(pkg_name)
|
|
517
|
-
if finder:
|
|
518
|
-
finder._enhance_classes_with_class_methods(result)
|
|
519
|
-
break
|
|
520
|
-
except Exception:
|
|
521
|
-
pass
|
|
522
|
-
return result
|
|
523
|
-
|
|
524
|
-
# Fast path: already imported (for non-fromlist imports)
|
|
525
|
-
# CRITICAL: For fromlist imports, enhance even if module is cached
|
|
526
|
-
if name in sys.modules:
|
|
527
|
-
result = _original_builtins_import(name, globals, locals, fromlist, level)
|
|
528
|
-
if fromlist:
|
|
529
|
-
module = sys.modules.get(name)
|
|
530
|
-
if module and isinstance(module, ModuleType):
|
|
531
|
-
try:
|
|
532
|
-
for pkg_name in _lazy_packages.keys():
|
|
533
|
-
if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
|
|
534
|
-
finder = _installed_hooks.get(pkg_name)
|
|
535
|
-
if finder:
|
|
536
|
-
finder._enhance_classes_with_class_methods(module)
|
|
537
|
-
break
|
|
538
|
-
except Exception:
|
|
539
|
-
pass
|
|
540
|
-
return result
|
|
541
|
-
|
|
542
|
-
# Fast path: known failure
|
|
543
|
-
if name in _failed_cache:
|
|
544
|
-
raise ImportError(f"No module named '{name}'")
|
|
545
|
-
|
|
546
|
-
# Fast path: skip stdlib/builtin modules (performance optimization)
|
|
547
|
-
if name in sys.builtin_module_names:
|
|
548
|
-
return _original_builtins_import(name, globals, locals, fromlist, level)
|
|
549
|
-
|
|
550
|
-
# Skip private/internal modules (performance optimization)
|
|
551
|
-
# But allow if it's a submodule of a registered package
|
|
552
|
-
if name.startswith('_'):
|
|
553
|
-
# Check if it's a submodule of a registered package
|
|
554
|
-
root_package = name.split('.')[0]
|
|
555
|
-
if root_package not in _lazy_packages:
|
|
556
|
-
return _original_builtins_import(name, globals, locals, fromlist, level)
|
|
557
|
-
|
|
558
|
-
# Skip test-related modules to avoid interfering with pytest
|
|
559
|
-
if name.startswith(('pytest', '_pytest', 'pluggy', '_pluggy')):
|
|
560
|
-
return _original_builtins_import(name, globals, locals, fromlist, level)
|
|
561
|
-
|
|
562
|
-
# Skip debugging/profiling modules
|
|
563
|
-
if name in ('tracemalloc', 'pdb', 'ipdb', 'debugpy', 'pydevd'):
|
|
564
|
-
return _original_builtins_import(name, globals, locals, fromlist, level)
|
|
565
|
-
|
|
566
|
-
try:
|
|
567
|
-
# Try normal import first
|
|
568
|
-
result = _original_builtins_import(name, globals, locals, fromlist, level)
|
|
569
|
-
# Success - cache it
|
|
570
|
-
_installed_cache.add(name)
|
|
571
|
-
# CRITICAL FIX: Enhance classes for fromlist imports (package/module agnostic)
|
|
572
|
-
# This ensures instance methods work on classes for ANY package/module structure
|
|
573
|
-
if result and isinstance(result, ModuleType) and fromlist:
|
|
574
|
-
try:
|
|
575
|
-
for pkg_name in _lazy_packages.keys():
|
|
576
|
-
if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
|
|
577
|
-
finder = _installed_hooks.get(pkg_name)
|
|
578
|
-
if finder:
|
|
579
|
-
finder._enhance_classes_with_class_methods(result)
|
|
580
|
-
break
|
|
581
|
-
except Exception:
|
|
582
|
-
pass
|
|
583
|
-
return result
|
|
584
|
-
except ImportError as e:
|
|
585
|
-
# Check if this package should be auto-installed
|
|
586
|
-
# ROOT CAUSE DEBUG: Log the exact import name and traceback to find where typos originate
|
|
587
|
-
if any(typo in name for typo in ['contrrib', 'msgpackk', 'msgppack', 'mmsgpack']):
|
|
588
|
-
import traceback
|
|
589
|
-
logger.warning(
|
|
590
|
-
f"[ROOT CAUSE] Typo detected in module name '{name}'. "
|
|
591
|
-
f"ImportError: {e}. "
|
|
592
|
-
f"Traceback:\n{''.join(traceback.format_stack()[-5:-1])}"
|
|
593
|
-
)
|
|
594
|
-
logger.debug(f"[AUTO-INSTALL] ImportError for '{name}': {e}")
|
|
595
|
-
should_install = _should_auto_install(name)
|
|
596
|
-
logger.debug(f"[AUTO-INSTALL] Should auto-install '{name}': {should_install}")
|
|
597
|
-
|
|
598
|
-
if should_install:
|
|
599
|
-
try:
|
|
600
|
-
logger.info(f"[AUTO-INSTALL] Attempting to install package for '{name}'")
|
|
601
|
-
if _try_install_package(name):
|
|
602
|
-
logger.info(f"[AUTO-INSTALL] Successfully installed package for '{name}', retrying import")
|
|
603
|
-
# Retry import after installation
|
|
604
|
-
try:
|
|
605
|
-
result = _original_builtins_import(name, globals, locals, fromlist, level)
|
|
606
|
-
_installed_cache.add(name)
|
|
607
|
-
logger.info(f"[AUTO-INSTALL] Successfully imported '{name}' after installation")
|
|
608
|
-
return result
|
|
609
|
-
except ImportError as retry_error:
|
|
610
|
-
logger.warning(f"[AUTO-INSTALL] Import still failed for '{name}' after installation: {retry_error}")
|
|
611
|
-
pass
|
|
612
|
-
else:
|
|
613
|
-
logger.warning(f"[AUTO-INSTALL] Installation attempt returned False for '{name}'")
|
|
614
|
-
except Exception as install_error:
|
|
615
|
-
# If installation fails, log it but don't crash - just raise the original ImportError
|
|
616
|
-
logger.error(f"[AUTO-INSTALL] Installation failed for '{name}': {install_error}", exc_info=True)
|
|
617
|
-
pass
|
|
618
|
-
else:
|
|
619
|
-
logger.debug(f"[AUTO-INSTALL] Not auto-installing '{name}' (not eligible)")
|
|
620
|
-
|
|
621
|
-
# Installation failed or not applicable - cache failure (but limit cache size)
|
|
622
|
-
if len(_failed_cache) < 1000: # Prevent unbounded growth
|
|
623
|
-
_failed_cache.add(name)
|
|
624
|
-
raise
|
|
625
|
-
except Exception:
|
|
626
|
-
# For any other exception, don't interfere - let it propagate
|
|
627
|
-
# This prevents the hook from breaking system functionality
|
|
628
|
-
raise
|
|
629
|
-
|
|
630
|
-
def install_global_import_hook() -> None:
|
|
631
|
-
"""
|
|
632
|
-
Install global builtins.__import__ hook for auto-install.
|
|
633
|
-
|
|
634
|
-
This hook intercepts ALL imports including module-level ones,
|
|
635
|
-
enabling auto-installation for registered packages.
|
|
636
|
-
"""
|
|
637
|
-
global _original_builtins_import, _global_import_hook_installed
|
|
638
|
-
|
|
639
|
-
with _global_import_hook_lock:
|
|
640
|
-
if _global_import_hook_installed:
|
|
641
|
-
logger.debug("Global import hook already installed")
|
|
642
|
-
return
|
|
643
|
-
|
|
644
|
-
if _original_builtins_import is None:
|
|
645
|
-
_original_builtins_import = builtins.__import__
|
|
646
|
-
|
|
647
|
-
builtins.__import__ = _intercepting_import
|
|
648
|
-
_global_import_hook_installed = True
|
|
649
|
-
logger.info("✅ Global builtins.__import__ hook installed for auto-install")
|
|
650
|
-
|
|
651
|
-
def uninstall_global_import_hook() -> None:
|
|
652
|
-
"""
|
|
653
|
-
Uninstall global builtins.__import__ hook.
|
|
654
|
-
|
|
655
|
-
Restores original builtins.__import__.
|
|
656
|
-
"""
|
|
657
|
-
global _original_builtins_import, _global_import_hook_installed
|
|
658
|
-
|
|
659
|
-
with _global_import_hook_lock:
|
|
660
|
-
if not _global_import_hook_installed:
|
|
661
|
-
return
|
|
662
|
-
|
|
663
|
-
if _original_builtins_import is not None:
|
|
664
|
-
builtins.__import__ = _original_builtins_import
|
|
665
|
-
_original_builtins_import = None
|
|
666
|
-
|
|
667
|
-
_global_import_hook_installed = False
|
|
668
|
-
logger.info("Global builtins.__import__ hook uninstalled")
|
|
669
|
-
|
|
670
|
-
def is_global_import_hook_installed() -> bool:
|
|
671
|
-
"""Check if global import hook is installed."""
|
|
672
|
-
return _global_import_hook_installed
|
|
673
|
-
|
|
674
|
-
def clear_global_import_caches() -> None:
|
|
675
|
-
"""
|
|
676
|
-
Clear global import hook caches (useful for testing).
|
|
677
|
-
|
|
678
|
-
Clears both installed and failed caches.
|
|
679
|
-
"""
|
|
680
|
-
global _installed_cache, _failed_cache
|
|
681
|
-
with _global_import_hook_lock:
|
|
682
|
-
_installed_cache.clear()
|
|
683
|
-
_failed_cache.clear()
|
|
684
|
-
logger.debug("Cleared global import hook caches")
|
|
685
|
-
|
|
686
|
-
def get_global_import_cache_stats() -> dict[str, Any]:
|
|
687
|
-
"""
|
|
688
|
-
Get statistics about global import hook caches.
|
|
689
|
-
|
|
690
|
-
Returns:
|
|
691
|
-
Dict with cache sizes and hit/miss information
|
|
692
|
-
"""
|
|
693
|
-
with _global_import_hook_lock:
|
|
694
|
-
return {
|
|
695
|
-
'installed_cache_size': len(_installed_cache),
|
|
696
|
-
'failed_cache_size': len(_failed_cache),
|
|
697
|
-
'registered_packages': list(_lazy_packages.keys()),
|
|
698
|
-
'hook_installed': _global_import_hook_installed,
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
# =============================================================================
|
|
702
|
-
# PREFIX TRIE (from prefix_trie.py)
|
|
703
|
-
# =============================================================================
|
|
704
|
-
|
|
705
|
-
class _PrefixTrie:
|
|
706
|
-
"""Trie data structure for prefix matching."""
|
|
707
|
-
|
|
708
|
-
__slots__ = ("_root",)
|
|
709
|
-
|
|
710
|
-
def __init__(self) -> None:
|
|
711
|
-
self._root: dict[str, dict[str, Any]] = {}
|
|
712
|
-
|
|
713
|
-
def add(self, prefix: str) -> None:
|
|
714
|
-
"""Add a prefix to the trie."""
|
|
715
|
-
node = self._root
|
|
716
|
-
for char in prefix:
|
|
717
|
-
node = node.setdefault(char, {})
|
|
718
|
-
node["_end"] = prefix
|
|
719
|
-
|
|
720
|
-
def iter_matches(self, value: str) -> tuple[str, ...]:
|
|
721
|
-
"""Find all matching prefixes for a given value."""
|
|
722
|
-
node = self._root
|
|
723
|
-
matches: list[str] = []
|
|
724
|
-
for char in value:
|
|
725
|
-
end_value = node.get("_end")
|
|
726
|
-
if end_value:
|
|
727
|
-
matches.append(end_value)
|
|
728
|
-
node = node.get(char)
|
|
729
|
-
if node is None:
|
|
730
|
-
break
|
|
731
|
-
else:
|
|
732
|
-
end_value = node.get("_end")
|
|
733
|
-
if end_value:
|
|
734
|
-
matches.append(end_value)
|
|
735
|
-
return tuple(matches)
|
|
736
|
-
|
|
737
|
-
# =============================================================================
|
|
738
|
-
# WATCHED REGISTRY (from watched_registry.py)
|
|
739
|
-
# =============================================================================
|
|
740
|
-
|
|
741
|
-
class WatchedPrefixRegistry:
|
|
742
|
-
"""Maintain watched prefixes and provide fast trie-based membership checks."""
|
|
743
|
-
|
|
744
|
-
__slots__ = (
|
|
745
|
-
"_lock",
|
|
746
|
-
"_prefix_refcounts",
|
|
747
|
-
"_owner_map",
|
|
748
|
-
"_prefixes",
|
|
749
|
-
"_trie",
|
|
750
|
-
"_dirty",
|
|
751
|
-
"_root_refcounts",
|
|
752
|
-
"_root_snapshot",
|
|
753
|
-
"_root_snapshot_dirty",
|
|
754
|
-
)
|
|
755
|
-
|
|
756
|
-
def __init__(self, initial: Optional[list[str]] = None) -> None:
|
|
757
|
-
self._lock = threading.RLock()
|
|
758
|
-
self._prefix_refcounts: Counter[str] = Counter()
|
|
759
|
-
self._owner_map: dict[str, set[str]] = {}
|
|
760
|
-
self._prefixes: set[str] = set()
|
|
761
|
-
self._trie = _PrefixTrie()
|
|
762
|
-
self._dirty = False
|
|
763
|
-
self._root_refcounts: Counter[str] = Counter()
|
|
764
|
-
self._root_snapshot: set[str] = set()
|
|
765
|
-
self._root_snapshot_dirty = False
|
|
766
|
-
if initial:
|
|
767
|
-
for prefix in initial:
|
|
768
|
-
self._register_manual(prefix)
|
|
769
|
-
|
|
770
|
-
def _register_manual(self, prefix: str) -> None:
|
|
771
|
-
normalized = _normalize_prefix(prefix)
|
|
772
|
-
if not normalized:
|
|
773
|
-
return
|
|
774
|
-
owner = "__manual__"
|
|
775
|
-
owners = self._owner_map.setdefault(owner, set())
|
|
776
|
-
if normalized in owners:
|
|
777
|
-
return
|
|
778
|
-
owners.add(normalized)
|
|
779
|
-
self._add_prefix(normalized)
|
|
780
|
-
|
|
781
|
-
def _add_prefix(self, prefix: str) -> None:
|
|
782
|
-
if not prefix:
|
|
783
|
-
return
|
|
784
|
-
self._prefix_refcounts[prefix] += 1
|
|
785
|
-
if self._prefix_refcounts[prefix] == 1:
|
|
786
|
-
self._prefixes.add(prefix)
|
|
787
|
-
self._dirty = True
|
|
788
|
-
root = prefix.split('.', 1)[0]
|
|
789
|
-
self._root_refcounts[root] += 1
|
|
790
|
-
self._root_snapshot_dirty = True
|
|
791
|
-
|
|
792
|
-
def _remove_prefix(self, prefix: str) -> None:
|
|
793
|
-
if prefix not in self._prefix_refcounts:
|
|
794
|
-
return
|
|
795
|
-
self._prefix_refcounts[prefix] -= 1
|
|
796
|
-
if self._prefix_refcounts[prefix] <= 0:
|
|
797
|
-
self._prefix_refcounts.pop(prefix, None)
|
|
798
|
-
self._prefixes.discard(prefix)
|
|
799
|
-
self._dirty = True
|
|
800
|
-
root = prefix.split('.', 1)[0]
|
|
801
|
-
self._root_refcounts[root] -= 1
|
|
802
|
-
if self._root_refcounts[root] <= 0:
|
|
803
|
-
self._root_refcounts.pop(root, None)
|
|
804
|
-
self._root_snapshot_dirty = True
|
|
805
|
-
|
|
806
|
-
def _ensure_trie(self) -> None:
|
|
807
|
-
if not self._dirty:
|
|
808
|
-
return
|
|
809
|
-
self._trie = _PrefixTrie()
|
|
810
|
-
for prefix in self._prefixes:
|
|
811
|
-
self._trie.add(prefix)
|
|
812
|
-
self._dirty = False
|
|
813
|
-
|
|
814
|
-
def add(self, prefix: str) -> None:
|
|
815
|
-
normalized = _normalize_prefix(prefix)
|
|
816
|
-
if not normalized:
|
|
817
|
-
return
|
|
818
|
-
with self._lock:
|
|
819
|
-
self._register_manual(normalized)
|
|
820
|
-
|
|
821
|
-
def is_empty(self) -> bool:
|
|
822
|
-
with self._lock:
|
|
823
|
-
return not self._prefixes
|
|
824
|
-
|
|
825
|
-
def register_package(self, package_name: str, prefixes: Iterable[str]) -> None:
|
|
826
|
-
owner_key = f"pkg::{package_name.lower()}"
|
|
827
|
-
normalized = {_normalize_prefix(p) for p in prefixes if _normalize_prefix(p)}
|
|
828
|
-
with self._lock:
|
|
829
|
-
current = self._owner_map.get(owner_key, set())
|
|
830
|
-
to_remove = current - normalized
|
|
831
|
-
to_add = normalized - current
|
|
832
|
-
|
|
833
|
-
for prefix in to_remove:
|
|
834
|
-
self._remove_prefix(prefix)
|
|
835
|
-
for prefix in to_add:
|
|
836
|
-
self._add_prefix(prefix)
|
|
837
|
-
|
|
838
|
-
if normalized:
|
|
839
|
-
self._owner_map[owner_key] = normalized
|
|
840
|
-
elif owner_key in self._owner_map:
|
|
841
|
-
self._owner_map.pop(owner_key, None)
|
|
842
|
-
|
|
843
|
-
def is_prefix_owned_by(self, package_name: str, prefix: str) -> bool:
|
|
844
|
-
normalized = _normalize_prefix(prefix)
|
|
845
|
-
owner_key = f"pkg::{package_name.lower()}"
|
|
846
|
-
with self._lock:
|
|
847
|
-
if normalized in self._owner_map.get("__manual__", set()):
|
|
848
|
-
return True
|
|
849
|
-
return normalized in self._owner_map.get(owner_key, set())
|
|
850
|
-
|
|
851
|
-
def get_matching_prefixes(self, module_name: str) -> tuple[str, ...]:
|
|
852
|
-
with self._lock:
|
|
853
|
-
if not self._prefixes:
|
|
854
|
-
return ()
|
|
855
|
-
self._ensure_trie()
|
|
856
|
-
return self._trie.iter_matches(module_name)
|
|
857
|
-
|
|
858
|
-
def has_root(self, root_name: str) -> bool:
|
|
859
|
-
snapshot = self._root_snapshot
|
|
860
|
-
if not self._root_snapshot_dirty:
|
|
861
|
-
return root_name in snapshot
|
|
862
|
-
with self._lock:
|
|
863
|
-
if self._root_snapshot_dirty:
|
|
864
|
-
self._root_snapshot = set(self._root_refcounts.keys())
|
|
865
|
-
self._root_snapshot_dirty = False
|
|
866
|
-
return root_name in self._root_snapshot
|
|
867
|
-
|
|
868
|
-
# Global registry instance
|
|
869
|
-
_DEFAULT_WATCHED_PREFIXES = tuple(
|
|
870
|
-
filter(
|
|
871
|
-
None,
|
|
872
|
-
os.environ.get(
|
|
873
|
-
"XWLAZY_LAZY_PREFIXES",
|
|
874
|
-
"",
|
|
875
|
-
).split(";"),
|
|
876
|
-
)
|
|
877
|
-
)
|
|
878
|
-
_watched_registry = WatchedPrefixRegistry(list(_DEFAULT_WATCHED_PREFIXES))
|
|
879
|
-
|
|
880
|
-
def get_watched_registry() -> WatchedPrefixRegistry:
|
|
881
|
-
"""Get the global watched prefix registry."""
|
|
882
|
-
return _watched_registry
|
|
883
|
-
|
|
884
|
-
# =============================================================================
|
|
885
|
-
# DEFERRED LOADER (from deferred_loader.py)
|
|
886
|
-
# =============================================================================
|
|
887
|
-
|
|
888
|
-
class _DeferredModuleLoader(importlib.abc.Loader):
|
|
889
|
-
"""Loader that simply returns a preconstructed module placeholder."""
|
|
890
|
-
|
|
891
|
-
def __init__(self, module: ModuleType) -> None:
|
|
892
|
-
self._module = module
|
|
893
|
-
|
|
894
|
-
def create_module(self, spec): # noqa: D401 - standard loader hook
|
|
895
|
-
return self._module
|
|
896
|
-
|
|
897
|
-
def exec_module(self, module): # noqa: D401 - nothing to execute
|
|
898
|
-
return None
|
|
899
|
-
|
|
900
|
-
# =============================================================================
|
|
901
|
-
# CACHE (from common.cache)
|
|
902
|
-
# =============================================================================
|
|
903
|
-
|
|
904
|
-
# MultiTierCache and BytecodeCache are now imported from ..common.cache
|
|
905
|
-
|
|
906
|
-
# =============================================================================
|
|
907
|
-
# PARALLEL UTILITIES (from parallel_utils.py)
|
|
908
|
-
# =============================================================================
|
|
909
|
-
|
|
910
|
-
class ParallelLoader:
|
|
911
|
-
"""Parallel module loader with smart dependency management."""
|
|
912
|
-
|
|
913
|
-
def __init__(self, max_workers: Optional[int] = None):
|
|
914
|
-
if max_workers is None:
|
|
915
|
-
max_workers = min(os.cpu_count() or 4, 8)
|
|
916
|
-
|
|
917
|
-
self._max_workers = max_workers
|
|
918
|
-
self._executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
|
|
919
|
-
self._lock = threading.RLock()
|
|
920
|
-
|
|
921
|
-
def _get_executor(self) -> concurrent.futures.ThreadPoolExecutor:
|
|
922
|
-
"""Get or create thread pool executor."""
|
|
923
|
-
with self._lock:
|
|
924
|
-
if self._executor is None:
|
|
925
|
-
self._executor = concurrent.futures.ThreadPoolExecutor(
|
|
926
|
-
max_workers=self._max_workers,
|
|
927
|
-
thread_name_prefix="xwlazy-parallel"
|
|
928
|
-
)
|
|
929
|
-
return self._executor
|
|
930
|
-
|
|
931
|
-
def load_modules_parallel(self, module_paths: list[str]) -> dict[str, Any]:
|
|
932
|
-
"""Load multiple modules in parallel."""
|
|
933
|
-
executor = self._get_executor()
|
|
934
|
-
results: dict[str, Any] = {}
|
|
935
|
-
|
|
936
|
-
def _load_module(module_path: str) -> tuple[str, Any, Optional[Exception]]:
|
|
937
|
-
try:
|
|
938
|
-
module = importlib.import_module(module_path)
|
|
939
|
-
return (module_path, module, None)
|
|
940
|
-
except Exception as e:
|
|
941
|
-
logger.debug(f"Failed to load {module_path} in parallel: {e}")
|
|
942
|
-
return (module_path, None, e)
|
|
943
|
-
|
|
944
|
-
futures = {executor.submit(_load_module, path): path for path in module_paths}
|
|
945
|
-
|
|
946
|
-
for future in concurrent.futures.as_completed(futures):
|
|
947
|
-
module_path, module, error = future.result()
|
|
948
|
-
results[module_path] = (module, error)
|
|
949
|
-
|
|
950
|
-
return results
|
|
951
|
-
|
|
952
|
-
def load_modules_with_priority(
|
|
953
|
-
self,
|
|
954
|
-
module_paths: list[tuple[str, int]]
|
|
955
|
-
) -> dict[str, Any]:
|
|
956
|
-
"""Load modules in parallel with priority ordering."""
|
|
957
|
-
sorted_modules = sorted(module_paths, key=lambda x: x[1], reverse=True)
|
|
958
|
-
module_list = [path for path, _ in sorted_modules]
|
|
959
|
-
return self.load_modules_parallel(module_list)
|
|
960
|
-
|
|
961
|
-
def shutdown(self, wait: bool = True) -> None:
|
|
962
|
-
"""Shutdown the executor."""
|
|
963
|
-
with self._lock:
|
|
964
|
-
if self._executor:
|
|
965
|
-
self._executor.shutdown(wait=wait)
|
|
966
|
-
self._executor = None
|
|
967
|
-
|
|
968
|
-
class DependencyGraph:
|
|
969
|
-
"""Manages module dependencies for optimal parallel loading."""
|
|
970
|
-
|
|
971
|
-
def __init__(self):
|
|
972
|
-
self._dependencies: dict[str, list[str]] = {}
|
|
973
|
-
self._reverse_deps: dict[str, list[str]] = {}
|
|
974
|
-
self._lock = threading.RLock()
|
|
975
|
-
|
|
976
|
-
def add_dependency(self, module: str, depends_on: list[str]) -> None:
|
|
977
|
-
"""Add dependencies for a module."""
|
|
978
|
-
with self._lock:
|
|
979
|
-
self._dependencies[module] = depends_on
|
|
980
|
-
for dep in depends_on:
|
|
981
|
-
if dep not in self._reverse_deps:
|
|
982
|
-
self._reverse_deps[dep] = []
|
|
983
|
-
if module not in self._reverse_deps[dep]:
|
|
984
|
-
self._reverse_deps[dep].append(module)
|
|
985
|
-
|
|
986
|
-
def get_load_order(self, modules: list[str]) -> list[list[str]]:
|
|
987
|
-
"""Get optimal load order for parallel loading (topological sort levels)."""
|
|
988
|
-
with self._lock:
|
|
989
|
-
in_degree: dict[str, int] = {m: 0 for m in modules}
|
|
990
|
-
for module, deps in self._dependencies.items():
|
|
991
|
-
if module in modules:
|
|
992
|
-
for dep in deps:
|
|
993
|
-
if dep in modules:
|
|
994
|
-
in_degree[module] += 1
|
|
995
|
-
|
|
996
|
-
levels: list[list[str]] = []
|
|
997
|
-
remaining = set(modules)
|
|
998
|
-
|
|
999
|
-
while remaining:
|
|
1000
|
-
current_level = [
|
|
1001
|
-
m for m in remaining
|
|
1002
|
-
if in_degree[m] == 0
|
|
1003
|
-
]
|
|
1004
|
-
|
|
1005
|
-
if not current_level:
|
|
1006
|
-
current_level = list(remaining)
|
|
1007
|
-
|
|
1008
|
-
levels.append(current_level)
|
|
1009
|
-
remaining -= set(current_level)
|
|
1010
|
-
|
|
1011
|
-
for module in current_level:
|
|
1012
|
-
for dependent in self._reverse_deps.get(module, []):
|
|
1013
|
-
if dependent in remaining:
|
|
1014
|
-
in_degree[dependent] = max(0, in_degree[dependent] - 1)
|
|
1015
|
-
|
|
1016
|
-
return levels
|
|
1017
|
-
|
|
1018
|
-
# =============================================================================
|
|
1019
|
-
# MODULE PATCHING (from module_patching.py)
|
|
1020
|
-
# =============================================================================
|
|
1021
|
-
|
|
1022
|
-
_original_import_module = importlib.import_module
|
|
1023
|
-
|
|
1024
|
-
def _lazy_aware_import_module(name: str, package: Optional[str] = None) -> ModuleType:
|
|
1025
|
-
"""Lazy-aware version of importlib.import_module."""
|
|
1026
|
-
if _is_import_in_progress(name):
|
|
1027
|
-
return _original_import_module(name, package)
|
|
1028
|
-
|
|
1029
|
-
_mark_import_started(name)
|
|
1030
|
-
try:
|
|
1031
|
-
return _original_import_module(name, package)
|
|
1032
|
-
finally:
|
|
1033
|
-
_mark_import_finished(name)
|
|
1034
|
-
|
|
1035
|
-
def _patch_import_module() -> None:
|
|
1036
|
-
"""
|
|
1037
|
-
Patch importlib.import_module to be lazy-aware.
|
|
1038
|
-
|
|
1039
|
-
WARNING: This performs global monkey-patching that affects ALL code in the Python process.
|
|
1040
|
-
This can cause conflicts with other libraries. Use sys.meta_path hooks instead.
|
|
1041
|
-
|
|
1042
|
-
This function is kept for backward compatibility but should be avoided in new code.
|
|
1043
|
-
"""
|
|
1044
|
-
# DISABLED: Dangerous monkey-patching disabled by default
|
|
1045
|
-
logger.warning(
|
|
1046
|
-
"_patch_import_module called - Dangerous monkey-patching is DISABLED. "
|
|
1047
|
-
"Use sys.meta_path hooks (install_import_hook) instead."
|
|
1048
|
-
)
|
|
1049
|
-
return
|
|
1050
|
-
# Original code (disabled):
|
|
1051
|
-
# importlib.import_module = _lazy_aware_import_module
|
|
1052
|
-
# logger.debug("Patched importlib.import_module to be lazy-aware")
|
|
1053
|
-
|
|
1054
|
-
def _unpatch_import_module() -> None:
|
|
1055
|
-
"""Restore original importlib.import_module."""
|
|
1056
|
-
importlib.import_module = _original_import_module
|
|
1057
|
-
logger.debug("Restored original importlib.import_module")
|
|
1058
|
-
|
|
1059
|
-
# =============================================================================
|
|
1060
|
-
# ARCHIVE IMPORTS (from archive_imports.py)
|
|
1061
|
-
# =============================================================================
|
|
1062
|
-
|
|
1063
|
-
_archive_path = None
|
|
1064
|
-
_archive_added = False
|
|
1065
|
-
|
|
1066
|
-
def get_archive_path() -> Path:
|
|
1067
|
-
"""Get the path to the _archive folder."""
|
|
1068
|
-
global _archive_path
|
|
1069
|
-
if _archive_path is None:
|
|
1070
|
-
current_file = Path(__file__)
|
|
1071
|
-
_archive_path = current_file.parent.parent.parent.parent.parent.parent / "_archive"
|
|
1072
|
-
return _archive_path
|
|
1073
|
-
|
|
1074
|
-
def ensure_archive_in_path() -> None:
|
|
1075
|
-
"""Ensure the archive folder is in sys.path for imports."""
|
|
1076
|
-
global _archive_added
|
|
1077
|
-
if not _archive_added:
|
|
1078
|
-
archive_path = get_archive_path()
|
|
1079
|
-
archive_str = str(archive_path)
|
|
1080
|
-
if archive_str not in sys.path:
|
|
1081
|
-
sys.path.insert(0, archive_str)
|
|
1082
|
-
_archive_added = True
|
|
1083
|
-
|
|
1084
|
-
def import_from_archive(module_name: str):
|
|
1085
|
-
"""Import a module from the archived lazy code."""
|
|
1086
|
-
ensure_archive_in_path()
|
|
1087
|
-
return __import__(module_name, fromlist=[''])
|
|
1088
|
-
|
|
1089
|
-
# =============================================================================
|
|
1090
|
-
# BOOTSTRAP (from bootstrap.py)
|
|
1091
|
-
# =============================================================================
|
|
1092
|
-
|
|
1093
|
-
def _env_enabled(env_value: Optional[str]) -> Optional[bool]:
|
|
1094
|
-
if not env_value:
|
|
1095
|
-
return None
|
|
1096
|
-
normalized = env_value.strip().lower()
|
|
1097
|
-
if normalized in ('true', '1', 'yes', 'on'):
|
|
1098
|
-
return True
|
|
1099
|
-
if normalized in ('false', '0', 'no', 'off'):
|
|
1100
|
-
return False
|
|
1101
|
-
return None
|
|
1102
|
-
|
|
1103
|
-
def bootstrap_lazy_mode(package_name: str) -> None:
|
|
1104
|
-
"""Detect whether lazy mode should be enabled for ``package_name`` and bootstrap hooks."""
|
|
1105
|
-
package_name = package_name.lower()
|
|
1106
|
-
env_value = os.environ.get(f"{package_name.upper()}_LAZY_INSTALL")
|
|
1107
|
-
env_enabled = _env_enabled(env_value)
|
|
1108
|
-
enabled = env_enabled
|
|
1109
|
-
|
|
1110
|
-
if enabled is None:
|
|
1111
|
-
from ..common.services.keyword_detection import _detect_lazy_installation
|
|
1112
|
-
enabled = _detect_lazy_installation(package_name)
|
|
1113
|
-
|
|
1114
|
-
if not enabled:
|
|
1115
|
-
return
|
|
1116
|
-
|
|
1117
|
-
from ..facade import config_package_lazy_install_enabled
|
|
1118
|
-
|
|
1119
|
-
config_package_lazy_install_enabled(
|
|
1120
|
-
package_name,
|
|
1121
|
-
enabled=True,
|
|
1122
|
-
install_hook=True,
|
|
1123
|
-
)
|
|
1124
|
-
|
|
1125
|
-
def bootstrap_lazy_mode_deferred(package_name: str) -> None:
|
|
1126
|
-
"""
|
|
1127
|
-
Schedule lazy mode bootstrap to run AFTER the calling package finishes importing.
|
|
1128
|
-
|
|
1129
|
-
WARNING: This function performs dangerous global monkey-patching of __builtins__.__import__.
|
|
1130
|
-
This can cause conflicts with other libraries (gevent, greenlet, debuggers, etc.).
|
|
1131
|
-
Consider using sys.meta_path hooks instead (install_import_hook) which is safer.
|
|
1132
|
-
|
|
1133
|
-
This function is kept for backward compatibility but should be avoided in new code.
|
|
1134
|
-
"""
|
|
1135
|
-
# DISABLED: Dangerous monkey-patching disabled by default
|
|
1136
|
-
# Uncomment only if absolutely necessary and you understand the risks
|
|
1137
|
-
logger.warning(
|
|
1138
|
-
f"bootstrap_lazy_mode_deferred called for {package_name} - "
|
|
1139
|
-
"Dangerous monkey-patching is DISABLED. Use install_import_hook() instead."
|
|
1140
|
-
)
|
|
1141
|
-
return
|
|
1142
|
-
|
|
1143
|
-
# Original code (disabled):
|
|
1144
|
-
# package_name_lower = package_name.lower()
|
|
1145
|
-
# package_module_name = f"exonware.{package_name_lower}"
|
|
1146
|
-
# original_import = __builtins__.__import__ if hasattr(__builtins__, '__import__') else __import__
|
|
1147
|
-
# ... (rest of dangerous code)
|
|
1148
|
-
|
|
1149
|
-
# =============================================================================
|
|
1150
|
-
# LAZY LOADER (from loader.py)
|
|
1151
|
-
# =============================================================================
|
|
1152
|
-
|
|
1153
|
-
class LazyLoader(AModuleHelper):
|
|
1154
|
-
"""Thread-safe lazy loader for modules with caching."""
|
|
1155
|
-
|
|
1156
|
-
def load_module(self, module_path: str = None) -> ModuleType:
|
|
1157
|
-
"""Thread-safe module loading with caching."""
|
|
1158
|
-
if module_path is None:
|
|
1159
|
-
module_path = self._module_path
|
|
1160
|
-
|
|
1161
|
-
if self._cached_module is not None:
|
|
1162
|
-
return self._cached_module
|
|
1163
|
-
|
|
1164
|
-
with self._lock:
|
|
1165
|
-
if self._cached_module is not None:
|
|
1166
|
-
return self._cached_module
|
|
1167
|
-
|
|
1168
|
-
if self._loading:
|
|
1169
|
-
raise ImportError(f"Circular import detected for {module_path}")
|
|
1170
|
-
|
|
1171
|
-
try:
|
|
1172
|
-
self._loading = True
|
|
1173
|
-
logger.debug(f"Lazy loading module: {module_path}")
|
|
1174
|
-
|
|
1175
|
-
self._cached_module = importlib.import_module(module_path)
|
|
1176
|
-
|
|
1177
|
-
logger.debug(f"Successfully loaded: {module_path}")
|
|
1178
|
-
return self._cached_module
|
|
1179
|
-
|
|
1180
|
-
except Exception as e:
|
|
1181
|
-
logger.error(f"Failed to load module {module_path}: {e}")
|
|
1182
|
-
raise ImportError(f"Failed to load {module_path}: {e}") from e
|
|
1183
|
-
finally:
|
|
1184
|
-
self._loading = False
|
|
1185
|
-
|
|
1186
|
-
def unload_module(self, module_path: str) -> None:
|
|
1187
|
-
"""Unload a module from cache."""
|
|
1188
|
-
with self._lock:
|
|
1189
|
-
if module_path == self._module_path:
|
|
1190
|
-
self._cached_module = None
|
|
1191
|
-
|
|
1192
|
-
def is_loaded(self) -> bool:
|
|
1193
|
-
"""Check if module is currently loaded."""
|
|
1194
|
-
return self._cached_module is not None
|
|
1195
|
-
|
|
1196
|
-
def __getattr__(self, name: str) -> Any:
|
|
1197
|
-
"""Get attribute from lazily loaded module."""
|
|
1198
|
-
module = self.load_module()
|
|
1199
|
-
try:
|
|
1200
|
-
return getattr(module, name)
|
|
1201
|
-
except AttributeError:
|
|
1202
|
-
raise AttributeError(
|
|
1203
|
-
f"module '{self._module_path}' has no attribute '{name}'"
|
|
1204
|
-
)
|
|
1205
|
-
|
|
1206
|
-
def __dir__(self) -> list:
|
|
1207
|
-
"""Return available attributes from loaded module."""
|
|
1208
|
-
module = self.load_module()
|
|
1209
|
-
return dir(module)
|
|
1210
|
-
|
|
1211
|
-
# =============================================================================
|
|
1212
|
-
# LAZY MODULE REGISTRY (from registry.py)
|
|
1213
|
-
# =============================================================================
|
|
1214
|
-
|
|
1215
|
-
class LazyModuleRegistry:
|
|
1216
|
-
"""Registry for managing lazy-loaded modules with performance tracking."""
|
|
1217
|
-
|
|
1218
|
-
__slots__ = ('_modules', '_load_times', '_lock', '_access_counts')
|
|
1219
|
-
|
|
1220
|
-
def __init__(self):
|
|
1221
|
-
self._modules: dict[str, LazyLoader] = {}
|
|
1222
|
-
self._load_times: dict[str, float] = {}
|
|
1223
|
-
self._access_counts: dict[str, int] = {}
|
|
1224
|
-
self._lock = threading.RLock()
|
|
1225
|
-
|
|
1226
|
-
def register_module(self, name: str, module_path: str) -> None:
|
|
1227
|
-
"""Register a module for lazy loading."""
|
|
1228
|
-
with self._lock:
|
|
1229
|
-
if name in self._modules:
|
|
1230
|
-
logger.warning(f"Module '{name}' already registered, overwriting")
|
|
1231
|
-
|
|
1232
|
-
self._modules[name] = LazyLoader(module_path)
|
|
1233
|
-
self._access_counts[name] = 0
|
|
1234
|
-
logger.debug(f"Registered lazy module: {name} -> {module_path}")
|
|
1235
|
-
|
|
1236
|
-
def get_module(self, name: str) -> LazyLoader:
|
|
1237
|
-
"""Get a lazy-loaded module."""
|
|
1238
|
-
with self._lock:
|
|
1239
|
-
if name not in self._modules:
|
|
1240
|
-
raise KeyError(f"Module '{name}' not registered")
|
|
1241
|
-
|
|
1242
|
-
self._access_counts[name] += 1
|
|
1243
|
-
return self._modules[name]
|
|
1244
|
-
|
|
1245
|
-
def preload_frequently_used(self, threshold: int = 5) -> None:
|
|
1246
|
-
"""Preload modules that are accessed frequently."""
|
|
1247
|
-
with self._lock:
|
|
1248
|
-
for name, count in self._access_counts.items():
|
|
1249
|
-
if count >= threshold:
|
|
1250
|
-
try:
|
|
1251
|
-
start_time = time.time()
|
|
1252
|
-
_ = self._modules[name].load_module()
|
|
1253
|
-
self._load_times[name] = time.time() - start_time
|
|
1254
|
-
log_event("hook", logger.info, f"Preloaded frequently used module: {name}")
|
|
1255
|
-
except Exception as e:
|
|
1256
|
-
logger.warning(f"Failed to preload {name}: {e}")
|
|
1257
|
-
|
|
1258
|
-
def get_stats(self) -> dict[str, Any]:
|
|
1259
|
-
"""Get loading statistics."""
|
|
1260
|
-
with self._lock:
|
|
1261
|
-
loaded_count = sum(
|
|
1262
|
-
1 for loader in self._modules.values()
|
|
1263
|
-
if loader.is_loaded()
|
|
1264
|
-
)
|
|
1265
|
-
|
|
1266
|
-
return {
|
|
1267
|
-
'total_registered': len(self._modules),
|
|
1268
|
-
'loaded_count': loaded_count,
|
|
1269
|
-
'unloaded_count': len(self._modules) - loaded_count,
|
|
1270
|
-
'access_counts': self._access_counts.copy(),
|
|
1271
|
-
'load_times': self._load_times.copy(),
|
|
1272
|
-
}
|
|
1273
|
-
|
|
1274
|
-
def clear_cache(self) -> None:
|
|
1275
|
-
"""Clear all cached modules."""
|
|
1276
|
-
with self._lock:
|
|
1277
|
-
for name, loader in self._modules.items():
|
|
1278
|
-
loader.unload_module(loader._module_path)
|
|
1279
|
-
log_event("config", logger.info, "Cleared all cached modules")
|
|
1280
|
-
|
|
1281
|
-
# =============================================================================
|
|
1282
|
-
# LAZY IMPORTER (from importer.py)
|
|
1283
|
-
# =============================================================================
|
|
1284
|
-
|
|
1285
|
-
class LazyImporter:
|
|
1286
|
-
"""
|
|
1287
|
-
Lazy importer that defers heavy module imports until first access.
|
|
1288
|
-
|
|
1289
|
-
Supports multiple load modes: NONE, AUTO, PRELOAD, BACKGROUND, CACHED,
|
|
1290
|
-
TURBO, ADAPTIVE, HYPERPARALLEL, STREAMING, ULTRA, INTELLIGENT.
|
|
1291
|
-
|
|
1292
|
-
ARCHITECTURAL NOTE: This class implements many modes for what should be a simple
|
|
1293
|
-
lazy loading mechanism. Modes like TURBO, ULTRA, and HYPERPARALLEL add significant
|
|
1294
|
-
complexity and may introduce concurrency issues with Python's import lock.
|
|
1295
|
-
Consider simplifying to just Lazy/Eager modes in future refactoring.
|
|
1296
|
-
"""
|
|
1297
|
-
|
|
1298
|
-
__slots__ = (
|
|
1299
|
-
'_enabled', '_load_mode', '_lazy_modules', '_loaded_modules', '_lock',
|
|
1300
|
-
'_access_counts', '_background_tasks', '_async_loop',
|
|
1301
|
-
'_multi_tier_cache', '_bytecode_cache', '_adaptive_learner',
|
|
1302
|
-
'_parallel_loader', '_dependency_graph', '_load_times',
|
|
1303
|
-
'_intelligent_selector', '_effective_mode', '_effective_install_mode'
|
|
1304
|
-
)
|
|
1305
|
-
|
|
1306
|
-
def __init__(self):
|
|
1307
|
-
"""Initialize lazy importer."""
|
|
1308
|
-
self._enabled = False
|
|
1309
|
-
self._load_mode = LazyLoadMode.NONE
|
|
1310
|
-
self._lazy_modules: dict[str, str] = {}
|
|
1311
|
-
self._loaded_modules: dict[str, ModuleType] = {}
|
|
1312
|
-
self._access_counts: dict[str, int] = {}
|
|
1313
|
-
self._load_times: dict[str, float] = {}
|
|
1314
|
-
self._background_tasks: dict[str, asyncio.Task] = {}
|
|
1315
|
-
self._async_loop: Optional[asyncio.AbstractEventLoop] = None
|
|
1316
|
-
|
|
1317
|
-
# Superior mode components
|
|
1318
|
-
self._multi_tier_cache: Optional[MultiTierCache] = None
|
|
1319
|
-
self._bytecode_cache: Optional[BytecodeCache] = None
|
|
1320
|
-
self._adaptive_learner: Optional[AdaptiveLearner] = None
|
|
1321
|
-
self._parallel_loader: Optional[ParallelLoader] = None
|
|
1322
|
-
self._dependency_graph: Optional[DependencyGraph] = None
|
|
1323
|
-
self._intelligent_selector: Optional[IntelligentModeSelector] = None
|
|
1324
|
-
|
|
1325
|
-
# Effective modes (for INTELLIGENT mode)
|
|
1326
|
-
self._effective_mode: Optional[LazyLoadMode] = None
|
|
1327
|
-
self._effective_install_mode = None
|
|
1328
|
-
|
|
1329
|
-
self._lock = threading.RLock()
|
|
1330
|
-
|
|
1331
|
-
def _ensure_async_loop(self) -> asyncio.AbstractEventLoop:
|
|
1332
|
-
"""Ensure async event loop is running for background loading."""
|
|
1333
|
-
if self._async_loop is not None and self._async_loop.is_running():
|
|
1334
|
-
return self._async_loop
|
|
1335
|
-
|
|
1336
|
-
with self._lock:
|
|
1337
|
-
if self._async_loop is None or not self._async_loop.is_running():
|
|
1338
|
-
loop_ready = threading.Event()
|
|
1339
|
-
|
|
1340
|
-
def _run_loop():
|
|
1341
|
-
loop = asyncio.new_event_loop()
|
|
1342
|
-
asyncio.set_event_loop(loop)
|
|
1343
|
-
self._async_loop = loop
|
|
1344
|
-
loop_ready.set()
|
|
1345
|
-
loop.run_forever()
|
|
1346
|
-
|
|
1347
|
-
thread = threading.Thread(target=_run_loop, daemon=True, name="xwlazy-loader-async")
|
|
1348
|
-
thread.start()
|
|
1349
|
-
|
|
1350
|
-
if not loop_ready.wait(timeout=5.0):
|
|
1351
|
-
raise RuntimeError("Failed to start async loop for lazy loader")
|
|
1352
|
-
|
|
1353
|
-
return self._async_loop
|
|
1354
|
-
|
|
1355
|
-
def enable(self, load_mode: LazyLoadMode = LazyLoadMode.AUTO) -> None:
|
|
1356
|
-
"""Enable lazy imports with specified load mode."""
|
|
1357
|
-
with self._lock:
|
|
1358
|
-
self._enabled = True
|
|
1359
|
-
self._load_mode = load_mode
|
|
1360
|
-
|
|
1361
|
-
# Initialize superior mode components
|
|
1362
|
-
if load_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA):
|
|
1363
|
-
self._multi_tier_cache = MultiTierCache(l1_size=1000, enable_l3=True)
|
|
1364
|
-
self._bytecode_cache = BytecodeCache()
|
|
1365
|
-
|
|
1366
|
-
if load_mode == LazyLoadMode.ADAPTIVE:
|
|
1367
|
-
self._adaptive_learner = AdaptiveLearner()
|
|
1368
|
-
self._multi_tier_cache = MultiTierCache(l1_size=1000, enable_l3=True)
|
|
1369
|
-
|
|
1370
|
-
if load_mode == LazyLoadMode.HYPERPARALLEL:
|
|
1371
|
-
max_workers = min(os.cpu_count() or 4, 8)
|
|
1372
|
-
self._parallel_loader = ParallelLoader(max_workers=max_workers)
|
|
1373
|
-
self._dependency_graph = DependencyGraph()
|
|
1374
|
-
|
|
1375
|
-
if load_mode == LazyLoadMode.STREAMING:
|
|
1376
|
-
self._ensure_async_loop()
|
|
1377
|
-
|
|
1378
|
-
if load_mode == LazyLoadMode.ULTRA:
|
|
1379
|
-
# ULTRA combines all optimizations
|
|
1380
|
-
if self._multi_tier_cache is None:
|
|
1381
|
-
self._multi_tier_cache = MultiTierCache(l1_size=2000, enable_l3=True)
|
|
1382
|
-
if self._bytecode_cache is None:
|
|
1383
|
-
self._bytecode_cache = BytecodeCache()
|
|
1384
|
-
if self._adaptive_learner is None:
|
|
1385
|
-
self._adaptive_learner = AdaptiveLearner()
|
|
1386
|
-
if self._parallel_loader is None:
|
|
1387
|
-
self._parallel_loader = ParallelLoader(max_workers=min(os.cpu_count() or 4, 8))
|
|
1388
|
-
if self._dependency_graph is None:
|
|
1389
|
-
self._dependency_graph = DependencyGraph()
|
|
1390
|
-
self._ensure_async_loop()
|
|
1391
|
-
|
|
1392
|
-
# INTELLIGENT mode: Initialize selector and determine initial mode
|
|
1393
|
-
if load_mode == LazyLoadMode.INTELLIGENT:
|
|
1394
|
-
self._intelligent_selector = IntelligentModeSelector()
|
|
1395
|
-
# Detect initial load level and get optimal mode
|
|
1396
|
-
initial_level = self._intelligent_selector.detect_load_level()
|
|
1397
|
-
self._effective_mode, self._effective_install_mode = self._intelligent_selector.get_optimal_mode(initial_level)
|
|
1398
|
-
logger.info(f"INTELLIGENT mode initialized: {initial_level.value} -> {self._effective_mode.value} + {self._effective_install_mode.value}")
|
|
1399
|
-
# Enable the effective mode recursively
|
|
1400
|
-
self.enable(self._effective_mode)
|
|
1401
|
-
return # Early return, effective mode is already enabled
|
|
1402
|
-
|
|
1403
|
-
# For PRELOAD/TURBO/ULTRA modes, preload modules
|
|
1404
|
-
if load_mode in (LazyLoadMode.PRELOAD, LazyLoadMode.TURBO, LazyLoadMode.ULTRA):
|
|
1405
|
-
self._preload_all_modules()
|
|
1406
|
-
# For BACKGROUND/STREAMING modes, ensure async loop is ready
|
|
1407
|
-
elif load_mode in (LazyLoadMode.BACKGROUND, LazyLoadMode.STREAMING):
|
|
1408
|
-
self._ensure_async_loop()
|
|
1409
|
-
|
|
1410
|
-
log_event("config", logger.info, f"Lazy imports enabled (mode: {load_mode.value})")
|
|
1411
|
-
|
|
1412
|
-
def disable(self) -> None:
|
|
1413
|
-
"""Disable lazy imports."""
|
|
1414
|
-
with self._lock:
|
|
1415
|
-
self._enabled = False
|
|
1416
|
-
|
|
1417
|
-
# Cleanup cache resources
|
|
1418
|
-
if self._multi_tier_cache:
|
|
1419
|
-
self._multi_tier_cache.shutdown()
|
|
1420
|
-
|
|
1421
|
-
log_event("config", logger.info, "Lazy imports disabled")
|
|
1422
|
-
|
|
1423
|
-
def is_enabled(self) -> bool:
|
|
1424
|
-
"""Check if lazy imports are enabled."""
|
|
1425
|
-
return self._enabled
|
|
1426
|
-
|
|
1427
|
-
def register_lazy_module(self, module_name: str, module_path: str = None) -> None:
|
|
1428
|
-
"""Register a module for lazy loading."""
|
|
1429
|
-
with self._lock:
|
|
1430
|
-
if module_path is None:
|
|
1431
|
-
module_path = module_name
|
|
1432
|
-
|
|
1433
|
-
self._lazy_modules[module_name] = module_path
|
|
1434
|
-
self._access_counts[module_name] = 0
|
|
1435
|
-
logger.debug(f"Registered lazy module: {module_name} -> {module_path}")
|
|
1436
|
-
|
|
1437
|
-
async def _background_load_module(self, module_name: str, module_path: str) -> ModuleType:
|
|
1438
|
-
"""Load module in background thread."""
|
|
1439
|
-
try:
|
|
1440
|
-
actual_module = importlib.import_module(module_path)
|
|
1441
|
-
with self._lock:
|
|
1442
|
-
self._loaded_modules[module_name] = actual_module
|
|
1443
|
-
self._access_counts[module_name] += 1
|
|
1444
|
-
logger.debug(f"Background loaded module: {module_name}")
|
|
1445
|
-
return actual_module
|
|
1446
|
-
except ImportError as e:
|
|
1447
|
-
logger.error(f"Failed to background load {module_name}: {e}")
|
|
1448
|
-
raise
|
|
1449
|
-
|
|
1450
|
-
def _preload_all_modules(self) -> None:
|
|
1451
|
-
"""Preload all registered modules using appropriate strategy based on mode."""
|
|
1452
|
-
if not self._lazy_modules:
|
|
1453
|
-
return
|
|
1454
|
-
|
|
1455
|
-
with self._lock:
|
|
1456
|
-
modules_to_load = [
|
|
1457
|
-
(name, path) for name, path in self._lazy_modules.items()
|
|
1458
|
-
if name not in self._loaded_modules
|
|
1459
|
-
]
|
|
1460
|
-
|
|
1461
|
-
if not modules_to_load:
|
|
1462
|
-
return
|
|
1463
|
-
|
|
1464
|
-
# HYPERPARALLEL/ULTRA: Use thread pool executor
|
|
1465
|
-
if self._load_mode in (LazyLoadMode.HYPERPARALLEL, LazyLoadMode.ULTRA) and self._parallel_loader:
|
|
1466
|
-
module_paths = [path for _, path in modules_to_load]
|
|
1467
|
-
results = self._parallel_loader.load_modules_parallel(module_paths)
|
|
1468
|
-
|
|
1469
|
-
with self._lock:
|
|
1470
|
-
for (name, path), (module, error) in zip(modules_to_load, results.items()):
|
|
1471
|
-
if module is not None:
|
|
1472
|
-
self._loaded_modules[name] = module
|
|
1473
|
-
self._access_counts[name] = 0
|
|
1474
|
-
if self._adaptive_learner:
|
|
1475
|
-
self._adaptive_learner.record_import(name, 0.0)
|
|
1476
|
-
|
|
1477
|
-
log_event("hook", logger.info, f"Parallel preloaded {len([r for r in results.values() if r[0] is not None])} modules")
|
|
1478
|
-
return
|
|
1479
|
-
|
|
1480
|
-
# TURBO/ULTRA: Preload with predictive caching
|
|
1481
|
-
if self._load_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._multi_tier_cache:
|
|
1482
|
-
# Get predictive modules to prioritize
|
|
1483
|
-
predictive_keys = self._multi_tier_cache.get_predictive_keys(limit=10)
|
|
1484
|
-
priority_modules = [(name, path) for name, path in modules_to_load if name in predictive_keys]
|
|
1485
|
-
normal_modules = [(name, path) for name, path in modules_to_load if name not in predictive_keys]
|
|
1486
|
-
modules_to_load = priority_modules + normal_modules
|
|
1487
|
-
|
|
1488
|
-
# ADAPTIVE: Preload based on learned patterns
|
|
1489
|
-
if self._load_mode == LazyLoadMode.ADAPTIVE and self._adaptive_learner:
|
|
1490
|
-
priority_modules = self._adaptive_learner.get_priority_modules(limit=10)
|
|
1491
|
-
priority_list = [(name, path) for name, path in modules_to_load if name in priority_modules]
|
|
1492
|
-
normal_list = [(name, path) for name, path in modules_to_load if name not in priority_modules]
|
|
1493
|
-
modules_to_load = priority_list + normal_list
|
|
1494
|
-
|
|
1495
|
-
# Default: Use asyncio for parallel loading
|
|
1496
|
-
loop = self._ensure_async_loop()
|
|
1497
|
-
|
|
1498
|
-
async def _preload_all():
|
|
1499
|
-
tasks = [
|
|
1500
|
-
self._background_load_module(name, path)
|
|
1501
|
-
for name, path in modules_to_load
|
|
1502
|
-
]
|
|
1503
|
-
if tasks:
|
|
1504
|
-
await asyncio.gather(*tasks, return_exceptions=True)
|
|
1505
|
-
log_event("hook", logger.info, f"Preloaded {len(tasks)} modules")
|
|
1506
|
-
|
|
1507
|
-
asyncio.run_coroutine_threadsafe(_preload_all(), loop)
|
|
1508
|
-
|
|
1509
|
-
def import_module(self, module_name: str, package_name: str = None) -> Any:
|
|
1510
|
-
"""Import a module with lazy loading."""
|
|
1511
|
-
start_time = time.time()
|
|
1512
|
-
|
|
1513
|
-
# Fast path: Check if already in sys.modules (lock-free read)
|
|
1514
|
-
if module_name in sys.modules:
|
|
1515
|
-
# Lock-free check first
|
|
1516
|
-
if module_name not in self._loaded_modules:
|
|
1517
|
-
with self._lock:
|
|
1518
|
-
# Double-check after acquiring lock
|
|
1519
|
-
if module_name not in self._loaded_modules:
|
|
1520
|
-
self._loaded_modules[module_name] = sys.modules[module_name]
|
|
1521
|
-
# Update access count (requires lock)
|
|
1522
|
-
with self._lock:
|
|
1523
|
-
self._access_counts[module_name] = self._access_counts.get(module_name, 0) + 1
|
|
1524
|
-
load_time = time.time() - start_time
|
|
1525
|
-
if self._adaptive_learner:
|
|
1526
|
-
self._adaptive_learner.record_import(module_name, load_time)
|
|
1527
|
-
if self._load_mode == LazyLoadMode.INTELLIGENT:
|
|
1528
|
-
self._total_import_time = getattr(self, '_total_import_time', 0.0) + load_time
|
|
1529
|
-
return sys.modules[module_name]
|
|
1530
|
-
|
|
1531
|
-
# Fast path: Check if already loaded (lock-free read)
|
|
1532
|
-
if module_name in self._loaded_modules:
|
|
1533
|
-
with self._lock:
|
|
1534
|
-
# Double-check and update
|
|
1535
|
-
if module_name in self._loaded_modules:
|
|
1536
|
-
self._access_counts[module_name] = self._access_counts.get(module_name, 0) + 1
|
|
1537
|
-
if self._adaptive_learner:
|
|
1538
|
-
self._adaptive_learner.record_import(module_name, 0.0)
|
|
1539
|
-
return self._loaded_modules[module_name]
|
|
1540
|
-
|
|
1541
|
-
# Check enabled state and get module path (requires lock)
|
|
1542
|
-
with self._lock:
|
|
1543
|
-
if not self._enabled or self._load_mode == LazyLoadMode.NONE:
|
|
1544
|
-
return importlib.import_module(module_name)
|
|
1545
|
-
|
|
1546
|
-
if module_name in self._lazy_modules:
|
|
1547
|
-
module_path = self._lazy_modules[module_name]
|
|
1548
|
-
else:
|
|
1549
|
-
return importlib.import_module(module_name)
|
|
1550
|
-
|
|
1551
|
-
# Update total import time for intelligent mode (initialization)
|
|
1552
|
-
if self._load_mode == LazyLoadMode.INTELLIGENT:
|
|
1553
|
-
if not hasattr(self, '_total_import_time'):
|
|
1554
|
-
self._total_import_time = 0.0
|
|
1555
|
-
|
|
1556
|
-
# INTELLIGENT mode: Check if mode switch is needed and determine effective mode
|
|
1557
|
-
effective_load_mode = self._load_mode
|
|
1558
|
-
if self._load_mode == LazyLoadMode.INTELLIGENT and self._intelligent_selector:
|
|
1559
|
-
# Throttle load level detection (cache for 0.1s to avoid excessive checks)
|
|
1560
|
-
current_time = time.time()
|
|
1561
|
-
last_check = getattr(self, '_last_load_level_check', 0.0)
|
|
1562
|
-
check_interval = 0.1 # 100ms throttle
|
|
1563
|
-
|
|
1564
|
-
if current_time - last_check >= check_interval:
|
|
1565
|
-
# Fast path: lock-free reads for stats
|
|
1566
|
-
module_count = len(self._loaded_modules) # Dict read is thread-safe
|
|
1567
|
-
total_import_time = getattr(self, '_total_import_time', 0.0)
|
|
1568
|
-
import_count = sum(self._access_counts.values()) # Dict read is thread-safe
|
|
1569
|
-
|
|
1570
|
-
# Cache psutil import and memory check (only check every 0.5s)
|
|
1571
|
-
last_memory_check = getattr(self, '_last_memory_check', 0.0)
|
|
1572
|
-
memory_mb = getattr(self, '_cached_memory_mb', 0.0)
|
|
1573
|
-
|
|
1574
|
-
if current_time - last_memory_check >= 0.5:
|
|
1575
|
-
try:
|
|
1576
|
-
import psutil
|
|
1577
|
-
process = psutil.Process()
|
|
1578
|
-
memory_mb = process.memory_info().rss / 1024 / 1024
|
|
1579
|
-
self._cached_memory_mb = memory_mb
|
|
1580
|
-
self._last_memory_check = current_time
|
|
1581
|
-
except Exception:
|
|
1582
|
-
memory_mb = 0.0
|
|
1583
|
-
|
|
1584
|
-
# Detect current load level (lock-free)
|
|
1585
|
-
detected_level = self._intelligent_selector.detect_load_level(
|
|
1586
|
-
module_count=module_count,
|
|
1587
|
-
total_import_time=total_import_time,
|
|
1588
|
-
import_count=import_count,
|
|
1589
|
-
memory_usage_mb=memory_mb
|
|
1590
|
-
)
|
|
1591
|
-
|
|
1592
|
-
# Check if mode switch is needed (requires lock for write)
|
|
1593
|
-
current_mode_tuple = (self._effective_mode or self._load_mode, self._effective_install_mode)
|
|
1594
|
-
if self._intelligent_selector.should_switch_mode(current_mode_tuple, detected_level):
|
|
1595
|
-
optimal_load, optimal_install = self._intelligent_selector.get_optimal_mode(detected_level)
|
|
1596
|
-
if optimal_load != self._effective_mode or optimal_install != self._effective_install_mode:
|
|
1597
|
-
with self._lock: # Only lock for mode switch
|
|
1598
|
-
if optimal_load != self._effective_mode or optimal_install != self._effective_install_mode:
|
|
1599
|
-
logger.info(f"INTELLIGENT mode switching: {detected_level.value} -> {optimal_load.value} + {optimal_install.value}")
|
|
1600
|
-
self._effective_mode = optimal_load
|
|
1601
|
-
self._effective_install_mode = optimal_install
|
|
1602
|
-
# Switch to optimal mode (re-enable with new mode)
|
|
1603
|
-
self.enable(optimal_load)
|
|
1604
|
-
|
|
1605
|
-
self._last_load_level_check = current_time
|
|
1606
|
-
|
|
1607
|
-
# Use effective mode for processing
|
|
1608
|
-
effective_load_mode = self._effective_mode or self._load_mode
|
|
1609
|
-
|
|
1610
|
-
# Use effective mode for all checks
|
|
1611
|
-
check_mode = effective_load_mode
|
|
1612
|
-
|
|
1613
|
-
# TURBO/ULTRA: Check multi-tier cache first
|
|
1614
|
-
if check_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._multi_tier_cache:
|
|
1615
|
-
cached_module = self._multi_tier_cache.get(module_name)
|
|
1616
|
-
if cached_module is not None:
|
|
1617
|
-
with self._lock:
|
|
1618
|
-
self._loaded_modules[module_name] = cached_module
|
|
1619
|
-
self._access_counts[module_name] += 1
|
|
1620
|
-
self._load_times[module_name] = time.time() - start_time
|
|
1621
|
-
if self._adaptive_learner:
|
|
1622
|
-
self._adaptive_learner.record_import(module_name, self._load_times[module_name])
|
|
1623
|
-
return cached_module
|
|
1624
|
-
|
|
1625
|
-
# ADAPTIVE: Check cache and predict next imports
|
|
1626
|
-
if check_mode == LazyLoadMode.ADAPTIVE:
|
|
1627
|
-
if self._multi_tier_cache:
|
|
1628
|
-
cached_module = self._multi_tier_cache.get(module_name)
|
|
1629
|
-
if cached_module is not None:
|
|
1630
|
-
with self._lock:
|
|
1631
|
-
self._loaded_modules[module_name] = cached_module
|
|
1632
|
-
self._access_counts[module_name] += 1
|
|
1633
|
-
load_time = time.time() - start_time
|
|
1634
|
-
self._load_times[module_name] = load_time
|
|
1635
|
-
if self._adaptive_learner:
|
|
1636
|
-
self._adaptive_learner.record_import(module_name, load_time)
|
|
1637
|
-
|
|
1638
|
-
# Predict and preload next likely imports
|
|
1639
|
-
if self._adaptive_learner:
|
|
1640
|
-
next_imports = self._adaptive_learner.predict_next_imports(module_name, limit=3)
|
|
1641
|
-
self._preload_predictive_modules(next_imports)
|
|
1642
|
-
|
|
1643
|
-
return cached_module
|
|
1644
|
-
|
|
1645
|
-
# Record import for learning
|
|
1646
|
-
with self._lock:
|
|
1647
|
-
if self._adaptive_learner:
|
|
1648
|
-
# Will be updated after load
|
|
1649
|
-
pass
|
|
1650
|
-
|
|
1651
|
-
# HYPERPARALLEL: Use parallel loading
|
|
1652
|
-
if check_mode == LazyLoadMode.HYPERPARALLEL and self._parallel_loader:
|
|
1653
|
-
results = self._parallel_loader.load_modules_parallel([module_path])
|
|
1654
|
-
module, error = results.get(module_path, (None, None))
|
|
1655
|
-
if module is not None:
|
|
1656
|
-
with self._lock:
|
|
1657
|
-
self._loaded_modules[module_name] = module
|
|
1658
|
-
self._access_counts[module_name] += 1
|
|
1659
|
-
self._load_times[module_name] = time.time() - start_time
|
|
1660
|
-
return module
|
|
1661
|
-
elif error:
|
|
1662
|
-
raise error
|
|
1663
|
-
|
|
1664
|
-
# STREAMING: Load asynchronously in background
|
|
1665
|
-
if check_mode == LazyLoadMode.STREAMING:
|
|
1666
|
-
return self._streaming_load(module_name, module_path)
|
|
1667
|
-
|
|
1668
|
-
# BACKGROUND mode: Load in background, return placeholder
|
|
1669
|
-
if check_mode == LazyLoadMode.BACKGROUND:
|
|
1670
|
-
return self._background_placeholder_load(module_name, module_path)
|
|
1671
|
-
|
|
1672
|
-
# TURBO/ULTRA: Load with bytecode cache
|
|
1673
|
-
actual_module = None
|
|
1674
|
-
if check_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._bytecode_cache:
|
|
1675
|
-
# Try to load from bytecode cache first
|
|
1676
|
-
bytecode = self._bytecode_cache.get_cached_bytecode(module_path)
|
|
1677
|
-
if bytecode is not None:
|
|
1678
|
-
try:
|
|
1679
|
-
# Load from bytecode
|
|
1680
|
-
code = compile(bytecode, f"<cached {module_path}>", "exec")
|
|
1681
|
-
actual_module = importlib.import_module(module_path)
|
|
1682
|
-
except Exception as e:
|
|
1683
|
-
logger.debug(f"Failed to load from bytecode cache: {e}")
|
|
1684
|
-
|
|
1685
|
-
# Load module (standard or cached)
|
|
1686
|
-
if actual_module is None:
|
|
1687
|
-
try:
|
|
1688
|
-
actual_module = importlib.import_module(module_path)
|
|
1689
|
-
|
|
1690
|
-
# Cache bytecode for TURBO/ULTRA
|
|
1691
|
-
if check_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._bytecode_cache:
|
|
1692
|
-
try:
|
|
1693
|
-
# Get compiled bytecode from module
|
|
1694
|
-
if hasattr(actual_module, '__file__') and actual_module.__file__:
|
|
1695
|
-
pyc_path = actual_module.__file__.replace('.py', '.pyc')
|
|
1696
|
-
if os.path.exists(pyc_path):
|
|
1697
|
-
with open(pyc_path, 'rb') as f:
|
|
1698
|
-
f.seek(16) # Skip header
|
|
1699
|
-
bytecode = f.read()
|
|
1700
|
-
self._bytecode_cache.cache_bytecode(module_path, bytecode)
|
|
1701
|
-
except Exception as e:
|
|
1702
|
-
logger.debug(f"Failed to cache bytecode: {e}")
|
|
1703
|
-
except ImportError as e:
|
|
1704
|
-
logger.error(f"Failed to lazy load {module_name}: {e}")
|
|
1705
|
-
raise
|
|
1706
|
-
|
|
1707
|
-
load_time = time.time() - start_time
|
|
1708
|
-
|
|
1709
|
-
with self._lock:
|
|
1710
|
-
self._loaded_modules[module_name] = actual_module
|
|
1711
|
-
self._access_counts[module_name] += 1
|
|
1712
|
-
self._load_times[module_name] = load_time
|
|
1713
|
-
|
|
1714
|
-
# Update total import time for intelligent mode
|
|
1715
|
-
if self._load_mode == LazyLoadMode.INTELLIGENT:
|
|
1716
|
-
self._total_import_time = getattr(self, '_total_import_time', 0.0) + load_time
|
|
1717
|
-
|
|
1718
|
-
# Cache in multi-tier cache for TURBO/ULTRA/ADAPTIVE
|
|
1719
|
-
if self._multi_tier_cache:
|
|
1720
|
-
self._multi_tier_cache.set(module_name, actual_module)
|
|
1721
|
-
|
|
1722
|
-
# Record for adaptive learning
|
|
1723
|
-
if self._adaptive_learner:
|
|
1724
|
-
self._adaptive_learner.record_import(module_name, load_time)
|
|
1725
|
-
|
|
1726
|
-
logger.debug(f"Lazy loaded module: {module_name} ({load_time*1000:.2f}ms)")
|
|
1727
|
-
|
|
1728
|
-
return actual_module
|
|
1729
|
-
|
|
1730
|
-
def _streaming_load(self, module_name: str, module_path: str) -> ModuleType:
|
|
1731
|
-
"""Load module asynchronously with streaming."""
|
|
1732
|
-
if module_name not in self._background_tasks or self._background_tasks[module_name].done():
|
|
1733
|
-
loop = self._ensure_async_loop()
|
|
1734
|
-
task = asyncio.run_coroutine_threadsafe(
|
|
1735
|
-
self._background_load_module(module_name, module_path),
|
|
1736
|
-
loop
|
|
1737
|
-
)
|
|
1738
|
-
self._background_tasks[module_name] = task
|
|
1739
|
-
|
|
1740
|
-
# Return placeholder that streams
|
|
1741
|
-
placeholder = ModuleType(module_name)
|
|
1742
|
-
placeholder.__path__ = []
|
|
1743
|
-
placeholder.__package__ = module_name
|
|
1744
|
-
|
|
1745
|
-
def _streaming_getattr(name):
|
|
1746
|
-
task = self._background_tasks.get(module_name)
|
|
1747
|
-
if task and not task.done():
|
|
1748
|
-
# Non-blocking check with short timeout
|
|
1749
|
-
try:
|
|
1750
|
-
task.result(timeout=0.01) # Very short timeout for streaming
|
|
1751
|
-
except Exception:
|
|
1752
|
-
pass # Still loading, continue
|
|
1753
|
-
|
|
1754
|
-
# Check if loaded now
|
|
1755
|
-
with self._lock:
|
|
1756
|
-
if module_name in self._loaded_modules:
|
|
1757
|
-
return getattr(self._loaded_modules[module_name], name)
|
|
1758
|
-
|
|
1759
|
-
# Still loading, wait for completion
|
|
1760
|
-
if task and not task.done():
|
|
1761
|
-
task.result(timeout=10.0)
|
|
1762
|
-
|
|
1763
|
-
with self._lock:
|
|
1764
|
-
if module_name in self._loaded_modules:
|
|
1765
|
-
return getattr(self._loaded_modules[module_name], name)
|
|
1766
|
-
raise AttributeError(f"module '{module_name}' has no attribute '{name}'")
|
|
1767
|
-
|
|
1768
|
-
placeholder.__getattr__ = _streaming_getattr # type: ignore[attr-defined]
|
|
1769
|
-
return placeholder
|
|
1770
|
-
|
|
1771
|
-
def _background_placeholder_load(self, module_name: str, module_path: str) -> ModuleType:
|
|
1772
|
-
"""Load module in background, return placeholder."""
|
|
1773
|
-
if module_name not in self._background_tasks or self._background_tasks[module_name].done():
|
|
1774
|
-
loop = self._ensure_async_loop()
|
|
1775
|
-
task = asyncio.run_coroutine_threadsafe(
|
|
1776
|
-
self._background_load_module(module_name, module_path),
|
|
1777
|
-
loop
|
|
1778
|
-
)
|
|
1779
|
-
self._background_tasks[module_name] = task
|
|
1780
|
-
|
|
1781
|
-
# Return placeholder module that will be replaced when loaded
|
|
1782
|
-
placeholder = ModuleType(module_name)
|
|
1783
|
-
placeholder.__path__ = []
|
|
1784
|
-
placeholder.__package__ = module_name
|
|
1785
|
-
|
|
1786
|
-
def _getattr(name):
|
|
1787
|
-
# Wait for background load to complete
|
|
1788
|
-
task = self._background_tasks.get(module_name)
|
|
1789
|
-
if task and not task.done():
|
|
1790
|
-
task.result(timeout=10.0) # Wait up to 10 seconds
|
|
1791
|
-
with self._lock:
|
|
1792
|
-
if module_name in self._loaded_modules:
|
|
1793
|
-
return getattr(self._loaded_modules[module_name], name)
|
|
1794
|
-
raise AttributeError(f"module '{module_name}' has no attribute '{name}'")
|
|
1795
|
-
|
|
1796
|
-
placeholder.__getattr__ = _getattr # type: ignore[attr-defined]
|
|
1797
|
-
return placeholder
|
|
1798
|
-
|
|
1799
|
-
def _preload_predictive_modules(self, module_names: list) -> None:
|
|
1800
|
-
"""Preload modules predicted to be needed soon."""
|
|
1801
|
-
if not module_names:
|
|
1802
|
-
return
|
|
1803
|
-
|
|
1804
|
-
with self._lock:
|
|
1805
|
-
modules_to_preload = [
|
|
1806
|
-
(name, self._lazy_modules[name])
|
|
1807
|
-
for name in module_names
|
|
1808
|
-
if name in self._lazy_modules and name not in self._loaded_modules
|
|
1809
|
-
]
|
|
1810
|
-
|
|
1811
|
-
if not modules_to_preload:
|
|
1812
|
-
return
|
|
1813
|
-
|
|
1814
|
-
# Preload in background
|
|
1815
|
-
loop = self._ensure_async_loop()
|
|
1816
|
-
|
|
1817
|
-
async def _preload_predictive():
|
|
1818
|
-
tasks = [
|
|
1819
|
-
self._background_load_module(name, path)
|
|
1820
|
-
for name, path in modules_to_preload
|
|
1821
|
-
]
|
|
1822
|
-
await asyncio.gather(*tasks, return_exceptions=True)
|
|
1823
|
-
|
|
1824
|
-
asyncio.run_coroutine_threadsafe(_preload_predictive(), loop)
|
|
1825
|
-
|
|
1826
|
-
def preload_module(self, module_name: str) -> bool:
|
|
1827
|
-
"""Preload a registered lazy module."""
|
|
1828
|
-
with self._lock:
|
|
1829
|
-
if module_name not in self._lazy_modules:
|
|
1830
|
-
logger.warning(f"Module {module_name} not registered for lazy loading")
|
|
1831
|
-
return False
|
|
1832
|
-
|
|
1833
|
-
try:
|
|
1834
|
-
self.import_module(module_name)
|
|
1835
|
-
log_event("hook", logger.info, f"Preloaded module: {module_name}")
|
|
1836
|
-
return True
|
|
1837
|
-
except Exception as e:
|
|
1838
|
-
logger.error(f"Failed to preload {module_name}: {e}")
|
|
1839
|
-
return False
|
|
1840
|
-
|
|
1841
|
-
def get_stats(self) -> dict[str, Any]:
|
|
1842
|
-
"""Get lazy import statistics."""
|
|
1843
|
-
with self._lock:
|
|
1844
|
-
return {
|
|
1845
|
-
'enabled': self._enabled,
|
|
1846
|
-
'registered_modules': list(self._lazy_modules.keys()),
|
|
1847
|
-
'loaded_modules': list(self._loaded_modules.keys()),
|
|
1848
|
-
'access_counts': self._access_counts.copy(),
|
|
1849
|
-
'total_registered': len(self._lazy_modules),
|
|
1850
|
-
'total_loaded': len(self._loaded_modules)
|
|
1851
|
-
}
|
|
1852
|
-
|
|
1853
|
-
# =============================================================================
|
|
1854
|
-
# IMPORT HOOK (from import_hook.py)
|
|
1855
|
-
# =============================================================================
|
|
1856
|
-
|
|
1857
|
-
class LazyImportHook(AModuleHelper):
|
|
1858
|
-
"""
|
|
1859
|
-
Import hook that intercepts ImportError and auto-installs packages.
|
|
1860
|
-
Performance optimized with zero overhead for successful imports.
|
|
1861
|
-
"""
|
|
1862
|
-
|
|
1863
|
-
__slots__ = AModuleHelper.__slots__
|
|
1864
|
-
|
|
1865
|
-
def handle_import_error(self, module_name: str) -> Optional[Any]:
|
|
1866
|
-
"""Handle ImportError by attempting to install and re-import."""
|
|
1867
|
-
if not self._enabled:
|
|
1868
|
-
return None
|
|
1869
|
-
|
|
1870
|
-
try:
|
|
1871
|
-
# Deferred import to avoid circular dependency
|
|
1872
|
-
from ..facade import lazy_import_with_install
|
|
1873
|
-
module, success = lazy_import_with_install(
|
|
1874
|
-
module_name,
|
|
1875
|
-
installer_package=self._package_name
|
|
1876
|
-
)
|
|
1877
|
-
return module if success else None
|
|
1878
|
-
except Exception:
|
|
1879
|
-
return None
|
|
1880
|
-
|
|
1881
|
-
def install_hook(self) -> None:
|
|
1882
|
-
"""Install the import hook into sys.meta_path."""
|
|
1883
|
-
install_import_hook(self._package_name)
|
|
1884
|
-
|
|
1885
|
-
def uninstall_hook(self) -> None:
|
|
1886
|
-
"""Uninstall the import hook from sys.meta_path."""
|
|
1887
|
-
uninstall_import_hook(self._package_name)
|
|
1888
|
-
|
|
1889
|
-
def is_installed(self) -> bool:
|
|
1890
|
-
"""Check if hook is installed."""
|
|
1891
|
-
return is_import_hook_installed(self._package_name)
|
|
1892
|
-
|
|
1893
|
-
# =============================================================================
|
|
1894
|
-
# META PATH FINDER (from meta_path_finder.py)
|
|
1895
|
-
# =============================================================================
|
|
1896
|
-
|
|
1897
|
-
# Wrapped class cache
|
|
1898
|
-
_WRAPPED_CLASS_CACHE: dict[str, set[str]] = defaultdict(set)
|
|
1899
|
-
_wrapped_cache_lock = threading.RLock()
|
|
1900
|
-
|
|
1901
|
-
# Default lazy methods
|
|
1902
|
-
_DEFAULT_LAZY_METHODS = tuple(
|
|
1903
|
-
filter(
|
|
1904
|
-
None,
|
|
1905
|
-
os.environ.get("XWLAZY_LAZY_METHODS", "").split(","),
|
|
1906
|
-
)
|
|
1907
|
-
)
|
|
1908
|
-
|
|
1909
|
-
# Lazy prefix method registry
|
|
1910
|
-
_lazy_prefix_method_registry: dict[str, tuple[str, ...]] = {}
|
|
1911
|
-
|
|
1912
|
-
# Package class hints
|
|
1913
|
-
_package_class_hints: dict[str, tuple[str, ...]] = {}
|
|
1914
|
-
_class_hint_lock = threading.RLock()
|
|
1915
|
-
|
|
1916
|
-
def _set_package_class_hints(package_name: str, hints: Iterable[str]) -> None:
|
|
1917
|
-
"""Set class hints for a package."""
|
|
1918
|
-
normalized: tuple[str, ...] = tuple(
|
|
1919
|
-
OrderedDict((hint.lower(), None) for hint in hints if hint).keys() # type: ignore[arg-type]
|
|
1920
|
-
)
|
|
1921
|
-
with _class_hint_lock:
|
|
1922
|
-
if normalized:
|
|
1923
|
-
_package_class_hints[package_name] = normalized
|
|
1924
|
-
else:
|
|
1925
|
-
_package_class_hints.pop(package_name, None)
|
|
1926
|
-
|
|
1927
|
-
def _get_package_class_hints(package_name: str) -> tuple[str, ...]:
|
|
1928
|
-
"""Get class hints for a package."""
|
|
1929
|
-
with _class_hint_lock:
|
|
1930
|
-
return _package_class_hints.get(package_name, ())
|
|
1931
|
-
|
|
1932
|
-
def _clear_all_package_class_hints() -> None:
|
|
1933
|
-
"""Clear all package class hints."""
|
|
1934
|
-
with _class_hint_lock:
|
|
1935
|
-
_package_class_hints.clear()
|
|
1936
|
-
|
|
1937
|
-
def register_lazy_module_methods(prefix: str, methods: tuple[str, ...]) -> None:
|
|
1938
|
-
"""Register method names to enhance for all classes under a module prefix."""
|
|
1939
|
-
prefix = prefix.strip()
|
|
1940
|
-
if not prefix:
|
|
1941
|
-
return
|
|
1942
|
-
|
|
1943
|
-
if not prefix.endswith("."):
|
|
1944
|
-
prefix += "."
|
|
1945
|
-
|
|
1946
|
-
_lazy_prefix_method_registry[prefix] = methods
|
|
1947
|
-
log_event("config", logger.info, f"Registered lazy module methods for prefix {prefix}: {methods}")
|
|
1948
|
-
|
|
1949
|
-
def _spec_for_existing_module(
|
|
1950
|
-
fullname: str,
|
|
1951
|
-
module: ModuleType,
|
|
1952
|
-
original_spec: Optional[importlib.machinery.ModuleSpec] = None,
|
|
1953
|
-
) -> importlib.machinery.ModuleSpec:
|
|
1954
|
-
"""Build a ModuleSpec whose loader simply returns an already-initialized module."""
|
|
1955
|
-
loader = _DeferredModuleLoader(module)
|
|
1956
|
-
spec = importlib.machinery.ModuleSpec(fullname, loader)
|
|
1957
|
-
if original_spec and original_spec.submodule_search_locations is not None:
|
|
1958
|
-
locations = list(original_spec.submodule_search_locations)
|
|
1959
|
-
spec.submodule_search_locations = locations
|
|
1960
|
-
if hasattr(module, "__path__"):
|
|
1961
|
-
module.__path__ = locations
|
|
1962
|
-
module.__loader__ = loader
|
|
1963
|
-
module.__spec__ = spec
|
|
1964
|
-
return spec
|
|
1965
|
-
|
|
1966
|
-
class LazyMetaPathFinder:
|
|
1967
|
-
"""
|
|
1968
|
-
Custom meta path finder that intercepts failed imports.
|
|
1969
|
-
Performance optimized - only triggers when import would fail anyway.
|
|
1970
|
-
"""
|
|
1971
|
-
|
|
1972
|
-
__slots__ = ('_package_name', '_enabled')
|
|
1973
|
-
|
|
1974
|
-
def __init__(self, package_name: str = 'default'):
|
|
1975
|
-
"""Initialize meta path finder."""
|
|
1976
|
-
self._package_name = package_name
|
|
1977
|
-
self._enabled = True
|
|
1978
|
-
|
|
1979
|
-
def _build_async_placeholder(
|
|
1980
|
-
self,
|
|
1981
|
-
fullname: str,
|
|
1982
|
-
installer: LazyInstaller,
|
|
1983
|
-
) -> Optional[importlib.machinery.ModuleSpec]:
|
|
1984
|
-
"""Create and register a deferred module placeholder for async installs."""
|
|
1985
|
-
handle = installer.ensure_async_install(fullname)
|
|
1986
|
-
if handle is None:
|
|
1987
|
-
return None
|
|
1988
|
-
|
|
1989
|
-
missing = ModuleNotFoundError(f"No module named '{fullname}'")
|
|
1990
|
-
deferred = DeferredImportError(fullname, missing, self._package_name, async_handle=handle)
|
|
1991
|
-
|
|
1992
|
-
module = ModuleType(fullname)
|
|
1993
|
-
loader = _DeferredModuleLoader(module)
|
|
1994
|
-
|
|
1995
|
-
def _resolve_real_module():
|
|
1996
|
-
real_module = deferred._try_install_and_import()
|
|
1997
|
-
sys.modules[fullname] = real_module
|
|
1998
|
-
module.__dict__.clear()
|
|
1999
|
-
module.__dict__.update(real_module.__dict__)
|
|
2000
|
-
module.__loader__ = getattr(real_module, "__loader__", loader)
|
|
2001
|
-
module.__spec__ = getattr(real_module, "__spec__", None)
|
|
2002
|
-
module.__path__ = getattr(real_module, "__path__", getattr(module, "__path__", []))
|
|
2003
|
-
module.__class__ = real_module.__class__
|
|
2004
|
-
try:
|
|
2005
|
-
spec_obj = getattr(real_module, "__spec__", None) or importlib.util.find_spec(fullname)
|
|
2006
|
-
if spec_obj is not None:
|
|
2007
|
-
_spec_cache_put(fullname, spec_obj)
|
|
2008
|
-
except (ValueError, AttributeError, ImportError):
|
|
2009
|
-
pass
|
|
2010
|
-
return real_module
|
|
2011
|
-
|
|
2012
|
-
def _module_getattr(name):
|
|
2013
|
-
real = _resolve_real_module()
|
|
2014
|
-
if name in module.__dict__:
|
|
2015
|
-
return module.__dict__[name]
|
|
2016
|
-
return getattr(real, name)
|
|
2017
|
-
|
|
2018
|
-
def _module_dir():
|
|
2019
|
-
try:
|
|
2020
|
-
real = _resolve_real_module()
|
|
2021
|
-
return dir(real)
|
|
2022
|
-
except Exception:
|
|
2023
|
-
return []
|
|
2024
|
-
|
|
2025
|
-
module.__getattr__ = _module_getattr # type: ignore[attr-defined]
|
|
2026
|
-
module.__dir__ = _module_dir # type: ignore[attr-defined]
|
|
2027
|
-
module.__loader__ = loader
|
|
2028
|
-
module.__package__ = fullname
|
|
2029
|
-
module.__path__ = []
|
|
2030
|
-
|
|
2031
|
-
spec = importlib.machinery.ModuleSpec(fullname, loader)
|
|
2032
|
-
spec.submodule_search_locations = []
|
|
2033
|
-
module.__spec__ = spec
|
|
2034
|
-
|
|
2035
|
-
sys.modules[fullname] = module
|
|
2036
|
-
log_event("hook", logger.info, f"⏳ [HOOK] Deferred import placeholder created for '{fullname}'")
|
|
2037
|
-
return spec
|
|
2038
|
-
|
|
2039
|
-
def find_module(self, fullname: str, path: Optional[str] = None):
|
|
2040
|
-
"""Find module - returns None to let standard import continue."""
|
|
2041
|
-
return None
|
|
2042
|
-
|
|
2043
|
-
def find_spec(self, fullname: str, path: Optional[str] = None, target=None):
|
|
2044
|
-
"""
|
|
2045
|
-
Find module spec - intercepts imports to enable two-stage lazy loading.
|
|
2046
|
-
|
|
2047
|
-
PERFORMANCE: Optimized for zero overhead on successful imports.
|
|
2048
|
-
"""
|
|
2049
|
-
# CRITICAL: Check installing state FIRST to prevent recursion during installation
|
|
2050
|
-
if getattr(_installing_state, 'active', False):
|
|
2051
|
-
logger.debug(f"[HOOK] Installation in progress, skipping {fullname} to prevent recursion")
|
|
2052
|
-
return None
|
|
2053
|
-
|
|
2054
|
-
# CRITICAL: Check if we're checking installation status to prevent infinite recursion
|
|
2055
|
-
if _is_checking_installation():
|
|
2056
|
-
logger.debug(f"[HOOK] Checking installation status, bypassing lazy finder for {fullname} to prevent recursion")
|
|
2057
|
-
return None
|
|
2058
|
-
|
|
2059
|
-
# Fast path 1: Hook disabled
|
|
2060
|
-
if not self._enabled:
|
|
2061
|
-
return None
|
|
2062
|
-
|
|
2063
|
-
# Fast path 2: Module already loaded - wrap it if needed
|
|
2064
|
-
if fullname in sys.modules:
|
|
2065
|
-
module = sys.modules[fullname]
|
|
2066
|
-
# Wrap classes in already-loaded modules to ensure auto-instantiation works
|
|
2067
|
-
if isinstance(module, ModuleType) and not getattr(module, '_xwlazy_wrapped', False):
|
|
2068
|
-
try:
|
|
2069
|
-
self._wrap_classes_for_auto_instantiation(module)
|
|
2070
|
-
module._xwlazy_wrapped = True # Mark as wrapped to avoid re-wrapping
|
|
2071
|
-
except Exception:
|
|
2072
|
-
pass
|
|
2073
|
-
return None
|
|
2074
|
-
|
|
2075
|
-
# Fast path 3: Skip C extension modules and internal modules
|
|
2076
|
-
# Also skip submodules that start with underscore (e.g., yaml._yaml)
|
|
2077
|
-
if fullname.startswith('_') or ('.' in fullname and fullname.split('.')[-1].startswith('_')):
|
|
2078
|
-
logger.debug(f"[HOOK] Skipping C extension/internal module {fullname}")
|
|
2079
|
-
return None
|
|
2080
|
-
|
|
2081
|
-
# Fast path 4: Check if parent package is partially initialized
|
|
2082
|
-
# CRITICAL: Skip ALL submodules of packages that are in sys.modules
|
|
2083
|
-
# This prevents circular import issues when a package imports its own submodules
|
|
2084
|
-
if '.' in fullname:
|
|
2085
|
-
parent_package = fullname.split('.', 1)[0]
|
|
2086
|
-
if parent_package in sys.modules:
|
|
2087
|
-
logger.debug(f"[HOOK] Skipping {fullname} - parent {parent_package} is in sys.modules (prevent circular import)")
|
|
2088
|
-
return None
|
|
2089
|
-
if _is_import_in_progress(parent_package):
|
|
2090
|
-
logger.debug(f"[HOOK] Skipping {fullname} - parent {parent_package} import in progress")
|
|
2091
|
-
return None
|
|
2092
|
-
|
|
2093
|
-
# ROOT CAUSE FIX: Check lazy install status FIRST
|
|
2094
|
-
root_name = fullname.split('.', 1)[0]
|
|
2095
|
-
_watched_registry = get_watched_registry()
|
|
2096
|
-
|
|
2097
|
-
# Check if lazy install is enabled
|
|
2098
|
-
lazy_install_enabled = LazyInstallConfig.is_enabled(self._package_name)
|
|
2099
|
-
install_mode = LazyInstallConfig.get_install_mode(self._package_name)
|
|
2100
|
-
|
|
2101
|
-
# If lazy install is disabled, only intercept watched modules
|
|
2102
|
-
if not lazy_install_enabled or install_mode == LazyInstallMode.NONE:
|
|
2103
|
-
if not _watched_registry.has_root(root_name):
|
|
2104
|
-
logger.debug(f"[HOOK] Module {fullname} not in watched registry and lazy install disabled, skipping interception")
|
|
2105
|
-
return None
|
|
2106
|
-
|
|
2107
|
-
# Check persistent installation cache FIRST
|
|
2108
|
-
try:
|
|
2109
|
-
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
2110
|
-
package_name = installer._dependency_mapper.get_package_name(root_name)
|
|
2111
|
-
|
|
2112
|
-
# ROOT CAUSE FIX: If this is the package we are managing, DO NOT skip interception!
|
|
2113
|
-
# We need to wrap it even if it is installed, so we can intercept its imports.
|
|
2114
|
-
should_skip = False
|
|
2115
|
-
if package_name == self._package_name or root_name == self._package_name:
|
|
2116
|
-
should_skip = False
|
|
2117
|
-
elif package_name:
|
|
2118
|
-
# Set flag to prevent recursion during installation check
|
|
2119
|
-
_set_checking_installation(True)
|
|
2120
|
-
try:
|
|
2121
|
-
if installer.is_package_installed(package_name):
|
|
2122
|
-
should_skip = True
|
|
2123
|
-
finally:
|
|
2124
|
-
_set_checking_installation(False)
|
|
2125
|
-
|
|
2126
|
-
if should_skip:
|
|
2127
|
-
logger.debug(f"[HOOK] Package {package_name} is installed (cache check), skipping interception of {fullname}")
|
|
2128
|
-
return None
|
|
2129
|
-
except Exception:
|
|
2130
|
-
pass
|
|
2131
|
-
|
|
2132
|
-
# Fast path 4: Cached spec
|
|
2133
|
-
cached_spec = _spec_cache_get(fullname)
|
|
2134
|
-
if cached_spec is not None:
|
|
2135
|
-
return cached_spec
|
|
2136
|
-
|
|
2137
|
-
# Fast path 5: Stdlib/builtin check
|
|
2138
|
-
if fullname.startswith('importlib') or fullname.startswith('_frozen_importlib'):
|
|
2139
|
-
return None
|
|
2140
|
-
|
|
2141
|
-
if '.' not in fullname:
|
|
2142
|
-
if DependencyMapper._is_stdlib_or_builtin(fullname):
|
|
2143
|
-
return None
|
|
2144
|
-
if fullname in DependencyMapper.DENY_LIST:
|
|
2145
|
-
return None
|
|
2146
|
-
|
|
2147
|
-
# Fast path 6: Import in progress
|
|
2148
|
-
# NOTE: We allow lazy install to proceed even if import is in progress,
|
|
2149
|
-
# because we need to install missing packages during import
|
|
2150
|
-
if _is_import_in_progress(fullname):
|
|
2151
|
-
# Only skip if lazy install is disabled (for watched modules, we still need to wrap)
|
|
2152
|
-
if not lazy_install_enabled and not _watched_registry.has_root(root_name):
|
|
2153
|
-
return None
|
|
2154
|
-
|
|
2155
|
-
# Only skip global importing state if lazy install is disabled
|
|
2156
|
-
# (lazy install needs to run even during imports to install missing packages)
|
|
2157
|
-
if getattr(_importing_state, 'active', False):
|
|
2158
|
-
if not lazy_install_enabled and not _watched_registry.has_root(root_name):
|
|
2159
|
-
return None
|
|
2160
|
-
|
|
2161
|
-
# Install mode check already done above
|
|
2162
|
-
matching_prefixes: tuple[str, ...] = ()
|
|
2163
|
-
if _watched_registry.has_root(root_name):
|
|
2164
|
-
matching_prefixes = _watched_registry.get_matching_prefixes(fullname)
|
|
2165
|
-
|
|
2166
|
-
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
2167
|
-
|
|
2168
|
-
# Two-stage lazy loading for serialization and archive modules
|
|
2169
|
-
if matching_prefixes:
|
|
2170
|
-
for prefix in matching_prefixes:
|
|
2171
|
-
if not _watched_registry.is_prefix_owned_by(self._package_name, prefix):
|
|
2172
|
-
continue
|
|
2173
|
-
if fullname.startswith(prefix):
|
|
2174
|
-
module_suffix = fullname[len(prefix):]
|
|
2175
|
-
|
|
2176
|
-
if module_suffix:
|
|
2177
|
-
log_event("hook", logger.info, f"[HOOK] Candidate for wrapping: {fullname}")
|
|
2178
|
-
|
|
2179
|
-
_mark_import_started(fullname)
|
|
2180
|
-
try:
|
|
2181
|
-
if getattr(_importing_state, 'active', False):
|
|
2182
|
-
logger.debug(f"[HOOK] Recursion guard active, skipping {fullname}")
|
|
2183
|
-
return None
|
|
2184
|
-
|
|
2185
|
-
try:
|
|
2186
|
-
logger.debug(f"[HOOK] Looking for spec: {fullname}")
|
|
2187
|
-
spec = _spec_cache_get(fullname)
|
|
2188
|
-
if spec is None:
|
|
2189
|
-
try:
|
|
2190
|
-
spec = importlib.util.find_spec(fullname)
|
|
2191
|
-
except (ValueError, AttributeError, ImportError):
|
|
2192
|
-
pass
|
|
2193
|
-
if spec is not None:
|
|
2194
|
-
_spec_cache_put(fullname, spec)
|
|
2195
|
-
if spec is not None:
|
|
2196
|
-
logger.debug(f"[HOOK] Spec found, trying normal import: {fullname}")
|
|
2197
|
-
_importing_state.active = True
|
|
2198
|
-
try:
|
|
2199
|
-
__import__(fullname)
|
|
2200
|
-
|
|
2201
|
-
module = sys.modules.get(fullname)
|
|
2202
|
-
if module:
|
|
2203
|
-
try:
|
|
2204
|
-
self._enhance_classes_with_class_methods(module)
|
|
2205
|
-
# Enable auto-instantiation for classes in this module
|
|
2206
|
-
self._wrap_classes_for_auto_instantiation(module)
|
|
2207
|
-
except Exception as enhance_exc:
|
|
2208
|
-
logger.debug(f"[HOOK] Could not enhance classes in {fullname}: {enhance_exc}")
|
|
2209
|
-
spec = _spec_for_existing_module(fullname, module, spec)
|
|
2210
|
-
log_event("hook", logger.info, f"✓ [HOOK] Module {fullname} imported successfully, no wrapping needed")
|
|
2211
|
-
if spec is not None:
|
|
2212
|
-
_spec_cache_put(fullname, spec)
|
|
2213
|
-
return spec
|
|
2214
|
-
return None
|
|
2215
|
-
finally:
|
|
2216
|
-
_importing_state.active = False
|
|
2217
|
-
except ImportError as e:
|
|
2218
|
-
if '.' not in module_suffix:
|
|
2219
|
-
log_event("hook", logger.info, f"⚠ [HOOK] Module {fullname} has missing dependencies, wrapping: {e}")
|
|
2220
|
-
wrapped_spec = self._wrap_serialization_module(fullname)
|
|
2221
|
-
if wrapped_spec is not None:
|
|
2222
|
-
log_event("hook", logger.info, f"✓ [HOOK] Successfully wrapped: {fullname}")
|
|
2223
|
-
return wrapped_spec
|
|
2224
|
-
logger.warning(f"✗ [HOOK] Failed to wrap: {fullname}")
|
|
2225
|
-
else:
|
|
2226
|
-
logger.debug(f"[HOOK] Import failed for nested module {fullname}: {e}")
|
|
2227
|
-
except (ModuleNotFoundError,) as e:
|
|
2228
|
-
logger.debug(f"[HOOK] Module {fullname} not found, skipping wrap: {e}")
|
|
2229
|
-
pass
|
|
2230
|
-
except Exception as e:
|
|
2231
|
-
logger.warning(f"[HOOK] Error checking module {fullname}: {e}")
|
|
2232
|
-
finally:
|
|
2233
|
-
_mark_import_finished(fullname)
|
|
2234
|
-
|
|
2235
|
-
return None
|
|
2236
|
-
|
|
2237
|
-
# If we had matching prefixes but didn't match any, continue to lazy install logic
|
|
2238
|
-
if matching_prefixes:
|
|
2239
|
-
logger.debug(f"[HOOK] {fullname} had matching prefixes but didn't match any, continuing to lazy install")
|
|
2240
|
-
|
|
2241
|
-
# For lazy installation, handle submodules by checking if parent package is installed
|
|
2242
|
-
if '.' in fullname:
|
|
2243
|
-
parent_package = fullname.split('.', 1)[0]
|
|
2244
|
-
if lazy_install_enabled:
|
|
2245
|
-
try:
|
|
2246
|
-
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
2247
|
-
package_name = installer._dependency_mapper.get_package_name(parent_package)
|
|
2248
|
-
if package_name:
|
|
2249
|
-
# Set flag to prevent recursion during installation check
|
|
2250
|
-
_set_checking_installation(True)
|
|
2251
|
-
try:
|
|
2252
|
-
if not installer.is_package_installed(package_name):
|
|
2253
|
-
logger.debug(f"[HOOK] Parent package {parent_package} not installed, intercepting parent")
|
|
2254
|
-
return self.find_spec(parent_package, path, target)
|
|
2255
|
-
finally:
|
|
2256
|
-
_set_checking_installation(False)
|
|
2257
|
-
except Exception:
|
|
2258
|
-
pass
|
|
2259
|
-
return None
|
|
2260
|
-
if DependencyMapper._is_stdlib_or_builtin(fullname):
|
|
2261
|
-
return None
|
|
2262
|
-
if fullname in DependencyMapper.DENY_LIST:
|
|
2263
|
-
return None
|
|
2264
|
-
|
|
2265
|
-
# ROOT CAUSE FIX: For lazy installation, intercept missing imports and install them
|
|
2266
|
-
logger.debug(f"[HOOK] Checking lazy install for {fullname}: enabled={lazy_install_enabled}, install_mode={install_mode}")
|
|
2267
|
-
if lazy_install_enabled:
|
|
2268
|
-
# Prevent infinite loops: Skip if already attempting import
|
|
2269
|
-
if _is_import_in_progress(fullname):
|
|
2270
|
-
logger.debug(f"[HOOK] Import {fullname} already in progress, skipping to prevent recursion")
|
|
2271
|
-
return None
|
|
2272
|
-
|
|
2273
|
-
# Prevent infinite loops: Check if we've already tried to install this package
|
|
2274
|
-
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
2275
|
-
package_name = installer._dependency_mapper.get_package_name(root_name)
|
|
2276
|
-
|
|
2277
|
-
# ROOT CAUSE FIX: Check if module is ALREADY importable BEFORE checking package installation
|
|
2278
|
-
# This handles cases where package name != module name (e.g., PyYAML -> yaml)
|
|
2279
|
-
try:
|
|
2280
|
-
# Try direct import first (most reliable check)
|
|
2281
|
-
if fullname in sys.modules:
|
|
2282
|
-
logger.debug(f"[HOOK] Module {fullname} already in sys.modules, skipping installation")
|
|
2283
|
-
return None
|
|
2284
|
-
|
|
2285
|
-
# Temporarily remove finders to check actual importability
|
|
2286
|
-
xwlazy_finder_names = {'LazyMetaPathFinder', 'LazyPathFinder', 'LazyLoader'}
|
|
2287
|
-
xwlazy_finders = [f for f in sys.meta_path if type(f).__name__ in xwlazy_finder_names]
|
|
2288
|
-
for finder in xwlazy_finders:
|
|
2289
|
-
try:
|
|
2290
|
-
sys.meta_path.remove(finder)
|
|
2291
|
-
except ValueError:
|
|
2292
|
-
pass
|
|
2293
|
-
|
|
2294
|
-
try:
|
|
2295
|
-
# Check spec without importing (avoids triggering module code execution)
|
|
2296
|
-
# This prevents circular import issues when checking importability
|
|
2297
|
-
spec = importlib.util.find_spec(fullname)
|
|
2298
|
-
if spec is not None and spec.loader is not None:
|
|
2299
|
-
logger.debug(f"[HOOK] Module {fullname} has valid spec, skipping installation")
|
|
2300
|
-
return None
|
|
2301
|
-
finally:
|
|
2302
|
-
# Restore finders
|
|
2303
|
-
for finder in reversed(xwlazy_finders):
|
|
2304
|
-
if finder not in sys.meta_path:
|
|
2305
|
-
sys.meta_path.insert(0, finder)
|
|
2306
|
-
except Exception as e:
|
|
2307
|
-
logger.debug(f"[HOOK] Importability check failed for {fullname}: {e}")
|
|
2308
|
-
# If check fails, proceed with installation attempt
|
|
2309
|
-
|
|
2310
|
-
if package_name:
|
|
2311
|
-
if package_name in installer.get_failed_packages():
|
|
2312
|
-
logger.debug(f"[HOOK] Package {package_name} previously failed installation, skipping {fullname}")
|
|
2313
|
-
return None
|
|
2314
|
-
|
|
2315
|
-
# Also check package installation status (for cache efficiency)
|
|
2316
|
-
# Set flag to prevent recursion during installation check
|
|
2317
|
-
_set_checking_installation(True)
|
|
2318
|
-
try:
|
|
2319
|
-
if installer.is_package_installed(package_name):
|
|
2320
|
-
logger.debug(f"[HOOK] Package {package_name} is already installed, skipping installation attempt for {fullname}")
|
|
2321
|
-
return None
|
|
2322
|
-
finally:
|
|
2323
|
-
_set_checking_installation(False)
|
|
2324
|
-
|
|
2325
|
-
_mark_import_started(fullname)
|
|
2326
|
-
try:
|
|
2327
|
-
# Guard against recursion when importing facade
|
|
2328
|
-
if 'exonware.xwlazy.facade' in sys.modules or 'xwlazy.facade' in sys.modules:
|
|
2329
|
-
from ..facade import lazy_import_with_install
|
|
2330
|
-
else:
|
|
2331
|
-
if _is_import_in_progress('exonware.xwlazy.facade') or _is_import_in_progress('xwlazy.facade'):
|
|
2332
|
-
logger.debug(f"[HOOK] Facade import in progress, skipping {fullname} to prevent recursion")
|
|
2333
|
-
return None
|
|
2334
|
-
from ..facade import lazy_import_with_install
|
|
2335
|
-
|
|
2336
|
-
# Log installation attempt (DEBUG level to reduce noise)
|
|
2337
|
-
if package_name:
|
|
2338
|
-
logger.debug(f"⏳ [HOOK] Attempting to install package '{package_name}' for module '{fullname}'")
|
|
2339
|
-
else:
|
|
2340
|
-
logger.debug(f"⏳ [HOOK] Attempting to install module '{fullname}' (no package mapping found)")
|
|
2341
|
-
|
|
2342
|
-
_importing_state.active = True
|
|
2343
|
-
try:
|
|
2344
|
-
module, success = lazy_import_with_install(
|
|
2345
|
-
fullname,
|
|
2346
|
-
installer_package=self._package_name
|
|
2347
|
-
)
|
|
2348
|
-
finally:
|
|
2349
|
-
_importing_state.active = False
|
|
2350
|
-
|
|
2351
|
-
if success and module:
|
|
2352
|
-
# Module was successfully installed and imported
|
|
2353
|
-
logger.debug(f"✅ [HOOK] Successfully installed and imported '{fullname}'")
|
|
2354
|
-
xwlazy_finder_names = {'LazyMetaPathFinder', 'LazyPathFinder', 'LazyLoader'}
|
|
2355
|
-
xwlazy_finders = [f for f in sys.meta_path if type(f).__name__ in xwlazy_finder_names]
|
|
2356
|
-
for finder in xwlazy_finders:
|
|
2357
|
-
try:
|
|
2358
|
-
sys.meta_path.remove(finder)
|
|
2359
|
-
except ValueError:
|
|
2360
|
-
pass
|
|
2361
|
-
|
|
2362
|
-
try:
|
|
2363
|
-
if fullname in sys.modules:
|
|
2364
|
-
del sys.modules[fullname]
|
|
2365
|
-
importlib.invalidate_caches()
|
|
2366
|
-
sys.path_importer_cache.clear()
|
|
2367
|
-
real_module = importlib.import_module(fullname)
|
|
2368
|
-
sys.modules[fullname] = real_module
|
|
2369
|
-
logger.debug(f"[HOOK] Successfully installed {fullname}, replaced module in sys.modules with real module")
|
|
2370
|
-
finally:
|
|
2371
|
-
for finder in reversed(xwlazy_finders):
|
|
2372
|
-
if finder not in sys.meta_path:
|
|
2373
|
-
sys.meta_path.insert(0, finder)
|
|
2374
|
-
return None
|
|
2375
|
-
else:
|
|
2376
|
-
# Only log warning if it's not a known missing package (reduce noise)
|
|
2377
|
-
if package_name and package_name not in installer.get_failed_packages():
|
|
2378
|
-
logger.debug(f"❌ [HOOK] Failed to install/import {fullname}")
|
|
2379
|
-
else:
|
|
2380
|
-
logger.debug(f"❌ [HOOK] Failed to install/import {fullname} (already marked as failed)")
|
|
2381
|
-
# Mark as failed to prevent infinite retry loops
|
|
2382
|
-
if package_name:
|
|
2383
|
-
installer._failed_packages.add(package_name)
|
|
2384
|
-
try:
|
|
2385
|
-
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
2386
|
-
# Force disable async install usage in engine
|
|
2387
|
-
use_async = False # installer.is_async_enabled()
|
|
2388
|
-
if use_async:
|
|
2389
|
-
placeholder = self._build_async_placeholder(fullname, installer)
|
|
2390
|
-
if placeholder is not None:
|
|
2391
|
-
return placeholder
|
|
2392
|
-
except Exception:
|
|
2393
|
-
pass
|
|
2394
|
-
# Return None to let Python handle the ImportError naturally
|
|
2395
|
-
return None
|
|
2396
|
-
except Exception as e:
|
|
2397
|
-
logger.error(f"❌ [HOOK] Lazy import hook failed for {fullname}: {e}", exc_info=True)
|
|
2398
|
-
# Mark as failed to prevent infinite retry loops
|
|
2399
|
-
if package_name:
|
|
2400
|
-
try:
|
|
2401
|
-
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
2402
|
-
installer._failed_packages.add(package_name)
|
|
2403
|
-
except Exception:
|
|
2404
|
-
pass
|
|
2405
|
-
return None
|
|
2406
|
-
finally:
|
|
2407
|
-
_mark_import_finished(fullname)
|
|
2408
|
-
|
|
2409
|
-
def _wrap_serialization_module(self, fullname: str):
|
|
2410
|
-
"""Wrap serialization module loading to defer missing dependencies."""
|
|
2411
|
-
log_event("hook", logger.info, f"[STAGE 1] Starting wrap of module: {fullname}")
|
|
2412
|
-
|
|
2413
|
-
try:
|
|
2414
|
-
logger.debug(f"[STAGE 1] Getting spec for: {fullname}")
|
|
2415
|
-
try:
|
|
2416
|
-
sys.meta_path.remove(self)
|
|
2417
|
-
except ValueError:
|
|
2418
|
-
pass
|
|
2419
|
-
try:
|
|
2420
|
-
spec = importlib.util.find_spec(fullname)
|
|
2421
|
-
finally:
|
|
2422
|
-
if self not in sys.meta_path:
|
|
2423
|
-
sys.meta_path.insert(0, self)
|
|
2424
|
-
if not spec or not spec.loader:
|
|
2425
|
-
logger.warning(f"[STAGE 1] No spec or loader for: {fullname}")
|
|
2426
|
-
return None
|
|
2427
|
-
|
|
2428
|
-
logger.debug(f"[STAGE 1] Creating module from spec: {fullname}")
|
|
2429
|
-
module = importlib.util.module_from_spec(spec)
|
|
2430
|
-
|
|
2431
|
-
deferred_imports = {}
|
|
2432
|
-
|
|
2433
|
-
logger.debug(f"[STAGE 1] Setting up import wrapper for: {fullname}")
|
|
2434
|
-
original_import = builtins.__import__
|
|
2435
|
-
|
|
2436
|
-
def capture_import_errors(name, *args, **kwargs):
|
|
2437
|
-
"""Intercept imports and defer ONLY external missing packages."""
|
|
2438
|
-
logger.debug(f"[STAGE 1] capture_import_errors: Trying to import '{name}' in {fullname}")
|
|
2439
|
-
|
|
2440
|
-
if _is_import_in_progress(name):
|
|
2441
|
-
logger.debug(f"[STAGE 1] Import '{name}' already in progress, using original_import")
|
|
2442
|
-
return original_import(name, *args, **kwargs)
|
|
2443
|
-
|
|
2444
|
-
_mark_import_started(name)
|
|
2445
|
-
try:
|
|
2446
|
-
result = original_import(name, *args, **kwargs)
|
|
2447
|
-
logger.debug(f"[STAGE 1] ✓ Successfully imported '{name}'")
|
|
2448
|
-
return result
|
|
2449
|
-
except ImportError as e:
|
|
2450
|
-
logger.debug(f"[STAGE 1] ✗ Import failed for '{name}': {e}")
|
|
2451
|
-
|
|
2452
|
-
host_alias = self._package_name or ""
|
|
2453
|
-
if name.startswith('exonware.') or (host_alias and name.startswith(f"{host_alias}.")):
|
|
2454
|
-
log_event("hook", logger.info, f"[STAGE 1] Letting internal import '{name}' fail normally (internal package)")
|
|
2455
|
-
raise
|
|
2456
|
-
|
|
2457
|
-
if '.' in name:
|
|
2458
|
-
log_event("hook", logger.info, f"[STAGE 1] Letting submodule '{name}' fail normally (has dots)")
|
|
2459
|
-
raise
|
|
2460
|
-
|
|
2461
|
-
log_event("hook", logger.info, f"⏳ [STAGE 1] DEFERRING missing external package '{name}' in {fullname}")
|
|
2462
|
-
async_handle = None
|
|
2463
|
-
try:
|
|
2464
|
-
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
2465
|
-
async_handle = installer.schedule_async_install(name)
|
|
2466
|
-
except Exception as schedule_exc:
|
|
2467
|
-
logger.debug(f"[STAGE 1] Async install scheduling failed for '{name}': {schedule_exc}")
|
|
2468
|
-
deferred = DeferredImportError(name, e, self._package_name, async_handle=async_handle)
|
|
2469
|
-
deferred_imports[name] = deferred
|
|
2470
|
-
|
|
2471
|
-
# ROOT CAUSE FIX: Register deferred object in sys.modules to prevent infinite import loops
|
|
2472
|
-
# If we don't do this, subsequent imports of the same missing module will trigger find_spec again
|
|
2473
|
-
sys.modules[name] = deferred
|
|
2474
|
-
|
|
2475
|
-
return deferred
|
|
2476
|
-
finally:
|
|
2477
|
-
_mark_import_finished(name)
|
|
2478
|
-
|
|
2479
|
-
logger.debug(f"[STAGE 1] Executing module with import wrapper: {fullname}")
|
|
2480
|
-
builtins.__import__ = capture_import_errors
|
|
2481
|
-
try:
|
|
2482
|
-
spec.loader.exec_module(module)
|
|
2483
|
-
logger.debug(f"[STAGE 1] Module execution completed: {fullname}")
|
|
2484
|
-
|
|
2485
|
-
if deferred_imports:
|
|
2486
|
-
log_event("hook", logger.info, f"✓ [STAGE 1] Module {fullname} loaded with {len(deferred_imports)} deferred imports: {list(deferred_imports.keys())}")
|
|
2487
|
-
self._replace_none_with_deferred(module, deferred_imports)
|
|
2488
|
-
self._wrap_module_classes(module, deferred_imports)
|
|
2489
|
-
else:
|
|
2490
|
-
log_event("hook", logger.info, f"✓ [STAGE 1] Module {fullname} loaded with NO deferred imports (all dependencies available)")
|
|
2491
|
-
|
|
2492
|
-
self._enhance_classes_with_class_methods(module)
|
|
2493
|
-
|
|
2494
|
-
# Enable auto-instantiation for classes in this module
|
|
2495
|
-
self._wrap_classes_for_auto_instantiation(module)
|
|
2496
|
-
|
|
2497
|
-
finally:
|
|
2498
|
-
logger.debug(f"[STAGE 1] Restoring original __import__")
|
|
2499
|
-
builtins.__import__ = original_import
|
|
2500
|
-
|
|
2501
|
-
logger.debug(f"[STAGE 1] Registering module in sys.modules: {fullname}")
|
|
2502
|
-
sys.modules[fullname] = module
|
|
2503
|
-
final_spec = _spec_for_existing_module(fullname, module, spec)
|
|
2504
|
-
_spec_cache_put(fullname, final_spec)
|
|
2505
|
-
log_event("hook", logger.info, f"✓ [STAGE 1] Successfully wrapped and registered: {fullname}")
|
|
2506
|
-
return final_spec
|
|
2507
|
-
|
|
2508
|
-
except Exception as e:
|
|
2509
|
-
logger.debug(f"Could not wrap {fullname}: {e}")
|
|
2510
|
-
return None
|
|
2511
|
-
|
|
2512
|
-
def _replace_none_with_deferred(self, module, deferred_imports: Dict):
|
|
2513
|
-
"""Replace None values in module namespace with deferred import proxies."""
|
|
2514
|
-
logger.debug(f"[STAGE 1] Replacing None with deferred imports in {module.__name__}")
|
|
2515
|
-
replaced_count = 0
|
|
2516
|
-
|
|
2517
|
-
for dep_name, deferred_import in deferred_imports.items():
|
|
2518
|
-
if hasattr(module, dep_name):
|
|
2519
|
-
current_value = getattr(module, dep_name)
|
|
2520
|
-
if current_value is None:
|
|
2521
|
-
log_event("hook", logger.info, f"[STAGE 1] Replacing {dep_name}=None with deferred import proxy in {module.__name__}")
|
|
2522
|
-
setattr(module, dep_name, deferred_import)
|
|
2523
|
-
replaced_count += 1
|
|
2524
|
-
|
|
2525
|
-
if replaced_count > 0:
|
|
2526
|
-
log_event("hook", logger.info, f"✓ [STAGE 1] Replaced {replaced_count} None values with deferred imports in {module.__name__}")
|
|
2527
|
-
|
|
2528
|
-
def _wrap_module_classes(self, module, deferred_imports: Dict):
|
|
2529
|
-
"""Wrap classes in a module that depend on deferred imports."""
|
|
2530
|
-
module_name = getattr(module, '__name__', '<unknown>')
|
|
2531
|
-
logger.debug(f"[STAGE 1] Wrapping classes in {module_name} (deferred: {list(deferred_imports.keys())})")
|
|
2532
|
-
module_file = (getattr(module, '__file__', '') or '').lower()
|
|
2533
|
-
lower_map = {dep_name.lower(): dep_name for dep_name in deferred_imports.keys()}
|
|
2534
|
-
class_hints = _get_package_class_hints(self._package_name)
|
|
2535
|
-
with _wrapped_cache_lock:
|
|
2536
|
-
already_wrapped = _WRAPPED_CLASS_CACHE.setdefault(module_name, set()).copy()
|
|
2537
|
-
pending_lower = {lower for lower in lower_map.keys() if lower_map[lower] not in already_wrapped}
|
|
2538
|
-
if not pending_lower:
|
|
2539
|
-
logger.debug(f"[STAGE 1] All deferred imports already wrapped for {module_name}")
|
|
2540
|
-
return
|
|
2541
|
-
dep_entries = [(lower, deferred_imports[lower_map[lower]]) for lower in pending_lower]
|
|
2542
|
-
wrapped_count = 0
|
|
2543
|
-
newly_wrapped: set[str] = set()
|
|
2544
|
-
|
|
2545
|
-
for name, obj in list(module.__dict__.items()):
|
|
2546
|
-
if not pending_lower:
|
|
2547
|
-
break
|
|
2548
|
-
if not isinstance(obj, type):
|
|
2549
|
-
continue
|
|
2550
|
-
lower_name = name.lower()
|
|
2551
|
-
if class_hints and not any(hint in lower_name for hint in class_hints):
|
|
2552
|
-
continue
|
|
2553
|
-
target_lower = None
|
|
2554
|
-
target_deferred = None
|
|
2555
|
-
for dep_lower, deferred in dep_entries:
|
|
2556
|
-
if dep_lower not in pending_lower:
|
|
2557
|
-
continue
|
|
2558
|
-
if dep_lower in lower_name or dep_lower in module_file:
|
|
2559
|
-
target_lower = dep_lower
|
|
2560
|
-
target_deferred = deferred
|
|
2561
|
-
break
|
|
2562
|
-
if target_deferred is None or target_lower is None:
|
|
2563
|
-
continue
|
|
2564
|
-
|
|
2565
|
-
logger.debug(f"[STAGE 1] Class '{name}' depends on deferred import, wrapping...")
|
|
2566
|
-
wrapped = self._create_lazy_class_wrapper(obj, target_deferred)
|
|
2567
|
-
module.__dict__[name] = wrapped
|
|
2568
|
-
wrapped_count += 1
|
|
2569
|
-
origin_name = lower_map.get(target_lower, target_lower)
|
|
2570
|
-
newly_wrapped.add(origin_name)
|
|
2571
|
-
pending_lower.discard(target_lower)
|
|
2572
|
-
log_event("hook", logger.info, f"✓ [STAGE 1] Wrapped class '{name}' in {module_name}")
|
|
2573
|
-
|
|
2574
|
-
if newly_wrapped:
|
|
2575
|
-
with _wrapped_cache_lock:
|
|
2576
|
-
cache = _WRAPPED_CLASS_CACHE.setdefault(module_name, set())
|
|
2577
|
-
cache.update(newly_wrapped)
|
|
2578
|
-
|
|
2579
|
-
log_event("hook", logger.info, f"[STAGE 1] Wrapped {wrapped_count} classes in {module_name}")
|
|
2580
|
-
|
|
2581
|
-
def _wrap_classes_for_auto_instantiation(self, module: ModuleType) -> None:
|
|
2582
|
-
"""
|
|
2583
|
-
Wrap classes in modules with AutoInstantiateProxy for auto-instantiation.
|
|
2584
|
-
|
|
2585
|
-
This enables: from module import Class as instance
|
|
2586
|
-
Then: instance.method() works automatically without manual instantiation.
|
|
2587
|
-
|
|
2588
|
-
We wrap classes directly in module.__dict__ AND add a __getattr__ fallback
|
|
2589
|
-
to catch any classes that might be accessed before wrapping completes.
|
|
2590
|
-
|
|
2591
|
-
We wrap ALL classes in the module's namespace, whether defined in this
|
|
2592
|
-
module or imported from submodules (like BsonSerializer imported in __init__.py).
|
|
2593
|
-
"""
|
|
2594
|
-
import inspect
|
|
2595
|
-
import types
|
|
2596
|
-
|
|
2597
|
-
# Store original __getattr__ if it exists
|
|
2598
|
-
original_getattr = getattr(module, '__getattr__', None)
|
|
2599
|
-
|
|
2600
|
-
wrapped_count = 0
|
|
2601
|
-
wrapped_classes = {} # Track wrapped classes for __getattr__
|
|
2602
|
-
|
|
2603
|
-
for name, obj in list(module.__dict__.items()):
|
|
2604
|
-
# Wrap classes that are in this module's namespace
|
|
2605
|
-
# This includes classes defined here AND classes imported from submodules
|
|
2606
|
-
if (inspect.isclass(obj) and
|
|
2607
|
-
not inspect.isbuiltin(obj) and
|
|
2608
|
-
hasattr(obj, '__module__')):
|
|
2609
|
-
# Skip if it's already a proxy
|
|
2610
|
-
if isinstance(obj, AutoInstantiateProxy):
|
|
2611
|
-
continue
|
|
2612
|
-
# Skip if it's a wrapper class
|
|
2613
|
-
if obj.__name__.startswith('Lazy'):
|
|
2614
|
-
continue
|
|
2615
|
-
# Skip if it's a type from typing or other special modules
|
|
2616
|
-
module_name = obj.__module__
|
|
2617
|
-
if module_name in ('typing', 'builtins', '__builtin__'):
|
|
2618
|
-
continue
|
|
2619
|
-
|
|
2620
|
-
logger.debug(f"[AUTO-INST] Wrapping class '{name}' ({module_name}) in {module.__name__} for auto-instantiation")
|
|
2621
|
-
proxy = AutoInstantiateProxy(obj)
|
|
2622
|
-
module.__dict__[name] = proxy
|
|
2623
|
-
wrapped_classes[name] = proxy
|
|
2624
|
-
wrapped_count += 1
|
|
2625
|
-
|
|
2626
|
-
# Add __getattr__ fallback to wrap classes on access (for classes added after initial load)
|
|
2627
|
-
if wrapped_count > 0 or True: # Always add __getattr__ for consistency
|
|
2628
|
-
def auto_instantiate_getattr(name: str):
|
|
2629
|
-
"""Module-level __getattr__ that wraps classes for auto-instantiation."""
|
|
2630
|
-
# First try original __getattr__
|
|
2631
|
-
if original_getattr:
|
|
2632
|
-
try:
|
|
2633
|
-
attr = original_getattr(name)
|
|
2634
|
-
# If it's a class, wrap it
|
|
2635
|
-
if inspect.isclass(attr) and not inspect.isbuiltin(attr):
|
|
2636
|
-
if not isinstance(attr, AutoInstantiateProxy):
|
|
2637
|
-
logger.debug(f"[AUTO-INST] Wrapping class '{name}' on access in {module.__name__}")
|
|
2638
|
-
proxy = AutoInstantiateProxy(attr)
|
|
2639
|
-
module.__dict__[name] = proxy # Cache it
|
|
2640
|
-
return proxy
|
|
2641
|
-
return attr
|
|
2642
|
-
except AttributeError:
|
|
2643
|
-
pass
|
|
2644
|
-
|
|
2645
|
-
# Get from __dict__ (might be unwrapped)
|
|
2646
|
-
if name in module.__dict__:
|
|
2647
|
-
attr = module.__dict__[name]
|
|
2648
|
-
# If it's an unwrapped class, wrap it now
|
|
2649
|
-
if (inspect.isclass(attr) and
|
|
2650
|
-
not inspect.isbuiltin(attr) and
|
|
2651
|
-
not isinstance(attr, AutoInstantiateProxy) and
|
|
2652
|
-
hasattr(attr, '__module__')):
|
|
2653
|
-
module_name = attr.__module__
|
|
2654
|
-
if module_name not in ('typing', 'builtins', '__builtin__'):
|
|
2655
|
-
logger.debug(f"[AUTO-INST] Wrapping class '{name}' on access in {module.__name__}")
|
|
2656
|
-
proxy = AutoInstantiateProxy(attr)
|
|
2657
|
-
module.__dict__[name] = proxy # Cache it
|
|
2658
|
-
return proxy
|
|
2659
|
-
return attr
|
|
2660
|
-
|
|
2661
|
-
raise AttributeError(f"module '{module.__name__}' has no attribute '{name}'")
|
|
2662
|
-
|
|
2663
|
-
# Only set __getattr__ if module doesn't already have a custom one
|
|
2664
|
-
if not hasattr(module, '__getattr__') or isinstance(getattr(module, '__getattr__', None), types.MethodType):
|
|
2665
|
-
module.__getattr__ = auto_instantiate_getattr # type: ignore[attr-defined]
|
|
2666
|
-
|
|
2667
|
-
if wrapped_count > 0:
|
|
2668
|
-
logger.debug(f"[AUTO-INST] Wrapped {wrapped_count} classes in {module.__name__} for auto-instantiation")
|
|
2669
|
-
|
|
2670
|
-
def _create_lazy_class_wrapper(self, original_class, deferred_import: DeferredImportError):
|
|
2671
|
-
"""Create a wrapper class that installs dependencies when instantiated."""
|
|
2672
|
-
class LazyClassWrapper:
|
|
2673
|
-
"""Lazy wrapper that installs dependencies on first instantiation."""
|
|
2674
|
-
|
|
2675
|
-
def __init__(self, *args, **kwargs):
|
|
2676
|
-
"""Install dependency and create real instance."""
|
|
2677
|
-
deferred_import._try_install_and_import()
|
|
2678
|
-
|
|
2679
|
-
real_module = importlib.reload(sys.modules[original_class.__module__])
|
|
2680
|
-
real_class = getattr(real_module, original_class.__name__)
|
|
2681
|
-
|
|
2682
|
-
real_instance = real_class(*args, **kwargs)
|
|
2683
|
-
self.__class__ = real_class
|
|
2684
|
-
self.__dict__ = real_instance.__dict__
|
|
2685
|
-
|
|
2686
|
-
def __repr__(self):
|
|
2687
|
-
return f"<Lazy{original_class.__name__}: will install dependencies on init>"
|
|
2688
|
-
|
|
2689
|
-
LazyClassWrapper.__name__ = f"Lazy{original_class.__name__}"
|
|
2690
|
-
LazyClassWrapper.__qualname__ = f"Lazy{original_class.__qualname__}"
|
|
2691
|
-
LazyClassWrapper.__module__ = original_class.__module__
|
|
2692
|
-
LazyClassWrapper.__doc__ = original_class.__doc__
|
|
2693
|
-
|
|
2694
|
-
return LazyClassWrapper
|
|
2695
|
-
|
|
2696
|
-
def _enhance_classes_with_class_methods(self, module):
|
|
2697
|
-
"""Enhance classes with lazy class methods - automatically detects ALL instance methods."""
|
|
2698
|
-
if module is None:
|
|
2699
|
-
return
|
|
2700
|
-
# Debug: Check if module has classes
|
|
2701
|
-
module_classes = [name for name, obj in module.__dict__.items() if isinstance(obj, type)]
|
|
2702
|
-
if not module_classes:
|
|
2703
|
-
return # No classes to enhance
|
|
2704
|
-
|
|
2705
|
-
# Get methods from registry (if any) for specific prefixes
|
|
2706
|
-
methods_to_apply: tuple[str, ...] = ()
|
|
2707
|
-
for prefix, methods in _lazy_prefix_method_registry.items():
|
|
2708
|
-
if module.__name__.startswith(prefix.rstrip('.')):
|
|
2709
|
-
methods_to_apply = methods
|
|
2710
|
-
break
|
|
2711
|
-
|
|
2712
|
-
# If no prefix match, use default methods from env var
|
|
2713
|
-
if not methods_to_apply:
|
|
2714
|
-
methods_to_apply = _DEFAULT_LAZY_METHODS
|
|
2715
|
-
|
|
2716
|
-
# CRITICAL FIX: If no specific methods registered, enhance ALL instance methods
|
|
2717
|
-
# This makes xwlazy handle ALL scenarios (classes, functions, instance methods)
|
|
2718
|
-
auto_detect_all = not methods_to_apply
|
|
2719
|
-
|
|
2720
|
-
enhanced = 0
|
|
2721
|
-
for name, obj in list(module.__dict__.items()):
|
|
2722
|
-
if not isinstance(obj, type):
|
|
2723
|
-
continue
|
|
2724
|
-
|
|
2725
|
-
# Get methods to enhance for this class
|
|
2726
|
-
if auto_detect_all:
|
|
2727
|
-
# Auto-detect instance methods defined DIRECTLY in this class (not inherited)
|
|
2728
|
-
# This prevents wrapping wrong methods from parent classes
|
|
2729
|
-
methods_to_wrap = []
|
|
2730
|
-
# Only check methods in this class's __dict__ to avoid inherited methods
|
|
2731
|
-
for attr_name, attr_value in obj.__dict__.items():
|
|
2732
|
-
if attr_name.startswith('_') and attr_name not in ('__init__', '__new__'):
|
|
2733
|
-
continue # Skip private methods except __init__ and __new__
|
|
2734
|
-
if not callable(attr_value):
|
|
2735
|
-
continue
|
|
2736
|
-
if isinstance(attr_value, (classmethod, staticmethod)):
|
|
2737
|
-
continue
|
|
2738
|
-
if getattr(attr_value, "__lazy_wrapped__", False):
|
|
2739
|
-
continue
|
|
2740
|
-
# Check if it's an instance method (has 'self' as first param)
|
|
2741
|
-
import inspect
|
|
2742
|
-
try:
|
|
2743
|
-
if inspect.isfunction(attr_value):
|
|
2744
|
-
sig = inspect.signature(attr_value)
|
|
2745
|
-
params = list(sig.parameters.keys())
|
|
2746
|
-
if params and params[0] == 'self':
|
|
2747
|
-
methods_to_wrap.append(attr_name)
|
|
2748
|
-
except (ValueError, TypeError):
|
|
2749
|
-
# Can't inspect signature, skip
|
|
2750
|
-
pass
|
|
2751
|
-
else:
|
|
2752
|
-
# Use specific methods from registry
|
|
2753
|
-
methods_to_wrap = list(methods_to_apply)
|
|
2754
|
-
|
|
2755
|
-
# Enhance each method
|
|
2756
|
-
for method_name in methods_to_wrap:
|
|
2757
|
-
try:
|
|
2758
|
-
# CRITICAL FIX: Only get from class __dict__ to ensure we get the RIGHT method
|
|
2759
|
-
original_func = obj.__dict__.get(method_name)
|
|
2760
|
-
if original_func is None:
|
|
2761
|
-
continue # Method not in class dict
|
|
2762
|
-
if not inspect.isfunction(original_func):
|
|
2763
|
-
continue
|
|
2764
|
-
if getattr(original_func, "__lazy_wrapped__", False):
|
|
2765
|
-
continue
|
|
2766
|
-
if isinstance(original_func, (classmethod, staticmethod)):
|
|
2767
|
-
continue
|
|
2768
|
-
if original_func.__name__ != method_name:
|
|
2769
|
-
continue
|
|
2770
|
-
# Verify it's an instance method
|
|
2771
|
-
try:
|
|
2772
|
-
params = list(inspect.signature(original_func).parameters.keys())
|
|
2773
|
-
if not params or params[0] != 'self':
|
|
2774
|
-
continue
|
|
2775
|
-
except Exception:
|
|
2776
|
-
continue
|
|
2777
|
-
# Create wrapper with proper closure
|
|
2778
|
-
class_obj, func_to_call = obj, original_func
|
|
2779
|
-
import functools
|
|
2780
|
-
def make_wrapper(fn, cls):
|
|
2781
|
-
@functools.wraps(fn)
|
|
2782
|
-
def wrapper(first_arg, *args, **kwargs):
|
|
2783
|
-
if isinstance(first_arg, cls):
|
|
2784
|
-
return fn(first_arg, *args, **kwargs)
|
|
2785
|
-
instance = cls()
|
|
2786
|
-
return fn(instance, first_arg, *args, **kwargs)
|
|
2787
|
-
return wrapper
|
|
2788
|
-
smart_wrapper = make_wrapper(func_to_call, class_obj)
|
|
2789
|
-
smart_wrapper.__lazy_wrapped__ = True
|
|
2790
|
-
setattr(obj, method_name, smart_wrapper)
|
|
2791
|
-
enhanced += 1
|
|
2792
|
-
except Exception as exc:
|
|
2793
|
-
# Silent skip - one line as requested (package/module agnostic)
|
|
2794
|
-
pass
|
|
2795
|
-
|
|
2796
|
-
if enhanced:
|
|
2797
|
-
log_event("enhance", logger.info, "✓ [LAZY ENHANCE] Enhanced %s methods in %s", enhanced, module.__name__)
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
class AutoInstantiateProxy:
|
|
2801
|
-
"""
|
|
2802
|
-
Proxy that auto-instantiates a class on first method call.
|
|
2803
|
-
|
|
2804
|
-
Enables: from module import Class as instance
|
|
2805
|
-
Then: instance.method() automatically creates Class() and calls method
|
|
2806
|
-
|
|
2807
|
-
This allows users to import classes with 'as' and use them directly
|
|
2808
|
-
without needing to manually instantiate.
|
|
2809
|
-
"""
|
|
2810
|
-
|
|
2811
|
-
def __init__(self, original_class):
|
|
2812
|
-
"""Initialize proxy with the class to wrap."""
|
|
2813
|
-
self._class = original_class
|
|
2814
|
-
self._instance = None
|
|
2815
|
-
self._is_instantiated = False
|
|
2816
|
-
|
|
2817
|
-
def _ensure_instantiated(self):
|
|
2818
|
-
"""Ensure the class is instantiated."""
|
|
2819
|
-
if not self._is_instantiated:
|
|
2820
|
-
self._instance = self._class()
|
|
2821
|
-
self._is_instantiated = True
|
|
2822
|
-
|
|
2823
|
-
def __getattr__(self, name: str):
|
|
2824
|
-
"""Get attribute from instance, instantiating if needed."""
|
|
2825
|
-
self._ensure_instantiated()
|
|
2826
|
-
return getattr(self._instance, name)
|
|
2827
|
-
|
|
2828
|
-
def __call__(self, *args, **kwargs):
|
|
2829
|
-
"""Allow calling the proxy directly (creates new instance)."""
|
|
2830
|
-
return self._class(*args, **kwargs)
|
|
2831
|
-
|
|
2832
|
-
def __repr__(self):
|
|
2833
|
-
if self._is_instantiated:
|
|
2834
|
-
return f"<AutoInstantiateProxy({self._class.__name__}): instantiated>"
|
|
2835
|
-
return f"<AutoInstantiateProxy({self._class.__name__}): will instantiate on first use>"
|
|
2836
|
-
|
|
2837
|
-
@property
|
|
2838
|
-
def __class__(self):
|
|
2839
|
-
"""Return the wrapped class for isinstance checks."""
|
|
2840
|
-
return self._class
|
|
2841
|
-
|
|
2842
|
-
# Registry of installed hooks per package
|
|
2843
|
-
_installed_hooks: dict[str, LazyMetaPathFinder] = {}
|
|
2844
|
-
_hook_lock = threading.RLock()
|
|
2845
|
-
|
|
2846
|
-
def install_import_hook(package_name: str = 'default') -> None:
|
|
2847
|
-
"""Install performant import hook for automatic lazy installation."""
|
|
2848
|
-
global _installed_hooks
|
|
2849
|
-
|
|
2850
|
-
log_event("hook", logger.info, f"[HOOK INSTALL] Installing import hook for package: {package_name}")
|
|
2851
|
-
|
|
2852
|
-
with _hook_lock:
|
|
2853
|
-
if package_name in _installed_hooks:
|
|
2854
|
-
log_event("hook", logger.info, f"[HOOK INSTALL] Import hook already installed for {package_name}")
|
|
2855
|
-
return
|
|
2856
|
-
|
|
2857
|
-
logger.debug(f"[HOOK INSTALL] Creating LazyMetaPathFinder for {package_name}")
|
|
2858
|
-
hook = LazyMetaPathFinder(package_name)
|
|
2859
|
-
|
|
2860
|
-
logger.debug(f"[HOOK INSTALL] Current sys.meta_path has {len(sys.meta_path)} entries")
|
|
2861
|
-
sys.meta_path.insert(0, hook)
|
|
2862
|
-
_installed_hooks[package_name] = hook
|
|
2863
|
-
|
|
2864
|
-
log_event("hook", logger.info, f"✅ [HOOK INSTALL] Lazy import hook installed for {package_name} (now {len(sys.meta_path)} meta_path entries)")
|
|
2865
|
-
|
|
2866
|
-
def uninstall_import_hook(package_name: str = 'default') -> None:
|
|
2867
|
-
"""Uninstall import hook for a package."""
|
|
2868
|
-
global _installed_hooks
|
|
2869
|
-
|
|
2870
|
-
with _hook_lock:
|
|
2871
|
-
if package_name in _installed_hooks:
|
|
2872
|
-
hook = _installed_hooks[package_name]
|
|
2873
|
-
try:
|
|
2874
|
-
sys.meta_path.remove(hook)
|
|
2875
|
-
except ValueError:
|
|
2876
|
-
pass
|
|
2877
|
-
del _installed_hooks[package_name]
|
|
2878
|
-
log_event("hook", logger.info, f"Lazy import hook uninstalled for {package_name}")
|
|
2879
|
-
|
|
2880
|
-
def is_import_hook_installed(package_name: str = 'default') -> bool:
|
|
2881
|
-
"""Check if import hook is installed for a package."""
|
|
2882
|
-
return package_name in _installed_hooks
|
|
2883
|
-
|
|
2884
|
-
def register_lazy_module_prefix(prefix: str) -> None:
|
|
2885
|
-
"""Register an import prefix for lazy wrapping."""
|
|
2886
|
-
_watched_registry = get_watched_registry()
|
|
2887
|
-
_watched_registry.add(prefix)
|
|
2888
|
-
normalized = _normalize_prefix(prefix)
|
|
2889
|
-
if normalized:
|
|
2890
|
-
log_event("config", logger.info, "Registered lazy module prefix: %s", normalized)
|
|
2891
|
-
|
|
2892
|
-
# =============================================================================
|
|
2893
|
-
# EXPORTS
|
|
2894
|
-
# =============================================================================
|
|
2895
|
-
|
|
2896
|
-
__all__ = [
|
|
2897
|
-
# Logging utilities
|
|
2898
|
-
'get_logger',
|
|
2899
|
-
'log_event',
|
|
2900
|
-
'print_formatted',
|
|
2901
|
-
'format_message',
|
|
2902
|
-
'is_log_category_enabled',
|
|
2903
|
-
'set_log_category',
|
|
2904
|
-
'set_log_categories',
|
|
2905
|
-
'get_log_categories',
|
|
2906
|
-
'XWLazyFormatter',
|
|
2907
|
-
# Import tracking
|
|
2908
|
-
'_is_import_in_progress',
|
|
2909
|
-
'_mark_import_started',
|
|
2910
|
-
'_mark_import_finished',
|
|
2911
|
-
'get_importing_state',
|
|
2912
|
-
'get_installing_state',
|
|
2913
|
-
'_installation_depth',
|
|
2914
|
-
'_installation_depth_lock',
|
|
2915
|
-
# Prefix trie
|
|
2916
|
-
'_PrefixTrie',
|
|
2917
|
-
# Watched registry
|
|
2918
|
-
'WatchedPrefixRegistry',
|
|
2919
|
-
'get_watched_registry',
|
|
2920
|
-
# Deferred loader
|
|
2921
|
-
'_DeferredModuleLoader',
|
|
2922
|
-
# Cache utilities
|
|
2923
|
-
# Parallel utilities
|
|
2924
|
-
'ParallelLoader',
|
|
2925
|
-
'DependencyGraph',
|
|
2926
|
-
# Module patching
|
|
2927
|
-
'_lazy_aware_import_module',
|
|
2928
|
-
'_patch_import_module',
|
|
2929
|
-
'_unpatch_import_module',
|
|
2930
|
-
# Archive imports
|
|
2931
|
-
'get_archive_path',
|
|
2932
|
-
'ensure_archive_in_path',
|
|
2933
|
-
'import_from_archive',
|
|
2934
|
-
# Bootstrap
|
|
2935
|
-
'bootstrap_lazy_mode',
|
|
2936
|
-
'bootstrap_lazy_mode_deferred',
|
|
2937
|
-
# Lazy loader
|
|
2938
|
-
'LazyLoader',
|
|
2939
|
-
# Lazy module registry
|
|
2940
|
-
'LazyModuleRegistry',
|
|
2941
|
-
# Lazy importer
|
|
2942
|
-
'LazyImporter',
|
|
2943
|
-
# Import hook
|
|
2944
|
-
'LazyImportHook',
|
|
2945
|
-
# Meta path finder
|
|
2946
|
-
'LazyMetaPathFinder',
|
|
2947
|
-
'install_import_hook',
|
|
2948
|
-
'uninstall_import_hook',
|
|
2949
|
-
'is_import_hook_installed',
|
|
2950
|
-
'register_lazy_module_prefix',
|
|
2951
|
-
'register_lazy_module_methods',
|
|
2952
|
-
'_set_package_class_hints',
|
|
2953
|
-
'_get_package_class_hints',
|
|
2954
|
-
'_clear_all_package_class_hints',
|
|
2955
|
-
'_spec_for_existing_module',
|
|
2956
|
-
# Global __import__ hook (for module-level imports)
|
|
2957
|
-
'register_lazy_package',
|
|
2958
|
-
'install_global_import_hook',
|
|
2959
|
-
'uninstall_global_import_hook',
|
|
2960
|
-
'is_global_import_hook_installed',
|
|
2961
|
-
'clear_global_import_caches',
|
|
2962
|
-
'get_global_import_cache_stats',
|
|
2963
|
-
]
|
|
2964
|
-
|