exonware-xwlazy 0.1.0.23__py3-none-any.whl → 1.0.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. exonware/__init__.py +85 -34
  2. exonware/xwlazy/version.py +5 -5
  3. exonware/xwlazy.py +2546 -0
  4. exonware/xwlazy_external_libs.toml +716 -0
  5. {exonware_xwlazy-0.1.0.23.dist-info → exonware_xwlazy-1.0.1.2.dist-info}/METADATA +5 -1
  6. exonware_xwlazy-1.0.1.2.dist-info/RECORD +8 -0
  7. exonware/xwlazy/__init__.py +0 -379
  8. exonware/xwlazy/common/__init__.py +0 -55
  9. exonware/xwlazy/common/base.py +0 -65
  10. exonware/xwlazy/common/cache.py +0 -504
  11. exonware/xwlazy/common/logger.py +0 -257
  12. exonware/xwlazy/common/services/__init__.py +0 -72
  13. exonware/xwlazy/common/services/dependency_mapper.py +0 -250
  14. exonware/xwlazy/common/services/install_async_utils.py +0 -170
  15. exonware/xwlazy/common/services/install_cache_utils.py +0 -245
  16. exonware/xwlazy/common/services/keyword_detection.py +0 -283
  17. exonware/xwlazy/common/services/spec_cache.py +0 -165
  18. exonware/xwlazy/common/services/state_manager.py +0 -84
  19. exonware/xwlazy/common/strategies/__init__.py +0 -28
  20. exonware/xwlazy/common/strategies/caching_dict.py +0 -44
  21. exonware/xwlazy/common/strategies/caching_installation.py +0 -88
  22. exonware/xwlazy/common/strategies/caching_lfu.py +0 -66
  23. exonware/xwlazy/common/strategies/caching_lru.py +0 -63
  24. exonware/xwlazy/common/strategies/caching_multitier.py +0 -59
  25. exonware/xwlazy/common/strategies/caching_ttl.py +0 -59
  26. exonware/xwlazy/common/utils.py +0 -142
  27. exonware/xwlazy/config.py +0 -193
  28. exonware/xwlazy/contracts.py +0 -1533
  29. exonware/xwlazy/defs.py +0 -378
  30. exonware/xwlazy/errors.py +0 -276
  31. exonware/xwlazy/facade.py +0 -1137
  32. exonware/xwlazy/host/__init__.py +0 -8
  33. exonware/xwlazy/host/conf.py +0 -16
  34. exonware/xwlazy/module/__init__.py +0 -18
  35. exonware/xwlazy/module/base.py +0 -622
  36. exonware/xwlazy/module/data.py +0 -17
  37. exonware/xwlazy/module/facade.py +0 -246
  38. exonware/xwlazy/module/importer_engine.py +0 -2964
  39. exonware/xwlazy/module/partial_module_detector.py +0 -275
  40. exonware/xwlazy/module/strategies/__init__.py +0 -22
  41. exonware/xwlazy/module/strategies/module_helper_lazy.py +0 -93
  42. exonware/xwlazy/module/strategies/module_helper_simple.py +0 -65
  43. exonware/xwlazy/module/strategies/module_manager_advanced.py +0 -111
  44. exonware/xwlazy/module/strategies/module_manager_simple.py +0 -95
  45. exonware/xwlazy/package/__init__.py +0 -18
  46. exonware/xwlazy/package/base.py +0 -863
  47. exonware/xwlazy/package/conf.py +0 -324
  48. exonware/xwlazy/package/data.py +0 -17
  49. exonware/xwlazy/package/facade.py +0 -480
  50. exonware/xwlazy/package/services/__init__.py +0 -84
  51. exonware/xwlazy/package/services/async_install_handle.py +0 -87
  52. exonware/xwlazy/package/services/config_manager.py +0 -249
  53. exonware/xwlazy/package/services/discovery.py +0 -435
  54. exonware/xwlazy/package/services/host_packages.py +0 -180
  55. exonware/xwlazy/package/services/install_async.py +0 -291
  56. exonware/xwlazy/package/services/install_cache.py +0 -145
  57. exonware/xwlazy/package/services/install_interactive.py +0 -59
  58. exonware/xwlazy/package/services/install_policy.py +0 -156
  59. exonware/xwlazy/package/services/install_registry.py +0 -54
  60. exonware/xwlazy/package/services/install_result.py +0 -17
  61. exonware/xwlazy/package/services/install_sbom.py +0 -153
  62. exonware/xwlazy/package/services/install_utils.py +0 -79
  63. exonware/xwlazy/package/services/installer_engine.py +0 -406
  64. exonware/xwlazy/package/services/lazy_installer.py +0 -803
  65. exonware/xwlazy/package/services/manifest.py +0 -503
  66. exonware/xwlazy/package/services/strategy_registry.py +0 -324
  67. exonware/xwlazy/package/strategies/__init__.py +0 -57
  68. exonware/xwlazy/package/strategies/package_discovery_file.py +0 -129
  69. exonware/xwlazy/package/strategies/package_discovery_hybrid.py +0 -84
  70. exonware/xwlazy/package/strategies/package_discovery_manifest.py +0 -101
  71. exonware/xwlazy/package/strategies/package_execution_async.py +0 -113
  72. exonware/xwlazy/package/strategies/package_execution_cached.py +0 -90
  73. exonware/xwlazy/package/strategies/package_execution_pip.py +0 -99
  74. exonware/xwlazy/package/strategies/package_execution_wheel.py +0 -106
  75. exonware/xwlazy/package/strategies/package_mapping_discovery_first.py +0 -100
  76. exonware/xwlazy/package/strategies/package_mapping_hybrid.py +0 -105
  77. exonware/xwlazy/package/strategies/package_mapping_manifest_first.py +0 -100
  78. exonware/xwlazy/package/strategies/package_policy_allow_list.py +0 -57
  79. exonware/xwlazy/package/strategies/package_policy_deny_list.py +0 -57
  80. exonware/xwlazy/package/strategies/package_policy_permissive.py +0 -46
  81. exonware/xwlazy/package/strategies/package_timing_clean.py +0 -67
  82. exonware/xwlazy/package/strategies/package_timing_full.py +0 -66
  83. exonware/xwlazy/package/strategies/package_timing_smart.py +0 -68
  84. exonware/xwlazy/package/strategies/package_timing_temporary.py +0 -66
  85. exonware/xwlazy/runtime/__init__.py +0 -18
  86. exonware/xwlazy/runtime/adaptive_learner.py +0 -129
  87. exonware/xwlazy/runtime/base.py +0 -274
  88. exonware/xwlazy/runtime/facade.py +0 -94
  89. exonware/xwlazy/runtime/intelligent_selector.py +0 -170
  90. exonware/xwlazy/runtime/metrics.py +0 -60
  91. exonware/xwlazy/runtime/performance.py +0 -37
  92. exonware_xwlazy-0.1.0.23.dist-info/RECORD +0 -93
  93. xwlazy/__init__.py +0 -14
  94. xwlazy/lazy.py +0 -30
  95. {exonware_xwlazy-0.1.0.23.dist-info → exonware_xwlazy-1.0.1.2.dist-info}/WHEEL +0 -0
  96. {exonware_xwlazy-0.1.0.23.dist-info → exonware_xwlazy-1.0.1.2.dist-info}/licenses/LICENSE +0 -0
@@ -1,2964 +0,0 @@
1
- """
2
- #exonware/xwlazy/src/exonware/xwlazy/module/importer_engine.py
3
-
4
- Import Engine - Unified engine for all import-related operations.
5
-
6
- Company: eXonware.com
7
- Author: Eng. Muhammad AlShehri
8
- Email: connect@exonware.com
9
-
10
- Generation Date: 15-Nov-2025
11
-
12
- This module provides unified import engine for all import-related functionality.
13
- All import-related functionality is centralized here.
14
-
15
- Merged from:
16
- - logging_utils.py (Logging utilities)
17
- - import_tracking.py (Import tracking)
18
- - prefix_trie.py (Prefix trie data structure)
19
- - watched_registry.py (Watched prefix registry)
20
- - deferred_loader.py (Deferred module loader)
21
- - cache_utils.py (Multi-tier cache and bytecode cache)
22
- - parallel_utils.py (Parallel loading utilities)
23
- - module_patching.py (Module patching utilities)
24
- - archive_imports.py (Archive import utilities)
25
- - bootstrap.py (Bootstrap utilities)
26
- - loader.py (Lazy loader)
27
- - registry.py (Lazy module registry)
28
- - importer.py (Lazy importer)
29
- - import_hook.py (Import hook)
30
- - meta_path_finder.py (Meta path finder)
31
-
32
- Features:
33
- - Unified import engine for all import operations
34
- - Multi-tier caching (L1/L2/L3)
35
- - Parallel loading support
36
- - Import tracking and circular import prevention
37
- - Watched prefix registry
38
- - Meta path finder for intercepting imports
39
- """
40
-
41
- # =============================================================================
42
- # IMPORTS
43
- # =============================================================================
44
-
45
- from __future__ import annotations
46
-
47
- import os
48
- import sys
49
- import json
50
- import time
51
- import asyncio
52
- import pickle
53
- import struct
54
- import builtins
55
- import atexit
56
- import logging
57
- import importlib
58
- import importlib.util
59
- import importlib.machinery
60
- import importlib.abc
61
- import threading
62
- import subprocess
63
- import concurrent.futures
64
- from pathlib import Path
65
- from types import ModuleType
66
- from typing import Optional, Any, Iterable, Callable
67
- from collections import OrderedDict, defaultdict, Counter, deque
68
- from queue import Queue
69
- from datetime import datetime
70
- from enum import Enum
71
-
72
- from ..defs import LazyLoadMode, LazyInstallMode
73
- from ..common.services.dependency_mapper import DependencyMapper
74
- from ..common.services.spec_cache import _spec_cache_get, _spec_cache_put
75
- from ..package.services import LazyInstallerRegistry, LazyInstaller
76
- from ..package.services.config_manager import LazyInstallConfig
77
- from ..package.services.manifest import _normalize_prefix
78
- from ..errors import DeferredImportError
79
- from .base import AModuleHelper
80
-
81
- # Import from common (logger and cache)
82
- from ..common.logger import get_logger, log_event
83
- from ..common.cache import MultiTierCache, BytecodeCache
84
-
85
- # Import from runtime folder (moved from module folder)
86
- from ..runtime.adaptive_learner import AdaptiveLearner
87
- from ..runtime.intelligent_selector import IntelligentModeSelector, LoadLevel
88
-
89
- # =============================================================================
90
- # LOGGER (from common.logger)
91
- # =============================================================================
92
-
93
- logger = get_logger("xwlazy.importer_engine")
94
-
95
- # =============================================================================
96
- # IMPORT TRACKING (from import_tracking.py)
97
- # =============================================================================
98
-
99
- _thread_local = threading.local()
100
- _importing = threading.local()
101
- _installing = threading.local()
102
-
103
- _installation_depth = 0
104
- _installation_depth_lock = threading.Lock()
105
-
106
- # Thread-local flag to prevent recursion during installation checks
107
- _checking_installation = threading.local()
108
-
109
- def _get_thread_imports() -> set[str]:
110
- """Get thread-local import set (creates if needed)."""
111
- if not hasattr(_thread_local, 'imports'):
112
- _thread_local.imports = set()
113
- return _thread_local.imports
114
-
115
- def _is_checking_installation() -> bool:
116
- """Check if we're currently checking installation status (to prevent recursion)."""
117
- return getattr(_checking_installation, 'active', False)
118
-
119
- def _set_checking_installation(value: bool) -> None:
120
- """Set the installation check flag."""
121
- _checking_installation.active = value
122
-
123
- def _is_import_in_progress(module_name: str) -> bool:
124
- """Check if a module import is currently in progress for this thread."""
125
- return module_name in _get_thread_imports()
126
-
127
- def _mark_import_started(module_name: str) -> None:
128
- """Mark a module import as started for this thread."""
129
- _get_thread_imports().add(module_name)
130
-
131
- def _mark_import_finished(module_name: str) -> None:
132
- """Mark a module import as finished for this thread."""
133
- _get_thread_imports().discard(module_name)
134
-
135
- def get_importing_state() -> threading.local:
136
- """Get thread-local importing state."""
137
- return _importing
138
-
139
- def get_installing_state() -> threading.local:
140
- """Get thread-local installing state."""
141
- return _installing
142
-
143
- # Thread-local storage for installation state
144
- _installing_state = get_installing_state()
145
- _importing_state = get_importing_state()
146
-
147
- # Global recursion depth counter to prevent infinite recursion
148
- _installation_depth = 0
149
- _installation_depth_lock = threading.Lock()
150
-
151
- # =============================================================================
152
- # GLOBAL __import__ HOOK (Critical for Module-Level Imports)
153
- # =============================================================================
154
-
155
- # Global state for builtins.__import__ hook
156
- _original_builtins_import: Optional[Callable] = None
157
- _global_import_hook_installed: bool = False
158
- _global_import_hook_lock = threading.RLock()
159
-
160
- # Fast-path caches for O(1) lookup
161
- _installed_cache: set[str] = set()
162
- _failed_cache: set[str] = set()
163
-
164
- # Registry of packages that should auto-install
165
- _lazy_packages: dict[str, Any] = {}
166
-
167
- def register_lazy_package(package_name: str, config: Optional[Any] = None) -> None:
168
- """
169
- Register a package for lazy loading/installation.
170
-
171
- Args:
172
- package_name: Package name to register
173
- config: Optional configuration object
174
- """
175
- with _global_import_hook_lock:
176
- _lazy_packages[package_name] = config or {}
177
- logger.debug(f"Registered lazy package: {package_name}")
178
-
179
- def _should_auto_install(module_name: str) -> bool:
180
- """
181
- Check if module should be auto-installed.
182
-
183
- Checks if:
184
- 1. Module's root package is registered, OR
185
- 2. Module maps to a known package (via DependencyMapper) AND packages are registered
186
-
187
- Args:
188
- module_name: Module name to check
189
-
190
- Returns:
191
- True if should auto-install, False otherwise
192
- """
193
- root_package = module_name.split('.')[0]
194
-
195
- # Fast path: root package is registered
196
- if root_package in _lazy_packages:
197
- return True
198
-
199
- # If no packages are registered, don't auto-install
200
- if not _lazy_packages:
201
- return False
202
-
203
- # Check if module maps to a known package via DependencyMapper
204
- # This handles cases like:
205
- # - 'yaml' -> 'PyYAML' (different name)
206
- # - 'msgpack' -> 'msgpack' (same name, but still a known dependency)
207
- # - 'bson' -> 'pymongo' (different name)
208
- try:
209
- # Use the first registered package to get the mapper context
210
- # All registered packages should have the same dependency mappings
211
- registered_package = next(iter(_lazy_packages.keys()), None)
212
- if registered_package:
213
- mapper = DependencyMapper(package_name=registered_package)
214
- else:
215
- mapper = DependencyMapper()
216
-
217
- package_name = mapper.get_package_name(module_name)
218
-
219
- # If DependencyMapper found a package name (not None), it's a known dependency
220
- # Allow installation attempt - the installer will verify if it's actually needed
221
- if package_name:
222
- logger.debug(f"[AUTO-INSTALL] Module '{module_name}' maps to package '{package_name}', allowing auto-install")
223
- return True
224
- else:
225
- logger.debug(f"[AUTO-INSTALL] Module '{module_name}' has no package mapping, skipping auto-install")
226
-
227
- except Exception as e:
228
- # If mapper fails, log and be conservative
229
- logger.debug(f"[AUTO-INSTALL] DependencyMapper failed for '{module_name}': {e}")
230
- pass
231
-
232
- return False
233
-
234
- def _try_install_package(module_name: str) -> bool:
235
- """
236
- Try to install package for missing module.
237
-
238
- Tries all registered packages until one succeeds.
239
-
240
- Args:
241
- module_name: Module name that failed to import
242
-
243
- Returns:
244
- True if installation successful, False otherwise
245
- """
246
- # Try root package first
247
- root_package = module_name.split('.')[0]
248
- if root_package in _lazy_packages:
249
- try:
250
- logger.info(f"[AUTO-INSTALL] Trying root package {root_package} for module {module_name}")
251
- installer = LazyInstallerRegistry.get_instance(root_package)
252
- logger.info(f"[AUTO-INSTALL] Installer for {root_package}: {installer is not None}, enabled={installer.is_enabled() if installer else False}")
253
-
254
- if installer is None:
255
- logger.warning(f"[AUTO-INSTALL] Installer for {root_package} is None")
256
- elif not installer.is_enabled():
257
- logger.warning(f"[AUTO-INSTALL] Installer for {root_package} is disabled")
258
- else:
259
- logger.info(f"Auto-installing missing package for module: {module_name} (via {root_package})")
260
- logger.info(f"[AUTO-INSTALL] Calling install_and_import('{module_name}')")
261
- try:
262
- module, success = installer.install_and_import(module_name)
263
- logger.info(f"[AUTO-INSTALL] install_and_import returned: module={module is not None}, success={success}")
264
- if success and module:
265
- _installed_cache.add(module_name)
266
- logger.info(f"[AUTO-INSTALL] Successfully installed and imported '{module_name}'")
267
- return True
268
- else:
269
- logger.warning(f"[AUTO-INSTALL] install_and_import failed for '{module_name}': success={success}, module={module is not None}")
270
- except Exception as install_exc:
271
- logger.error(f"[AUTO-INSTALL] install_and_import raised exception: {install_exc}", exc_info=True)
272
- raise
273
- except Exception as e:
274
- logger.error(f"[AUTO-INSTALL] Exception installing via root package {root_package}: {e}", exc_info=True)
275
-
276
- # Try all registered packages (for dependencies like 'yaml' when 'xwsystem' is registered)
277
- for registered_package in _lazy_packages.keys():
278
- if registered_package == root_package:
279
- continue # Already tried
280
-
281
- try:
282
- logger.debug(f"[AUTO-INSTALL] Trying to get installer for {registered_package}")
283
- installer = LazyInstallerRegistry.get_instance(registered_package)
284
- logger.debug(f"[AUTO-INSTALL] Installer retrieved: {installer is not None}")
285
-
286
- if installer is None:
287
- logger.warning(f"[AUTO-INSTALL] Installer for {registered_package} is None")
288
- continue
289
-
290
- if not installer.is_enabled():
291
- logger.warning(f"[AUTO-INSTALL] Installer for {registered_package} is disabled")
292
- continue
293
-
294
- logger.info(f"Auto-installing missing package for module: {module_name} (via {registered_package})")
295
- logger.info(f"[AUTO-INSTALL] Calling install_and_import('{module_name}')")
296
- try:
297
- module, success = installer.install_and_import(module_name)
298
- logger.info(f"[AUTO-INSTALL] install_and_import returned: module={module is not None}, success={success}")
299
- except Exception as install_exc:
300
- logger.error(f"[AUTO-INSTALL] install_and_import raised exception: {install_exc}", exc_info=True)
301
- raise
302
-
303
- if success and module:
304
- _installed_cache.add(module_name)
305
- logger.info(f"[AUTO-INSTALL] Successfully installed and imported '{module_name}'")
306
- return True
307
- else:
308
- logger.warning(f"[AUTO-INSTALL] install_and_import failed for '{module_name}': success={success}, module={module is not None}")
309
- except Exception as e:
310
- logger.error(f"[AUTO-INSTALL] Exception installing via {registered_package}: {e}", exc_info=True)
311
- continue
312
-
313
- logger.warning(f"[AUTO-INSTALL] Failed to install package for module '{module_name}' via all registered packages")
314
- return False
315
-
316
- def _intercepting_import(name: str, globals=None, locals=None, fromlist=(), level=0):
317
- """
318
- Intercept ALL imports including module-level ones.
319
-
320
- This is the global builtins.__import__ replacement that catches
321
- ALL imports, including those at module level during package initialization.
322
-
323
- CRITICAL: Skip relative imports (level > 0) - they must use normal import path.
324
- Relative imports are package/module agnostic and should not be intercepted.
325
-
326
- CRITICAL: When fromlist is non-empty (e.g., "from module import Class"),
327
- Python's import machinery needs direct access to module attributes via getattr().
328
- We MUST return the actual module object without any wrapping.
329
- """
330
- # CRITICAL FIX: Handle relative imports (level > 0)
331
- # Relative imports like "from .common import" have level=1
332
- # We must use normal import BUT still enhance the module after import (package/module agnostic)
333
- if level > 0:
334
- result = _original_builtins_import(name, globals, locals, fromlist, level)
335
- # CRITICAL: Enhance modules imported via relative imports (package/module agnostic)
336
- # This ensures instance methods work on classes for ANY package/module structure
337
- if result and isinstance(result, ModuleType) and fromlist:
338
- try:
339
- for pkg_name in _lazy_packages.keys():
340
- if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
341
- finder = _installed_hooks.get(pkg_name)
342
- if finder:
343
- finder._enhance_classes_with_class_methods(result)
344
- break
345
- except Exception:
346
- pass
347
- return result
348
-
349
- # CRITICAL FIX: Handle fromlist imports specially
350
- # When fromlist is present (e.g., "from module import Class"),
351
- # Python needs direct access to module attributes via getattr(module, 'Class')
352
- # We MUST return the actual module without any wrapping or modification
353
- if fromlist:
354
- # Import partial module detector (lazy import to avoid circular dependency)
355
- try:
356
- from .partial_module_detector import (
357
- is_partially_initialized as _default_is_partially_initialized,
358
- mark_module_importing,
359
- unmark_module_importing,
360
- DetectionStrategy,
361
- PartialModuleDetector
362
- )
363
- # Allow strategy override via environment variable for testing
364
- strategy_name = os.environ.get('XWLAZY_PARTIAL_DETECTION_STRATEGY', 'hybrid')
365
- try:
366
- strategy = DetectionStrategy(strategy_name)
367
- detector = PartialModuleDetector(strategy)
368
- is_partially_initialized = lambda n, m: detector.is_partially_initialized(n, m)
369
- except (ValueError, AttributeError):
370
- # Use default detector (HYBRID strategy)
371
- is_partially_initialized = _default_is_partially_initialized
372
- except ImportError:
373
- # Fallback if detector not available
374
- def is_partially_initialized(name, mod):
375
- return False
376
- def mark_module_importing(name):
377
- pass
378
- def unmark_module_importing(name):
379
- pass
380
-
381
- # Mark this module as being imported (for tracking)
382
- mark_module_importing(name)
383
-
384
- try:
385
- # Fast path: already imported and in sys.modules
386
- # CRITICAL: For fromlist imports, we must be VERY conservative
387
- # Only return early if we're CERTAIN the module is fully initialized
388
- # and NOT currently being imported
389
- if name in sys.modules:
390
- module = sys.modules[name]
391
- # Ensure module is fully loaded (not a placeholder or lazy wrapper)
392
- # Check if it's a real ModuleType (not a proxy/wrapper)
393
- if isinstance(module, ModuleType):
394
- # CRITICAL: Check if module is partially initialized
395
- # We must NOT return a partially initialized module
396
- # For fromlist imports, be extra conservative - only return if
397
- # module is definitely fully loaded AND not currently importing
398
- if not is_partially_initialized(name, module):
399
- # Additional check: verify module has meaningful content
400
- # (not just metadata attributes)
401
- module_dict = getattr(module, '__dict__', {})
402
- metadata_attrs = {'__name__', '__loader__', '__spec__', '__package__', '__file__', '__path__', '__cached__'}
403
- content_attrs = set(module_dict.keys()) - metadata_attrs
404
-
405
- # Only return if module has actual content (classes, functions, etc.)
406
- # OR if it's a namespace package (has __path__)
407
- has_content = len(content_attrs) > 0
408
- is_namespace = hasattr(module, '__path__')
409
-
410
- if has_content or is_namespace:
411
- # Check if it's a placeholder by looking for common placeholder patterns
412
- is_placeholder = (
413
- hasattr(module, '__getattr__') and
414
- not hasattr(module, '__file__') and
415
- not hasattr(module, '__path__') # Namespace packages don't have __file__
416
- )
417
- if not is_placeholder:
418
- # Module is fully loaded with content, return it directly
419
- # CRITICAL FIX: Enhance classes before returning
420
- try:
421
- # Find which package this module belongs to
422
- for pkg_name in _lazy_packages.keys():
423
- # Check if module name starts with package name or "exonware." + package name
424
- if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
425
- finder = _installed_hooks.get(pkg_name)
426
- if finder:
427
- finder._enhance_classes_with_class_methods(module)
428
- break
429
- except Exception:
430
- pass
431
- unmark_module_importing(name)
432
- return module
433
- # Module is partially initialized or has no content - fall through to normal import
434
- # If it's a placeholder, partially initialized, or has no content, fall through to normal import
435
- finally:
436
- # Always unmark when done (even if exception occurs)
437
- unmark_module_importing(name)
438
-
439
- # For fromlist imports, use normal import path to ensure classes/functions
440
- # are accessible via getattr() - Python's import machinery handles extraction
441
- try:
442
- # Mark as importing before calling original import
443
- try:
444
- from .partial_module_detector import mark_module_importing, unmark_module_importing
445
- mark_module_importing(name)
446
- except ImportError:
447
- pass
448
-
449
- try:
450
- result = _original_builtins_import(name, globals, locals, fromlist, level)
451
- # Cache success but return actual module (no wrapping)
452
- _installed_cache.add(name)
453
- # CRITICAL FIX: Enhance classes in the module for class-level method access
454
- # This makes instance methods callable on classes (e.g., BsonSerializer.encode(data))
455
- if result and isinstance(result, ModuleType):
456
- try:
457
- # Find which package this module belongs to
458
- for pkg_name in _lazy_packages.keys():
459
- # Check if module name starts with package name or "exonware." + package name
460
- if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
461
- finder = _installed_hooks.get(pkg_name)
462
- if finder:
463
- finder._enhance_classes_with_class_methods(result)
464
- break
465
- except Exception as e:
466
- # Enhancement failed - don't break the import, but log for debugging
467
- logger.debug(f"Enhancement failed for {name}: {e}", exc_info=True)
468
- pass
469
- return result
470
- finally:
471
- # Always unmark when done
472
- try:
473
- from .partial_module_detector import unmark_module_importing
474
- unmark_module_importing(name)
475
- except ImportError:
476
- pass
477
- except ImportError as e:
478
- # Only try auto-install if needed
479
- if _should_auto_install(name):
480
- try:
481
- if _try_install_package(name):
482
- # Retry import after installation - return actual module
483
- result = _original_builtins_import(name, globals, locals, fromlist, level)
484
- _installed_cache.add(name)
485
- # CRITICAL FIX: Enhance classes in the module for class-level method access
486
- if result and isinstance(result, ModuleType):
487
- try:
488
- # Find which package this module belongs to
489
- for pkg_name in _lazy_packages.keys():
490
- # Check if module name starts with package name or "exonware." + package name
491
- if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
492
- finder = _installed_hooks.get(pkg_name)
493
- if finder:
494
- finder._enhance_classes_with_class_methods(result)
495
- break
496
- except Exception:
497
- pass
498
- return result
499
- except Exception:
500
- # If installation fails, don't crash - just raise the original ImportError
501
- pass
502
-
503
- # Installation failed or not applicable - cache failure (but limit cache size)
504
- if len(_failed_cache) < 1000: # Prevent unbounded growth
505
- _failed_cache.add(name)
506
- raise
507
-
508
- # Fast path: cached as installed (but still enhance for fromlist)
509
- if name in _installed_cache:
510
- result = _original_builtins_import(name, globals, locals, fromlist, level)
511
- # CRITICAL: Enhance even cached modules for fromlist imports (package/module agnostic)
512
- if fromlist and result and isinstance(result, ModuleType):
513
- try:
514
- for pkg_name in _lazy_packages.keys():
515
- if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
516
- finder = _installed_hooks.get(pkg_name)
517
- if finder:
518
- finder._enhance_classes_with_class_methods(result)
519
- break
520
- except Exception:
521
- pass
522
- return result
523
-
524
- # Fast path: already imported (for non-fromlist imports)
525
- # CRITICAL: For fromlist imports, enhance even if module is cached
526
- if name in sys.modules:
527
- result = _original_builtins_import(name, globals, locals, fromlist, level)
528
- if fromlist:
529
- module = sys.modules.get(name)
530
- if module and isinstance(module, ModuleType):
531
- try:
532
- for pkg_name in _lazy_packages.keys():
533
- if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
534
- finder = _installed_hooks.get(pkg_name)
535
- if finder:
536
- finder._enhance_classes_with_class_methods(module)
537
- break
538
- except Exception:
539
- pass
540
- return result
541
-
542
- # Fast path: known failure
543
- if name in _failed_cache:
544
- raise ImportError(f"No module named '{name}'")
545
-
546
- # Fast path: skip stdlib/builtin modules (performance optimization)
547
- if name in sys.builtin_module_names:
548
- return _original_builtins_import(name, globals, locals, fromlist, level)
549
-
550
- # Skip private/internal modules (performance optimization)
551
- # But allow if it's a submodule of a registered package
552
- if name.startswith('_'):
553
- # Check if it's a submodule of a registered package
554
- root_package = name.split('.')[0]
555
- if root_package not in _lazy_packages:
556
- return _original_builtins_import(name, globals, locals, fromlist, level)
557
-
558
- # Skip test-related modules to avoid interfering with pytest
559
- if name.startswith(('pytest', '_pytest', 'pluggy', '_pluggy')):
560
- return _original_builtins_import(name, globals, locals, fromlist, level)
561
-
562
- # Skip debugging/profiling modules
563
- if name in ('tracemalloc', 'pdb', 'ipdb', 'debugpy', 'pydevd'):
564
- return _original_builtins_import(name, globals, locals, fromlist, level)
565
-
566
- try:
567
- # Try normal import first
568
- result = _original_builtins_import(name, globals, locals, fromlist, level)
569
- # Success - cache it
570
- _installed_cache.add(name)
571
- # CRITICAL FIX: Enhance classes for fromlist imports (package/module agnostic)
572
- # This ensures instance methods work on classes for ANY package/module structure
573
- if result and isinstance(result, ModuleType) and fromlist:
574
- try:
575
- for pkg_name in _lazy_packages.keys():
576
- if name.startswith(pkg_name) or name.startswith(f"exonware.{pkg_name}"):
577
- finder = _installed_hooks.get(pkg_name)
578
- if finder:
579
- finder._enhance_classes_with_class_methods(result)
580
- break
581
- except Exception:
582
- pass
583
- return result
584
- except ImportError as e:
585
- # Check if this package should be auto-installed
586
- # ROOT CAUSE DEBUG: Log the exact import name and traceback to find where typos originate
587
- if any(typo in name for typo in ['contrrib', 'msgpackk', 'msgppack', 'mmsgpack']):
588
- import traceback
589
- logger.warning(
590
- f"[ROOT CAUSE] Typo detected in module name '{name}'. "
591
- f"ImportError: {e}. "
592
- f"Traceback:\n{''.join(traceback.format_stack()[-5:-1])}"
593
- )
594
- logger.debug(f"[AUTO-INSTALL] ImportError for '{name}': {e}")
595
- should_install = _should_auto_install(name)
596
- logger.debug(f"[AUTO-INSTALL] Should auto-install '{name}': {should_install}")
597
-
598
- if should_install:
599
- try:
600
- logger.info(f"[AUTO-INSTALL] Attempting to install package for '{name}'")
601
- if _try_install_package(name):
602
- logger.info(f"[AUTO-INSTALL] Successfully installed package for '{name}', retrying import")
603
- # Retry import after installation
604
- try:
605
- result = _original_builtins_import(name, globals, locals, fromlist, level)
606
- _installed_cache.add(name)
607
- logger.info(f"[AUTO-INSTALL] Successfully imported '{name}' after installation")
608
- return result
609
- except ImportError as retry_error:
610
- logger.warning(f"[AUTO-INSTALL] Import still failed for '{name}' after installation: {retry_error}")
611
- pass
612
- else:
613
- logger.warning(f"[AUTO-INSTALL] Installation attempt returned False for '{name}'")
614
- except Exception as install_error:
615
- # If installation fails, log it but don't crash - just raise the original ImportError
616
- logger.error(f"[AUTO-INSTALL] Installation failed for '{name}': {install_error}", exc_info=True)
617
- pass
618
- else:
619
- logger.debug(f"[AUTO-INSTALL] Not auto-installing '{name}' (not eligible)")
620
-
621
- # Installation failed or not applicable - cache failure (but limit cache size)
622
- if len(_failed_cache) < 1000: # Prevent unbounded growth
623
- _failed_cache.add(name)
624
- raise
625
- except Exception:
626
- # For any other exception, don't interfere - let it propagate
627
- # This prevents the hook from breaking system functionality
628
- raise
629
-
630
- def install_global_import_hook() -> None:
631
- """
632
- Install global builtins.__import__ hook for auto-install.
633
-
634
- This hook intercepts ALL imports including module-level ones,
635
- enabling auto-installation for registered packages.
636
- """
637
- global _original_builtins_import, _global_import_hook_installed
638
-
639
- with _global_import_hook_lock:
640
- if _global_import_hook_installed:
641
- logger.debug("Global import hook already installed")
642
- return
643
-
644
- if _original_builtins_import is None:
645
- _original_builtins_import = builtins.__import__
646
-
647
- builtins.__import__ = _intercepting_import
648
- _global_import_hook_installed = True
649
- logger.info("✅ Global builtins.__import__ hook installed for auto-install")
650
-
651
- def uninstall_global_import_hook() -> None:
652
- """
653
- Uninstall global builtins.__import__ hook.
654
-
655
- Restores original builtins.__import__.
656
- """
657
- global _original_builtins_import, _global_import_hook_installed
658
-
659
- with _global_import_hook_lock:
660
- if not _global_import_hook_installed:
661
- return
662
-
663
- if _original_builtins_import is not None:
664
- builtins.__import__ = _original_builtins_import
665
- _original_builtins_import = None
666
-
667
- _global_import_hook_installed = False
668
- logger.info("Global builtins.__import__ hook uninstalled")
669
-
670
- def is_global_import_hook_installed() -> bool:
671
- """Check if global import hook is installed."""
672
- return _global_import_hook_installed
673
-
674
- def clear_global_import_caches() -> None:
675
- """
676
- Clear global import hook caches (useful for testing).
677
-
678
- Clears both installed and failed caches.
679
- """
680
- global _installed_cache, _failed_cache
681
- with _global_import_hook_lock:
682
- _installed_cache.clear()
683
- _failed_cache.clear()
684
- logger.debug("Cleared global import hook caches")
685
-
686
- def get_global_import_cache_stats() -> dict[str, Any]:
687
- """
688
- Get statistics about global import hook caches.
689
-
690
- Returns:
691
- Dict with cache sizes and hit/miss information
692
- """
693
- with _global_import_hook_lock:
694
- return {
695
- 'installed_cache_size': len(_installed_cache),
696
- 'failed_cache_size': len(_failed_cache),
697
- 'registered_packages': list(_lazy_packages.keys()),
698
- 'hook_installed': _global_import_hook_installed,
699
- }
700
-
701
- # =============================================================================
702
- # PREFIX TRIE (from prefix_trie.py)
703
- # =============================================================================
704
-
705
- class _PrefixTrie:
706
- """Trie data structure for prefix matching."""
707
-
708
- __slots__ = ("_root",)
709
-
710
- def __init__(self) -> None:
711
- self._root: dict[str, dict[str, Any]] = {}
712
-
713
- def add(self, prefix: str) -> None:
714
- """Add a prefix to the trie."""
715
- node = self._root
716
- for char in prefix:
717
- node = node.setdefault(char, {})
718
- node["_end"] = prefix
719
-
720
- def iter_matches(self, value: str) -> tuple[str, ...]:
721
- """Find all matching prefixes for a given value."""
722
- node = self._root
723
- matches: list[str] = []
724
- for char in value:
725
- end_value = node.get("_end")
726
- if end_value:
727
- matches.append(end_value)
728
- node = node.get(char)
729
- if node is None:
730
- break
731
- else:
732
- end_value = node.get("_end")
733
- if end_value:
734
- matches.append(end_value)
735
- return tuple(matches)
736
-
737
- # =============================================================================
738
- # WATCHED REGISTRY (from watched_registry.py)
739
- # =============================================================================
740
-
741
- class WatchedPrefixRegistry:
742
- """Maintain watched prefixes and provide fast trie-based membership checks."""
743
-
744
- __slots__ = (
745
- "_lock",
746
- "_prefix_refcounts",
747
- "_owner_map",
748
- "_prefixes",
749
- "_trie",
750
- "_dirty",
751
- "_root_refcounts",
752
- "_root_snapshot",
753
- "_root_snapshot_dirty",
754
- )
755
-
756
- def __init__(self, initial: Optional[list[str]] = None) -> None:
757
- self._lock = threading.RLock()
758
- self._prefix_refcounts: Counter[str] = Counter()
759
- self._owner_map: dict[str, set[str]] = {}
760
- self._prefixes: set[str] = set()
761
- self._trie = _PrefixTrie()
762
- self._dirty = False
763
- self._root_refcounts: Counter[str] = Counter()
764
- self._root_snapshot: set[str] = set()
765
- self._root_snapshot_dirty = False
766
- if initial:
767
- for prefix in initial:
768
- self._register_manual(prefix)
769
-
770
- def _register_manual(self, prefix: str) -> None:
771
- normalized = _normalize_prefix(prefix)
772
- if not normalized:
773
- return
774
- owner = "__manual__"
775
- owners = self._owner_map.setdefault(owner, set())
776
- if normalized in owners:
777
- return
778
- owners.add(normalized)
779
- self._add_prefix(normalized)
780
-
781
- def _add_prefix(self, prefix: str) -> None:
782
- if not prefix:
783
- return
784
- self._prefix_refcounts[prefix] += 1
785
- if self._prefix_refcounts[prefix] == 1:
786
- self._prefixes.add(prefix)
787
- self._dirty = True
788
- root = prefix.split('.', 1)[0]
789
- self._root_refcounts[root] += 1
790
- self._root_snapshot_dirty = True
791
-
792
- def _remove_prefix(self, prefix: str) -> None:
793
- if prefix not in self._prefix_refcounts:
794
- return
795
- self._prefix_refcounts[prefix] -= 1
796
- if self._prefix_refcounts[prefix] <= 0:
797
- self._prefix_refcounts.pop(prefix, None)
798
- self._prefixes.discard(prefix)
799
- self._dirty = True
800
- root = prefix.split('.', 1)[0]
801
- self._root_refcounts[root] -= 1
802
- if self._root_refcounts[root] <= 0:
803
- self._root_refcounts.pop(root, None)
804
- self._root_snapshot_dirty = True
805
-
806
- def _ensure_trie(self) -> None:
807
- if not self._dirty:
808
- return
809
- self._trie = _PrefixTrie()
810
- for prefix in self._prefixes:
811
- self._trie.add(prefix)
812
- self._dirty = False
813
-
814
- def add(self, prefix: str) -> None:
815
- normalized = _normalize_prefix(prefix)
816
- if not normalized:
817
- return
818
- with self._lock:
819
- self._register_manual(normalized)
820
-
821
- def is_empty(self) -> bool:
822
- with self._lock:
823
- return not self._prefixes
824
-
825
- def register_package(self, package_name: str, prefixes: Iterable[str]) -> None:
826
- owner_key = f"pkg::{package_name.lower()}"
827
- normalized = {_normalize_prefix(p) for p in prefixes if _normalize_prefix(p)}
828
- with self._lock:
829
- current = self._owner_map.get(owner_key, set())
830
- to_remove = current - normalized
831
- to_add = normalized - current
832
-
833
- for prefix in to_remove:
834
- self._remove_prefix(prefix)
835
- for prefix in to_add:
836
- self._add_prefix(prefix)
837
-
838
- if normalized:
839
- self._owner_map[owner_key] = normalized
840
- elif owner_key in self._owner_map:
841
- self._owner_map.pop(owner_key, None)
842
-
843
- def is_prefix_owned_by(self, package_name: str, prefix: str) -> bool:
844
- normalized = _normalize_prefix(prefix)
845
- owner_key = f"pkg::{package_name.lower()}"
846
- with self._lock:
847
- if normalized in self._owner_map.get("__manual__", set()):
848
- return True
849
- return normalized in self._owner_map.get(owner_key, set())
850
-
851
- def get_matching_prefixes(self, module_name: str) -> tuple[str, ...]:
852
- with self._lock:
853
- if not self._prefixes:
854
- return ()
855
- self._ensure_trie()
856
- return self._trie.iter_matches(module_name)
857
-
858
- def has_root(self, root_name: str) -> bool:
859
- snapshot = self._root_snapshot
860
- if not self._root_snapshot_dirty:
861
- return root_name in snapshot
862
- with self._lock:
863
- if self._root_snapshot_dirty:
864
- self._root_snapshot = set(self._root_refcounts.keys())
865
- self._root_snapshot_dirty = False
866
- return root_name in self._root_snapshot
867
-
868
- # Global registry instance
869
- _DEFAULT_WATCHED_PREFIXES = tuple(
870
- filter(
871
- None,
872
- os.environ.get(
873
- "XWLAZY_LAZY_PREFIXES",
874
- "",
875
- ).split(";"),
876
- )
877
- )
878
- _watched_registry = WatchedPrefixRegistry(list(_DEFAULT_WATCHED_PREFIXES))
879
-
880
- def get_watched_registry() -> WatchedPrefixRegistry:
881
- """Get the global watched prefix registry."""
882
- return _watched_registry
883
-
884
- # =============================================================================
885
- # DEFERRED LOADER (from deferred_loader.py)
886
- # =============================================================================
887
-
888
- class _DeferredModuleLoader(importlib.abc.Loader):
889
- """Loader that simply returns a preconstructed module placeholder."""
890
-
891
- def __init__(self, module: ModuleType) -> None:
892
- self._module = module
893
-
894
- def create_module(self, spec): # noqa: D401 - standard loader hook
895
- return self._module
896
-
897
- def exec_module(self, module): # noqa: D401 - nothing to execute
898
- return None
899
-
900
- # =============================================================================
901
- # CACHE (from common.cache)
902
- # =============================================================================
903
-
904
- # MultiTierCache and BytecodeCache are now imported from ..common.cache
905
-
906
- # =============================================================================
907
- # PARALLEL UTILITIES (from parallel_utils.py)
908
- # =============================================================================
909
-
910
- class ParallelLoader:
911
- """Parallel module loader with smart dependency management."""
912
-
913
- def __init__(self, max_workers: Optional[int] = None):
914
- if max_workers is None:
915
- max_workers = min(os.cpu_count() or 4, 8)
916
-
917
- self._max_workers = max_workers
918
- self._executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
919
- self._lock = threading.RLock()
920
-
921
- def _get_executor(self) -> concurrent.futures.ThreadPoolExecutor:
922
- """Get or create thread pool executor."""
923
- with self._lock:
924
- if self._executor is None:
925
- self._executor = concurrent.futures.ThreadPoolExecutor(
926
- max_workers=self._max_workers,
927
- thread_name_prefix="xwlazy-parallel"
928
- )
929
- return self._executor
930
-
931
- def load_modules_parallel(self, module_paths: list[str]) -> dict[str, Any]:
932
- """Load multiple modules in parallel."""
933
- executor = self._get_executor()
934
- results: dict[str, Any] = {}
935
-
936
- def _load_module(module_path: str) -> tuple[str, Any, Optional[Exception]]:
937
- try:
938
- module = importlib.import_module(module_path)
939
- return (module_path, module, None)
940
- except Exception as e:
941
- logger.debug(f"Failed to load {module_path} in parallel: {e}")
942
- return (module_path, None, e)
943
-
944
- futures = {executor.submit(_load_module, path): path for path in module_paths}
945
-
946
- for future in concurrent.futures.as_completed(futures):
947
- module_path, module, error = future.result()
948
- results[module_path] = (module, error)
949
-
950
- return results
951
-
952
- def load_modules_with_priority(
953
- self,
954
- module_paths: list[tuple[str, int]]
955
- ) -> dict[str, Any]:
956
- """Load modules in parallel with priority ordering."""
957
- sorted_modules = sorted(module_paths, key=lambda x: x[1], reverse=True)
958
- module_list = [path for path, _ in sorted_modules]
959
- return self.load_modules_parallel(module_list)
960
-
961
- def shutdown(self, wait: bool = True) -> None:
962
- """Shutdown the executor."""
963
- with self._lock:
964
- if self._executor:
965
- self._executor.shutdown(wait=wait)
966
- self._executor = None
967
-
968
- class DependencyGraph:
969
- """Manages module dependencies for optimal parallel loading."""
970
-
971
- def __init__(self):
972
- self._dependencies: dict[str, list[str]] = {}
973
- self._reverse_deps: dict[str, list[str]] = {}
974
- self._lock = threading.RLock()
975
-
976
- def add_dependency(self, module: str, depends_on: list[str]) -> None:
977
- """Add dependencies for a module."""
978
- with self._lock:
979
- self._dependencies[module] = depends_on
980
- for dep in depends_on:
981
- if dep not in self._reverse_deps:
982
- self._reverse_deps[dep] = []
983
- if module not in self._reverse_deps[dep]:
984
- self._reverse_deps[dep].append(module)
985
-
986
- def get_load_order(self, modules: list[str]) -> list[list[str]]:
987
- """Get optimal load order for parallel loading (topological sort levels)."""
988
- with self._lock:
989
- in_degree: dict[str, int] = {m: 0 for m in modules}
990
- for module, deps in self._dependencies.items():
991
- if module in modules:
992
- for dep in deps:
993
- if dep in modules:
994
- in_degree[module] += 1
995
-
996
- levels: list[list[str]] = []
997
- remaining = set(modules)
998
-
999
- while remaining:
1000
- current_level = [
1001
- m for m in remaining
1002
- if in_degree[m] == 0
1003
- ]
1004
-
1005
- if not current_level:
1006
- current_level = list(remaining)
1007
-
1008
- levels.append(current_level)
1009
- remaining -= set(current_level)
1010
-
1011
- for module in current_level:
1012
- for dependent in self._reverse_deps.get(module, []):
1013
- if dependent in remaining:
1014
- in_degree[dependent] = max(0, in_degree[dependent] - 1)
1015
-
1016
- return levels
1017
-
1018
- # =============================================================================
1019
- # MODULE PATCHING (from module_patching.py)
1020
- # =============================================================================
1021
-
1022
- _original_import_module = importlib.import_module
1023
-
1024
- def _lazy_aware_import_module(name: str, package: Optional[str] = None) -> ModuleType:
1025
- """Lazy-aware version of importlib.import_module."""
1026
- if _is_import_in_progress(name):
1027
- return _original_import_module(name, package)
1028
-
1029
- _mark_import_started(name)
1030
- try:
1031
- return _original_import_module(name, package)
1032
- finally:
1033
- _mark_import_finished(name)
1034
-
1035
- def _patch_import_module() -> None:
1036
- """
1037
- Patch importlib.import_module to be lazy-aware.
1038
-
1039
- WARNING: This performs global monkey-patching that affects ALL code in the Python process.
1040
- This can cause conflicts with other libraries. Use sys.meta_path hooks instead.
1041
-
1042
- This function is kept for backward compatibility but should be avoided in new code.
1043
- """
1044
- # DISABLED: Dangerous monkey-patching disabled by default
1045
- logger.warning(
1046
- "_patch_import_module called - Dangerous monkey-patching is DISABLED. "
1047
- "Use sys.meta_path hooks (install_import_hook) instead."
1048
- )
1049
- return
1050
- # Original code (disabled):
1051
- # importlib.import_module = _lazy_aware_import_module
1052
- # logger.debug("Patched importlib.import_module to be lazy-aware")
1053
-
1054
- def _unpatch_import_module() -> None:
1055
- """Restore original importlib.import_module."""
1056
- importlib.import_module = _original_import_module
1057
- logger.debug("Restored original importlib.import_module")
1058
-
1059
- # =============================================================================
1060
- # ARCHIVE IMPORTS (from archive_imports.py)
1061
- # =============================================================================
1062
-
1063
- _archive_path = None
1064
- _archive_added = False
1065
-
1066
- def get_archive_path() -> Path:
1067
- """Get the path to the _archive folder."""
1068
- global _archive_path
1069
- if _archive_path is None:
1070
- current_file = Path(__file__)
1071
- _archive_path = current_file.parent.parent.parent.parent.parent.parent / "_archive"
1072
- return _archive_path
1073
-
1074
- def ensure_archive_in_path() -> None:
1075
- """Ensure the archive folder is in sys.path for imports."""
1076
- global _archive_added
1077
- if not _archive_added:
1078
- archive_path = get_archive_path()
1079
- archive_str = str(archive_path)
1080
- if archive_str not in sys.path:
1081
- sys.path.insert(0, archive_str)
1082
- _archive_added = True
1083
-
1084
- def import_from_archive(module_name: str):
1085
- """Import a module from the archived lazy code."""
1086
- ensure_archive_in_path()
1087
- return __import__(module_name, fromlist=[''])
1088
-
1089
- # =============================================================================
1090
- # BOOTSTRAP (from bootstrap.py)
1091
- # =============================================================================
1092
-
1093
- def _env_enabled(env_value: Optional[str]) -> Optional[bool]:
1094
- if not env_value:
1095
- return None
1096
- normalized = env_value.strip().lower()
1097
- if normalized in ('true', '1', 'yes', 'on'):
1098
- return True
1099
- if normalized in ('false', '0', 'no', 'off'):
1100
- return False
1101
- return None
1102
-
1103
- def bootstrap_lazy_mode(package_name: str) -> None:
1104
- """Detect whether lazy mode should be enabled for ``package_name`` and bootstrap hooks."""
1105
- package_name = package_name.lower()
1106
- env_value = os.environ.get(f"{package_name.upper()}_LAZY_INSTALL")
1107
- env_enabled = _env_enabled(env_value)
1108
- enabled = env_enabled
1109
-
1110
- if enabled is None:
1111
- from ..common.services.keyword_detection import _detect_lazy_installation
1112
- enabled = _detect_lazy_installation(package_name)
1113
-
1114
- if not enabled:
1115
- return
1116
-
1117
- from ..facade import config_package_lazy_install_enabled
1118
-
1119
- config_package_lazy_install_enabled(
1120
- package_name,
1121
- enabled=True,
1122
- install_hook=True,
1123
- )
1124
-
1125
- def bootstrap_lazy_mode_deferred(package_name: str) -> None:
1126
- """
1127
- Schedule lazy mode bootstrap to run AFTER the calling package finishes importing.
1128
-
1129
- WARNING: This function performs dangerous global monkey-patching of __builtins__.__import__.
1130
- This can cause conflicts with other libraries (gevent, greenlet, debuggers, etc.).
1131
- Consider using sys.meta_path hooks instead (install_import_hook) which is safer.
1132
-
1133
- This function is kept for backward compatibility but should be avoided in new code.
1134
- """
1135
- # DISABLED: Dangerous monkey-patching disabled by default
1136
- # Uncomment only if absolutely necessary and you understand the risks
1137
- logger.warning(
1138
- f"bootstrap_lazy_mode_deferred called for {package_name} - "
1139
- "Dangerous monkey-patching is DISABLED. Use install_import_hook() instead."
1140
- )
1141
- return
1142
-
1143
- # Original code (disabled):
1144
- # package_name_lower = package_name.lower()
1145
- # package_module_name = f"exonware.{package_name_lower}"
1146
- # original_import = __builtins__.__import__ if hasattr(__builtins__, '__import__') else __import__
1147
- # ... (rest of dangerous code)
1148
-
1149
- # =============================================================================
1150
- # LAZY LOADER (from loader.py)
1151
- # =============================================================================
1152
-
1153
- class LazyLoader(AModuleHelper):
1154
- """Thread-safe lazy loader for modules with caching."""
1155
-
1156
- def load_module(self, module_path: str = None) -> ModuleType:
1157
- """Thread-safe module loading with caching."""
1158
- if module_path is None:
1159
- module_path = self._module_path
1160
-
1161
- if self._cached_module is not None:
1162
- return self._cached_module
1163
-
1164
- with self._lock:
1165
- if self._cached_module is not None:
1166
- return self._cached_module
1167
-
1168
- if self._loading:
1169
- raise ImportError(f"Circular import detected for {module_path}")
1170
-
1171
- try:
1172
- self._loading = True
1173
- logger.debug(f"Lazy loading module: {module_path}")
1174
-
1175
- self._cached_module = importlib.import_module(module_path)
1176
-
1177
- logger.debug(f"Successfully loaded: {module_path}")
1178
- return self._cached_module
1179
-
1180
- except Exception as e:
1181
- logger.error(f"Failed to load module {module_path}: {e}")
1182
- raise ImportError(f"Failed to load {module_path}: {e}") from e
1183
- finally:
1184
- self._loading = False
1185
-
1186
- def unload_module(self, module_path: str) -> None:
1187
- """Unload a module from cache."""
1188
- with self._lock:
1189
- if module_path == self._module_path:
1190
- self._cached_module = None
1191
-
1192
- def is_loaded(self) -> bool:
1193
- """Check if module is currently loaded."""
1194
- return self._cached_module is not None
1195
-
1196
- def __getattr__(self, name: str) -> Any:
1197
- """Get attribute from lazily loaded module."""
1198
- module = self.load_module()
1199
- try:
1200
- return getattr(module, name)
1201
- except AttributeError:
1202
- raise AttributeError(
1203
- f"module '{self._module_path}' has no attribute '{name}'"
1204
- )
1205
-
1206
- def __dir__(self) -> list:
1207
- """Return available attributes from loaded module."""
1208
- module = self.load_module()
1209
- return dir(module)
1210
-
1211
- # =============================================================================
1212
- # LAZY MODULE REGISTRY (from registry.py)
1213
- # =============================================================================
1214
-
1215
- class LazyModuleRegistry:
1216
- """Registry for managing lazy-loaded modules with performance tracking."""
1217
-
1218
- __slots__ = ('_modules', '_load_times', '_lock', '_access_counts')
1219
-
1220
- def __init__(self):
1221
- self._modules: dict[str, LazyLoader] = {}
1222
- self._load_times: dict[str, float] = {}
1223
- self._access_counts: dict[str, int] = {}
1224
- self._lock = threading.RLock()
1225
-
1226
- def register_module(self, name: str, module_path: str) -> None:
1227
- """Register a module for lazy loading."""
1228
- with self._lock:
1229
- if name in self._modules:
1230
- logger.warning(f"Module '{name}' already registered, overwriting")
1231
-
1232
- self._modules[name] = LazyLoader(module_path)
1233
- self._access_counts[name] = 0
1234
- logger.debug(f"Registered lazy module: {name} -> {module_path}")
1235
-
1236
- def get_module(self, name: str) -> LazyLoader:
1237
- """Get a lazy-loaded module."""
1238
- with self._lock:
1239
- if name not in self._modules:
1240
- raise KeyError(f"Module '{name}' not registered")
1241
-
1242
- self._access_counts[name] += 1
1243
- return self._modules[name]
1244
-
1245
- def preload_frequently_used(self, threshold: int = 5) -> None:
1246
- """Preload modules that are accessed frequently."""
1247
- with self._lock:
1248
- for name, count in self._access_counts.items():
1249
- if count >= threshold:
1250
- try:
1251
- start_time = time.time()
1252
- _ = self._modules[name].load_module()
1253
- self._load_times[name] = time.time() - start_time
1254
- log_event("hook", logger.info, f"Preloaded frequently used module: {name}")
1255
- except Exception as e:
1256
- logger.warning(f"Failed to preload {name}: {e}")
1257
-
1258
- def get_stats(self) -> dict[str, Any]:
1259
- """Get loading statistics."""
1260
- with self._lock:
1261
- loaded_count = sum(
1262
- 1 for loader in self._modules.values()
1263
- if loader.is_loaded()
1264
- )
1265
-
1266
- return {
1267
- 'total_registered': len(self._modules),
1268
- 'loaded_count': loaded_count,
1269
- 'unloaded_count': len(self._modules) - loaded_count,
1270
- 'access_counts': self._access_counts.copy(),
1271
- 'load_times': self._load_times.copy(),
1272
- }
1273
-
1274
- def clear_cache(self) -> None:
1275
- """Clear all cached modules."""
1276
- with self._lock:
1277
- for name, loader in self._modules.items():
1278
- loader.unload_module(loader._module_path)
1279
- log_event("config", logger.info, "Cleared all cached modules")
1280
-
1281
- # =============================================================================
1282
- # LAZY IMPORTER (from importer.py)
1283
- # =============================================================================
1284
-
1285
- class LazyImporter:
1286
- """
1287
- Lazy importer that defers heavy module imports until first access.
1288
-
1289
- Supports multiple load modes: NONE, AUTO, PRELOAD, BACKGROUND, CACHED,
1290
- TURBO, ADAPTIVE, HYPERPARALLEL, STREAMING, ULTRA, INTELLIGENT.
1291
-
1292
- ARCHITECTURAL NOTE: This class implements many modes for what should be a simple
1293
- lazy loading mechanism. Modes like TURBO, ULTRA, and HYPERPARALLEL add significant
1294
- complexity and may introduce concurrency issues with Python's import lock.
1295
- Consider simplifying to just Lazy/Eager modes in future refactoring.
1296
- """
1297
-
1298
- __slots__ = (
1299
- '_enabled', '_load_mode', '_lazy_modules', '_loaded_modules', '_lock',
1300
- '_access_counts', '_background_tasks', '_async_loop',
1301
- '_multi_tier_cache', '_bytecode_cache', '_adaptive_learner',
1302
- '_parallel_loader', '_dependency_graph', '_load_times',
1303
- '_intelligent_selector', '_effective_mode', '_effective_install_mode'
1304
- )
1305
-
1306
- def __init__(self):
1307
- """Initialize lazy importer."""
1308
- self._enabled = False
1309
- self._load_mode = LazyLoadMode.NONE
1310
- self._lazy_modules: dict[str, str] = {}
1311
- self._loaded_modules: dict[str, ModuleType] = {}
1312
- self._access_counts: dict[str, int] = {}
1313
- self._load_times: dict[str, float] = {}
1314
- self._background_tasks: dict[str, asyncio.Task] = {}
1315
- self._async_loop: Optional[asyncio.AbstractEventLoop] = None
1316
-
1317
- # Superior mode components
1318
- self._multi_tier_cache: Optional[MultiTierCache] = None
1319
- self._bytecode_cache: Optional[BytecodeCache] = None
1320
- self._adaptive_learner: Optional[AdaptiveLearner] = None
1321
- self._parallel_loader: Optional[ParallelLoader] = None
1322
- self._dependency_graph: Optional[DependencyGraph] = None
1323
- self._intelligent_selector: Optional[IntelligentModeSelector] = None
1324
-
1325
- # Effective modes (for INTELLIGENT mode)
1326
- self._effective_mode: Optional[LazyLoadMode] = None
1327
- self._effective_install_mode = None
1328
-
1329
- self._lock = threading.RLock()
1330
-
1331
- def _ensure_async_loop(self) -> asyncio.AbstractEventLoop:
1332
- """Ensure async event loop is running for background loading."""
1333
- if self._async_loop is not None and self._async_loop.is_running():
1334
- return self._async_loop
1335
-
1336
- with self._lock:
1337
- if self._async_loop is None or not self._async_loop.is_running():
1338
- loop_ready = threading.Event()
1339
-
1340
- def _run_loop():
1341
- loop = asyncio.new_event_loop()
1342
- asyncio.set_event_loop(loop)
1343
- self._async_loop = loop
1344
- loop_ready.set()
1345
- loop.run_forever()
1346
-
1347
- thread = threading.Thread(target=_run_loop, daemon=True, name="xwlazy-loader-async")
1348
- thread.start()
1349
-
1350
- if not loop_ready.wait(timeout=5.0):
1351
- raise RuntimeError("Failed to start async loop for lazy loader")
1352
-
1353
- return self._async_loop
1354
-
1355
- def enable(self, load_mode: LazyLoadMode = LazyLoadMode.AUTO) -> None:
1356
- """Enable lazy imports with specified load mode."""
1357
- with self._lock:
1358
- self._enabled = True
1359
- self._load_mode = load_mode
1360
-
1361
- # Initialize superior mode components
1362
- if load_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA):
1363
- self._multi_tier_cache = MultiTierCache(l1_size=1000, enable_l3=True)
1364
- self._bytecode_cache = BytecodeCache()
1365
-
1366
- if load_mode == LazyLoadMode.ADAPTIVE:
1367
- self._adaptive_learner = AdaptiveLearner()
1368
- self._multi_tier_cache = MultiTierCache(l1_size=1000, enable_l3=True)
1369
-
1370
- if load_mode == LazyLoadMode.HYPERPARALLEL:
1371
- max_workers = min(os.cpu_count() or 4, 8)
1372
- self._parallel_loader = ParallelLoader(max_workers=max_workers)
1373
- self._dependency_graph = DependencyGraph()
1374
-
1375
- if load_mode == LazyLoadMode.STREAMING:
1376
- self._ensure_async_loop()
1377
-
1378
- if load_mode == LazyLoadMode.ULTRA:
1379
- # ULTRA combines all optimizations
1380
- if self._multi_tier_cache is None:
1381
- self._multi_tier_cache = MultiTierCache(l1_size=2000, enable_l3=True)
1382
- if self._bytecode_cache is None:
1383
- self._bytecode_cache = BytecodeCache()
1384
- if self._adaptive_learner is None:
1385
- self._adaptive_learner = AdaptiveLearner()
1386
- if self._parallel_loader is None:
1387
- self._parallel_loader = ParallelLoader(max_workers=min(os.cpu_count() or 4, 8))
1388
- if self._dependency_graph is None:
1389
- self._dependency_graph = DependencyGraph()
1390
- self._ensure_async_loop()
1391
-
1392
- # INTELLIGENT mode: Initialize selector and determine initial mode
1393
- if load_mode == LazyLoadMode.INTELLIGENT:
1394
- self._intelligent_selector = IntelligentModeSelector()
1395
- # Detect initial load level and get optimal mode
1396
- initial_level = self._intelligent_selector.detect_load_level()
1397
- self._effective_mode, self._effective_install_mode = self._intelligent_selector.get_optimal_mode(initial_level)
1398
- logger.info(f"INTELLIGENT mode initialized: {initial_level.value} -> {self._effective_mode.value} + {self._effective_install_mode.value}")
1399
- # Enable the effective mode recursively
1400
- self.enable(self._effective_mode)
1401
- return # Early return, effective mode is already enabled
1402
-
1403
- # For PRELOAD/TURBO/ULTRA modes, preload modules
1404
- if load_mode in (LazyLoadMode.PRELOAD, LazyLoadMode.TURBO, LazyLoadMode.ULTRA):
1405
- self._preload_all_modules()
1406
- # For BACKGROUND/STREAMING modes, ensure async loop is ready
1407
- elif load_mode in (LazyLoadMode.BACKGROUND, LazyLoadMode.STREAMING):
1408
- self._ensure_async_loop()
1409
-
1410
- log_event("config", logger.info, f"Lazy imports enabled (mode: {load_mode.value})")
1411
-
1412
- def disable(self) -> None:
1413
- """Disable lazy imports."""
1414
- with self._lock:
1415
- self._enabled = False
1416
-
1417
- # Cleanup cache resources
1418
- if self._multi_tier_cache:
1419
- self._multi_tier_cache.shutdown()
1420
-
1421
- log_event("config", logger.info, "Lazy imports disabled")
1422
-
1423
- def is_enabled(self) -> bool:
1424
- """Check if lazy imports are enabled."""
1425
- return self._enabled
1426
-
1427
- def register_lazy_module(self, module_name: str, module_path: str = None) -> None:
1428
- """Register a module for lazy loading."""
1429
- with self._lock:
1430
- if module_path is None:
1431
- module_path = module_name
1432
-
1433
- self._lazy_modules[module_name] = module_path
1434
- self._access_counts[module_name] = 0
1435
- logger.debug(f"Registered lazy module: {module_name} -> {module_path}")
1436
-
1437
- async def _background_load_module(self, module_name: str, module_path: str) -> ModuleType:
1438
- """Load module in background thread."""
1439
- try:
1440
- actual_module = importlib.import_module(module_path)
1441
- with self._lock:
1442
- self._loaded_modules[module_name] = actual_module
1443
- self._access_counts[module_name] += 1
1444
- logger.debug(f"Background loaded module: {module_name}")
1445
- return actual_module
1446
- except ImportError as e:
1447
- logger.error(f"Failed to background load {module_name}: {e}")
1448
- raise
1449
-
1450
- def _preload_all_modules(self) -> None:
1451
- """Preload all registered modules using appropriate strategy based on mode."""
1452
- if not self._lazy_modules:
1453
- return
1454
-
1455
- with self._lock:
1456
- modules_to_load = [
1457
- (name, path) for name, path in self._lazy_modules.items()
1458
- if name not in self._loaded_modules
1459
- ]
1460
-
1461
- if not modules_to_load:
1462
- return
1463
-
1464
- # HYPERPARALLEL/ULTRA: Use thread pool executor
1465
- if self._load_mode in (LazyLoadMode.HYPERPARALLEL, LazyLoadMode.ULTRA) and self._parallel_loader:
1466
- module_paths = [path for _, path in modules_to_load]
1467
- results = self._parallel_loader.load_modules_parallel(module_paths)
1468
-
1469
- with self._lock:
1470
- for (name, path), (module, error) in zip(modules_to_load, results.items()):
1471
- if module is not None:
1472
- self._loaded_modules[name] = module
1473
- self._access_counts[name] = 0
1474
- if self._adaptive_learner:
1475
- self._adaptive_learner.record_import(name, 0.0)
1476
-
1477
- log_event("hook", logger.info, f"Parallel preloaded {len([r for r in results.values() if r[0] is not None])} modules")
1478
- return
1479
-
1480
- # TURBO/ULTRA: Preload with predictive caching
1481
- if self._load_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._multi_tier_cache:
1482
- # Get predictive modules to prioritize
1483
- predictive_keys = self._multi_tier_cache.get_predictive_keys(limit=10)
1484
- priority_modules = [(name, path) for name, path in modules_to_load if name in predictive_keys]
1485
- normal_modules = [(name, path) for name, path in modules_to_load if name not in predictive_keys]
1486
- modules_to_load = priority_modules + normal_modules
1487
-
1488
- # ADAPTIVE: Preload based on learned patterns
1489
- if self._load_mode == LazyLoadMode.ADAPTIVE and self._adaptive_learner:
1490
- priority_modules = self._adaptive_learner.get_priority_modules(limit=10)
1491
- priority_list = [(name, path) for name, path in modules_to_load if name in priority_modules]
1492
- normal_list = [(name, path) for name, path in modules_to_load if name not in priority_modules]
1493
- modules_to_load = priority_list + normal_list
1494
-
1495
- # Default: Use asyncio for parallel loading
1496
- loop = self._ensure_async_loop()
1497
-
1498
- async def _preload_all():
1499
- tasks = [
1500
- self._background_load_module(name, path)
1501
- for name, path in modules_to_load
1502
- ]
1503
- if tasks:
1504
- await asyncio.gather(*tasks, return_exceptions=True)
1505
- log_event("hook", logger.info, f"Preloaded {len(tasks)} modules")
1506
-
1507
- asyncio.run_coroutine_threadsafe(_preload_all(), loop)
1508
-
1509
- def import_module(self, module_name: str, package_name: str = None) -> Any:
1510
- """Import a module with lazy loading."""
1511
- start_time = time.time()
1512
-
1513
- # Fast path: Check if already in sys.modules (lock-free read)
1514
- if module_name in sys.modules:
1515
- # Lock-free check first
1516
- if module_name not in self._loaded_modules:
1517
- with self._lock:
1518
- # Double-check after acquiring lock
1519
- if module_name not in self._loaded_modules:
1520
- self._loaded_modules[module_name] = sys.modules[module_name]
1521
- # Update access count (requires lock)
1522
- with self._lock:
1523
- self._access_counts[module_name] = self._access_counts.get(module_name, 0) + 1
1524
- load_time = time.time() - start_time
1525
- if self._adaptive_learner:
1526
- self._adaptive_learner.record_import(module_name, load_time)
1527
- if self._load_mode == LazyLoadMode.INTELLIGENT:
1528
- self._total_import_time = getattr(self, '_total_import_time', 0.0) + load_time
1529
- return sys.modules[module_name]
1530
-
1531
- # Fast path: Check if already loaded (lock-free read)
1532
- if module_name in self._loaded_modules:
1533
- with self._lock:
1534
- # Double-check and update
1535
- if module_name in self._loaded_modules:
1536
- self._access_counts[module_name] = self._access_counts.get(module_name, 0) + 1
1537
- if self._adaptive_learner:
1538
- self._adaptive_learner.record_import(module_name, 0.0)
1539
- return self._loaded_modules[module_name]
1540
-
1541
- # Check enabled state and get module path (requires lock)
1542
- with self._lock:
1543
- if not self._enabled or self._load_mode == LazyLoadMode.NONE:
1544
- return importlib.import_module(module_name)
1545
-
1546
- if module_name in self._lazy_modules:
1547
- module_path = self._lazy_modules[module_name]
1548
- else:
1549
- return importlib.import_module(module_name)
1550
-
1551
- # Update total import time for intelligent mode (initialization)
1552
- if self._load_mode == LazyLoadMode.INTELLIGENT:
1553
- if not hasattr(self, '_total_import_time'):
1554
- self._total_import_time = 0.0
1555
-
1556
- # INTELLIGENT mode: Check if mode switch is needed and determine effective mode
1557
- effective_load_mode = self._load_mode
1558
- if self._load_mode == LazyLoadMode.INTELLIGENT and self._intelligent_selector:
1559
- # Throttle load level detection (cache for 0.1s to avoid excessive checks)
1560
- current_time = time.time()
1561
- last_check = getattr(self, '_last_load_level_check', 0.0)
1562
- check_interval = 0.1 # 100ms throttle
1563
-
1564
- if current_time - last_check >= check_interval:
1565
- # Fast path: lock-free reads for stats
1566
- module_count = len(self._loaded_modules) # Dict read is thread-safe
1567
- total_import_time = getattr(self, '_total_import_time', 0.0)
1568
- import_count = sum(self._access_counts.values()) # Dict read is thread-safe
1569
-
1570
- # Cache psutil import and memory check (only check every 0.5s)
1571
- last_memory_check = getattr(self, '_last_memory_check', 0.0)
1572
- memory_mb = getattr(self, '_cached_memory_mb', 0.0)
1573
-
1574
- if current_time - last_memory_check >= 0.5:
1575
- try:
1576
- import psutil
1577
- process = psutil.Process()
1578
- memory_mb = process.memory_info().rss / 1024 / 1024
1579
- self._cached_memory_mb = memory_mb
1580
- self._last_memory_check = current_time
1581
- except Exception:
1582
- memory_mb = 0.0
1583
-
1584
- # Detect current load level (lock-free)
1585
- detected_level = self._intelligent_selector.detect_load_level(
1586
- module_count=module_count,
1587
- total_import_time=total_import_time,
1588
- import_count=import_count,
1589
- memory_usage_mb=memory_mb
1590
- )
1591
-
1592
- # Check if mode switch is needed (requires lock for write)
1593
- current_mode_tuple = (self._effective_mode or self._load_mode, self._effective_install_mode)
1594
- if self._intelligent_selector.should_switch_mode(current_mode_tuple, detected_level):
1595
- optimal_load, optimal_install = self._intelligent_selector.get_optimal_mode(detected_level)
1596
- if optimal_load != self._effective_mode or optimal_install != self._effective_install_mode:
1597
- with self._lock: # Only lock for mode switch
1598
- if optimal_load != self._effective_mode or optimal_install != self._effective_install_mode:
1599
- logger.info(f"INTELLIGENT mode switching: {detected_level.value} -> {optimal_load.value} + {optimal_install.value}")
1600
- self._effective_mode = optimal_load
1601
- self._effective_install_mode = optimal_install
1602
- # Switch to optimal mode (re-enable with new mode)
1603
- self.enable(optimal_load)
1604
-
1605
- self._last_load_level_check = current_time
1606
-
1607
- # Use effective mode for processing
1608
- effective_load_mode = self._effective_mode or self._load_mode
1609
-
1610
- # Use effective mode for all checks
1611
- check_mode = effective_load_mode
1612
-
1613
- # TURBO/ULTRA: Check multi-tier cache first
1614
- if check_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._multi_tier_cache:
1615
- cached_module = self._multi_tier_cache.get(module_name)
1616
- if cached_module is not None:
1617
- with self._lock:
1618
- self._loaded_modules[module_name] = cached_module
1619
- self._access_counts[module_name] += 1
1620
- self._load_times[module_name] = time.time() - start_time
1621
- if self._adaptive_learner:
1622
- self._adaptive_learner.record_import(module_name, self._load_times[module_name])
1623
- return cached_module
1624
-
1625
- # ADAPTIVE: Check cache and predict next imports
1626
- if check_mode == LazyLoadMode.ADAPTIVE:
1627
- if self._multi_tier_cache:
1628
- cached_module = self._multi_tier_cache.get(module_name)
1629
- if cached_module is not None:
1630
- with self._lock:
1631
- self._loaded_modules[module_name] = cached_module
1632
- self._access_counts[module_name] += 1
1633
- load_time = time.time() - start_time
1634
- self._load_times[module_name] = load_time
1635
- if self._adaptive_learner:
1636
- self._adaptive_learner.record_import(module_name, load_time)
1637
-
1638
- # Predict and preload next likely imports
1639
- if self._adaptive_learner:
1640
- next_imports = self._adaptive_learner.predict_next_imports(module_name, limit=3)
1641
- self._preload_predictive_modules(next_imports)
1642
-
1643
- return cached_module
1644
-
1645
- # Record import for learning
1646
- with self._lock:
1647
- if self._adaptive_learner:
1648
- # Will be updated after load
1649
- pass
1650
-
1651
- # HYPERPARALLEL: Use parallel loading
1652
- if check_mode == LazyLoadMode.HYPERPARALLEL and self._parallel_loader:
1653
- results = self._parallel_loader.load_modules_parallel([module_path])
1654
- module, error = results.get(module_path, (None, None))
1655
- if module is not None:
1656
- with self._lock:
1657
- self._loaded_modules[module_name] = module
1658
- self._access_counts[module_name] += 1
1659
- self._load_times[module_name] = time.time() - start_time
1660
- return module
1661
- elif error:
1662
- raise error
1663
-
1664
- # STREAMING: Load asynchronously in background
1665
- if check_mode == LazyLoadMode.STREAMING:
1666
- return self._streaming_load(module_name, module_path)
1667
-
1668
- # BACKGROUND mode: Load in background, return placeholder
1669
- if check_mode == LazyLoadMode.BACKGROUND:
1670
- return self._background_placeholder_load(module_name, module_path)
1671
-
1672
- # TURBO/ULTRA: Load with bytecode cache
1673
- actual_module = None
1674
- if check_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._bytecode_cache:
1675
- # Try to load from bytecode cache first
1676
- bytecode = self._bytecode_cache.get_cached_bytecode(module_path)
1677
- if bytecode is not None:
1678
- try:
1679
- # Load from bytecode
1680
- code = compile(bytecode, f"<cached {module_path}>", "exec")
1681
- actual_module = importlib.import_module(module_path)
1682
- except Exception as e:
1683
- logger.debug(f"Failed to load from bytecode cache: {e}")
1684
-
1685
- # Load module (standard or cached)
1686
- if actual_module is None:
1687
- try:
1688
- actual_module = importlib.import_module(module_path)
1689
-
1690
- # Cache bytecode for TURBO/ULTRA
1691
- if check_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._bytecode_cache:
1692
- try:
1693
- # Get compiled bytecode from module
1694
- if hasattr(actual_module, '__file__') and actual_module.__file__:
1695
- pyc_path = actual_module.__file__.replace('.py', '.pyc')
1696
- if os.path.exists(pyc_path):
1697
- with open(pyc_path, 'rb') as f:
1698
- f.seek(16) # Skip header
1699
- bytecode = f.read()
1700
- self._bytecode_cache.cache_bytecode(module_path, bytecode)
1701
- except Exception as e:
1702
- logger.debug(f"Failed to cache bytecode: {e}")
1703
- except ImportError as e:
1704
- logger.error(f"Failed to lazy load {module_name}: {e}")
1705
- raise
1706
-
1707
- load_time = time.time() - start_time
1708
-
1709
- with self._lock:
1710
- self._loaded_modules[module_name] = actual_module
1711
- self._access_counts[module_name] += 1
1712
- self._load_times[module_name] = load_time
1713
-
1714
- # Update total import time for intelligent mode
1715
- if self._load_mode == LazyLoadMode.INTELLIGENT:
1716
- self._total_import_time = getattr(self, '_total_import_time', 0.0) + load_time
1717
-
1718
- # Cache in multi-tier cache for TURBO/ULTRA/ADAPTIVE
1719
- if self._multi_tier_cache:
1720
- self._multi_tier_cache.set(module_name, actual_module)
1721
-
1722
- # Record for adaptive learning
1723
- if self._adaptive_learner:
1724
- self._adaptive_learner.record_import(module_name, load_time)
1725
-
1726
- logger.debug(f"Lazy loaded module: {module_name} ({load_time*1000:.2f}ms)")
1727
-
1728
- return actual_module
1729
-
1730
- def _streaming_load(self, module_name: str, module_path: str) -> ModuleType:
1731
- """Load module asynchronously with streaming."""
1732
- if module_name not in self._background_tasks or self._background_tasks[module_name].done():
1733
- loop = self._ensure_async_loop()
1734
- task = asyncio.run_coroutine_threadsafe(
1735
- self._background_load_module(module_name, module_path),
1736
- loop
1737
- )
1738
- self._background_tasks[module_name] = task
1739
-
1740
- # Return placeholder that streams
1741
- placeholder = ModuleType(module_name)
1742
- placeholder.__path__ = []
1743
- placeholder.__package__ = module_name
1744
-
1745
- def _streaming_getattr(name):
1746
- task = self._background_tasks.get(module_name)
1747
- if task and not task.done():
1748
- # Non-blocking check with short timeout
1749
- try:
1750
- task.result(timeout=0.01) # Very short timeout for streaming
1751
- except Exception:
1752
- pass # Still loading, continue
1753
-
1754
- # Check if loaded now
1755
- with self._lock:
1756
- if module_name in self._loaded_modules:
1757
- return getattr(self._loaded_modules[module_name], name)
1758
-
1759
- # Still loading, wait for completion
1760
- if task and not task.done():
1761
- task.result(timeout=10.0)
1762
-
1763
- with self._lock:
1764
- if module_name in self._loaded_modules:
1765
- return getattr(self._loaded_modules[module_name], name)
1766
- raise AttributeError(f"module '{module_name}' has no attribute '{name}'")
1767
-
1768
- placeholder.__getattr__ = _streaming_getattr # type: ignore[attr-defined]
1769
- return placeholder
1770
-
1771
- def _background_placeholder_load(self, module_name: str, module_path: str) -> ModuleType:
1772
- """Load module in background, return placeholder."""
1773
- if module_name not in self._background_tasks or self._background_tasks[module_name].done():
1774
- loop = self._ensure_async_loop()
1775
- task = asyncio.run_coroutine_threadsafe(
1776
- self._background_load_module(module_name, module_path),
1777
- loop
1778
- )
1779
- self._background_tasks[module_name] = task
1780
-
1781
- # Return placeholder module that will be replaced when loaded
1782
- placeholder = ModuleType(module_name)
1783
- placeholder.__path__ = []
1784
- placeholder.__package__ = module_name
1785
-
1786
- def _getattr(name):
1787
- # Wait for background load to complete
1788
- task = self._background_tasks.get(module_name)
1789
- if task and not task.done():
1790
- task.result(timeout=10.0) # Wait up to 10 seconds
1791
- with self._lock:
1792
- if module_name in self._loaded_modules:
1793
- return getattr(self._loaded_modules[module_name], name)
1794
- raise AttributeError(f"module '{module_name}' has no attribute '{name}'")
1795
-
1796
- placeholder.__getattr__ = _getattr # type: ignore[attr-defined]
1797
- return placeholder
1798
-
1799
- def _preload_predictive_modules(self, module_names: list) -> None:
1800
- """Preload modules predicted to be needed soon."""
1801
- if not module_names:
1802
- return
1803
-
1804
- with self._lock:
1805
- modules_to_preload = [
1806
- (name, self._lazy_modules[name])
1807
- for name in module_names
1808
- if name in self._lazy_modules and name not in self._loaded_modules
1809
- ]
1810
-
1811
- if not modules_to_preload:
1812
- return
1813
-
1814
- # Preload in background
1815
- loop = self._ensure_async_loop()
1816
-
1817
- async def _preload_predictive():
1818
- tasks = [
1819
- self._background_load_module(name, path)
1820
- for name, path in modules_to_preload
1821
- ]
1822
- await asyncio.gather(*tasks, return_exceptions=True)
1823
-
1824
- asyncio.run_coroutine_threadsafe(_preload_predictive(), loop)
1825
-
1826
- def preload_module(self, module_name: str) -> bool:
1827
- """Preload a registered lazy module."""
1828
- with self._lock:
1829
- if module_name not in self._lazy_modules:
1830
- logger.warning(f"Module {module_name} not registered for lazy loading")
1831
- return False
1832
-
1833
- try:
1834
- self.import_module(module_name)
1835
- log_event("hook", logger.info, f"Preloaded module: {module_name}")
1836
- return True
1837
- except Exception as e:
1838
- logger.error(f"Failed to preload {module_name}: {e}")
1839
- return False
1840
-
1841
- def get_stats(self) -> dict[str, Any]:
1842
- """Get lazy import statistics."""
1843
- with self._lock:
1844
- return {
1845
- 'enabled': self._enabled,
1846
- 'registered_modules': list(self._lazy_modules.keys()),
1847
- 'loaded_modules': list(self._loaded_modules.keys()),
1848
- 'access_counts': self._access_counts.copy(),
1849
- 'total_registered': len(self._lazy_modules),
1850
- 'total_loaded': len(self._loaded_modules)
1851
- }
1852
-
1853
- # =============================================================================
1854
- # IMPORT HOOK (from import_hook.py)
1855
- # =============================================================================
1856
-
1857
- class LazyImportHook(AModuleHelper):
1858
- """
1859
- Import hook that intercepts ImportError and auto-installs packages.
1860
- Performance optimized with zero overhead for successful imports.
1861
- """
1862
-
1863
- __slots__ = AModuleHelper.__slots__
1864
-
1865
- def handle_import_error(self, module_name: str) -> Optional[Any]:
1866
- """Handle ImportError by attempting to install and re-import."""
1867
- if not self._enabled:
1868
- return None
1869
-
1870
- try:
1871
- # Deferred import to avoid circular dependency
1872
- from ..facade import lazy_import_with_install
1873
- module, success = lazy_import_with_install(
1874
- module_name,
1875
- installer_package=self._package_name
1876
- )
1877
- return module if success else None
1878
- except Exception:
1879
- return None
1880
-
1881
- def install_hook(self) -> None:
1882
- """Install the import hook into sys.meta_path."""
1883
- install_import_hook(self._package_name)
1884
-
1885
- def uninstall_hook(self) -> None:
1886
- """Uninstall the import hook from sys.meta_path."""
1887
- uninstall_import_hook(self._package_name)
1888
-
1889
- def is_installed(self) -> bool:
1890
- """Check if hook is installed."""
1891
- return is_import_hook_installed(self._package_name)
1892
-
1893
- # =============================================================================
1894
- # META PATH FINDER (from meta_path_finder.py)
1895
- # =============================================================================
1896
-
1897
- # Wrapped class cache
1898
- _WRAPPED_CLASS_CACHE: dict[str, set[str]] = defaultdict(set)
1899
- _wrapped_cache_lock = threading.RLock()
1900
-
1901
- # Default lazy methods
1902
- _DEFAULT_LAZY_METHODS = tuple(
1903
- filter(
1904
- None,
1905
- os.environ.get("XWLAZY_LAZY_METHODS", "").split(","),
1906
- )
1907
- )
1908
-
1909
- # Lazy prefix method registry
1910
- _lazy_prefix_method_registry: dict[str, tuple[str, ...]] = {}
1911
-
1912
- # Package class hints
1913
- _package_class_hints: dict[str, tuple[str, ...]] = {}
1914
- _class_hint_lock = threading.RLock()
1915
-
1916
- def _set_package_class_hints(package_name: str, hints: Iterable[str]) -> None:
1917
- """Set class hints for a package."""
1918
- normalized: tuple[str, ...] = tuple(
1919
- OrderedDict((hint.lower(), None) for hint in hints if hint).keys() # type: ignore[arg-type]
1920
- )
1921
- with _class_hint_lock:
1922
- if normalized:
1923
- _package_class_hints[package_name] = normalized
1924
- else:
1925
- _package_class_hints.pop(package_name, None)
1926
-
1927
- def _get_package_class_hints(package_name: str) -> tuple[str, ...]:
1928
- """Get class hints for a package."""
1929
- with _class_hint_lock:
1930
- return _package_class_hints.get(package_name, ())
1931
-
1932
- def _clear_all_package_class_hints() -> None:
1933
- """Clear all package class hints."""
1934
- with _class_hint_lock:
1935
- _package_class_hints.clear()
1936
-
1937
- def register_lazy_module_methods(prefix: str, methods: tuple[str, ...]) -> None:
1938
- """Register method names to enhance for all classes under a module prefix."""
1939
- prefix = prefix.strip()
1940
- if not prefix:
1941
- return
1942
-
1943
- if not prefix.endswith("."):
1944
- prefix += "."
1945
-
1946
- _lazy_prefix_method_registry[prefix] = methods
1947
- log_event("config", logger.info, f"Registered lazy module methods for prefix {prefix}: {methods}")
1948
-
1949
- def _spec_for_existing_module(
1950
- fullname: str,
1951
- module: ModuleType,
1952
- original_spec: Optional[importlib.machinery.ModuleSpec] = None,
1953
- ) -> importlib.machinery.ModuleSpec:
1954
- """Build a ModuleSpec whose loader simply returns an already-initialized module."""
1955
- loader = _DeferredModuleLoader(module)
1956
- spec = importlib.machinery.ModuleSpec(fullname, loader)
1957
- if original_spec and original_spec.submodule_search_locations is not None:
1958
- locations = list(original_spec.submodule_search_locations)
1959
- spec.submodule_search_locations = locations
1960
- if hasattr(module, "__path__"):
1961
- module.__path__ = locations
1962
- module.__loader__ = loader
1963
- module.__spec__ = spec
1964
- return spec
1965
-
1966
- class LazyMetaPathFinder:
1967
- """
1968
- Custom meta path finder that intercepts failed imports.
1969
- Performance optimized - only triggers when import would fail anyway.
1970
- """
1971
-
1972
- __slots__ = ('_package_name', '_enabled')
1973
-
1974
- def __init__(self, package_name: str = 'default'):
1975
- """Initialize meta path finder."""
1976
- self._package_name = package_name
1977
- self._enabled = True
1978
-
1979
- def _build_async_placeholder(
1980
- self,
1981
- fullname: str,
1982
- installer: LazyInstaller,
1983
- ) -> Optional[importlib.machinery.ModuleSpec]:
1984
- """Create and register a deferred module placeholder for async installs."""
1985
- handle = installer.ensure_async_install(fullname)
1986
- if handle is None:
1987
- return None
1988
-
1989
- missing = ModuleNotFoundError(f"No module named '{fullname}'")
1990
- deferred = DeferredImportError(fullname, missing, self._package_name, async_handle=handle)
1991
-
1992
- module = ModuleType(fullname)
1993
- loader = _DeferredModuleLoader(module)
1994
-
1995
- def _resolve_real_module():
1996
- real_module = deferred._try_install_and_import()
1997
- sys.modules[fullname] = real_module
1998
- module.__dict__.clear()
1999
- module.__dict__.update(real_module.__dict__)
2000
- module.__loader__ = getattr(real_module, "__loader__", loader)
2001
- module.__spec__ = getattr(real_module, "__spec__", None)
2002
- module.__path__ = getattr(real_module, "__path__", getattr(module, "__path__", []))
2003
- module.__class__ = real_module.__class__
2004
- try:
2005
- spec_obj = getattr(real_module, "__spec__", None) or importlib.util.find_spec(fullname)
2006
- if spec_obj is not None:
2007
- _spec_cache_put(fullname, spec_obj)
2008
- except (ValueError, AttributeError, ImportError):
2009
- pass
2010
- return real_module
2011
-
2012
- def _module_getattr(name):
2013
- real = _resolve_real_module()
2014
- if name in module.__dict__:
2015
- return module.__dict__[name]
2016
- return getattr(real, name)
2017
-
2018
- def _module_dir():
2019
- try:
2020
- real = _resolve_real_module()
2021
- return dir(real)
2022
- except Exception:
2023
- return []
2024
-
2025
- module.__getattr__ = _module_getattr # type: ignore[attr-defined]
2026
- module.__dir__ = _module_dir # type: ignore[attr-defined]
2027
- module.__loader__ = loader
2028
- module.__package__ = fullname
2029
- module.__path__ = []
2030
-
2031
- spec = importlib.machinery.ModuleSpec(fullname, loader)
2032
- spec.submodule_search_locations = []
2033
- module.__spec__ = spec
2034
-
2035
- sys.modules[fullname] = module
2036
- log_event("hook", logger.info, f"⏳ [HOOK] Deferred import placeholder created for '{fullname}'")
2037
- return spec
2038
-
2039
- def find_module(self, fullname: str, path: Optional[str] = None):
2040
- """Find module - returns None to let standard import continue."""
2041
- return None
2042
-
2043
- def find_spec(self, fullname: str, path: Optional[str] = None, target=None):
2044
- """
2045
- Find module spec - intercepts imports to enable two-stage lazy loading.
2046
-
2047
- PERFORMANCE: Optimized for zero overhead on successful imports.
2048
- """
2049
- # CRITICAL: Check installing state FIRST to prevent recursion during installation
2050
- if getattr(_installing_state, 'active', False):
2051
- logger.debug(f"[HOOK] Installation in progress, skipping {fullname} to prevent recursion")
2052
- return None
2053
-
2054
- # CRITICAL: Check if we're checking installation status to prevent infinite recursion
2055
- if _is_checking_installation():
2056
- logger.debug(f"[HOOK] Checking installation status, bypassing lazy finder for {fullname} to prevent recursion")
2057
- return None
2058
-
2059
- # Fast path 1: Hook disabled
2060
- if not self._enabled:
2061
- return None
2062
-
2063
- # Fast path 2: Module already loaded - wrap it if needed
2064
- if fullname in sys.modules:
2065
- module = sys.modules[fullname]
2066
- # Wrap classes in already-loaded modules to ensure auto-instantiation works
2067
- if isinstance(module, ModuleType) and not getattr(module, '_xwlazy_wrapped', False):
2068
- try:
2069
- self._wrap_classes_for_auto_instantiation(module)
2070
- module._xwlazy_wrapped = True # Mark as wrapped to avoid re-wrapping
2071
- except Exception:
2072
- pass
2073
- return None
2074
-
2075
- # Fast path 3: Skip C extension modules and internal modules
2076
- # Also skip submodules that start with underscore (e.g., yaml._yaml)
2077
- if fullname.startswith('_') or ('.' in fullname and fullname.split('.')[-1].startswith('_')):
2078
- logger.debug(f"[HOOK] Skipping C extension/internal module {fullname}")
2079
- return None
2080
-
2081
- # Fast path 4: Check if parent package is partially initialized
2082
- # CRITICAL: Skip ALL submodules of packages that are in sys.modules
2083
- # This prevents circular import issues when a package imports its own submodules
2084
- if '.' in fullname:
2085
- parent_package = fullname.split('.', 1)[0]
2086
- if parent_package in sys.modules:
2087
- logger.debug(f"[HOOK] Skipping {fullname} - parent {parent_package} is in sys.modules (prevent circular import)")
2088
- return None
2089
- if _is_import_in_progress(parent_package):
2090
- logger.debug(f"[HOOK] Skipping {fullname} - parent {parent_package} import in progress")
2091
- return None
2092
-
2093
- # ROOT CAUSE FIX: Check lazy install status FIRST
2094
- root_name = fullname.split('.', 1)[0]
2095
- _watched_registry = get_watched_registry()
2096
-
2097
- # Check if lazy install is enabled
2098
- lazy_install_enabled = LazyInstallConfig.is_enabled(self._package_name)
2099
- install_mode = LazyInstallConfig.get_install_mode(self._package_name)
2100
-
2101
- # If lazy install is disabled, only intercept watched modules
2102
- if not lazy_install_enabled or install_mode == LazyInstallMode.NONE:
2103
- if not _watched_registry.has_root(root_name):
2104
- logger.debug(f"[HOOK] Module {fullname} not in watched registry and lazy install disabled, skipping interception")
2105
- return None
2106
-
2107
- # Check persistent installation cache FIRST
2108
- try:
2109
- installer = LazyInstallerRegistry.get_instance(self._package_name)
2110
- package_name = installer._dependency_mapper.get_package_name(root_name)
2111
-
2112
- # ROOT CAUSE FIX: If this is the package we are managing, DO NOT skip interception!
2113
- # We need to wrap it even if it is installed, so we can intercept its imports.
2114
- should_skip = False
2115
- if package_name == self._package_name or root_name == self._package_name:
2116
- should_skip = False
2117
- elif package_name:
2118
- # Set flag to prevent recursion during installation check
2119
- _set_checking_installation(True)
2120
- try:
2121
- if installer.is_package_installed(package_name):
2122
- should_skip = True
2123
- finally:
2124
- _set_checking_installation(False)
2125
-
2126
- if should_skip:
2127
- logger.debug(f"[HOOK] Package {package_name} is installed (cache check), skipping interception of {fullname}")
2128
- return None
2129
- except Exception:
2130
- pass
2131
-
2132
- # Fast path 4: Cached spec
2133
- cached_spec = _spec_cache_get(fullname)
2134
- if cached_spec is not None:
2135
- return cached_spec
2136
-
2137
- # Fast path 5: Stdlib/builtin check
2138
- if fullname.startswith('importlib') or fullname.startswith('_frozen_importlib'):
2139
- return None
2140
-
2141
- if '.' not in fullname:
2142
- if DependencyMapper._is_stdlib_or_builtin(fullname):
2143
- return None
2144
- if fullname in DependencyMapper.DENY_LIST:
2145
- return None
2146
-
2147
- # Fast path 6: Import in progress
2148
- # NOTE: We allow lazy install to proceed even if import is in progress,
2149
- # because we need to install missing packages during import
2150
- if _is_import_in_progress(fullname):
2151
- # Only skip if lazy install is disabled (for watched modules, we still need to wrap)
2152
- if not lazy_install_enabled and not _watched_registry.has_root(root_name):
2153
- return None
2154
-
2155
- # Only skip global importing state if lazy install is disabled
2156
- # (lazy install needs to run even during imports to install missing packages)
2157
- if getattr(_importing_state, 'active', False):
2158
- if not lazy_install_enabled and not _watched_registry.has_root(root_name):
2159
- return None
2160
-
2161
- # Install mode check already done above
2162
- matching_prefixes: tuple[str, ...] = ()
2163
- if _watched_registry.has_root(root_name):
2164
- matching_prefixes = _watched_registry.get_matching_prefixes(fullname)
2165
-
2166
- installer = LazyInstallerRegistry.get_instance(self._package_name)
2167
-
2168
- # Two-stage lazy loading for serialization and archive modules
2169
- if matching_prefixes:
2170
- for prefix in matching_prefixes:
2171
- if not _watched_registry.is_prefix_owned_by(self._package_name, prefix):
2172
- continue
2173
- if fullname.startswith(prefix):
2174
- module_suffix = fullname[len(prefix):]
2175
-
2176
- if module_suffix:
2177
- log_event("hook", logger.info, f"[HOOK] Candidate for wrapping: {fullname}")
2178
-
2179
- _mark_import_started(fullname)
2180
- try:
2181
- if getattr(_importing_state, 'active', False):
2182
- logger.debug(f"[HOOK] Recursion guard active, skipping {fullname}")
2183
- return None
2184
-
2185
- try:
2186
- logger.debug(f"[HOOK] Looking for spec: {fullname}")
2187
- spec = _spec_cache_get(fullname)
2188
- if spec is None:
2189
- try:
2190
- spec = importlib.util.find_spec(fullname)
2191
- except (ValueError, AttributeError, ImportError):
2192
- pass
2193
- if spec is not None:
2194
- _spec_cache_put(fullname, spec)
2195
- if spec is not None:
2196
- logger.debug(f"[HOOK] Spec found, trying normal import: {fullname}")
2197
- _importing_state.active = True
2198
- try:
2199
- __import__(fullname)
2200
-
2201
- module = sys.modules.get(fullname)
2202
- if module:
2203
- try:
2204
- self._enhance_classes_with_class_methods(module)
2205
- # Enable auto-instantiation for classes in this module
2206
- self._wrap_classes_for_auto_instantiation(module)
2207
- except Exception as enhance_exc:
2208
- logger.debug(f"[HOOK] Could not enhance classes in {fullname}: {enhance_exc}")
2209
- spec = _spec_for_existing_module(fullname, module, spec)
2210
- log_event("hook", logger.info, f"✓ [HOOK] Module {fullname} imported successfully, no wrapping needed")
2211
- if spec is not None:
2212
- _spec_cache_put(fullname, spec)
2213
- return spec
2214
- return None
2215
- finally:
2216
- _importing_state.active = False
2217
- except ImportError as e:
2218
- if '.' not in module_suffix:
2219
- log_event("hook", logger.info, f"⚠ [HOOK] Module {fullname} has missing dependencies, wrapping: {e}")
2220
- wrapped_spec = self._wrap_serialization_module(fullname)
2221
- if wrapped_spec is not None:
2222
- log_event("hook", logger.info, f"✓ [HOOK] Successfully wrapped: {fullname}")
2223
- return wrapped_spec
2224
- logger.warning(f"✗ [HOOK] Failed to wrap: {fullname}")
2225
- else:
2226
- logger.debug(f"[HOOK] Import failed for nested module {fullname}: {e}")
2227
- except (ModuleNotFoundError,) as e:
2228
- logger.debug(f"[HOOK] Module {fullname} not found, skipping wrap: {e}")
2229
- pass
2230
- except Exception as e:
2231
- logger.warning(f"[HOOK] Error checking module {fullname}: {e}")
2232
- finally:
2233
- _mark_import_finished(fullname)
2234
-
2235
- return None
2236
-
2237
- # If we had matching prefixes but didn't match any, continue to lazy install logic
2238
- if matching_prefixes:
2239
- logger.debug(f"[HOOK] {fullname} had matching prefixes but didn't match any, continuing to lazy install")
2240
-
2241
- # For lazy installation, handle submodules by checking if parent package is installed
2242
- if '.' in fullname:
2243
- parent_package = fullname.split('.', 1)[0]
2244
- if lazy_install_enabled:
2245
- try:
2246
- installer = LazyInstallerRegistry.get_instance(self._package_name)
2247
- package_name = installer._dependency_mapper.get_package_name(parent_package)
2248
- if package_name:
2249
- # Set flag to prevent recursion during installation check
2250
- _set_checking_installation(True)
2251
- try:
2252
- if not installer.is_package_installed(package_name):
2253
- logger.debug(f"[HOOK] Parent package {parent_package} not installed, intercepting parent")
2254
- return self.find_spec(parent_package, path, target)
2255
- finally:
2256
- _set_checking_installation(False)
2257
- except Exception:
2258
- pass
2259
- return None
2260
- if DependencyMapper._is_stdlib_or_builtin(fullname):
2261
- return None
2262
- if fullname in DependencyMapper.DENY_LIST:
2263
- return None
2264
-
2265
- # ROOT CAUSE FIX: For lazy installation, intercept missing imports and install them
2266
- logger.debug(f"[HOOK] Checking lazy install for {fullname}: enabled={lazy_install_enabled}, install_mode={install_mode}")
2267
- if lazy_install_enabled:
2268
- # Prevent infinite loops: Skip if already attempting import
2269
- if _is_import_in_progress(fullname):
2270
- logger.debug(f"[HOOK] Import {fullname} already in progress, skipping to prevent recursion")
2271
- return None
2272
-
2273
- # Prevent infinite loops: Check if we've already tried to install this package
2274
- installer = LazyInstallerRegistry.get_instance(self._package_name)
2275
- package_name = installer._dependency_mapper.get_package_name(root_name)
2276
-
2277
- # ROOT CAUSE FIX: Check if module is ALREADY importable BEFORE checking package installation
2278
- # This handles cases where package name != module name (e.g., PyYAML -> yaml)
2279
- try:
2280
- # Try direct import first (most reliable check)
2281
- if fullname in sys.modules:
2282
- logger.debug(f"[HOOK] Module {fullname} already in sys.modules, skipping installation")
2283
- return None
2284
-
2285
- # Temporarily remove finders to check actual importability
2286
- xwlazy_finder_names = {'LazyMetaPathFinder', 'LazyPathFinder', 'LazyLoader'}
2287
- xwlazy_finders = [f for f in sys.meta_path if type(f).__name__ in xwlazy_finder_names]
2288
- for finder in xwlazy_finders:
2289
- try:
2290
- sys.meta_path.remove(finder)
2291
- except ValueError:
2292
- pass
2293
-
2294
- try:
2295
- # Check spec without importing (avoids triggering module code execution)
2296
- # This prevents circular import issues when checking importability
2297
- spec = importlib.util.find_spec(fullname)
2298
- if spec is not None and spec.loader is not None:
2299
- logger.debug(f"[HOOK] Module {fullname} has valid spec, skipping installation")
2300
- return None
2301
- finally:
2302
- # Restore finders
2303
- for finder in reversed(xwlazy_finders):
2304
- if finder not in sys.meta_path:
2305
- sys.meta_path.insert(0, finder)
2306
- except Exception as e:
2307
- logger.debug(f"[HOOK] Importability check failed for {fullname}: {e}")
2308
- # If check fails, proceed with installation attempt
2309
-
2310
- if package_name:
2311
- if package_name in installer.get_failed_packages():
2312
- logger.debug(f"[HOOK] Package {package_name} previously failed installation, skipping {fullname}")
2313
- return None
2314
-
2315
- # Also check package installation status (for cache efficiency)
2316
- # Set flag to prevent recursion during installation check
2317
- _set_checking_installation(True)
2318
- try:
2319
- if installer.is_package_installed(package_name):
2320
- logger.debug(f"[HOOK] Package {package_name} is already installed, skipping installation attempt for {fullname}")
2321
- return None
2322
- finally:
2323
- _set_checking_installation(False)
2324
-
2325
- _mark_import_started(fullname)
2326
- try:
2327
- # Guard against recursion when importing facade
2328
- if 'exonware.xwlazy.facade' in sys.modules or 'xwlazy.facade' in sys.modules:
2329
- from ..facade import lazy_import_with_install
2330
- else:
2331
- if _is_import_in_progress('exonware.xwlazy.facade') or _is_import_in_progress('xwlazy.facade'):
2332
- logger.debug(f"[HOOK] Facade import in progress, skipping {fullname} to prevent recursion")
2333
- return None
2334
- from ..facade import lazy_import_with_install
2335
-
2336
- # Log installation attempt (DEBUG level to reduce noise)
2337
- if package_name:
2338
- logger.debug(f"⏳ [HOOK] Attempting to install package '{package_name}' for module '{fullname}'")
2339
- else:
2340
- logger.debug(f"⏳ [HOOK] Attempting to install module '{fullname}' (no package mapping found)")
2341
-
2342
- _importing_state.active = True
2343
- try:
2344
- module, success = lazy_import_with_install(
2345
- fullname,
2346
- installer_package=self._package_name
2347
- )
2348
- finally:
2349
- _importing_state.active = False
2350
-
2351
- if success and module:
2352
- # Module was successfully installed and imported
2353
- logger.debug(f"✅ [HOOK] Successfully installed and imported '{fullname}'")
2354
- xwlazy_finder_names = {'LazyMetaPathFinder', 'LazyPathFinder', 'LazyLoader'}
2355
- xwlazy_finders = [f for f in sys.meta_path if type(f).__name__ in xwlazy_finder_names]
2356
- for finder in xwlazy_finders:
2357
- try:
2358
- sys.meta_path.remove(finder)
2359
- except ValueError:
2360
- pass
2361
-
2362
- try:
2363
- if fullname in sys.modules:
2364
- del sys.modules[fullname]
2365
- importlib.invalidate_caches()
2366
- sys.path_importer_cache.clear()
2367
- real_module = importlib.import_module(fullname)
2368
- sys.modules[fullname] = real_module
2369
- logger.debug(f"[HOOK] Successfully installed {fullname}, replaced module in sys.modules with real module")
2370
- finally:
2371
- for finder in reversed(xwlazy_finders):
2372
- if finder not in sys.meta_path:
2373
- sys.meta_path.insert(0, finder)
2374
- return None
2375
- else:
2376
- # Only log warning if it's not a known missing package (reduce noise)
2377
- if package_name and package_name not in installer.get_failed_packages():
2378
- logger.debug(f"❌ [HOOK] Failed to install/import {fullname}")
2379
- else:
2380
- logger.debug(f"❌ [HOOK] Failed to install/import {fullname} (already marked as failed)")
2381
- # Mark as failed to prevent infinite retry loops
2382
- if package_name:
2383
- installer._failed_packages.add(package_name)
2384
- try:
2385
- installer = LazyInstallerRegistry.get_instance(self._package_name)
2386
- # Force disable async install usage in engine
2387
- use_async = False # installer.is_async_enabled()
2388
- if use_async:
2389
- placeholder = self._build_async_placeholder(fullname, installer)
2390
- if placeholder is not None:
2391
- return placeholder
2392
- except Exception:
2393
- pass
2394
- # Return None to let Python handle the ImportError naturally
2395
- return None
2396
- except Exception as e:
2397
- logger.error(f"❌ [HOOK] Lazy import hook failed for {fullname}: {e}", exc_info=True)
2398
- # Mark as failed to prevent infinite retry loops
2399
- if package_name:
2400
- try:
2401
- installer = LazyInstallerRegistry.get_instance(self._package_name)
2402
- installer._failed_packages.add(package_name)
2403
- except Exception:
2404
- pass
2405
- return None
2406
- finally:
2407
- _mark_import_finished(fullname)
2408
-
2409
- def _wrap_serialization_module(self, fullname: str):
2410
- """Wrap serialization module loading to defer missing dependencies."""
2411
- log_event("hook", logger.info, f"[STAGE 1] Starting wrap of module: {fullname}")
2412
-
2413
- try:
2414
- logger.debug(f"[STAGE 1] Getting spec for: {fullname}")
2415
- try:
2416
- sys.meta_path.remove(self)
2417
- except ValueError:
2418
- pass
2419
- try:
2420
- spec = importlib.util.find_spec(fullname)
2421
- finally:
2422
- if self not in sys.meta_path:
2423
- sys.meta_path.insert(0, self)
2424
- if not spec or not spec.loader:
2425
- logger.warning(f"[STAGE 1] No spec or loader for: {fullname}")
2426
- return None
2427
-
2428
- logger.debug(f"[STAGE 1] Creating module from spec: {fullname}")
2429
- module = importlib.util.module_from_spec(spec)
2430
-
2431
- deferred_imports = {}
2432
-
2433
- logger.debug(f"[STAGE 1] Setting up import wrapper for: {fullname}")
2434
- original_import = builtins.__import__
2435
-
2436
- def capture_import_errors(name, *args, **kwargs):
2437
- """Intercept imports and defer ONLY external missing packages."""
2438
- logger.debug(f"[STAGE 1] capture_import_errors: Trying to import '{name}' in {fullname}")
2439
-
2440
- if _is_import_in_progress(name):
2441
- logger.debug(f"[STAGE 1] Import '{name}' already in progress, using original_import")
2442
- return original_import(name, *args, **kwargs)
2443
-
2444
- _mark_import_started(name)
2445
- try:
2446
- result = original_import(name, *args, **kwargs)
2447
- logger.debug(f"[STAGE 1] ✓ Successfully imported '{name}'")
2448
- return result
2449
- except ImportError as e:
2450
- logger.debug(f"[STAGE 1] ✗ Import failed for '{name}': {e}")
2451
-
2452
- host_alias = self._package_name or ""
2453
- if name.startswith('exonware.') or (host_alias and name.startswith(f"{host_alias}.")):
2454
- log_event("hook", logger.info, f"[STAGE 1] Letting internal import '{name}' fail normally (internal package)")
2455
- raise
2456
-
2457
- if '.' in name:
2458
- log_event("hook", logger.info, f"[STAGE 1] Letting submodule '{name}' fail normally (has dots)")
2459
- raise
2460
-
2461
- log_event("hook", logger.info, f"⏳ [STAGE 1] DEFERRING missing external package '{name}' in {fullname}")
2462
- async_handle = None
2463
- try:
2464
- installer = LazyInstallerRegistry.get_instance(self._package_name)
2465
- async_handle = installer.schedule_async_install(name)
2466
- except Exception as schedule_exc:
2467
- logger.debug(f"[STAGE 1] Async install scheduling failed for '{name}': {schedule_exc}")
2468
- deferred = DeferredImportError(name, e, self._package_name, async_handle=async_handle)
2469
- deferred_imports[name] = deferred
2470
-
2471
- # ROOT CAUSE FIX: Register deferred object in sys.modules to prevent infinite import loops
2472
- # If we don't do this, subsequent imports of the same missing module will trigger find_spec again
2473
- sys.modules[name] = deferred
2474
-
2475
- return deferred
2476
- finally:
2477
- _mark_import_finished(name)
2478
-
2479
- logger.debug(f"[STAGE 1] Executing module with import wrapper: {fullname}")
2480
- builtins.__import__ = capture_import_errors
2481
- try:
2482
- spec.loader.exec_module(module)
2483
- logger.debug(f"[STAGE 1] Module execution completed: {fullname}")
2484
-
2485
- if deferred_imports:
2486
- log_event("hook", logger.info, f"✓ [STAGE 1] Module {fullname} loaded with {len(deferred_imports)} deferred imports: {list(deferred_imports.keys())}")
2487
- self._replace_none_with_deferred(module, deferred_imports)
2488
- self._wrap_module_classes(module, deferred_imports)
2489
- else:
2490
- log_event("hook", logger.info, f"✓ [STAGE 1] Module {fullname} loaded with NO deferred imports (all dependencies available)")
2491
-
2492
- self._enhance_classes_with_class_methods(module)
2493
-
2494
- # Enable auto-instantiation for classes in this module
2495
- self._wrap_classes_for_auto_instantiation(module)
2496
-
2497
- finally:
2498
- logger.debug(f"[STAGE 1] Restoring original __import__")
2499
- builtins.__import__ = original_import
2500
-
2501
- logger.debug(f"[STAGE 1] Registering module in sys.modules: {fullname}")
2502
- sys.modules[fullname] = module
2503
- final_spec = _spec_for_existing_module(fullname, module, spec)
2504
- _spec_cache_put(fullname, final_spec)
2505
- log_event("hook", logger.info, f"✓ [STAGE 1] Successfully wrapped and registered: {fullname}")
2506
- return final_spec
2507
-
2508
- except Exception as e:
2509
- logger.debug(f"Could not wrap {fullname}: {e}")
2510
- return None
2511
-
2512
- def _replace_none_with_deferred(self, module, deferred_imports: Dict):
2513
- """Replace None values in module namespace with deferred import proxies."""
2514
- logger.debug(f"[STAGE 1] Replacing None with deferred imports in {module.__name__}")
2515
- replaced_count = 0
2516
-
2517
- for dep_name, deferred_import in deferred_imports.items():
2518
- if hasattr(module, dep_name):
2519
- current_value = getattr(module, dep_name)
2520
- if current_value is None:
2521
- log_event("hook", logger.info, f"[STAGE 1] Replacing {dep_name}=None with deferred import proxy in {module.__name__}")
2522
- setattr(module, dep_name, deferred_import)
2523
- replaced_count += 1
2524
-
2525
- if replaced_count > 0:
2526
- log_event("hook", logger.info, f"✓ [STAGE 1] Replaced {replaced_count} None values with deferred imports in {module.__name__}")
2527
-
2528
- def _wrap_module_classes(self, module, deferred_imports: Dict):
2529
- """Wrap classes in a module that depend on deferred imports."""
2530
- module_name = getattr(module, '__name__', '<unknown>')
2531
- logger.debug(f"[STAGE 1] Wrapping classes in {module_name} (deferred: {list(deferred_imports.keys())})")
2532
- module_file = (getattr(module, '__file__', '') or '').lower()
2533
- lower_map = {dep_name.lower(): dep_name for dep_name in deferred_imports.keys()}
2534
- class_hints = _get_package_class_hints(self._package_name)
2535
- with _wrapped_cache_lock:
2536
- already_wrapped = _WRAPPED_CLASS_CACHE.setdefault(module_name, set()).copy()
2537
- pending_lower = {lower for lower in lower_map.keys() if lower_map[lower] not in already_wrapped}
2538
- if not pending_lower:
2539
- logger.debug(f"[STAGE 1] All deferred imports already wrapped for {module_name}")
2540
- return
2541
- dep_entries = [(lower, deferred_imports[lower_map[lower]]) for lower in pending_lower]
2542
- wrapped_count = 0
2543
- newly_wrapped: set[str] = set()
2544
-
2545
- for name, obj in list(module.__dict__.items()):
2546
- if not pending_lower:
2547
- break
2548
- if not isinstance(obj, type):
2549
- continue
2550
- lower_name = name.lower()
2551
- if class_hints and not any(hint in lower_name for hint in class_hints):
2552
- continue
2553
- target_lower = None
2554
- target_deferred = None
2555
- for dep_lower, deferred in dep_entries:
2556
- if dep_lower not in pending_lower:
2557
- continue
2558
- if dep_lower in lower_name or dep_lower in module_file:
2559
- target_lower = dep_lower
2560
- target_deferred = deferred
2561
- break
2562
- if target_deferred is None or target_lower is None:
2563
- continue
2564
-
2565
- logger.debug(f"[STAGE 1] Class '{name}' depends on deferred import, wrapping...")
2566
- wrapped = self._create_lazy_class_wrapper(obj, target_deferred)
2567
- module.__dict__[name] = wrapped
2568
- wrapped_count += 1
2569
- origin_name = lower_map.get(target_lower, target_lower)
2570
- newly_wrapped.add(origin_name)
2571
- pending_lower.discard(target_lower)
2572
- log_event("hook", logger.info, f"✓ [STAGE 1] Wrapped class '{name}' in {module_name}")
2573
-
2574
- if newly_wrapped:
2575
- with _wrapped_cache_lock:
2576
- cache = _WRAPPED_CLASS_CACHE.setdefault(module_name, set())
2577
- cache.update(newly_wrapped)
2578
-
2579
- log_event("hook", logger.info, f"[STAGE 1] Wrapped {wrapped_count} classes in {module_name}")
2580
-
2581
- def _wrap_classes_for_auto_instantiation(self, module: ModuleType) -> None:
2582
- """
2583
- Wrap classes in modules with AutoInstantiateProxy for auto-instantiation.
2584
-
2585
- This enables: from module import Class as instance
2586
- Then: instance.method() works automatically without manual instantiation.
2587
-
2588
- We wrap classes directly in module.__dict__ AND add a __getattr__ fallback
2589
- to catch any classes that might be accessed before wrapping completes.
2590
-
2591
- We wrap ALL classes in the module's namespace, whether defined in this
2592
- module or imported from submodules (like BsonSerializer imported in __init__.py).
2593
- """
2594
- import inspect
2595
- import types
2596
-
2597
- # Store original __getattr__ if it exists
2598
- original_getattr = getattr(module, '__getattr__', None)
2599
-
2600
- wrapped_count = 0
2601
- wrapped_classes = {} # Track wrapped classes for __getattr__
2602
-
2603
- for name, obj in list(module.__dict__.items()):
2604
- # Wrap classes that are in this module's namespace
2605
- # This includes classes defined here AND classes imported from submodules
2606
- if (inspect.isclass(obj) and
2607
- not inspect.isbuiltin(obj) and
2608
- hasattr(obj, '__module__')):
2609
- # Skip if it's already a proxy
2610
- if isinstance(obj, AutoInstantiateProxy):
2611
- continue
2612
- # Skip if it's a wrapper class
2613
- if obj.__name__.startswith('Lazy'):
2614
- continue
2615
- # Skip if it's a type from typing or other special modules
2616
- module_name = obj.__module__
2617
- if module_name in ('typing', 'builtins', '__builtin__'):
2618
- continue
2619
-
2620
- logger.debug(f"[AUTO-INST] Wrapping class '{name}' ({module_name}) in {module.__name__} for auto-instantiation")
2621
- proxy = AutoInstantiateProxy(obj)
2622
- module.__dict__[name] = proxy
2623
- wrapped_classes[name] = proxy
2624
- wrapped_count += 1
2625
-
2626
- # Add __getattr__ fallback to wrap classes on access (for classes added after initial load)
2627
- if wrapped_count > 0 or True: # Always add __getattr__ for consistency
2628
- def auto_instantiate_getattr(name: str):
2629
- """Module-level __getattr__ that wraps classes for auto-instantiation."""
2630
- # First try original __getattr__
2631
- if original_getattr:
2632
- try:
2633
- attr = original_getattr(name)
2634
- # If it's a class, wrap it
2635
- if inspect.isclass(attr) and not inspect.isbuiltin(attr):
2636
- if not isinstance(attr, AutoInstantiateProxy):
2637
- logger.debug(f"[AUTO-INST] Wrapping class '{name}' on access in {module.__name__}")
2638
- proxy = AutoInstantiateProxy(attr)
2639
- module.__dict__[name] = proxy # Cache it
2640
- return proxy
2641
- return attr
2642
- except AttributeError:
2643
- pass
2644
-
2645
- # Get from __dict__ (might be unwrapped)
2646
- if name in module.__dict__:
2647
- attr = module.__dict__[name]
2648
- # If it's an unwrapped class, wrap it now
2649
- if (inspect.isclass(attr) and
2650
- not inspect.isbuiltin(attr) and
2651
- not isinstance(attr, AutoInstantiateProxy) and
2652
- hasattr(attr, '__module__')):
2653
- module_name = attr.__module__
2654
- if module_name not in ('typing', 'builtins', '__builtin__'):
2655
- logger.debug(f"[AUTO-INST] Wrapping class '{name}' on access in {module.__name__}")
2656
- proxy = AutoInstantiateProxy(attr)
2657
- module.__dict__[name] = proxy # Cache it
2658
- return proxy
2659
- return attr
2660
-
2661
- raise AttributeError(f"module '{module.__name__}' has no attribute '{name}'")
2662
-
2663
- # Only set __getattr__ if module doesn't already have a custom one
2664
- if not hasattr(module, '__getattr__') or isinstance(getattr(module, '__getattr__', None), types.MethodType):
2665
- module.__getattr__ = auto_instantiate_getattr # type: ignore[attr-defined]
2666
-
2667
- if wrapped_count > 0:
2668
- logger.debug(f"[AUTO-INST] Wrapped {wrapped_count} classes in {module.__name__} for auto-instantiation")
2669
-
2670
- def _create_lazy_class_wrapper(self, original_class, deferred_import: DeferredImportError):
2671
- """Create a wrapper class that installs dependencies when instantiated."""
2672
- class LazyClassWrapper:
2673
- """Lazy wrapper that installs dependencies on first instantiation."""
2674
-
2675
- def __init__(self, *args, **kwargs):
2676
- """Install dependency and create real instance."""
2677
- deferred_import._try_install_and_import()
2678
-
2679
- real_module = importlib.reload(sys.modules[original_class.__module__])
2680
- real_class = getattr(real_module, original_class.__name__)
2681
-
2682
- real_instance = real_class(*args, **kwargs)
2683
- self.__class__ = real_class
2684
- self.__dict__ = real_instance.__dict__
2685
-
2686
- def __repr__(self):
2687
- return f"<Lazy{original_class.__name__}: will install dependencies on init>"
2688
-
2689
- LazyClassWrapper.__name__ = f"Lazy{original_class.__name__}"
2690
- LazyClassWrapper.__qualname__ = f"Lazy{original_class.__qualname__}"
2691
- LazyClassWrapper.__module__ = original_class.__module__
2692
- LazyClassWrapper.__doc__ = original_class.__doc__
2693
-
2694
- return LazyClassWrapper
2695
-
2696
- def _enhance_classes_with_class_methods(self, module):
2697
- """Enhance classes with lazy class methods - automatically detects ALL instance methods."""
2698
- if module is None:
2699
- return
2700
- # Debug: Check if module has classes
2701
- module_classes = [name for name, obj in module.__dict__.items() if isinstance(obj, type)]
2702
- if not module_classes:
2703
- return # No classes to enhance
2704
-
2705
- # Get methods from registry (if any) for specific prefixes
2706
- methods_to_apply: tuple[str, ...] = ()
2707
- for prefix, methods in _lazy_prefix_method_registry.items():
2708
- if module.__name__.startswith(prefix.rstrip('.')):
2709
- methods_to_apply = methods
2710
- break
2711
-
2712
- # If no prefix match, use default methods from env var
2713
- if not methods_to_apply:
2714
- methods_to_apply = _DEFAULT_LAZY_METHODS
2715
-
2716
- # CRITICAL FIX: If no specific methods registered, enhance ALL instance methods
2717
- # This makes xwlazy handle ALL scenarios (classes, functions, instance methods)
2718
- auto_detect_all = not methods_to_apply
2719
-
2720
- enhanced = 0
2721
- for name, obj in list(module.__dict__.items()):
2722
- if not isinstance(obj, type):
2723
- continue
2724
-
2725
- # Get methods to enhance for this class
2726
- if auto_detect_all:
2727
- # Auto-detect instance methods defined DIRECTLY in this class (not inherited)
2728
- # This prevents wrapping wrong methods from parent classes
2729
- methods_to_wrap = []
2730
- # Only check methods in this class's __dict__ to avoid inherited methods
2731
- for attr_name, attr_value in obj.__dict__.items():
2732
- if attr_name.startswith('_') and attr_name not in ('__init__', '__new__'):
2733
- continue # Skip private methods except __init__ and __new__
2734
- if not callable(attr_value):
2735
- continue
2736
- if isinstance(attr_value, (classmethod, staticmethod)):
2737
- continue
2738
- if getattr(attr_value, "__lazy_wrapped__", False):
2739
- continue
2740
- # Check if it's an instance method (has 'self' as first param)
2741
- import inspect
2742
- try:
2743
- if inspect.isfunction(attr_value):
2744
- sig = inspect.signature(attr_value)
2745
- params = list(sig.parameters.keys())
2746
- if params and params[0] == 'self':
2747
- methods_to_wrap.append(attr_name)
2748
- except (ValueError, TypeError):
2749
- # Can't inspect signature, skip
2750
- pass
2751
- else:
2752
- # Use specific methods from registry
2753
- methods_to_wrap = list(methods_to_apply)
2754
-
2755
- # Enhance each method
2756
- for method_name in methods_to_wrap:
2757
- try:
2758
- # CRITICAL FIX: Only get from class __dict__ to ensure we get the RIGHT method
2759
- original_func = obj.__dict__.get(method_name)
2760
- if original_func is None:
2761
- continue # Method not in class dict
2762
- if not inspect.isfunction(original_func):
2763
- continue
2764
- if getattr(original_func, "__lazy_wrapped__", False):
2765
- continue
2766
- if isinstance(original_func, (classmethod, staticmethod)):
2767
- continue
2768
- if original_func.__name__ != method_name:
2769
- continue
2770
- # Verify it's an instance method
2771
- try:
2772
- params = list(inspect.signature(original_func).parameters.keys())
2773
- if not params or params[0] != 'self':
2774
- continue
2775
- except Exception:
2776
- continue
2777
- # Create wrapper with proper closure
2778
- class_obj, func_to_call = obj, original_func
2779
- import functools
2780
- def make_wrapper(fn, cls):
2781
- @functools.wraps(fn)
2782
- def wrapper(first_arg, *args, **kwargs):
2783
- if isinstance(first_arg, cls):
2784
- return fn(first_arg, *args, **kwargs)
2785
- instance = cls()
2786
- return fn(instance, first_arg, *args, **kwargs)
2787
- return wrapper
2788
- smart_wrapper = make_wrapper(func_to_call, class_obj)
2789
- smart_wrapper.__lazy_wrapped__ = True
2790
- setattr(obj, method_name, smart_wrapper)
2791
- enhanced += 1
2792
- except Exception as exc:
2793
- # Silent skip - one line as requested (package/module agnostic)
2794
- pass
2795
-
2796
- if enhanced:
2797
- log_event("enhance", logger.info, "✓ [LAZY ENHANCE] Enhanced %s methods in %s", enhanced, module.__name__)
2798
-
2799
-
2800
- class AutoInstantiateProxy:
2801
- """
2802
- Proxy that auto-instantiates a class on first method call.
2803
-
2804
- Enables: from module import Class as instance
2805
- Then: instance.method() automatically creates Class() and calls method
2806
-
2807
- This allows users to import classes with 'as' and use them directly
2808
- without needing to manually instantiate.
2809
- """
2810
-
2811
- def __init__(self, original_class):
2812
- """Initialize proxy with the class to wrap."""
2813
- self._class = original_class
2814
- self._instance = None
2815
- self._is_instantiated = False
2816
-
2817
- def _ensure_instantiated(self):
2818
- """Ensure the class is instantiated."""
2819
- if not self._is_instantiated:
2820
- self._instance = self._class()
2821
- self._is_instantiated = True
2822
-
2823
- def __getattr__(self, name: str):
2824
- """Get attribute from instance, instantiating if needed."""
2825
- self._ensure_instantiated()
2826
- return getattr(self._instance, name)
2827
-
2828
- def __call__(self, *args, **kwargs):
2829
- """Allow calling the proxy directly (creates new instance)."""
2830
- return self._class(*args, **kwargs)
2831
-
2832
- def __repr__(self):
2833
- if self._is_instantiated:
2834
- return f"<AutoInstantiateProxy({self._class.__name__}): instantiated>"
2835
- return f"<AutoInstantiateProxy({self._class.__name__}): will instantiate on first use>"
2836
-
2837
- @property
2838
- def __class__(self):
2839
- """Return the wrapped class for isinstance checks."""
2840
- return self._class
2841
-
2842
- # Registry of installed hooks per package
2843
- _installed_hooks: dict[str, LazyMetaPathFinder] = {}
2844
- _hook_lock = threading.RLock()
2845
-
2846
- def install_import_hook(package_name: str = 'default') -> None:
2847
- """Install performant import hook for automatic lazy installation."""
2848
- global _installed_hooks
2849
-
2850
- log_event("hook", logger.info, f"[HOOK INSTALL] Installing import hook for package: {package_name}")
2851
-
2852
- with _hook_lock:
2853
- if package_name in _installed_hooks:
2854
- log_event("hook", logger.info, f"[HOOK INSTALL] Import hook already installed for {package_name}")
2855
- return
2856
-
2857
- logger.debug(f"[HOOK INSTALL] Creating LazyMetaPathFinder for {package_name}")
2858
- hook = LazyMetaPathFinder(package_name)
2859
-
2860
- logger.debug(f"[HOOK INSTALL] Current sys.meta_path has {len(sys.meta_path)} entries")
2861
- sys.meta_path.insert(0, hook)
2862
- _installed_hooks[package_name] = hook
2863
-
2864
- log_event("hook", logger.info, f"✅ [HOOK INSTALL] Lazy import hook installed for {package_name} (now {len(sys.meta_path)} meta_path entries)")
2865
-
2866
- def uninstall_import_hook(package_name: str = 'default') -> None:
2867
- """Uninstall import hook for a package."""
2868
- global _installed_hooks
2869
-
2870
- with _hook_lock:
2871
- if package_name in _installed_hooks:
2872
- hook = _installed_hooks[package_name]
2873
- try:
2874
- sys.meta_path.remove(hook)
2875
- except ValueError:
2876
- pass
2877
- del _installed_hooks[package_name]
2878
- log_event("hook", logger.info, f"Lazy import hook uninstalled for {package_name}")
2879
-
2880
- def is_import_hook_installed(package_name: str = 'default') -> bool:
2881
- """Check if import hook is installed for a package."""
2882
- return package_name in _installed_hooks
2883
-
2884
- def register_lazy_module_prefix(prefix: str) -> None:
2885
- """Register an import prefix for lazy wrapping."""
2886
- _watched_registry = get_watched_registry()
2887
- _watched_registry.add(prefix)
2888
- normalized = _normalize_prefix(prefix)
2889
- if normalized:
2890
- log_event("config", logger.info, "Registered lazy module prefix: %s", normalized)
2891
-
2892
- # =============================================================================
2893
- # EXPORTS
2894
- # =============================================================================
2895
-
2896
- __all__ = [
2897
- # Logging utilities
2898
- 'get_logger',
2899
- 'log_event',
2900
- 'print_formatted',
2901
- 'format_message',
2902
- 'is_log_category_enabled',
2903
- 'set_log_category',
2904
- 'set_log_categories',
2905
- 'get_log_categories',
2906
- 'XWLazyFormatter',
2907
- # Import tracking
2908
- '_is_import_in_progress',
2909
- '_mark_import_started',
2910
- '_mark_import_finished',
2911
- 'get_importing_state',
2912
- 'get_installing_state',
2913
- '_installation_depth',
2914
- '_installation_depth_lock',
2915
- # Prefix trie
2916
- '_PrefixTrie',
2917
- # Watched registry
2918
- 'WatchedPrefixRegistry',
2919
- 'get_watched_registry',
2920
- # Deferred loader
2921
- '_DeferredModuleLoader',
2922
- # Cache utilities
2923
- # Parallel utilities
2924
- 'ParallelLoader',
2925
- 'DependencyGraph',
2926
- # Module patching
2927
- '_lazy_aware_import_module',
2928
- '_patch_import_module',
2929
- '_unpatch_import_module',
2930
- # Archive imports
2931
- 'get_archive_path',
2932
- 'ensure_archive_in_path',
2933
- 'import_from_archive',
2934
- # Bootstrap
2935
- 'bootstrap_lazy_mode',
2936
- 'bootstrap_lazy_mode_deferred',
2937
- # Lazy loader
2938
- 'LazyLoader',
2939
- # Lazy module registry
2940
- 'LazyModuleRegistry',
2941
- # Lazy importer
2942
- 'LazyImporter',
2943
- # Import hook
2944
- 'LazyImportHook',
2945
- # Meta path finder
2946
- 'LazyMetaPathFinder',
2947
- 'install_import_hook',
2948
- 'uninstall_import_hook',
2949
- 'is_import_hook_installed',
2950
- 'register_lazy_module_prefix',
2951
- 'register_lazy_module_methods',
2952
- '_set_package_class_hints',
2953
- '_get_package_class_hints',
2954
- '_clear_all_package_class_hints',
2955
- '_spec_for_existing_module',
2956
- # Global __import__ hook (for module-level imports)
2957
- 'register_lazy_package',
2958
- 'install_global_import_hook',
2959
- 'uninstall_global_import_hook',
2960
- 'is_global_import_hook_installed',
2961
- 'clear_global_import_caches',
2962
- 'get_global_import_cache_stats',
2963
- ]
2964
-