exonware-xwlazy 0.1.0.22__py3-none-any.whl → 1.0.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. exonware/__init__.py +86 -16
  2. exonware/xwlazy/version.py +5 -5
  3. exonware/xwlazy.py +2546 -0
  4. exonware/xwlazy_external_libs.toml +716 -0
  5. {exonware_xwlazy-0.1.0.22.dist-info → exonware_xwlazy-1.0.1.2.dist-info}/METADATA +6 -6
  6. exonware_xwlazy-1.0.1.2.dist-info/RECORD +8 -0
  7. exonware/xwlazy/__init__.py +0 -367
  8. exonware/xwlazy/common/__init__.py +0 -47
  9. exonware/xwlazy/common/base.py +0 -56
  10. exonware/xwlazy/common/cache.py +0 -504
  11. exonware/xwlazy/common/logger.py +0 -257
  12. exonware/xwlazy/common/services/__init__.py +0 -72
  13. exonware/xwlazy/common/services/dependency_mapper.py +0 -232
  14. exonware/xwlazy/common/services/install_async_utils.py +0 -165
  15. exonware/xwlazy/common/services/install_cache_utils.py +0 -245
  16. exonware/xwlazy/common/services/keyword_detection.py +0 -283
  17. exonware/xwlazy/common/services/spec_cache.py +0 -165
  18. exonware/xwlazy/common/services/state_manager.py +0 -84
  19. exonware/xwlazy/common/strategies/__init__.py +0 -28
  20. exonware/xwlazy/common/strategies/caching_dict.py +0 -44
  21. exonware/xwlazy/common/strategies/caching_installation.py +0 -88
  22. exonware/xwlazy/common/strategies/caching_lfu.py +0 -66
  23. exonware/xwlazy/common/strategies/caching_lru.py +0 -63
  24. exonware/xwlazy/common/strategies/caching_multitier.py +0 -59
  25. exonware/xwlazy/common/strategies/caching_ttl.py +0 -59
  26. exonware/xwlazy/config.py +0 -193
  27. exonware/xwlazy/contracts.py +0 -1396
  28. exonware/xwlazy/defs.py +0 -378
  29. exonware/xwlazy/errors.py +0 -276
  30. exonware/xwlazy/facade.py +0 -991
  31. exonware/xwlazy/module/__init__.py +0 -18
  32. exonware/xwlazy/module/base.py +0 -565
  33. exonware/xwlazy/module/data.py +0 -17
  34. exonware/xwlazy/module/facade.py +0 -246
  35. exonware/xwlazy/module/importer_engine.py +0 -2117
  36. exonware/xwlazy/module/strategies/__init__.py +0 -22
  37. exonware/xwlazy/module/strategies/module_helper_lazy.py +0 -93
  38. exonware/xwlazy/module/strategies/module_helper_simple.py +0 -65
  39. exonware/xwlazy/module/strategies/module_manager_advanced.py +0 -111
  40. exonware/xwlazy/module/strategies/module_manager_simple.py +0 -95
  41. exonware/xwlazy/package/__init__.py +0 -18
  42. exonware/xwlazy/package/base.py +0 -798
  43. exonware/xwlazy/package/conf.py +0 -324
  44. exonware/xwlazy/package/data.py +0 -17
  45. exonware/xwlazy/package/facade.py +0 -480
  46. exonware/xwlazy/package/services/__init__.py +0 -84
  47. exonware/xwlazy/package/services/async_install_handle.py +0 -87
  48. exonware/xwlazy/package/services/config_manager.py +0 -245
  49. exonware/xwlazy/package/services/discovery.py +0 -370
  50. exonware/xwlazy/package/services/host_packages.py +0 -145
  51. exonware/xwlazy/package/services/install_async.py +0 -277
  52. exonware/xwlazy/package/services/install_cache.py +0 -145
  53. exonware/xwlazy/package/services/install_interactive.py +0 -59
  54. exonware/xwlazy/package/services/install_policy.py +0 -156
  55. exonware/xwlazy/package/services/install_registry.py +0 -54
  56. exonware/xwlazy/package/services/install_result.py +0 -17
  57. exonware/xwlazy/package/services/install_sbom.py +0 -153
  58. exonware/xwlazy/package/services/install_utils.py +0 -79
  59. exonware/xwlazy/package/services/installer_engine.py +0 -406
  60. exonware/xwlazy/package/services/lazy_installer.py +0 -718
  61. exonware/xwlazy/package/services/manifest.py +0 -496
  62. exonware/xwlazy/package/services/strategy_registry.py +0 -186
  63. exonware/xwlazy/package/strategies/__init__.py +0 -57
  64. exonware/xwlazy/package/strategies/package_discovery_file.py +0 -129
  65. exonware/xwlazy/package/strategies/package_discovery_hybrid.py +0 -84
  66. exonware/xwlazy/package/strategies/package_discovery_manifest.py +0 -101
  67. exonware/xwlazy/package/strategies/package_execution_async.py +0 -113
  68. exonware/xwlazy/package/strategies/package_execution_cached.py +0 -90
  69. exonware/xwlazy/package/strategies/package_execution_pip.py +0 -99
  70. exonware/xwlazy/package/strategies/package_execution_wheel.py +0 -106
  71. exonware/xwlazy/package/strategies/package_mapping_discovery_first.py +0 -100
  72. exonware/xwlazy/package/strategies/package_mapping_hybrid.py +0 -105
  73. exonware/xwlazy/package/strategies/package_mapping_manifest_first.py +0 -100
  74. exonware/xwlazy/package/strategies/package_policy_allow_list.py +0 -57
  75. exonware/xwlazy/package/strategies/package_policy_deny_list.py +0 -57
  76. exonware/xwlazy/package/strategies/package_policy_permissive.py +0 -46
  77. exonware/xwlazy/package/strategies/package_timing_clean.py +0 -67
  78. exonware/xwlazy/package/strategies/package_timing_full.py +0 -66
  79. exonware/xwlazy/package/strategies/package_timing_smart.py +0 -68
  80. exonware/xwlazy/package/strategies/package_timing_temporary.py +0 -66
  81. exonware/xwlazy/runtime/__init__.py +0 -18
  82. exonware/xwlazy/runtime/adaptive_learner.py +0 -129
  83. exonware/xwlazy/runtime/base.py +0 -274
  84. exonware/xwlazy/runtime/facade.py +0 -94
  85. exonware/xwlazy/runtime/intelligent_selector.py +0 -170
  86. exonware/xwlazy/runtime/metrics.py +0 -60
  87. exonware/xwlazy/runtime/performance.py +0 -37
  88. exonware_xwlazy-0.1.0.22.dist-info/RECORD +0 -87
  89. {exonware_xwlazy-0.1.0.22.dist-info → exonware_xwlazy-1.0.1.2.dist-info}/WHEEL +0 -0
  90. {exonware_xwlazy-0.1.0.22.dist-info → exonware_xwlazy-1.0.1.2.dist-info}/licenses/LICENSE +0 -0
@@ -1,2117 +0,0 @@
1
- """
2
- #exonware/xwlazy/src/exonware/xwlazy/module/importer_engine.py
3
-
4
- Import Engine - Unified engine for all import-related operations.
5
-
6
- Company: eXonware.com
7
- Author: Eng. Muhammad AlShehri
8
- Email: connect@exonware.com
9
-
10
- Generation Date: 15-Nov-2025
11
-
12
- This module provides unified import engine for all import-related functionality.
13
- All import-related functionality is centralized here.
14
-
15
- Merged from:
16
- - logging_utils.py (Logging utilities)
17
- - import_tracking.py (Import tracking)
18
- - prefix_trie.py (Prefix trie data structure)
19
- - watched_registry.py (Watched prefix registry)
20
- - deferred_loader.py (Deferred module loader)
21
- - cache_utils.py (Multi-tier cache and bytecode cache)
22
- - parallel_utils.py (Parallel loading utilities)
23
- - module_patching.py (Module patching utilities)
24
- - archive_imports.py (Archive import utilities)
25
- - bootstrap.py (Bootstrap utilities)
26
- - loader.py (Lazy loader)
27
- - registry.py (Lazy module registry)
28
- - importer.py (Lazy importer)
29
- - import_hook.py (Import hook)
30
- - meta_path_finder.py (Meta path finder)
31
-
32
- Features:
33
- - Unified import engine for all import operations
34
- - Multi-tier caching (L1/L2/L3)
35
- - Parallel loading support
36
- - Import tracking and circular import prevention
37
- - Watched prefix registry
38
- - Meta path finder for intercepting imports
39
- """
40
-
41
- # =============================================================================
42
- # IMPORTS
43
- # =============================================================================
44
-
45
- from __future__ import annotations
46
-
47
- import os
48
- import sys
49
- import json
50
- import time
51
- import asyncio
52
- import pickle
53
- import struct
54
- import builtins
55
- import atexit
56
- import logging
57
- import importlib
58
- import importlib.util
59
- import importlib.machinery
60
- import importlib.abc
61
- import threading
62
- import subprocess
63
- import concurrent.futures
64
- from pathlib import Path
65
- from types import ModuleType
66
- from typing import Dict, List, Optional, Set, Tuple, Any, Iterable, Callable
67
- from collections import OrderedDict, defaultdict, Counter, deque
68
- from queue import Queue
69
- from datetime import datetime
70
- from enum import Enum
71
-
72
- from ..defs import LazyLoadMode, LazyInstallMode
73
- from ..common.services.dependency_mapper import DependencyMapper
74
- from ..common.services.spec_cache import _spec_cache_get, _spec_cache_put
75
- from ..package.services import LazyInstallerRegistry, LazyInstaller
76
- from ..package.services.config_manager import LazyInstallConfig
77
- from ..package.services.manifest import _normalize_prefix
78
- from ..errors import DeferredImportError
79
- from .base import AModuleHelper
80
-
81
- # Import from common (logger and cache)
82
- from ..common.logger import get_logger, log_event
83
- from ..common.cache import MultiTierCache, BytecodeCache
84
-
85
- # Import from runtime folder (moved from module folder)
86
- from ..runtime.adaptive_learner import AdaptiveLearner
87
- from ..runtime.intelligent_selector import IntelligentModeSelector, LoadLevel
88
-
89
- # =============================================================================
90
- # LOGGER (from common.logger)
91
- # =============================================================================
92
-
93
- logger = get_logger("xwlazy.importer_engine")
94
-
95
- # =============================================================================
96
- # IMPORT TRACKING (from import_tracking.py)
97
- # =============================================================================
98
-
99
- _thread_local = threading.local()
100
- _importing = threading.local()
101
- _installing = threading.local()
102
-
103
- _installation_depth = 0
104
- _installation_depth_lock = threading.Lock()
105
-
106
- def _get_thread_imports() -> Set[str]:
107
- """Get thread-local import set (creates if needed)."""
108
- if not hasattr(_thread_local, 'imports'):
109
- _thread_local.imports = set()
110
- return _thread_local.imports
111
-
112
- def _is_import_in_progress(module_name: str) -> bool:
113
- """Check if a module import is currently in progress for this thread."""
114
- return module_name in _get_thread_imports()
115
-
116
- def _mark_import_started(module_name: str) -> None:
117
- """Mark a module import as started for this thread."""
118
- _get_thread_imports().add(module_name)
119
-
120
- def _mark_import_finished(module_name: str) -> None:
121
- """Mark a module import as finished for this thread."""
122
- _get_thread_imports().discard(module_name)
123
-
124
- def get_importing_state() -> threading.local:
125
- """Get thread-local importing state."""
126
- return _importing
127
-
128
- def get_installing_state() -> threading.local:
129
- """Get thread-local installing state."""
130
- return _installing
131
-
132
- # Thread-local storage for installation state
133
- _installing_state = get_installing_state()
134
- _importing_state = get_importing_state()
135
-
136
- # Global recursion depth counter to prevent infinite recursion
137
- _installation_depth = 0
138
- _installation_depth_lock = threading.Lock()
139
-
140
- # =============================================================================
141
- # PREFIX TRIE (from prefix_trie.py)
142
- # =============================================================================
143
-
144
- class _PrefixTrie:
145
- """Trie data structure for prefix matching."""
146
-
147
- __slots__ = ("_root",)
148
-
149
- def __init__(self) -> None:
150
- self._root: Dict[str, Dict[str, Any]] = {}
151
-
152
- def add(self, prefix: str) -> None:
153
- """Add a prefix to the trie."""
154
- node = self._root
155
- for char in prefix:
156
- node = node.setdefault(char, {})
157
- node["_end"] = prefix
158
-
159
- def iter_matches(self, value: str) -> Tuple[str, ...]:
160
- """Find all matching prefixes for a given value."""
161
- node = self._root
162
- matches: List[str] = []
163
- for char in value:
164
- end_value = node.get("_end")
165
- if end_value:
166
- matches.append(end_value)
167
- node = node.get(char)
168
- if node is None:
169
- break
170
- else:
171
- end_value = node.get("_end")
172
- if end_value:
173
- matches.append(end_value)
174
- return tuple(matches)
175
-
176
- # =============================================================================
177
- # WATCHED REGISTRY (from watched_registry.py)
178
- # =============================================================================
179
-
180
- class WatchedPrefixRegistry:
181
- """Maintain watched prefixes and provide fast trie-based membership checks."""
182
-
183
- __slots__ = (
184
- "_lock",
185
- "_prefix_refcounts",
186
- "_owner_map",
187
- "_prefixes",
188
- "_trie",
189
- "_dirty",
190
- "_root_refcounts",
191
- "_root_snapshot",
192
- "_root_snapshot_dirty",
193
- )
194
-
195
- def __init__(self, initial: Optional[List[str]] = None) -> None:
196
- self._lock = threading.RLock()
197
- self._prefix_refcounts: Counter[str] = Counter()
198
- self._owner_map: Dict[str, Set[str]] = {}
199
- self._prefixes: Set[str] = set()
200
- self._trie = _PrefixTrie()
201
- self._dirty = False
202
- self._root_refcounts: Counter[str] = Counter()
203
- self._root_snapshot: Set[str] = set()
204
- self._root_snapshot_dirty = False
205
- if initial:
206
- for prefix in initial:
207
- self._register_manual(prefix)
208
-
209
- def _register_manual(self, prefix: str) -> None:
210
- normalized = _normalize_prefix(prefix)
211
- if not normalized:
212
- return
213
- owner = "__manual__"
214
- owners = self._owner_map.setdefault(owner, set())
215
- if normalized in owners:
216
- return
217
- owners.add(normalized)
218
- self._add_prefix(normalized)
219
-
220
- def _add_prefix(self, prefix: str) -> None:
221
- if not prefix:
222
- return
223
- self._prefix_refcounts[prefix] += 1
224
- if self._prefix_refcounts[prefix] == 1:
225
- self._prefixes.add(prefix)
226
- self._dirty = True
227
- root = prefix.split('.', 1)[0]
228
- self._root_refcounts[root] += 1
229
- self._root_snapshot_dirty = True
230
-
231
- def _remove_prefix(self, prefix: str) -> None:
232
- if prefix not in self._prefix_refcounts:
233
- return
234
- self._prefix_refcounts[prefix] -= 1
235
- if self._prefix_refcounts[prefix] <= 0:
236
- self._prefix_refcounts.pop(prefix, None)
237
- self._prefixes.discard(prefix)
238
- self._dirty = True
239
- root = prefix.split('.', 1)[0]
240
- self._root_refcounts[root] -= 1
241
- if self._root_refcounts[root] <= 0:
242
- self._root_refcounts.pop(root, None)
243
- self._root_snapshot_dirty = True
244
-
245
- def _ensure_trie(self) -> None:
246
- if not self._dirty:
247
- return
248
- self._trie = _PrefixTrie()
249
- for prefix in self._prefixes:
250
- self._trie.add(prefix)
251
- self._dirty = False
252
-
253
- def add(self, prefix: str) -> None:
254
- normalized = _normalize_prefix(prefix)
255
- if not normalized:
256
- return
257
- with self._lock:
258
- self._register_manual(normalized)
259
-
260
- def is_empty(self) -> bool:
261
- with self._lock:
262
- return not self._prefixes
263
-
264
- def register_package(self, package_name: str, prefixes: Iterable[str]) -> None:
265
- owner_key = f"pkg::{package_name.lower()}"
266
- normalized = {_normalize_prefix(p) for p in prefixes if _normalize_prefix(p)}
267
- with self._lock:
268
- current = self._owner_map.get(owner_key, set())
269
- to_remove = current - normalized
270
- to_add = normalized - current
271
-
272
- for prefix in to_remove:
273
- self._remove_prefix(prefix)
274
- for prefix in to_add:
275
- self._add_prefix(prefix)
276
-
277
- if normalized:
278
- self._owner_map[owner_key] = normalized
279
- elif owner_key in self._owner_map:
280
- self._owner_map.pop(owner_key, None)
281
-
282
- def is_prefix_owned_by(self, package_name: str, prefix: str) -> bool:
283
- normalized = _normalize_prefix(prefix)
284
- owner_key = f"pkg::{package_name.lower()}"
285
- with self._lock:
286
- if normalized in self._owner_map.get("__manual__", set()):
287
- return True
288
- return normalized in self._owner_map.get(owner_key, set())
289
-
290
- def get_matching_prefixes(self, module_name: str) -> Tuple[str, ...]:
291
- with self._lock:
292
- if not self._prefixes:
293
- return ()
294
- self._ensure_trie()
295
- return self._trie.iter_matches(module_name)
296
-
297
- def has_root(self, root_name: str) -> bool:
298
- snapshot = self._root_snapshot
299
- if not self._root_snapshot_dirty:
300
- return root_name in snapshot
301
- with self._lock:
302
- if self._root_snapshot_dirty:
303
- self._root_snapshot = set(self._root_refcounts.keys())
304
- self._root_snapshot_dirty = False
305
- return root_name in self._root_snapshot
306
-
307
- # Global registry instance
308
- _DEFAULT_WATCHED_PREFIXES = tuple(
309
- filter(
310
- None,
311
- os.environ.get(
312
- "XWLAZY_LAZY_PREFIXES",
313
- "",
314
- ).split(";"),
315
- )
316
- )
317
- _watched_registry = WatchedPrefixRegistry(list(_DEFAULT_WATCHED_PREFIXES))
318
-
319
- def get_watched_registry() -> WatchedPrefixRegistry:
320
- """Get the global watched prefix registry."""
321
- return _watched_registry
322
-
323
- # =============================================================================
324
- # DEFERRED LOADER (from deferred_loader.py)
325
- # =============================================================================
326
-
327
- class _DeferredModuleLoader(importlib.abc.Loader):
328
- """Loader that simply returns a preconstructed module placeholder."""
329
-
330
- def __init__(self, module: ModuleType) -> None:
331
- self._module = module
332
-
333
- def create_module(self, spec): # noqa: D401 - standard loader hook
334
- return self._module
335
-
336
- def exec_module(self, module): # noqa: D401 - nothing to execute
337
- return None
338
-
339
- # =============================================================================
340
- # CACHE (from common.cache)
341
- # =============================================================================
342
-
343
- # MultiTierCache and BytecodeCache are now imported from ..common.cache
344
-
345
- # =============================================================================
346
- # PARALLEL UTILITIES (from parallel_utils.py)
347
- # =============================================================================
348
-
349
- class ParallelLoader:
350
- """Parallel module loader with smart dependency management."""
351
-
352
- def __init__(self, max_workers: Optional[int] = None):
353
- if max_workers is None:
354
- max_workers = min(os.cpu_count() or 4, 8)
355
-
356
- self._max_workers = max_workers
357
- self._executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
358
- self._lock = threading.RLock()
359
-
360
- def _get_executor(self) -> concurrent.futures.ThreadPoolExecutor:
361
- """Get or create thread pool executor."""
362
- with self._lock:
363
- if self._executor is None:
364
- self._executor = concurrent.futures.ThreadPoolExecutor(
365
- max_workers=self._max_workers,
366
- thread_name_prefix="xwlazy-parallel"
367
- )
368
- return self._executor
369
-
370
- def load_modules_parallel(self, module_paths: List[str]) -> Dict[str, Any]:
371
- """Load multiple modules in parallel."""
372
- executor = self._get_executor()
373
- results: Dict[str, Any] = {}
374
-
375
- def _load_module(module_path: str) -> Tuple[str, Any, Optional[Exception]]:
376
- try:
377
- module = importlib.import_module(module_path)
378
- return (module_path, module, None)
379
- except Exception as e:
380
- logger.debug(f"Failed to load {module_path} in parallel: {e}")
381
- return (module_path, None, e)
382
-
383
- futures = {executor.submit(_load_module, path): path for path in module_paths}
384
-
385
- for future in concurrent.futures.as_completed(futures):
386
- module_path, module, error = future.result()
387
- results[module_path] = (module, error)
388
-
389
- return results
390
-
391
- def load_modules_with_priority(
392
- self,
393
- module_paths: List[Tuple[str, int]]
394
- ) -> Dict[str, Any]:
395
- """Load modules in parallel with priority ordering."""
396
- sorted_modules = sorted(module_paths, key=lambda x: x[1], reverse=True)
397
- module_list = [path for path, _ in sorted_modules]
398
- return self.load_modules_parallel(module_list)
399
-
400
- def shutdown(self, wait: bool = True) -> None:
401
- """Shutdown the executor."""
402
- with self._lock:
403
- if self._executor:
404
- self._executor.shutdown(wait=wait)
405
- self._executor = None
406
-
407
- class DependencyGraph:
408
- """Manages module dependencies for optimal parallel loading."""
409
-
410
- def __init__(self):
411
- self._dependencies: Dict[str, List[str]] = {}
412
- self._reverse_deps: Dict[str, List[str]] = {}
413
- self._lock = threading.RLock()
414
-
415
- def add_dependency(self, module: str, depends_on: List[str]) -> None:
416
- """Add dependencies for a module."""
417
- with self._lock:
418
- self._dependencies[module] = depends_on
419
- for dep in depends_on:
420
- if dep not in self._reverse_deps:
421
- self._reverse_deps[dep] = []
422
- if module not in self._reverse_deps[dep]:
423
- self._reverse_deps[dep].append(module)
424
-
425
- def get_load_order(self, modules: List[str]) -> List[List[str]]:
426
- """Get optimal load order for parallel loading (topological sort levels)."""
427
- with self._lock:
428
- in_degree: Dict[str, int] = {m: 0 for m in modules}
429
- for module, deps in self._dependencies.items():
430
- if module in modules:
431
- for dep in deps:
432
- if dep in modules:
433
- in_degree[module] += 1
434
-
435
- levels: List[List[str]] = []
436
- remaining = set(modules)
437
-
438
- while remaining:
439
- current_level = [
440
- m for m in remaining
441
- if in_degree[m] == 0
442
- ]
443
-
444
- if not current_level:
445
- current_level = list(remaining)
446
-
447
- levels.append(current_level)
448
- remaining -= set(current_level)
449
-
450
- for module in current_level:
451
- for dependent in self._reverse_deps.get(module, []):
452
- if dependent in remaining:
453
- in_degree[dependent] = max(0, in_degree[dependent] - 1)
454
-
455
- return levels
456
-
457
- # =============================================================================
458
- # MODULE PATCHING (from module_patching.py)
459
- # =============================================================================
460
-
461
- _original_import_module = importlib.import_module
462
-
463
- def _lazy_aware_import_module(name: str, package: Optional[str] = None) -> ModuleType:
464
- """Lazy-aware version of importlib.import_module."""
465
- if _is_import_in_progress(name):
466
- return _original_import_module(name, package)
467
-
468
- _mark_import_started(name)
469
- try:
470
- return _original_import_module(name, package)
471
- finally:
472
- _mark_import_finished(name)
473
-
474
- def _patch_import_module() -> None:
475
- """Patch importlib.import_module to be lazy-aware."""
476
- importlib.import_module = _lazy_aware_import_module
477
- logger.debug("Patched importlib.import_module to be lazy-aware")
478
-
479
- def _unpatch_import_module() -> None:
480
- """Restore original importlib.import_module."""
481
- importlib.import_module = _original_import_module
482
- logger.debug("Restored original importlib.import_module")
483
-
484
- # =============================================================================
485
- # ARCHIVE IMPORTS (from archive_imports.py)
486
- # =============================================================================
487
-
488
- _archive_path = None
489
- _archive_added = False
490
-
491
- def get_archive_path() -> Path:
492
- """Get the path to the _archive folder."""
493
- global _archive_path
494
- if _archive_path is None:
495
- current_file = Path(__file__)
496
- _archive_path = current_file.parent.parent.parent.parent.parent.parent / "_archive"
497
- return _archive_path
498
-
499
- def ensure_archive_in_path() -> None:
500
- """Ensure the archive folder is in sys.path for imports."""
501
- global _archive_added
502
- if not _archive_added:
503
- archive_path = get_archive_path()
504
- archive_str = str(archive_path)
505
- if archive_str not in sys.path:
506
- sys.path.insert(0, archive_str)
507
- _archive_added = True
508
-
509
- def import_from_archive(module_name: str):
510
- """Import a module from the archived lazy code."""
511
- ensure_archive_in_path()
512
- return __import__(module_name, fromlist=[''])
513
-
514
- # =============================================================================
515
- # BOOTSTRAP (from bootstrap.py)
516
- # =============================================================================
517
-
518
- def _env_enabled(env_value: Optional[str]) -> Optional[bool]:
519
- if not env_value:
520
- return None
521
- normalized = env_value.strip().lower()
522
- if normalized in ('true', '1', 'yes', 'on'):
523
- return True
524
- if normalized in ('false', '0', 'no', 'off'):
525
- return False
526
- return None
527
-
528
- def bootstrap_lazy_mode(package_name: str) -> None:
529
- """Detect whether lazy mode should be enabled for ``package_name`` and bootstrap hooks."""
530
- package_name = package_name.lower()
531
- env_value = os.environ.get(f"{package_name.upper()}_LAZY_INSTALL")
532
- env_enabled = _env_enabled(env_value)
533
- enabled = env_enabled
534
-
535
- if enabled is None:
536
- from ...common.services import _detect_lazy_installation
537
- enabled = _detect_lazy_installation(package_name)
538
-
539
- if not enabled:
540
- return
541
-
542
- from ..facade import config_package_lazy_install_enabled
543
-
544
- config_package_lazy_install_enabled(
545
- package_name,
546
- enabled=True,
547
- install_hook=True,
548
- )
549
-
550
- def bootstrap_lazy_mode_deferred(package_name: str) -> None:
551
- """Schedule lazy mode bootstrap to run AFTER the calling package finishes importing."""
552
- package_name_lower = package_name.lower()
553
- package_module_name = f"exonware.{package_name_lower}"
554
-
555
- original_import = __builtins__.__import__ if hasattr(__builtins__, '__import__') else __import__
556
-
557
- def _import_hook(name, *args, **kwargs):
558
- result = original_import(name, *args, **kwargs)
559
-
560
- if name == package_module_name or name.startswith(f"{package_module_name}."):
561
- if package_module_name in sys.modules:
562
- import threading
563
- def _install_hook():
564
- if hasattr(__builtins__, '__import__'):
565
- __builtins__.__import__ = original_import
566
- else:
567
- import builtins
568
- builtins.__import__ = original_import
569
- bootstrap_lazy_mode(package_name_lower)
570
-
571
- threading.Timer(0.0, _install_hook).start()
572
-
573
- return result
574
-
575
- if hasattr(__builtins__, '__import__'):
576
- __builtins__.__import__ = _import_hook
577
- else:
578
- import builtins
579
- builtins.__import__ = _import_hook
580
-
581
- # =============================================================================
582
- # LAZY LOADER (from loader.py)
583
- # =============================================================================
584
-
585
- class LazyLoader(AModuleHelper):
586
- """Thread-safe lazy loader for modules with caching."""
587
-
588
- def load_module(self, module_path: str = None) -> ModuleType:
589
- """Thread-safe module loading with caching."""
590
- if module_path is None:
591
- module_path = self._module_path
592
-
593
- if self._cached_module is not None:
594
- return self._cached_module
595
-
596
- with self._lock:
597
- if self._cached_module is not None:
598
- return self._cached_module
599
-
600
- if self._loading:
601
- raise ImportError(f"Circular import detected for {module_path}")
602
-
603
- try:
604
- self._loading = True
605
- logger.debug(f"Lazy loading module: {module_path}")
606
-
607
- self._cached_module = importlib.import_module(module_path)
608
-
609
- logger.debug(f"Successfully loaded: {module_path}")
610
- return self._cached_module
611
-
612
- except Exception as e:
613
- logger.error(f"Failed to load module {module_path}: {e}")
614
- raise ImportError(f"Failed to load {module_path}: {e}") from e
615
- finally:
616
- self._loading = False
617
-
618
- def unload_module(self, module_path: str) -> None:
619
- """Unload a module from cache."""
620
- with self._lock:
621
- if module_path == self._module_path:
622
- self._cached_module = None
623
-
624
- def is_loaded(self) -> bool:
625
- """Check if module is currently loaded."""
626
- return self._cached_module is not None
627
-
628
- def __getattr__(self, name: str) -> Any:
629
- """Get attribute from lazily loaded module."""
630
- module = self.load_module()
631
- try:
632
- return getattr(module, name)
633
- except AttributeError:
634
- raise AttributeError(
635
- f"module '{self._module_path}' has no attribute '{name}'"
636
- )
637
-
638
- def __dir__(self) -> list:
639
- """Return available attributes from loaded module."""
640
- module = self.load_module()
641
- return dir(module)
642
-
643
- # =============================================================================
644
- # LAZY MODULE REGISTRY (from registry.py)
645
- # =============================================================================
646
-
647
- class LazyModuleRegistry:
648
- """Registry for managing lazy-loaded modules with performance tracking."""
649
-
650
- __slots__ = ('_modules', '_load_times', '_lock', '_access_counts')
651
-
652
- def __init__(self):
653
- self._modules: Dict[str, LazyLoader] = {}
654
- self._load_times: Dict[str, float] = {}
655
- self._access_counts: Dict[str, int] = {}
656
- self._lock = threading.RLock()
657
-
658
- def register_module(self, name: str, module_path: str) -> None:
659
- """Register a module for lazy loading."""
660
- with self._lock:
661
- if name in self._modules:
662
- logger.warning(f"Module '{name}' already registered, overwriting")
663
-
664
- self._modules[name] = LazyLoader(module_path)
665
- self._access_counts[name] = 0
666
- logger.debug(f"Registered lazy module: {name} -> {module_path}")
667
-
668
- def get_module(self, name: str) -> LazyLoader:
669
- """Get a lazy-loaded module."""
670
- with self._lock:
671
- if name not in self._modules:
672
- raise KeyError(f"Module '{name}' not registered")
673
-
674
- self._access_counts[name] += 1
675
- return self._modules[name]
676
-
677
- def preload_frequently_used(self, threshold: int = 5) -> None:
678
- """Preload modules that are accessed frequently."""
679
- with self._lock:
680
- for name, count in self._access_counts.items():
681
- if count >= threshold:
682
- try:
683
- start_time = time.time()
684
- _ = self._modules[name].load_module()
685
- self._load_times[name] = time.time() - start_time
686
- log_event("hook", logger.info, f"Preloaded frequently used module: {name}")
687
- except Exception as e:
688
- logger.warning(f"Failed to preload {name}: {e}")
689
-
690
- def get_stats(self) -> Dict[str, Any]:
691
- """Get loading statistics."""
692
- with self._lock:
693
- loaded_count = sum(
694
- 1 for loader in self._modules.values()
695
- if loader.is_loaded()
696
- )
697
-
698
- return {
699
- 'total_registered': len(self._modules),
700
- 'loaded_count': loaded_count,
701
- 'unloaded_count': len(self._modules) - loaded_count,
702
- 'access_counts': self._access_counts.copy(),
703
- 'load_times': self._load_times.copy(),
704
- }
705
-
706
- def clear_cache(self) -> None:
707
- """Clear all cached modules."""
708
- with self._lock:
709
- for name, loader in self._modules.items():
710
- loader.unload_module(loader._module_path)
711
- log_event("config", logger.info, "Cleared all cached modules")
712
-
713
- # =============================================================================
714
- # LAZY IMPORTER (from importer.py)
715
- # =============================================================================
716
-
717
- class LazyImporter:
718
- """
719
- Lazy importer that defers heavy module imports until first access.
720
- Supports multiple load modes: NONE, AUTO, PRELOAD, BACKGROUND, CACHED,
721
- TURBO, ADAPTIVE, HYPERPARALLEL, STREAMING, ULTRA, INTELLIGENT.
722
- """
723
-
724
- __slots__ = (
725
- '_enabled', '_load_mode', '_lazy_modules', '_loaded_modules', '_lock',
726
- '_access_counts', '_background_tasks', '_async_loop',
727
- '_multi_tier_cache', '_bytecode_cache', '_adaptive_learner',
728
- '_parallel_loader', '_dependency_graph', '_load_times',
729
- '_intelligent_selector', '_effective_mode', '_effective_install_mode'
730
- )
731
-
732
- def __init__(self):
733
- """Initialize lazy importer."""
734
- self._enabled = False
735
- self._load_mode = LazyLoadMode.NONE
736
- self._lazy_modules: Dict[str, str] = {}
737
- self._loaded_modules: Dict[str, ModuleType] = {}
738
- self._access_counts: Dict[str, int] = {}
739
- self._load_times: Dict[str, float] = {}
740
- self._background_tasks: Dict[str, asyncio.Task] = {}
741
- self._async_loop: Optional[asyncio.AbstractEventLoop] = None
742
-
743
- # Superior mode components
744
- self._multi_tier_cache: Optional[MultiTierCache] = None
745
- self._bytecode_cache: Optional[BytecodeCache] = None
746
- self._adaptive_learner: Optional[AdaptiveLearner] = None
747
- self._parallel_loader: Optional[ParallelLoader] = None
748
- self._dependency_graph: Optional[DependencyGraph] = None
749
- self._intelligent_selector: Optional[IntelligentModeSelector] = None
750
-
751
- # Effective modes (for INTELLIGENT mode)
752
- self._effective_mode: Optional[LazyLoadMode] = None
753
- self._effective_install_mode = None
754
-
755
- self._lock = threading.RLock()
756
-
757
- def _ensure_async_loop(self) -> asyncio.AbstractEventLoop:
758
- """Ensure async event loop is running for background loading."""
759
- if self._async_loop is not None and self._async_loop.is_running():
760
- return self._async_loop
761
-
762
- with self._lock:
763
- if self._async_loop is None or not self._async_loop.is_running():
764
- loop_ready = threading.Event()
765
-
766
- def _run_loop():
767
- loop = asyncio.new_event_loop()
768
- asyncio.set_event_loop(loop)
769
- self._async_loop = loop
770
- loop_ready.set()
771
- loop.run_forever()
772
-
773
- thread = threading.Thread(target=_run_loop, daemon=True, name="xwlazy-loader-async")
774
- thread.start()
775
-
776
- if not loop_ready.wait(timeout=5.0):
777
- raise RuntimeError("Failed to start async loop for lazy loader")
778
-
779
- return self._async_loop
780
-
781
- def enable(self, load_mode: LazyLoadMode = LazyLoadMode.AUTO) -> None:
782
- """Enable lazy imports with specified load mode."""
783
- with self._lock:
784
- self._enabled = True
785
- self._load_mode = load_mode
786
-
787
- # Initialize superior mode components
788
- if load_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA):
789
- self._multi_tier_cache = MultiTierCache(l1_size=1000, enable_l3=True)
790
- self._bytecode_cache = BytecodeCache()
791
-
792
- if load_mode == LazyLoadMode.ADAPTIVE:
793
- self._adaptive_learner = AdaptiveLearner()
794
- self._multi_tier_cache = MultiTierCache(l1_size=1000, enable_l3=True)
795
-
796
- if load_mode == LazyLoadMode.HYPERPARALLEL:
797
- max_workers = min(os.cpu_count() or 4, 8)
798
- self._parallel_loader = ParallelLoader(max_workers=max_workers)
799
- self._dependency_graph = DependencyGraph()
800
-
801
- if load_mode == LazyLoadMode.STREAMING:
802
- self._ensure_async_loop()
803
-
804
- if load_mode == LazyLoadMode.ULTRA:
805
- # ULTRA combines all optimizations
806
- if self._multi_tier_cache is None:
807
- self._multi_tier_cache = MultiTierCache(l1_size=2000, enable_l3=True)
808
- if self._bytecode_cache is None:
809
- self._bytecode_cache = BytecodeCache()
810
- if self._adaptive_learner is None:
811
- self._adaptive_learner = AdaptiveLearner()
812
- if self._parallel_loader is None:
813
- self._parallel_loader = ParallelLoader(max_workers=min(os.cpu_count() or 4, 8))
814
- if self._dependency_graph is None:
815
- self._dependency_graph = DependencyGraph()
816
- self._ensure_async_loop()
817
-
818
- # INTELLIGENT mode: Initialize selector and determine initial mode
819
- if load_mode == LazyLoadMode.INTELLIGENT:
820
- self._intelligent_selector = IntelligentModeSelector()
821
- # Detect initial load level and get optimal mode
822
- initial_level = self._intelligent_selector.detect_load_level()
823
- self._effective_mode, self._effective_install_mode = self._intelligent_selector.get_optimal_mode(initial_level)
824
- logger.info(f"INTELLIGENT mode initialized: {initial_level.value} -> {self._effective_mode.value} + {self._effective_install_mode.value}")
825
- # Enable the effective mode recursively
826
- self.enable(self._effective_mode)
827
- return # Early return, effective mode is already enabled
828
-
829
- # For PRELOAD/TURBO/ULTRA modes, preload modules
830
- if load_mode in (LazyLoadMode.PRELOAD, LazyLoadMode.TURBO, LazyLoadMode.ULTRA):
831
- self._preload_all_modules()
832
- # For BACKGROUND/STREAMING modes, ensure async loop is ready
833
- elif load_mode in (LazyLoadMode.BACKGROUND, LazyLoadMode.STREAMING):
834
- self._ensure_async_loop()
835
-
836
- log_event("config", logger.info, f"Lazy imports enabled (mode: {load_mode.value})")
837
-
838
- def disable(self) -> None:
839
- """Disable lazy imports."""
840
- with self._lock:
841
- self._enabled = False
842
-
843
- # Cleanup cache resources
844
- if self._multi_tier_cache:
845
- self._multi_tier_cache.shutdown()
846
-
847
- log_event("config", logger.info, "Lazy imports disabled")
848
-
849
- def is_enabled(self) -> bool:
850
- """Check if lazy imports are enabled."""
851
- return self._enabled
852
-
853
- def register_lazy_module(self, module_name: str, module_path: str = None) -> None:
854
- """Register a module for lazy loading."""
855
- with self._lock:
856
- if module_path is None:
857
- module_path = module_name
858
-
859
- self._lazy_modules[module_name] = module_path
860
- self._access_counts[module_name] = 0
861
- logger.debug(f"Registered lazy module: {module_name} -> {module_path}")
862
-
863
- async def _background_load_module(self, module_name: str, module_path: str) -> ModuleType:
864
- """Load module in background thread."""
865
- try:
866
- actual_module = importlib.import_module(module_path)
867
- with self._lock:
868
- self._loaded_modules[module_name] = actual_module
869
- self._access_counts[module_name] += 1
870
- logger.debug(f"Background loaded module: {module_name}")
871
- return actual_module
872
- except ImportError as e:
873
- logger.error(f"Failed to background load {module_name}: {e}")
874
- raise
875
-
876
- def _preload_all_modules(self) -> None:
877
- """Preload all registered modules using appropriate strategy based on mode."""
878
- if not self._lazy_modules:
879
- return
880
-
881
- with self._lock:
882
- modules_to_load = [
883
- (name, path) for name, path in self._lazy_modules.items()
884
- if name not in self._loaded_modules
885
- ]
886
-
887
- if not modules_to_load:
888
- return
889
-
890
- # HYPERPARALLEL/ULTRA: Use thread pool executor
891
- if self._load_mode in (LazyLoadMode.HYPERPARALLEL, LazyLoadMode.ULTRA) and self._parallel_loader:
892
- module_paths = [path for _, path in modules_to_load]
893
- results = self._parallel_loader.load_modules_parallel(module_paths)
894
-
895
- with self._lock:
896
- for (name, path), (module, error) in zip(modules_to_load, results.items()):
897
- if module is not None:
898
- self._loaded_modules[name] = module
899
- self._access_counts[name] = 0
900
- if self._adaptive_learner:
901
- self._adaptive_learner.record_import(name, 0.0)
902
-
903
- log_event("hook", logger.info, f"Parallel preloaded {len([r for r in results.values() if r[0] is not None])} modules")
904
- return
905
-
906
- # TURBO/ULTRA: Preload with predictive caching
907
- if self._load_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._multi_tier_cache:
908
- # Get predictive modules to prioritize
909
- predictive_keys = self._multi_tier_cache.get_predictive_keys(limit=10)
910
- priority_modules = [(name, path) for name, path in modules_to_load if name in predictive_keys]
911
- normal_modules = [(name, path) for name, path in modules_to_load if name not in predictive_keys]
912
- modules_to_load = priority_modules + normal_modules
913
-
914
- # ADAPTIVE: Preload based on learned patterns
915
- if self._load_mode == LazyLoadMode.ADAPTIVE and self._adaptive_learner:
916
- priority_modules = self._adaptive_learner.get_priority_modules(limit=10)
917
- priority_list = [(name, path) for name, path in modules_to_load if name in priority_modules]
918
- normal_list = [(name, path) for name, path in modules_to_load if name not in priority_modules]
919
- modules_to_load = priority_list + normal_list
920
-
921
- # Default: Use asyncio for parallel loading
922
- loop = self._ensure_async_loop()
923
-
924
- async def _preload_all():
925
- tasks = [
926
- self._background_load_module(name, path)
927
- for name, path in modules_to_load
928
- ]
929
- if tasks:
930
- await asyncio.gather(*tasks, return_exceptions=True)
931
- log_event("hook", logger.info, f"Preloaded {len(tasks)} modules")
932
-
933
- asyncio.run_coroutine_threadsafe(_preload_all(), loop)
934
-
935
- def import_module(self, module_name: str, package_name: str = None) -> Any:
936
- """Import a module with lazy loading."""
937
- start_time = time.time()
938
-
939
- # Fast path: Check if already in sys.modules (lock-free read)
940
- if module_name in sys.modules:
941
- # Lock-free check first
942
- if module_name not in self._loaded_modules:
943
- with self._lock:
944
- # Double-check after acquiring lock
945
- if module_name not in self._loaded_modules:
946
- self._loaded_modules[module_name] = sys.modules[module_name]
947
- # Update access count (requires lock)
948
- with self._lock:
949
- self._access_counts[module_name] = self._access_counts.get(module_name, 0) + 1
950
- load_time = time.time() - start_time
951
- if self._adaptive_learner:
952
- self._adaptive_learner.record_import(module_name, load_time)
953
- if self._load_mode == LazyLoadMode.INTELLIGENT:
954
- self._total_import_time = getattr(self, '_total_import_time', 0.0) + load_time
955
- return sys.modules[module_name]
956
-
957
- # Fast path: Check if already loaded (lock-free read)
958
- if module_name in self._loaded_modules:
959
- with self._lock:
960
- # Double-check and update
961
- if module_name in self._loaded_modules:
962
- self._access_counts[module_name] = self._access_counts.get(module_name, 0) + 1
963
- if self._adaptive_learner:
964
- self._adaptive_learner.record_import(module_name, 0.0)
965
- return self._loaded_modules[module_name]
966
-
967
- # Check enabled state and get module path (requires lock)
968
- with self._lock:
969
- if not self._enabled or self._load_mode == LazyLoadMode.NONE:
970
- return importlib.import_module(module_name)
971
-
972
- if module_name in self._lazy_modules:
973
- module_path = self._lazy_modules[module_name]
974
- else:
975
- return importlib.import_module(module_name)
976
-
977
- # Update total import time for intelligent mode (initialization)
978
- if self._load_mode == LazyLoadMode.INTELLIGENT:
979
- if not hasattr(self, '_total_import_time'):
980
- self._total_import_time = 0.0
981
-
982
- # INTELLIGENT mode: Check if mode switch is needed and determine effective mode
983
- effective_load_mode = self._load_mode
984
- if self._load_mode == LazyLoadMode.INTELLIGENT and self._intelligent_selector:
985
- # Throttle load level detection (cache for 0.1s to avoid excessive checks)
986
- current_time = time.time()
987
- last_check = getattr(self, '_last_load_level_check', 0.0)
988
- check_interval = 0.1 # 100ms throttle
989
-
990
- if current_time - last_check >= check_interval:
991
- # Fast path: lock-free reads for stats
992
- module_count = len(self._loaded_modules) # Dict read is thread-safe
993
- total_import_time = getattr(self, '_total_import_time', 0.0)
994
- import_count = sum(self._access_counts.values()) # Dict read is thread-safe
995
-
996
- # Cache psutil import and memory check (only check every 0.5s)
997
- last_memory_check = getattr(self, '_last_memory_check', 0.0)
998
- memory_mb = getattr(self, '_cached_memory_mb', 0.0)
999
-
1000
- if current_time - last_memory_check >= 0.5:
1001
- try:
1002
- import psutil
1003
- process = psutil.Process()
1004
- memory_mb = process.memory_info().rss / 1024 / 1024
1005
- self._cached_memory_mb = memory_mb
1006
- self._last_memory_check = current_time
1007
- except Exception:
1008
- memory_mb = 0.0
1009
-
1010
- # Detect current load level (lock-free)
1011
- detected_level = self._intelligent_selector.detect_load_level(
1012
- module_count=module_count,
1013
- total_import_time=total_import_time,
1014
- import_count=import_count,
1015
- memory_usage_mb=memory_mb
1016
- )
1017
-
1018
- # Check if mode switch is needed (requires lock for write)
1019
- current_mode_tuple = (self._effective_mode or self._load_mode, self._effective_install_mode)
1020
- if self._intelligent_selector.should_switch_mode(current_mode_tuple, detected_level):
1021
- optimal_load, optimal_install = self._intelligent_selector.get_optimal_mode(detected_level)
1022
- if optimal_load != self._effective_mode or optimal_install != self._effective_install_mode:
1023
- with self._lock: # Only lock for mode switch
1024
- if optimal_load != self._effective_mode or optimal_install != self._effective_install_mode:
1025
- logger.info(f"INTELLIGENT mode switching: {detected_level.value} -> {optimal_load.value} + {optimal_install.value}")
1026
- self._effective_mode = optimal_load
1027
- self._effective_install_mode = optimal_install
1028
- # Switch to optimal mode (re-enable with new mode)
1029
- self.enable(optimal_load)
1030
-
1031
- self._last_load_level_check = current_time
1032
-
1033
- # Use effective mode for processing
1034
- effective_load_mode = self._effective_mode or self._load_mode
1035
-
1036
- # Use effective mode for all checks
1037
- check_mode = effective_load_mode
1038
-
1039
- # TURBO/ULTRA: Check multi-tier cache first
1040
- if check_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._multi_tier_cache:
1041
- cached_module = self._multi_tier_cache.get(module_name)
1042
- if cached_module is not None:
1043
- with self._lock:
1044
- self._loaded_modules[module_name] = cached_module
1045
- self._access_counts[module_name] += 1
1046
- self._load_times[module_name] = time.time() - start_time
1047
- if self._adaptive_learner:
1048
- self._adaptive_learner.record_import(module_name, self._load_times[module_name])
1049
- return cached_module
1050
-
1051
- # ADAPTIVE: Check cache and predict next imports
1052
- if check_mode == LazyLoadMode.ADAPTIVE:
1053
- if self._multi_tier_cache:
1054
- cached_module = self._multi_tier_cache.get(module_name)
1055
- if cached_module is not None:
1056
- with self._lock:
1057
- self._loaded_modules[module_name] = cached_module
1058
- self._access_counts[module_name] += 1
1059
- load_time = time.time() - start_time
1060
- self._load_times[module_name] = load_time
1061
- if self._adaptive_learner:
1062
- self._adaptive_learner.record_import(module_name, load_time)
1063
-
1064
- # Predict and preload next likely imports
1065
- if self._adaptive_learner:
1066
- next_imports = self._adaptive_learner.predict_next_imports(module_name, limit=3)
1067
- self._preload_predictive_modules(next_imports)
1068
-
1069
- return cached_module
1070
-
1071
- # Record import for learning
1072
- with self._lock:
1073
- if self._adaptive_learner:
1074
- # Will be updated after load
1075
- pass
1076
-
1077
- # HYPERPARALLEL: Use parallel loading
1078
- if check_mode == LazyLoadMode.HYPERPARALLEL and self._parallel_loader:
1079
- results = self._parallel_loader.load_modules_parallel([module_path])
1080
- module, error = results.get(module_path, (None, None))
1081
- if module is not None:
1082
- with self._lock:
1083
- self._loaded_modules[module_name] = module
1084
- self._access_counts[module_name] += 1
1085
- self._load_times[module_name] = time.time() - start_time
1086
- return module
1087
- elif error:
1088
- raise error
1089
-
1090
- # STREAMING: Load asynchronously in background
1091
- if check_mode == LazyLoadMode.STREAMING:
1092
- return self._streaming_load(module_name, module_path)
1093
-
1094
- # BACKGROUND mode: Load in background, return placeholder
1095
- if check_mode == LazyLoadMode.BACKGROUND:
1096
- return self._background_placeholder_load(module_name, module_path)
1097
-
1098
- # TURBO/ULTRA: Load with bytecode cache
1099
- actual_module = None
1100
- if check_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._bytecode_cache:
1101
- # Try to load from bytecode cache first
1102
- bytecode = self._bytecode_cache.get_cached_bytecode(module_path)
1103
- if bytecode is not None:
1104
- try:
1105
- # Load from bytecode
1106
- code = compile(bytecode, f"<cached {module_path}>", "exec")
1107
- actual_module = importlib.import_module(module_path)
1108
- except Exception as e:
1109
- logger.debug(f"Failed to load from bytecode cache: {e}")
1110
-
1111
- # Load module (standard or cached)
1112
- if actual_module is None:
1113
- try:
1114
- actual_module = importlib.import_module(module_path)
1115
-
1116
- # Cache bytecode for TURBO/ULTRA
1117
- if check_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._bytecode_cache:
1118
- try:
1119
- # Get compiled bytecode from module
1120
- if hasattr(actual_module, '__file__') and actual_module.__file__:
1121
- pyc_path = actual_module.__file__.replace('.py', '.pyc')
1122
- if os.path.exists(pyc_path):
1123
- with open(pyc_path, 'rb') as f:
1124
- f.seek(16) # Skip header
1125
- bytecode = f.read()
1126
- self._bytecode_cache.cache_bytecode(module_path, bytecode)
1127
- except Exception as e:
1128
- logger.debug(f"Failed to cache bytecode: {e}")
1129
- except ImportError as e:
1130
- logger.error(f"Failed to lazy load {module_name}: {e}")
1131
- raise
1132
-
1133
- load_time = time.time() - start_time
1134
-
1135
- with self._lock:
1136
- self._loaded_modules[module_name] = actual_module
1137
- self._access_counts[module_name] += 1
1138
- self._load_times[module_name] = load_time
1139
-
1140
- # Update total import time for intelligent mode
1141
- if self._load_mode == LazyLoadMode.INTELLIGENT:
1142
- self._total_import_time = getattr(self, '_total_import_time', 0.0) + load_time
1143
-
1144
- # Cache in multi-tier cache for TURBO/ULTRA/ADAPTIVE
1145
- if self._multi_tier_cache:
1146
- self._multi_tier_cache.set(module_name, actual_module)
1147
-
1148
- # Record for adaptive learning
1149
- if self._adaptive_learner:
1150
- self._adaptive_learner.record_import(module_name, load_time)
1151
-
1152
- logger.debug(f"Lazy loaded module: {module_name} ({load_time*1000:.2f}ms)")
1153
-
1154
- return actual_module
1155
-
1156
- def _streaming_load(self, module_name: str, module_path: str) -> ModuleType:
1157
- """Load module asynchronously with streaming."""
1158
- if module_name not in self._background_tasks or self._background_tasks[module_name].done():
1159
- loop = self._ensure_async_loop()
1160
- task = asyncio.run_coroutine_threadsafe(
1161
- self._background_load_module(module_name, module_path),
1162
- loop
1163
- )
1164
- self._background_tasks[module_name] = task
1165
-
1166
- # Return placeholder that streams
1167
- placeholder = ModuleType(module_name)
1168
- placeholder.__path__ = []
1169
- placeholder.__package__ = module_name
1170
-
1171
- def _streaming_getattr(name):
1172
- task = self._background_tasks.get(module_name)
1173
- if task and not task.done():
1174
- # Non-blocking check with short timeout
1175
- try:
1176
- task.result(timeout=0.01) # Very short timeout for streaming
1177
- except Exception:
1178
- pass # Still loading, continue
1179
-
1180
- # Check if loaded now
1181
- with self._lock:
1182
- if module_name in self._loaded_modules:
1183
- return getattr(self._loaded_modules[module_name], name)
1184
-
1185
- # Still loading, wait for completion
1186
- if task and not task.done():
1187
- task.result(timeout=10.0)
1188
-
1189
- with self._lock:
1190
- if module_name in self._loaded_modules:
1191
- return getattr(self._loaded_modules[module_name], name)
1192
- raise AttributeError(f"module '{module_name}' has no attribute '{name}'")
1193
-
1194
- placeholder.__getattr__ = _streaming_getattr # type: ignore[attr-defined]
1195
- return placeholder
1196
-
1197
- def _background_placeholder_load(self, module_name: str, module_path: str) -> ModuleType:
1198
- """Load module in background, return placeholder."""
1199
- if module_name not in self._background_tasks or self._background_tasks[module_name].done():
1200
- loop = self._ensure_async_loop()
1201
- task = asyncio.run_coroutine_threadsafe(
1202
- self._background_load_module(module_name, module_path),
1203
- loop
1204
- )
1205
- self._background_tasks[module_name] = task
1206
-
1207
- # Return placeholder module that will be replaced when loaded
1208
- placeholder = ModuleType(module_name)
1209
- placeholder.__path__ = []
1210
- placeholder.__package__ = module_name
1211
-
1212
- def _getattr(name):
1213
- # Wait for background load to complete
1214
- task = self._background_tasks.get(module_name)
1215
- if task and not task.done():
1216
- task.result(timeout=10.0) # Wait up to 10 seconds
1217
- with self._lock:
1218
- if module_name in self._loaded_modules:
1219
- return getattr(self._loaded_modules[module_name], name)
1220
- raise AttributeError(f"module '{module_name}' has no attribute '{name}'")
1221
-
1222
- placeholder.__getattr__ = _getattr # type: ignore[attr-defined]
1223
- return placeholder
1224
-
1225
- def _preload_predictive_modules(self, module_names: list) -> None:
1226
- """Preload modules predicted to be needed soon."""
1227
- if not module_names:
1228
- return
1229
-
1230
- with self._lock:
1231
- modules_to_preload = [
1232
- (name, self._lazy_modules[name])
1233
- for name in module_names
1234
- if name in self._lazy_modules and name not in self._loaded_modules
1235
- ]
1236
-
1237
- if not modules_to_preload:
1238
- return
1239
-
1240
- # Preload in background
1241
- loop = self._ensure_async_loop()
1242
-
1243
- async def _preload_predictive():
1244
- tasks = [
1245
- self._background_load_module(name, path)
1246
- for name, path in modules_to_preload
1247
- ]
1248
- await asyncio.gather(*tasks, return_exceptions=True)
1249
-
1250
- asyncio.run_coroutine_threadsafe(_preload_predictive(), loop)
1251
-
1252
- def preload_module(self, module_name: str) -> bool:
1253
- """Preload a registered lazy module."""
1254
- with self._lock:
1255
- if module_name not in self._lazy_modules:
1256
- logger.warning(f"Module {module_name} not registered for lazy loading")
1257
- return False
1258
-
1259
- try:
1260
- self.import_module(module_name)
1261
- log_event("hook", logger.info, f"Preloaded module: {module_name}")
1262
- return True
1263
- except Exception as e:
1264
- logger.error(f"Failed to preload {module_name}: {e}")
1265
- return False
1266
-
1267
- def get_stats(self) -> Dict[str, Any]:
1268
- """Get lazy import statistics."""
1269
- with self._lock:
1270
- return {
1271
- 'enabled': self._enabled,
1272
- 'registered_modules': list(self._lazy_modules.keys()),
1273
- 'loaded_modules': list(self._loaded_modules.keys()),
1274
- 'access_counts': self._access_counts.copy(),
1275
- 'total_registered': len(self._lazy_modules),
1276
- 'total_loaded': len(self._loaded_modules)
1277
- }
1278
-
1279
- # =============================================================================
1280
- # IMPORT HOOK (from import_hook.py)
1281
- # =============================================================================
1282
-
1283
- class LazyImportHook(AModuleHelper):
1284
- """
1285
- Import hook that intercepts ImportError and auto-installs packages.
1286
- Performance optimized with zero overhead for successful imports.
1287
- """
1288
-
1289
- __slots__ = AModuleHelper.__slots__
1290
-
1291
- def handle_import_error(self, module_name: str) -> Optional[Any]:
1292
- """Handle ImportError by attempting to install and re-import."""
1293
- if not self._enabled:
1294
- return None
1295
-
1296
- try:
1297
- # Deferred import to avoid circular dependency
1298
- from ..facade import lazy_import_with_install
1299
- module, success = lazy_import_with_install(
1300
- module_name,
1301
- installer_package=self._package_name
1302
- )
1303
- return module if success else None
1304
- except Exception:
1305
- return None
1306
-
1307
- def install_hook(self) -> None:
1308
- """Install the import hook into sys.meta_path."""
1309
- install_import_hook(self._package_name)
1310
-
1311
- def uninstall_hook(self) -> None:
1312
- """Uninstall the import hook from sys.meta_path."""
1313
- uninstall_import_hook(self._package_name)
1314
-
1315
- def is_installed(self) -> bool:
1316
- """Check if hook is installed."""
1317
- return is_import_hook_installed(self._package_name)
1318
-
1319
- # =============================================================================
1320
- # META PATH FINDER (from meta_path_finder.py)
1321
- # =============================================================================
1322
-
1323
- # Wrapped class cache
1324
- _WRAPPED_CLASS_CACHE: Dict[str, Set[str]] = defaultdict(set)
1325
- _wrapped_cache_lock = threading.RLock()
1326
-
1327
- # Default lazy methods
1328
- _DEFAULT_LAZY_METHODS = tuple(
1329
- filter(
1330
- None,
1331
- os.environ.get("XWLAZY_LAZY_METHODS", "").split(","),
1332
- )
1333
- )
1334
-
1335
- # Lazy prefix method registry
1336
- _lazy_prefix_method_registry: Dict[str, Tuple[str, ...]] = {}
1337
-
1338
- # Package class hints
1339
- _package_class_hints: Dict[str, Tuple[str, ...]] = {}
1340
- _class_hint_lock = threading.RLock()
1341
-
1342
- def _set_package_class_hints(package_name: str, hints: Iterable[str]) -> None:
1343
- """Set class hints for a package."""
1344
- normalized: Tuple[str, ...] = tuple(
1345
- OrderedDict((hint.lower(), None) for hint in hints if hint).keys() # type: ignore[arg-type]
1346
- )
1347
- with _class_hint_lock:
1348
- if normalized:
1349
- _package_class_hints[package_name] = normalized
1350
- else:
1351
- _package_class_hints.pop(package_name, None)
1352
-
1353
- def _get_package_class_hints(package_name: str) -> Tuple[str, ...]:
1354
- """Get class hints for a package."""
1355
- with _class_hint_lock:
1356
- return _package_class_hints.get(package_name, ())
1357
-
1358
- def _clear_all_package_class_hints() -> None:
1359
- """Clear all package class hints."""
1360
- with _class_hint_lock:
1361
- _package_class_hints.clear()
1362
-
1363
- def register_lazy_module_methods(prefix: str, methods: Tuple[str, ...]) -> None:
1364
- """Register method names to enhance for all classes under a module prefix."""
1365
- prefix = prefix.strip()
1366
- if not prefix:
1367
- return
1368
-
1369
- if not prefix.endswith("."):
1370
- prefix += "."
1371
-
1372
- _lazy_prefix_method_registry[prefix] = methods
1373
- log_event("config", logger.info, f"Registered lazy module methods for prefix {prefix}: {methods}")
1374
-
1375
- def _spec_for_existing_module(
1376
- fullname: str,
1377
- module: ModuleType,
1378
- original_spec: Optional[importlib.machinery.ModuleSpec] = None,
1379
- ) -> importlib.machinery.ModuleSpec:
1380
- """Build a ModuleSpec whose loader simply returns an already-initialized module."""
1381
- loader = _DeferredModuleLoader(module)
1382
- spec = importlib.machinery.ModuleSpec(fullname, loader)
1383
- if original_spec and original_spec.submodule_search_locations is not None:
1384
- locations = list(original_spec.submodule_search_locations)
1385
- spec.submodule_search_locations = locations
1386
- if hasattr(module, "__path__"):
1387
- module.__path__ = locations
1388
- module.__loader__ = loader
1389
- module.__spec__ = spec
1390
- return spec
1391
-
1392
- class LazyMetaPathFinder:
1393
- """
1394
- Custom meta path finder that intercepts failed imports.
1395
- Performance optimized - only triggers when import would fail anyway.
1396
- """
1397
-
1398
- __slots__ = ('_package_name', '_enabled')
1399
-
1400
- def __init__(self, package_name: str = 'default'):
1401
- """Initialize meta path finder."""
1402
- self._package_name = package_name
1403
- self._enabled = True
1404
-
1405
- def _build_async_placeholder(
1406
- self,
1407
- fullname: str,
1408
- installer: LazyInstaller,
1409
- ) -> Optional[importlib.machinery.ModuleSpec]:
1410
- """Create and register a deferred module placeholder for async installs."""
1411
- handle = installer.ensure_async_install(fullname)
1412
- if handle is None:
1413
- return None
1414
-
1415
- missing = ModuleNotFoundError(f"No module named '{fullname}'")
1416
- deferred = DeferredImportError(fullname, missing, self._package_name, async_handle=handle)
1417
-
1418
- module = ModuleType(fullname)
1419
- loader = _DeferredModuleLoader(module)
1420
-
1421
- def _resolve_real_module():
1422
- real_module = deferred._try_install_and_import()
1423
- sys.modules[fullname] = real_module
1424
- module.__dict__.clear()
1425
- module.__dict__.update(real_module.__dict__)
1426
- module.__loader__ = getattr(real_module, "__loader__", loader)
1427
- module.__spec__ = getattr(real_module, "__spec__", None)
1428
- module.__path__ = getattr(real_module, "__path__", getattr(module, "__path__", []))
1429
- module.__class__ = real_module.__class__
1430
- try:
1431
- spec_obj = getattr(real_module, "__spec__", None) or importlib.util.find_spec(fullname)
1432
- if spec_obj is not None:
1433
- _spec_cache_put(fullname, spec_obj)
1434
- except (ValueError, AttributeError, ImportError):
1435
- pass
1436
- return real_module
1437
-
1438
- def _module_getattr(name):
1439
- real = _resolve_real_module()
1440
- if name in module.__dict__:
1441
- return module.__dict__[name]
1442
- return getattr(real, name)
1443
-
1444
- def _module_dir():
1445
- try:
1446
- real = _resolve_real_module()
1447
- return dir(real)
1448
- except Exception:
1449
- return []
1450
-
1451
- module.__getattr__ = _module_getattr # type: ignore[attr-defined]
1452
- module.__dir__ = _module_dir # type: ignore[attr-defined]
1453
- module.__loader__ = loader
1454
- module.__package__ = fullname
1455
- module.__path__ = []
1456
-
1457
- spec = importlib.machinery.ModuleSpec(fullname, loader)
1458
- spec.submodule_search_locations = []
1459
- module.__spec__ = spec
1460
-
1461
- sys.modules[fullname] = module
1462
- log_event("hook", logger.info, f"⏳ [HOOK] Deferred import placeholder created for '{fullname}'")
1463
- return spec
1464
-
1465
- def find_module(self, fullname: str, path: Optional[str] = None):
1466
- """Find module - returns None to let standard import continue."""
1467
- return None
1468
-
1469
- def find_spec(self, fullname: str, path: Optional[str] = None, target=None):
1470
- """
1471
- Find module spec - intercepts imports to enable two-stage lazy loading.
1472
-
1473
- PERFORMANCE: Optimized for zero overhead on successful imports.
1474
- """
1475
- # Debug logging for msgpack to trace why it's not being intercepted
1476
- if fullname == 'msgpack':
1477
- logger.info(f"[HOOK] find_spec called for msgpack, enabled={self._enabled}, in_sys_modules={fullname in sys.modules}, installing={getattr(_installing_state, 'active', False)}, importing={getattr(_importing_state, 'active', False)}")
1478
-
1479
- # CRITICAL: Check installing state FIRST to prevent recursion during installation
1480
- if getattr(_installing_state, 'active', False):
1481
- if fullname == 'msgpack':
1482
- logger.info(f"[HOOK] Installation in progress, skipping msgpack")
1483
- logger.debug(f"[HOOK] Installation in progress, skipping {fullname} to prevent recursion")
1484
- return None
1485
-
1486
- # Fast path 1: Hook disabled
1487
- if not self._enabled:
1488
- if fullname == 'msgpack':
1489
- logger.info(f"[HOOK] Hook disabled, skipping msgpack")
1490
- return None
1491
-
1492
- # Fast path 2: Module already loaded or partially initialized
1493
- if fullname in sys.modules:
1494
- if fullname == 'msgpack':
1495
- logger.info(f"[HOOK] msgpack already in sys.modules, skipping")
1496
- return None
1497
-
1498
- # Fast path 3: Skip C extension modules and internal modules
1499
- if fullname.startswith('_'):
1500
- logger.debug(f"[HOOK] Skipping C extension/internal module {fullname}")
1501
- return None
1502
-
1503
- # Fast path 4: Check if parent package is partially initialized
1504
- if '.' in fullname:
1505
- parent_package = fullname.split('.', 1)[0]
1506
- if parent_package in sys.modules:
1507
- logger.debug(f"[HOOK] Skipping {fullname} - parent {parent_package} is partially initialized")
1508
- return None
1509
- if _is_import_in_progress(parent_package):
1510
- logger.debug(f"[HOOK] Skipping {fullname} - parent {parent_package} import in progress")
1511
- return None
1512
-
1513
- # ROOT CAUSE FIX: Check lazy install status FIRST
1514
- root_name = fullname.split('.', 1)[0]
1515
- _watched_registry = get_watched_registry()
1516
-
1517
- # Check if lazy install is enabled
1518
- lazy_install_enabled = LazyInstallConfig.is_enabled(self._package_name)
1519
- install_mode = LazyInstallConfig.get_install_mode(self._package_name)
1520
-
1521
- # If lazy install is disabled, only intercept watched modules
1522
- if not lazy_install_enabled or install_mode == LazyInstallMode.NONE:
1523
- if not _watched_registry.has_root(root_name):
1524
- logger.debug(f"[HOOK] Module {fullname} not in watched registry and lazy install disabled, skipping interception")
1525
- return None
1526
-
1527
- # Check persistent installation cache FIRST
1528
- try:
1529
- installer = LazyInstallerRegistry.get_instance(self._package_name)
1530
- package_name = installer._dependency_mapper.get_package_name(root_name)
1531
- if package_name and installer.is_package_installed(package_name):
1532
- logger.debug(f"[HOOK] Package {package_name} is installed (cache check), skipping interception of {fullname}")
1533
- return None
1534
- except Exception:
1535
- pass
1536
-
1537
- # Fast path 4: Cached spec
1538
- cached_spec = _spec_cache_get(fullname)
1539
- if cached_spec is not None:
1540
- return cached_spec
1541
-
1542
- # Fast path 5: Stdlib/builtin check
1543
- if fullname.startswith('importlib') or fullname.startswith('_frozen_importlib'):
1544
- return None
1545
-
1546
- if '.' not in fullname:
1547
- if DependencyMapper._is_stdlib_or_builtin(fullname):
1548
- return None
1549
- if fullname in DependencyMapper.DENY_LIST:
1550
- return None
1551
-
1552
- # Fast path 6: Import in progress
1553
- if _is_import_in_progress(fullname):
1554
- if fullname == 'msgpack':
1555
- logger.info(f"[HOOK] msgpack import in progress, skipping")
1556
- return None
1557
-
1558
- if getattr(_importing_state, 'active', False):
1559
- if fullname == 'msgpack':
1560
- logger.info(f"[HOOK] Global importing active, skipping msgpack")
1561
- return None
1562
-
1563
- # Install mode check already done above
1564
- matching_prefixes: Tuple[str, ...] = ()
1565
- if _watched_registry.has_root(root_name):
1566
- matching_prefixes = _watched_registry.get_matching_prefixes(fullname)
1567
-
1568
- # Debug: Check if msgpack reaches this point
1569
- if fullname == 'msgpack':
1570
- logger.debug(f"[HOOK] msgpack: matching_prefixes={matching_prefixes}, has_root={_watched_registry.has_root(root_name)}")
1571
-
1572
- installer = LazyInstallerRegistry.get_instance(self._package_name)
1573
-
1574
- # Two-stage lazy loading for serialization and archive modules
1575
- if matching_prefixes:
1576
- for prefix in matching_prefixes:
1577
- if not _watched_registry.is_prefix_owned_by(self._package_name, prefix):
1578
- continue
1579
- if fullname.startswith(prefix):
1580
- module_suffix = fullname[len(prefix):]
1581
-
1582
- if module_suffix:
1583
- log_event("hook", logger.info, f"[HOOK] Candidate for wrapping: {fullname}")
1584
-
1585
- _mark_import_started(fullname)
1586
- try:
1587
- if getattr(_importing_state, 'active', False):
1588
- logger.debug(f"[HOOK] Recursion guard active, skipping {fullname}")
1589
- return None
1590
-
1591
- try:
1592
- logger.debug(f"[HOOK] Looking for spec: {fullname}")
1593
- spec = _spec_cache_get(fullname)
1594
- if spec is None:
1595
- try:
1596
- spec = importlib.util.find_spec(fullname)
1597
- except (ValueError, AttributeError, ImportError):
1598
- pass
1599
- if spec is not None:
1600
- _spec_cache_put(fullname, spec)
1601
- if spec is not None:
1602
- logger.debug(f"[HOOK] Spec found, trying normal import: {fullname}")
1603
- _importing_state.active = True
1604
- try:
1605
- __import__(fullname)
1606
-
1607
- module = sys.modules.get(fullname)
1608
- if module:
1609
- try:
1610
- self._enhance_classes_with_class_methods(module)
1611
- except Exception as enhance_exc:
1612
- logger.debug(f"[HOOK] Could not enhance classes in {fullname}: {enhance_exc}")
1613
- spec = _spec_for_existing_module(fullname, module, spec)
1614
- log_event("hook", logger.info, f"✓ [HOOK] Module {fullname} imported successfully, no wrapping needed")
1615
- if spec is not None:
1616
- _spec_cache_put(fullname, spec)
1617
- return spec
1618
- return None
1619
- finally:
1620
- _importing_state.active = False
1621
- except ImportError as e:
1622
- if '.' not in module_suffix:
1623
- log_event("hook", logger.info, f"⚠ [HOOK] Module {fullname} has missing dependencies, wrapping: {e}")
1624
- wrapped_spec = self._wrap_serialization_module(fullname)
1625
- if wrapped_spec is not None:
1626
- log_event("hook", logger.info, f"✓ [HOOK] Successfully wrapped: {fullname}")
1627
- return wrapped_spec
1628
- logger.warning(f"✗ [HOOK] Failed to wrap: {fullname}")
1629
- else:
1630
- logger.debug(f"[HOOK] Import failed for nested module {fullname}: {e}")
1631
- except (ModuleNotFoundError,) as e:
1632
- logger.debug(f"[HOOK] Module {fullname} not found, skipping wrap: {e}")
1633
- pass
1634
- except Exception as e:
1635
- logger.warning(f"[HOOK] Error checking module {fullname}: {e}")
1636
- finally:
1637
- _mark_import_finished(fullname)
1638
-
1639
- return None
1640
-
1641
- # If we had matching prefixes but didn't match any, continue to lazy install logic
1642
- if matching_prefixes:
1643
- logger.debug(f"[HOOK] {fullname} had matching prefixes but didn't match any, continuing to lazy install")
1644
-
1645
- # For lazy installation, handle submodules by checking if parent package is installed
1646
- if '.' in fullname:
1647
- parent_package = fullname.split('.', 1)[0]
1648
- if lazy_install_enabled:
1649
- try:
1650
- installer = LazyInstallerRegistry.get_instance(self._package_name)
1651
- package_name = installer._dependency_mapper.get_package_name(parent_package)
1652
- if package_name and not installer.is_package_installed(package_name):
1653
- logger.debug(f"[HOOK] Parent package {parent_package} not installed, intercepting parent")
1654
- return self.find_spec(parent_package, path, target)
1655
- except Exception:
1656
- pass
1657
- return None
1658
- if DependencyMapper._is_stdlib_or_builtin(fullname):
1659
- return None
1660
- if fullname in DependencyMapper.DENY_LIST:
1661
- return None
1662
-
1663
- # ROOT CAUSE FIX: For lazy installation, intercept missing imports and install them
1664
- logger.debug(f"[HOOK] Checking lazy install for {fullname}: enabled={lazy_install_enabled}, install_mode={install_mode}")
1665
- if lazy_install_enabled:
1666
- if _is_import_in_progress(fullname):
1667
- return None
1668
-
1669
- _mark_import_started(fullname)
1670
- try:
1671
- # Guard against recursion when importing facade
1672
- if 'exonware.xwlazy.facade' in sys.modules or 'xwlazy.facade' in sys.modules:
1673
- from ..facade import lazy_import_with_install
1674
- else:
1675
- if _is_import_in_progress('exonware.xwlazy.facade') or _is_import_in_progress('xwlazy.facade'):
1676
- logger.debug(f"[HOOK] Facade import in progress, skipping {fullname} to prevent recursion")
1677
- return None
1678
- from ..facade import lazy_import_with_install
1679
-
1680
- _importing_state.active = True
1681
- try:
1682
- module, success = lazy_import_with_install(
1683
- fullname,
1684
- installer_package=self._package_name
1685
- )
1686
- finally:
1687
- _importing_state.active = False
1688
-
1689
- if success and module:
1690
- # Module was successfully installed and imported
1691
- xwlazy_finder_names = {'LazyMetaPathFinder', 'LazyPathFinder', 'LazyLoader'}
1692
- xwlazy_finders = [f for f in sys.meta_path if type(f).__name__ in xwlazy_finder_names]
1693
- for finder in xwlazy_finders:
1694
- try:
1695
- sys.meta_path.remove(finder)
1696
- except ValueError:
1697
- pass
1698
-
1699
- try:
1700
- if fullname in sys.modules:
1701
- del sys.modules[fullname]
1702
- importlib.invalidate_caches()
1703
- sys.path_importer_cache.clear()
1704
- real_module = importlib.import_module(fullname)
1705
- sys.modules[fullname] = real_module
1706
- logger.debug(f"[HOOK] Successfully installed {fullname}, replaced module in sys.modules with real module")
1707
- finally:
1708
- for finder in reversed(xwlazy_finders):
1709
- if finder not in sys.meta_path:
1710
- sys.meta_path.insert(0, finder)
1711
- return None
1712
- else:
1713
- logger.debug(f"[HOOK] Failed to install/import {fullname}")
1714
- try:
1715
- installer = LazyInstallerRegistry.get_instance(self._package_name)
1716
- if installer.is_async_enabled():
1717
- placeholder = self._build_async_placeholder(fullname, installer)
1718
- if placeholder is not None:
1719
- return placeholder
1720
- except Exception:
1721
- pass
1722
- return None
1723
- except Exception as e:
1724
- logger.debug(f"Lazy import hook failed for {fullname}: {e}")
1725
- return None
1726
- finally:
1727
- _mark_import_finished(fullname)
1728
-
1729
- def _wrap_serialization_module(self, fullname: str):
1730
- """Wrap serialization module loading to defer missing dependencies."""
1731
- log_event("hook", logger.info, f"[STAGE 1] Starting wrap of module: {fullname}")
1732
-
1733
- try:
1734
- logger.debug(f"[STAGE 1] Getting spec for: {fullname}")
1735
- try:
1736
- sys.meta_path.remove(self)
1737
- except ValueError:
1738
- pass
1739
- try:
1740
- spec = importlib.util.find_spec(fullname)
1741
- finally:
1742
- if self not in sys.meta_path:
1743
- sys.meta_path.insert(0, self)
1744
- if not spec or not spec.loader:
1745
- logger.warning(f"[STAGE 1] No spec or loader for: {fullname}")
1746
- return None
1747
-
1748
- logger.debug(f"[STAGE 1] Creating module from spec: {fullname}")
1749
- module = importlib.util.module_from_spec(spec)
1750
-
1751
- deferred_imports = {}
1752
-
1753
- logger.debug(f"[STAGE 1] Setting up import wrapper for: {fullname}")
1754
- original_import = builtins.__import__
1755
-
1756
- def capture_import_errors(name, *args, **kwargs):
1757
- """Intercept imports and defer ONLY external missing packages."""
1758
- logger.debug(f"[STAGE 1] capture_import_errors: Trying to import '{name}' in {fullname}")
1759
-
1760
- if _is_import_in_progress(name):
1761
- logger.debug(f"[STAGE 1] Import '{name}' already in progress, using original_import")
1762
- return original_import(name, *args, **kwargs)
1763
-
1764
- _mark_import_started(name)
1765
- try:
1766
- result = original_import(name, *args, **kwargs)
1767
- logger.debug(f"[STAGE 1] ✓ Successfully imported '{name}'")
1768
- return result
1769
- except ImportError as e:
1770
- logger.debug(f"[STAGE 1] ✗ Import failed for '{name}': {e}")
1771
-
1772
- host_alias = self._package_name or ""
1773
- if name.startswith('exonware.') or (host_alias and name.startswith(f"{host_alias}.")):
1774
- log_event("hook", logger.info, f"[STAGE 1] Letting internal import '{name}' fail normally (internal package)")
1775
- raise
1776
-
1777
- if '.' in name:
1778
- log_event("hook", logger.info, f"[STAGE 1] Letting submodule '{name}' fail normally (has dots)")
1779
- raise
1780
-
1781
- log_event("hook", logger.info, f"⏳ [STAGE 1] DEFERRING missing external package '{name}' in {fullname}")
1782
- async_handle = None
1783
- try:
1784
- installer = LazyInstallerRegistry.get_instance(self._package_name)
1785
- async_handle = installer.schedule_async_install(name)
1786
- except Exception as schedule_exc:
1787
- logger.debug(f"[STAGE 1] Async install scheduling failed for '{name}': {schedule_exc}")
1788
- deferred = DeferredImportError(name, e, self._package_name, async_handle=async_handle)
1789
- deferred_imports[name] = deferred
1790
- return deferred
1791
- finally:
1792
- _mark_import_finished(name)
1793
-
1794
- logger.debug(f"[STAGE 1] Executing module with import wrapper: {fullname}")
1795
- builtins.__import__ = capture_import_errors
1796
- try:
1797
- spec.loader.exec_module(module)
1798
- logger.debug(f"[STAGE 1] Module execution completed: {fullname}")
1799
-
1800
- if deferred_imports:
1801
- log_event("hook", logger.info, f"✓ [STAGE 1] Module {fullname} loaded with {len(deferred_imports)} deferred imports: {list(deferred_imports.keys())}")
1802
- self._replace_none_with_deferred(module, deferred_imports)
1803
- self._wrap_module_classes(module, deferred_imports)
1804
- else:
1805
- log_event("hook", logger.info, f"✓ [STAGE 1] Module {fullname} loaded with NO deferred imports (all dependencies available)")
1806
-
1807
- self._enhance_classes_with_class_methods(module)
1808
-
1809
- finally:
1810
- logger.debug(f"[STAGE 1] Restoring original __import__")
1811
- builtins.__import__ = original_import
1812
-
1813
- logger.debug(f"[STAGE 1] Registering module in sys.modules: {fullname}")
1814
- sys.modules[fullname] = module
1815
- final_spec = _spec_for_existing_module(fullname, module, spec)
1816
- _spec_cache_put(fullname, final_spec)
1817
- log_event("hook", logger.info, f"✓ [STAGE 1] Successfully wrapped and registered: {fullname}")
1818
- return final_spec
1819
-
1820
- except Exception as e:
1821
- logger.debug(f"Could not wrap {fullname}: {e}")
1822
- return None
1823
-
1824
- def _replace_none_with_deferred(self, module, deferred_imports: Dict):
1825
- """Replace None values in module namespace with deferred import proxies."""
1826
- logger.debug(f"[STAGE 1] Replacing None with deferred imports in {module.__name__}")
1827
- replaced_count = 0
1828
-
1829
- for dep_name, deferred_import in deferred_imports.items():
1830
- if hasattr(module, dep_name):
1831
- current_value = getattr(module, dep_name)
1832
- if current_value is None:
1833
- log_event("hook", logger.info, f"[STAGE 1] Replacing {dep_name}=None with deferred import proxy in {module.__name__}")
1834
- setattr(module, dep_name, deferred_import)
1835
- replaced_count += 1
1836
-
1837
- if replaced_count > 0:
1838
- log_event("hook", logger.info, f"✓ [STAGE 1] Replaced {replaced_count} None values with deferred imports in {module.__name__}")
1839
-
1840
- def _wrap_module_classes(self, module, deferred_imports: Dict):
1841
- """Wrap classes in a module that depend on deferred imports."""
1842
- module_name = getattr(module, '__name__', '<unknown>')
1843
- logger.debug(f"[STAGE 1] Wrapping classes in {module_name} (deferred: {list(deferred_imports.keys())})")
1844
- module_file = (getattr(module, '__file__', '') or '').lower()
1845
- lower_map = {dep_name.lower(): dep_name for dep_name in deferred_imports.keys()}
1846
- class_hints = _get_package_class_hints(self._package_name)
1847
- with _wrapped_cache_lock:
1848
- already_wrapped = _WRAPPED_CLASS_CACHE.setdefault(module_name, set()).copy()
1849
- pending_lower = {lower for lower in lower_map.keys() if lower_map[lower] not in already_wrapped}
1850
- if not pending_lower:
1851
- logger.debug(f"[STAGE 1] All deferred imports already wrapped for {module_name}")
1852
- return
1853
- dep_entries = [(lower, deferred_imports[lower_map[lower]]) for lower in pending_lower]
1854
- wrapped_count = 0
1855
- newly_wrapped: Set[str] = set()
1856
-
1857
- for name, obj in list(module.__dict__.items()):
1858
- if not pending_lower:
1859
- break
1860
- if not isinstance(obj, type):
1861
- continue
1862
- lower_name = name.lower()
1863
- if class_hints and not any(hint in lower_name for hint in class_hints):
1864
- continue
1865
- target_lower = None
1866
- target_deferred = None
1867
- for dep_lower, deferred in dep_entries:
1868
- if dep_lower not in pending_lower:
1869
- continue
1870
- if dep_lower in lower_name or dep_lower in module_file:
1871
- target_lower = dep_lower
1872
- target_deferred = deferred
1873
- break
1874
- if target_deferred is None or target_lower is None:
1875
- continue
1876
-
1877
- logger.debug(f"[STAGE 1] Class '{name}' depends on deferred import, wrapping...")
1878
- wrapped = self._create_lazy_class_wrapper(obj, target_deferred)
1879
- module.__dict__[name] = wrapped
1880
- wrapped_count += 1
1881
- origin_name = lower_map.get(target_lower, target_lower)
1882
- newly_wrapped.add(origin_name)
1883
- pending_lower.discard(target_lower)
1884
- log_event("hook", logger.info, f"✓ [STAGE 1] Wrapped class '{name}' in {module_name}")
1885
-
1886
- if newly_wrapped:
1887
- with _wrapped_cache_lock:
1888
- cache = _WRAPPED_CLASS_CACHE.setdefault(module_name, set())
1889
- cache.update(newly_wrapped)
1890
-
1891
- log_event("hook", logger.info, f"[STAGE 1] Wrapped {wrapped_count} classes in {module_name}")
1892
-
1893
- def _enhance_classes_with_class_methods(self, module):
1894
- """Enhance classes that registered lazy class methods."""
1895
- if module is None:
1896
- return
1897
-
1898
- methods_to_apply: Tuple[str, ...] = ()
1899
- for prefix, methods in _lazy_prefix_method_registry.items():
1900
- if module.__name__.startswith(prefix.rstrip('.')):
1901
- methods_to_apply = methods
1902
- break
1903
-
1904
- if not methods_to_apply:
1905
- methods_to_apply = _DEFAULT_LAZY_METHODS
1906
-
1907
- if not methods_to_apply:
1908
- return
1909
-
1910
- enhanced = 0
1911
- for name, obj in list(module.__dict__.items()):
1912
- if not isinstance(obj, type):
1913
- continue
1914
- for method_name in methods_to_apply:
1915
- attr = obj.__dict__.get(method_name)
1916
- if attr is None:
1917
- continue
1918
- if getattr(attr, "__lazy_wrapped__", False):
1919
- continue
1920
- if not callable(attr):
1921
- continue
1922
-
1923
- if isinstance(attr, (classmethod, staticmethod)):
1924
- continue
1925
-
1926
- import inspect
1927
- try:
1928
- sig = inspect.signature(attr)
1929
- params = list(sig.parameters.keys())
1930
- if params and params[0] == 'self':
1931
- logger.debug(
1932
- "[LAZY ENHANCE] Wrapping instance method %s.%s.%s for class-level access",
1933
- module.__name__,
1934
- name,
1935
- method_name,
1936
- )
1937
- except Exception:
1938
- pass
1939
-
1940
- try:
1941
- original_func = attr
1942
-
1943
- def class_method_wrapper(func):
1944
- def _class_call(cls, *args, **kwargs):
1945
- instance = cls()
1946
- return func(instance, *args, **kwargs)
1947
- _class_call.__name__ = getattr(func, '__name__', 'lazy_method')
1948
- _class_call.__doc__ = func.__doc__
1949
- _class_call.__lazy_wrapped__ = True
1950
- return _class_call
1951
-
1952
- setattr(
1953
- obj,
1954
- method_name,
1955
- classmethod(class_method_wrapper(original_func)),
1956
- )
1957
- enhanced += 1
1958
- logger.debug(
1959
- "[LAZY ENHANCE] Added class-level %s() to %s.%s",
1960
- method_name,
1961
- module.__name__,
1962
- name,
1963
- )
1964
- except Exception as exc:
1965
- logger.debug(
1966
- "[LAZY ENHANCE] Skipped %s.%s.%s: %s",
1967
- module.__name__,
1968
- name,
1969
- method_name,
1970
- exc,
1971
- )
1972
-
1973
- if enhanced:
1974
- log_event("enhance", logger.info, "✓ [LAZY ENHANCE] Added %s convenience methods in %s", enhanced, module.__name__)
1975
-
1976
- def _create_lazy_class_wrapper(self, original_class, deferred_import: DeferredImportError):
1977
- """Create a wrapper class that installs dependencies when instantiated."""
1978
- class LazyClassWrapper:
1979
- """Lazy wrapper that installs dependencies on first instantiation."""
1980
-
1981
- def __init__(self, *args, **kwargs):
1982
- """Install dependency and create real instance."""
1983
- deferred_import._try_install_and_import()
1984
-
1985
- real_module = importlib.reload(sys.modules[original_class.__module__])
1986
- real_class = getattr(real_module, original_class.__name__)
1987
-
1988
- real_instance = real_class(*args, **kwargs)
1989
- self.__class__ = real_class
1990
- self.__dict__ = real_instance.__dict__
1991
-
1992
- def __repr__(self):
1993
- return f"<Lazy{original_class.__name__}: will install dependencies on init>"
1994
-
1995
- LazyClassWrapper.__name__ = f"Lazy{original_class.__name__}"
1996
- LazyClassWrapper.__qualname__ = f"Lazy{original_class.__qualname__}"
1997
- LazyClassWrapper.__module__ = original_class.__module__
1998
- LazyClassWrapper.__doc__ = original_class.__doc__
1999
-
2000
- return LazyClassWrapper
2001
-
2002
- # Registry of installed hooks per package
2003
- _installed_hooks: Dict[str, LazyMetaPathFinder] = {}
2004
- _hook_lock = threading.RLock()
2005
-
2006
- def install_import_hook(package_name: str = 'default') -> None:
2007
- """Install performant import hook for automatic lazy installation."""
2008
- global _installed_hooks
2009
-
2010
- log_event("hook", logger.info, f"[HOOK INSTALL] Installing import hook for package: {package_name}")
2011
-
2012
- with _hook_lock:
2013
- if package_name in _installed_hooks:
2014
- log_event("hook", logger.info, f"[HOOK INSTALL] Import hook already installed for {package_name}")
2015
- return
2016
-
2017
- logger.debug(f"[HOOK INSTALL] Creating LazyMetaPathFinder for {package_name}")
2018
- hook = LazyMetaPathFinder(package_name)
2019
-
2020
- logger.debug(f"[HOOK INSTALL] Current sys.meta_path has {len(sys.meta_path)} entries")
2021
- sys.meta_path.insert(0, hook)
2022
- _installed_hooks[package_name] = hook
2023
-
2024
- log_event("hook", logger.info, f"✅ [HOOK INSTALL] Lazy import hook installed for {package_name} (now {len(sys.meta_path)} meta_path entries)")
2025
-
2026
- def uninstall_import_hook(package_name: str = 'default') -> None:
2027
- """Uninstall import hook for a package."""
2028
- global _installed_hooks
2029
-
2030
- with _hook_lock:
2031
- if package_name in _installed_hooks:
2032
- hook = _installed_hooks[package_name]
2033
- try:
2034
- sys.meta_path.remove(hook)
2035
- except ValueError:
2036
- pass
2037
- del _installed_hooks[package_name]
2038
- log_event("hook", logger.info, f"Lazy import hook uninstalled for {package_name}")
2039
-
2040
- def is_import_hook_installed(package_name: str = 'default') -> bool:
2041
- """Check if import hook is installed for a package."""
2042
- return package_name in _installed_hooks
2043
-
2044
- def register_lazy_module_prefix(prefix: str) -> None:
2045
- """Register an import prefix for lazy wrapping."""
2046
- _watched_registry = get_watched_registry()
2047
- _watched_registry.add(prefix)
2048
- normalized = _normalize_prefix(prefix)
2049
- if normalized:
2050
- log_event("config", logger.info, "Registered lazy module prefix: %s", normalized)
2051
-
2052
- # =============================================================================
2053
- # EXPORTS
2054
- # =============================================================================
2055
-
2056
- __all__ = [
2057
- # Logging utilities
2058
- 'get_logger',
2059
- 'log_event',
2060
- 'print_formatted',
2061
- 'format_message',
2062
- 'is_log_category_enabled',
2063
- 'set_log_category',
2064
- 'set_log_categories',
2065
- 'get_log_categories',
2066
- 'XWLazyFormatter',
2067
- # Import tracking
2068
- '_is_import_in_progress',
2069
- '_mark_import_started',
2070
- '_mark_import_finished',
2071
- 'get_importing_state',
2072
- 'get_installing_state',
2073
- '_installation_depth',
2074
- '_installation_depth_lock',
2075
- # Prefix trie
2076
- '_PrefixTrie',
2077
- # Watched registry
2078
- 'WatchedPrefixRegistry',
2079
- 'get_watched_registry',
2080
- # Deferred loader
2081
- '_DeferredModuleLoader',
2082
- # Cache utilities
2083
- # Parallel utilities
2084
- 'ParallelLoader',
2085
- 'DependencyGraph',
2086
- # Module patching
2087
- '_lazy_aware_import_module',
2088
- '_patch_import_module',
2089
- '_unpatch_import_module',
2090
- # Archive imports
2091
- 'get_archive_path',
2092
- 'ensure_archive_in_path',
2093
- 'import_from_archive',
2094
- # Bootstrap
2095
- 'bootstrap_lazy_mode',
2096
- 'bootstrap_lazy_mode_deferred',
2097
- # Lazy loader
2098
- 'LazyLoader',
2099
- # Lazy module registry
2100
- 'LazyModuleRegistry',
2101
- # Lazy importer
2102
- 'LazyImporter',
2103
- # Import hook
2104
- 'LazyImportHook',
2105
- # Meta path finder
2106
- 'LazyMetaPathFinder',
2107
- 'install_import_hook',
2108
- 'uninstall_import_hook',
2109
- 'is_import_hook_installed',
2110
- 'register_lazy_module_prefix',
2111
- 'register_lazy_module_methods',
2112
- '_set_package_class_hints',
2113
- '_get_package_class_hints',
2114
- '_clear_all_package_class_hints',
2115
- '_spec_for_existing_module',
2116
- ]
2117
-