exonware-xwlazy 0.1.0.10__py3-none-any.whl → 0.1.0.19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- exonware/__init__.py +22 -0
- exonware/xwlazy/__init__.py +0 -0
- exonware/xwlazy/common/__init__.py +47 -0
- exonware/xwlazy/common/base.py +58 -0
- exonware/xwlazy/common/cache.py +506 -0
- exonware/xwlazy/common/logger.py +268 -0
- exonware/xwlazy/common/services/__init__.py +72 -0
- exonware/xwlazy/common/services/dependency_mapper.py +234 -0
- exonware/xwlazy/common/services/install_async_utils.py +169 -0
- exonware/xwlazy/common/services/install_cache_utils.py +257 -0
- exonware/xwlazy/common/services/keyword_detection.py +292 -0
- exonware/xwlazy/common/services/spec_cache.py +173 -0
- exonware/xwlazy/common/services/state_manager.py +86 -0
- exonware/xwlazy/common/strategies/__init__.py +28 -0
- exonware/xwlazy/common/strategies/caching_dict.py +45 -0
- exonware/xwlazy/common/strategies/caching_installation.py +89 -0
- exonware/xwlazy/common/strategies/caching_lfu.py +67 -0
- exonware/xwlazy/common/strategies/caching_lru.py +64 -0
- exonware/xwlazy/common/strategies/caching_multitier.py +60 -0
- exonware/xwlazy/common/strategies/caching_ttl.py +60 -0
- exonware/xwlazy/config.py +195 -0
- exonware/xwlazy/contracts.py +1410 -0
- exonware/xwlazy/defs.py +397 -0
- exonware/xwlazy/errors.py +284 -0
- exonware/xwlazy/facade.py +1049 -0
- exonware/xwlazy/module/__init__.py +18 -0
- exonware/xwlazy/module/base.py +569 -0
- exonware/xwlazy/module/data.py +17 -0
- exonware/xwlazy/module/facade.py +247 -0
- exonware/xwlazy/module/importer_engine.py +2161 -0
- exonware/xwlazy/module/strategies/__init__.py +22 -0
- exonware/xwlazy/module/strategies/module_helper_lazy.py +94 -0
- exonware/xwlazy/module/strategies/module_helper_simple.py +66 -0
- exonware/xwlazy/module/strategies/module_manager_advanced.py +112 -0
- exonware/xwlazy/module/strategies/module_manager_simple.py +96 -0
- exonware/xwlazy/package/__init__.py +18 -0
- exonware/xwlazy/package/base.py +807 -0
- exonware/xwlazy/package/conf.py +331 -0
- exonware/xwlazy/package/data.py +17 -0
- exonware/xwlazy/package/facade.py +481 -0
- exonware/xwlazy/package/services/__init__.py +84 -0
- exonware/xwlazy/package/services/async_install_handle.py +89 -0
- exonware/xwlazy/package/services/config_manager.py +246 -0
- exonware/xwlazy/package/services/discovery.py +374 -0
- exonware/xwlazy/package/services/host_packages.py +149 -0
- exonware/xwlazy/package/services/install_async.py +278 -0
- exonware/xwlazy/package/services/install_cache.py +146 -0
- exonware/xwlazy/package/services/install_interactive.py +60 -0
- exonware/xwlazy/package/services/install_policy.py +158 -0
- exonware/xwlazy/package/services/install_registry.py +56 -0
- exonware/xwlazy/package/services/install_result.py +17 -0
- exonware/xwlazy/package/services/install_sbom.py +154 -0
- exonware/xwlazy/package/services/install_utils.py +83 -0
- exonware/xwlazy/package/services/installer_engine.py +408 -0
- exonware/xwlazy/package/services/lazy_installer.py +720 -0
- exonware/xwlazy/package/services/manifest.py +506 -0
- exonware/xwlazy/package/services/strategy_registry.py +188 -0
- exonware/xwlazy/package/strategies/__init__.py +57 -0
- exonware/xwlazy/package/strategies/package_discovery_file.py +130 -0
- exonware/xwlazy/package/strategies/package_discovery_hybrid.py +85 -0
- exonware/xwlazy/package/strategies/package_discovery_manifest.py +102 -0
- exonware/xwlazy/package/strategies/package_execution_async.py +114 -0
- exonware/xwlazy/package/strategies/package_execution_cached.py +91 -0
- exonware/xwlazy/package/strategies/package_execution_pip.py +100 -0
- exonware/xwlazy/package/strategies/package_execution_wheel.py +107 -0
- exonware/xwlazy/package/strategies/package_mapping_discovery_first.py +101 -0
- exonware/xwlazy/package/strategies/package_mapping_hybrid.py +106 -0
- exonware/xwlazy/package/strategies/package_mapping_manifest_first.py +101 -0
- exonware/xwlazy/package/strategies/package_policy_allow_list.py +58 -0
- exonware/xwlazy/package/strategies/package_policy_deny_list.py +58 -0
- exonware/xwlazy/package/strategies/package_policy_permissive.py +47 -0
- exonware/xwlazy/package/strategies/package_timing_clean.py +68 -0
- exonware/xwlazy/package/strategies/package_timing_full.py +67 -0
- exonware/xwlazy/package/strategies/package_timing_smart.py +69 -0
- exonware/xwlazy/package/strategies/package_timing_temporary.py +67 -0
- exonware/xwlazy/runtime/__init__.py +18 -0
- exonware/xwlazy/runtime/adaptive_learner.py +131 -0
- exonware/xwlazy/runtime/base.py +276 -0
- exonware/xwlazy/runtime/facade.py +95 -0
- exonware/xwlazy/runtime/intelligent_selector.py +173 -0
- exonware/xwlazy/runtime/metrics.py +64 -0
- exonware/xwlazy/runtime/performance.py +39 -0
- exonware/xwlazy/version.py +2 -2
- exonware_xwlazy-0.1.0.19.dist-info/METADATA +456 -0
- exonware_xwlazy-0.1.0.19.dist-info/RECORD +87 -0
- exonware_xwlazy-0.1.0.10.dist-info/METADATA +0 -0
- exonware_xwlazy-0.1.0.10.dist-info/RECORD +0 -6
- {exonware_xwlazy-0.1.0.10.dist-info → exonware_xwlazy-0.1.0.19.dist-info}/WHEEL +0 -0
- {exonware_xwlazy-0.1.0.10.dist-info → exonware_xwlazy-0.1.0.19.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,2161 @@
|
|
|
1
|
+
"""
|
|
2
|
+
#exonware/xwlazy/src/exonware/xwlazy/module/importer_engine.py
|
|
3
|
+
|
|
4
|
+
Import Engine - Unified engine for all import-related operations.
|
|
5
|
+
|
|
6
|
+
Company: eXonware.com
|
|
7
|
+
Author: Eng. Muhammad AlShehri
|
|
8
|
+
Email: connect@exonware.com
|
|
9
|
+
Version: 0.1.0.19
|
|
10
|
+
Generation Date: 15-Nov-2025
|
|
11
|
+
|
|
12
|
+
This module provides unified import engine for all import-related functionality.
|
|
13
|
+
All import-related functionality is centralized here.
|
|
14
|
+
|
|
15
|
+
Merged from:
|
|
16
|
+
- logging_utils.py (Logging utilities)
|
|
17
|
+
- import_tracking.py (Import tracking)
|
|
18
|
+
- prefix_trie.py (Prefix trie data structure)
|
|
19
|
+
- watched_registry.py (Watched prefix registry)
|
|
20
|
+
- deferred_loader.py (Deferred module loader)
|
|
21
|
+
- cache_utils.py (Multi-tier cache and bytecode cache)
|
|
22
|
+
- parallel_utils.py (Parallel loading utilities)
|
|
23
|
+
- module_patching.py (Module patching utilities)
|
|
24
|
+
- archive_imports.py (Archive import utilities)
|
|
25
|
+
- bootstrap.py (Bootstrap utilities)
|
|
26
|
+
- loader.py (Lazy loader)
|
|
27
|
+
- registry.py (Lazy module registry)
|
|
28
|
+
- importer.py (Lazy importer)
|
|
29
|
+
- import_hook.py (Import hook)
|
|
30
|
+
- meta_path_finder.py (Meta path finder)
|
|
31
|
+
|
|
32
|
+
Features:
|
|
33
|
+
- Unified import engine for all import operations
|
|
34
|
+
- Multi-tier caching (L1/L2/L3)
|
|
35
|
+
- Parallel loading support
|
|
36
|
+
- Import tracking and circular import prevention
|
|
37
|
+
- Watched prefix registry
|
|
38
|
+
- Meta path finder for intercepting imports
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
# =============================================================================
|
|
42
|
+
# IMPORTS
|
|
43
|
+
# =============================================================================
|
|
44
|
+
|
|
45
|
+
from __future__ import annotations
|
|
46
|
+
|
|
47
|
+
import os
|
|
48
|
+
import sys
|
|
49
|
+
import json
|
|
50
|
+
import time
|
|
51
|
+
import asyncio
|
|
52
|
+
import pickle
|
|
53
|
+
import struct
|
|
54
|
+
import builtins
|
|
55
|
+
import atexit
|
|
56
|
+
import logging
|
|
57
|
+
import importlib
|
|
58
|
+
import importlib.util
|
|
59
|
+
import importlib.machinery
|
|
60
|
+
import importlib.abc
|
|
61
|
+
import threading
|
|
62
|
+
import subprocess
|
|
63
|
+
import concurrent.futures
|
|
64
|
+
from pathlib import Path
|
|
65
|
+
from types import ModuleType
|
|
66
|
+
from typing import Dict, List, Optional, Set, Tuple, Any, Iterable, Callable
|
|
67
|
+
from collections import OrderedDict, defaultdict, Counter, deque
|
|
68
|
+
from queue import Queue
|
|
69
|
+
from datetime import datetime
|
|
70
|
+
from enum import Enum
|
|
71
|
+
|
|
72
|
+
from ..defs import LazyLoadMode, LazyInstallMode
|
|
73
|
+
from ..common.services.dependency_mapper import DependencyMapper
|
|
74
|
+
from ..common.services.spec_cache import _spec_cache_get, _spec_cache_put
|
|
75
|
+
from ..package.services import LazyInstallerRegistry, LazyInstaller
|
|
76
|
+
from ..package.services.config_manager import LazyInstallConfig
|
|
77
|
+
from ..package.services.manifest import _normalize_prefix
|
|
78
|
+
from ..errors import DeferredImportError
|
|
79
|
+
from .base import AModuleHelper
|
|
80
|
+
|
|
81
|
+
# Import from common (logger and cache)
|
|
82
|
+
from ..common.logger import get_logger, log_event
|
|
83
|
+
from ..common.cache import MultiTierCache, BytecodeCache
|
|
84
|
+
|
|
85
|
+
# Import from runtime folder (moved from module folder)
|
|
86
|
+
from ..runtime.adaptive_learner import AdaptiveLearner
|
|
87
|
+
from ..runtime.intelligent_selector import IntelligentModeSelector, LoadLevel
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
# =============================================================================
|
|
91
|
+
# LOGGER (from common.logger)
|
|
92
|
+
# =============================================================================
|
|
93
|
+
|
|
94
|
+
logger = get_logger("xwlazy.importer_engine")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
# =============================================================================
|
|
98
|
+
# IMPORT TRACKING (from import_tracking.py)
|
|
99
|
+
# =============================================================================
|
|
100
|
+
|
|
101
|
+
_thread_local = threading.local()
|
|
102
|
+
_importing = threading.local()
|
|
103
|
+
_installing = threading.local()
|
|
104
|
+
|
|
105
|
+
_installation_depth = 0
|
|
106
|
+
_installation_depth_lock = threading.Lock()
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _get_thread_imports() -> Set[str]:
|
|
110
|
+
"""Get thread-local import set (creates if needed)."""
|
|
111
|
+
if not hasattr(_thread_local, 'imports'):
|
|
112
|
+
_thread_local.imports = set()
|
|
113
|
+
return _thread_local.imports
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _is_import_in_progress(module_name: str) -> bool:
|
|
117
|
+
"""Check if a module import is currently in progress for this thread."""
|
|
118
|
+
return module_name in _get_thread_imports()
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _mark_import_started(module_name: str) -> None:
|
|
122
|
+
"""Mark a module import as started for this thread."""
|
|
123
|
+
_get_thread_imports().add(module_name)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _mark_import_finished(module_name: str) -> None:
|
|
127
|
+
"""Mark a module import as finished for this thread."""
|
|
128
|
+
_get_thread_imports().discard(module_name)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def get_importing_state() -> threading.local:
|
|
132
|
+
"""Get thread-local importing state."""
|
|
133
|
+
return _importing
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def get_installing_state() -> threading.local:
|
|
137
|
+
"""Get thread-local installing state."""
|
|
138
|
+
return _installing
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
# Thread-local storage for installation state
|
|
142
|
+
_installing_state = get_installing_state()
|
|
143
|
+
_importing_state = get_importing_state()
|
|
144
|
+
|
|
145
|
+
# Global recursion depth counter to prevent infinite recursion
|
|
146
|
+
_installation_depth = 0
|
|
147
|
+
_installation_depth_lock = threading.Lock()
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
# =============================================================================
|
|
151
|
+
# PREFIX TRIE (from prefix_trie.py)
|
|
152
|
+
# =============================================================================
|
|
153
|
+
|
|
154
|
+
class _PrefixTrie:
|
|
155
|
+
"""Trie data structure for prefix matching."""
|
|
156
|
+
|
|
157
|
+
__slots__ = ("_root",)
|
|
158
|
+
|
|
159
|
+
def __init__(self) -> None:
|
|
160
|
+
self._root: Dict[str, Dict[str, Any]] = {}
|
|
161
|
+
|
|
162
|
+
def add(self, prefix: str) -> None:
|
|
163
|
+
"""Add a prefix to the trie."""
|
|
164
|
+
node = self._root
|
|
165
|
+
for char in prefix:
|
|
166
|
+
node = node.setdefault(char, {})
|
|
167
|
+
node["_end"] = prefix
|
|
168
|
+
|
|
169
|
+
def iter_matches(self, value: str) -> Tuple[str, ...]:
|
|
170
|
+
"""Find all matching prefixes for a given value."""
|
|
171
|
+
node = self._root
|
|
172
|
+
matches: List[str] = []
|
|
173
|
+
for char in value:
|
|
174
|
+
end_value = node.get("_end")
|
|
175
|
+
if end_value:
|
|
176
|
+
matches.append(end_value)
|
|
177
|
+
node = node.get(char)
|
|
178
|
+
if node is None:
|
|
179
|
+
break
|
|
180
|
+
else:
|
|
181
|
+
end_value = node.get("_end")
|
|
182
|
+
if end_value:
|
|
183
|
+
matches.append(end_value)
|
|
184
|
+
return tuple(matches)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
# =============================================================================
|
|
188
|
+
# WATCHED REGISTRY (from watched_registry.py)
|
|
189
|
+
# =============================================================================
|
|
190
|
+
|
|
191
|
+
class WatchedPrefixRegistry:
|
|
192
|
+
"""Maintain watched prefixes and provide fast trie-based membership checks."""
|
|
193
|
+
|
|
194
|
+
__slots__ = (
|
|
195
|
+
"_lock",
|
|
196
|
+
"_prefix_refcounts",
|
|
197
|
+
"_owner_map",
|
|
198
|
+
"_prefixes",
|
|
199
|
+
"_trie",
|
|
200
|
+
"_dirty",
|
|
201
|
+
"_root_refcounts",
|
|
202
|
+
"_root_snapshot",
|
|
203
|
+
"_root_snapshot_dirty",
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
def __init__(self, initial: Optional[List[str]] = None) -> None:
|
|
207
|
+
self._lock = threading.RLock()
|
|
208
|
+
self._prefix_refcounts: Counter[str] = Counter()
|
|
209
|
+
self._owner_map: Dict[str, Set[str]] = {}
|
|
210
|
+
self._prefixes: Set[str] = set()
|
|
211
|
+
self._trie = _PrefixTrie()
|
|
212
|
+
self._dirty = False
|
|
213
|
+
self._root_refcounts: Counter[str] = Counter()
|
|
214
|
+
self._root_snapshot: Set[str] = set()
|
|
215
|
+
self._root_snapshot_dirty = False
|
|
216
|
+
if initial:
|
|
217
|
+
for prefix in initial:
|
|
218
|
+
self._register_manual(prefix)
|
|
219
|
+
|
|
220
|
+
def _register_manual(self, prefix: str) -> None:
|
|
221
|
+
normalized = _normalize_prefix(prefix)
|
|
222
|
+
if not normalized:
|
|
223
|
+
return
|
|
224
|
+
owner = "__manual__"
|
|
225
|
+
owners = self._owner_map.setdefault(owner, set())
|
|
226
|
+
if normalized in owners:
|
|
227
|
+
return
|
|
228
|
+
owners.add(normalized)
|
|
229
|
+
self._add_prefix(normalized)
|
|
230
|
+
|
|
231
|
+
def _add_prefix(self, prefix: str) -> None:
|
|
232
|
+
if not prefix:
|
|
233
|
+
return
|
|
234
|
+
self._prefix_refcounts[prefix] += 1
|
|
235
|
+
if self._prefix_refcounts[prefix] == 1:
|
|
236
|
+
self._prefixes.add(prefix)
|
|
237
|
+
self._dirty = True
|
|
238
|
+
root = prefix.split('.', 1)[0]
|
|
239
|
+
self._root_refcounts[root] += 1
|
|
240
|
+
self._root_snapshot_dirty = True
|
|
241
|
+
|
|
242
|
+
def _remove_prefix(self, prefix: str) -> None:
|
|
243
|
+
if prefix not in self._prefix_refcounts:
|
|
244
|
+
return
|
|
245
|
+
self._prefix_refcounts[prefix] -= 1
|
|
246
|
+
if self._prefix_refcounts[prefix] <= 0:
|
|
247
|
+
self._prefix_refcounts.pop(prefix, None)
|
|
248
|
+
self._prefixes.discard(prefix)
|
|
249
|
+
self._dirty = True
|
|
250
|
+
root = prefix.split('.', 1)[0]
|
|
251
|
+
self._root_refcounts[root] -= 1
|
|
252
|
+
if self._root_refcounts[root] <= 0:
|
|
253
|
+
self._root_refcounts.pop(root, None)
|
|
254
|
+
self._root_snapshot_dirty = True
|
|
255
|
+
|
|
256
|
+
def _ensure_trie(self) -> None:
|
|
257
|
+
if not self._dirty:
|
|
258
|
+
return
|
|
259
|
+
self._trie = _PrefixTrie()
|
|
260
|
+
for prefix in self._prefixes:
|
|
261
|
+
self._trie.add(prefix)
|
|
262
|
+
self._dirty = False
|
|
263
|
+
|
|
264
|
+
def add(self, prefix: str) -> None:
|
|
265
|
+
normalized = _normalize_prefix(prefix)
|
|
266
|
+
if not normalized:
|
|
267
|
+
return
|
|
268
|
+
with self._lock:
|
|
269
|
+
self._register_manual(normalized)
|
|
270
|
+
|
|
271
|
+
def is_empty(self) -> bool:
|
|
272
|
+
with self._lock:
|
|
273
|
+
return not self._prefixes
|
|
274
|
+
|
|
275
|
+
def register_package(self, package_name: str, prefixes: Iterable[str]) -> None:
|
|
276
|
+
owner_key = f"pkg::{package_name.lower()}"
|
|
277
|
+
normalized = {_normalize_prefix(p) for p in prefixes if _normalize_prefix(p)}
|
|
278
|
+
with self._lock:
|
|
279
|
+
current = self._owner_map.get(owner_key, set())
|
|
280
|
+
to_remove = current - normalized
|
|
281
|
+
to_add = normalized - current
|
|
282
|
+
|
|
283
|
+
for prefix in to_remove:
|
|
284
|
+
self._remove_prefix(prefix)
|
|
285
|
+
for prefix in to_add:
|
|
286
|
+
self._add_prefix(prefix)
|
|
287
|
+
|
|
288
|
+
if normalized:
|
|
289
|
+
self._owner_map[owner_key] = normalized
|
|
290
|
+
elif owner_key in self._owner_map:
|
|
291
|
+
self._owner_map.pop(owner_key, None)
|
|
292
|
+
|
|
293
|
+
def is_prefix_owned_by(self, package_name: str, prefix: str) -> bool:
|
|
294
|
+
normalized = _normalize_prefix(prefix)
|
|
295
|
+
owner_key = f"pkg::{package_name.lower()}"
|
|
296
|
+
with self._lock:
|
|
297
|
+
if normalized in self._owner_map.get("__manual__", set()):
|
|
298
|
+
return True
|
|
299
|
+
return normalized in self._owner_map.get(owner_key, set())
|
|
300
|
+
|
|
301
|
+
def get_matching_prefixes(self, module_name: str) -> Tuple[str, ...]:
|
|
302
|
+
with self._lock:
|
|
303
|
+
if not self._prefixes:
|
|
304
|
+
return ()
|
|
305
|
+
self._ensure_trie()
|
|
306
|
+
return self._trie.iter_matches(module_name)
|
|
307
|
+
|
|
308
|
+
def has_root(self, root_name: str) -> bool:
|
|
309
|
+
snapshot = self._root_snapshot
|
|
310
|
+
if not self._root_snapshot_dirty:
|
|
311
|
+
return root_name in snapshot
|
|
312
|
+
with self._lock:
|
|
313
|
+
if self._root_snapshot_dirty:
|
|
314
|
+
self._root_snapshot = set(self._root_refcounts.keys())
|
|
315
|
+
self._root_snapshot_dirty = False
|
|
316
|
+
return root_name in self._root_snapshot
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
# Global registry instance
|
|
320
|
+
_DEFAULT_WATCHED_PREFIXES = tuple(
|
|
321
|
+
filter(
|
|
322
|
+
None,
|
|
323
|
+
os.environ.get(
|
|
324
|
+
"XWLAZY_LAZY_PREFIXES",
|
|
325
|
+
"",
|
|
326
|
+
).split(";"),
|
|
327
|
+
)
|
|
328
|
+
)
|
|
329
|
+
_watched_registry = WatchedPrefixRegistry(list(_DEFAULT_WATCHED_PREFIXES))
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def get_watched_registry() -> WatchedPrefixRegistry:
|
|
333
|
+
"""Get the global watched prefix registry."""
|
|
334
|
+
return _watched_registry
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
# =============================================================================
|
|
338
|
+
# DEFERRED LOADER (from deferred_loader.py)
|
|
339
|
+
# =============================================================================
|
|
340
|
+
|
|
341
|
+
class _DeferredModuleLoader(importlib.abc.Loader):
|
|
342
|
+
"""Loader that simply returns a preconstructed module placeholder."""
|
|
343
|
+
|
|
344
|
+
def __init__(self, module: ModuleType) -> None:
|
|
345
|
+
self._module = module
|
|
346
|
+
|
|
347
|
+
def create_module(self, spec): # noqa: D401 - standard loader hook
|
|
348
|
+
return self._module
|
|
349
|
+
|
|
350
|
+
def exec_module(self, module): # noqa: D401 - nothing to execute
|
|
351
|
+
return None
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
# =============================================================================
|
|
355
|
+
# CACHE (from common.cache)
|
|
356
|
+
# =============================================================================
|
|
357
|
+
|
|
358
|
+
# MultiTierCache and BytecodeCache are now imported from ..common.cache
|
|
359
|
+
|
|
360
|
+
# =============================================================================
|
|
361
|
+
# PARALLEL UTILITIES (from parallel_utils.py)
|
|
362
|
+
# =============================================================================
|
|
363
|
+
|
|
364
|
+
class ParallelLoader:
|
|
365
|
+
"""Parallel module loader with smart dependency management."""
|
|
366
|
+
|
|
367
|
+
def __init__(self, max_workers: Optional[int] = None):
|
|
368
|
+
if max_workers is None:
|
|
369
|
+
max_workers = min(os.cpu_count() or 4, 8)
|
|
370
|
+
|
|
371
|
+
self._max_workers = max_workers
|
|
372
|
+
self._executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
|
|
373
|
+
self._lock = threading.RLock()
|
|
374
|
+
|
|
375
|
+
def _get_executor(self) -> concurrent.futures.ThreadPoolExecutor:
|
|
376
|
+
"""Get or create thread pool executor."""
|
|
377
|
+
with self._lock:
|
|
378
|
+
if self._executor is None:
|
|
379
|
+
self._executor = concurrent.futures.ThreadPoolExecutor(
|
|
380
|
+
max_workers=self._max_workers,
|
|
381
|
+
thread_name_prefix="xwlazy-parallel"
|
|
382
|
+
)
|
|
383
|
+
return self._executor
|
|
384
|
+
|
|
385
|
+
def load_modules_parallel(self, module_paths: List[str]) -> Dict[str, Any]:
|
|
386
|
+
"""Load multiple modules in parallel."""
|
|
387
|
+
executor = self._get_executor()
|
|
388
|
+
results: Dict[str, Any] = {}
|
|
389
|
+
|
|
390
|
+
def _load_module(module_path: str) -> Tuple[str, Any, Optional[Exception]]:
|
|
391
|
+
try:
|
|
392
|
+
module = importlib.import_module(module_path)
|
|
393
|
+
return (module_path, module, None)
|
|
394
|
+
except Exception as e:
|
|
395
|
+
logger.debug(f"Failed to load {module_path} in parallel: {e}")
|
|
396
|
+
return (module_path, None, e)
|
|
397
|
+
|
|
398
|
+
futures = {executor.submit(_load_module, path): path for path in module_paths}
|
|
399
|
+
|
|
400
|
+
for future in concurrent.futures.as_completed(futures):
|
|
401
|
+
module_path, module, error = future.result()
|
|
402
|
+
results[module_path] = (module, error)
|
|
403
|
+
|
|
404
|
+
return results
|
|
405
|
+
|
|
406
|
+
def load_modules_with_priority(
|
|
407
|
+
self,
|
|
408
|
+
module_paths: List[Tuple[str, int]]
|
|
409
|
+
) -> Dict[str, Any]:
|
|
410
|
+
"""Load modules in parallel with priority ordering."""
|
|
411
|
+
sorted_modules = sorted(module_paths, key=lambda x: x[1], reverse=True)
|
|
412
|
+
module_list = [path for path, _ in sorted_modules]
|
|
413
|
+
return self.load_modules_parallel(module_list)
|
|
414
|
+
|
|
415
|
+
def shutdown(self, wait: bool = True) -> None:
|
|
416
|
+
"""Shutdown the executor."""
|
|
417
|
+
with self._lock:
|
|
418
|
+
if self._executor:
|
|
419
|
+
self._executor.shutdown(wait=wait)
|
|
420
|
+
self._executor = None
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
class DependencyGraph:
|
|
424
|
+
"""Manages module dependencies for optimal parallel loading."""
|
|
425
|
+
|
|
426
|
+
def __init__(self):
|
|
427
|
+
self._dependencies: Dict[str, List[str]] = {}
|
|
428
|
+
self._reverse_deps: Dict[str, List[str]] = {}
|
|
429
|
+
self._lock = threading.RLock()
|
|
430
|
+
|
|
431
|
+
def add_dependency(self, module: str, depends_on: List[str]) -> None:
|
|
432
|
+
"""Add dependencies for a module."""
|
|
433
|
+
with self._lock:
|
|
434
|
+
self._dependencies[module] = depends_on
|
|
435
|
+
for dep in depends_on:
|
|
436
|
+
if dep not in self._reverse_deps:
|
|
437
|
+
self._reverse_deps[dep] = []
|
|
438
|
+
if module not in self._reverse_deps[dep]:
|
|
439
|
+
self._reverse_deps[dep].append(module)
|
|
440
|
+
|
|
441
|
+
def get_load_order(self, modules: List[str]) -> List[List[str]]:
|
|
442
|
+
"""Get optimal load order for parallel loading (topological sort levels)."""
|
|
443
|
+
with self._lock:
|
|
444
|
+
in_degree: Dict[str, int] = {m: 0 for m in modules}
|
|
445
|
+
for module, deps in self._dependencies.items():
|
|
446
|
+
if module in modules:
|
|
447
|
+
for dep in deps:
|
|
448
|
+
if dep in modules:
|
|
449
|
+
in_degree[module] += 1
|
|
450
|
+
|
|
451
|
+
levels: List[List[str]] = []
|
|
452
|
+
remaining = set(modules)
|
|
453
|
+
|
|
454
|
+
while remaining:
|
|
455
|
+
current_level = [
|
|
456
|
+
m for m in remaining
|
|
457
|
+
if in_degree[m] == 0
|
|
458
|
+
]
|
|
459
|
+
|
|
460
|
+
if not current_level:
|
|
461
|
+
current_level = list(remaining)
|
|
462
|
+
|
|
463
|
+
levels.append(current_level)
|
|
464
|
+
remaining -= set(current_level)
|
|
465
|
+
|
|
466
|
+
for module in current_level:
|
|
467
|
+
for dependent in self._reverse_deps.get(module, []):
|
|
468
|
+
if dependent in remaining:
|
|
469
|
+
in_degree[dependent] = max(0, in_degree[dependent] - 1)
|
|
470
|
+
|
|
471
|
+
return levels
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
# =============================================================================
|
|
475
|
+
# MODULE PATCHING (from module_patching.py)
|
|
476
|
+
# =============================================================================
|
|
477
|
+
|
|
478
|
+
_original_import_module = importlib.import_module
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _lazy_aware_import_module(name: str, package: Optional[str] = None) -> ModuleType:
|
|
482
|
+
"""Lazy-aware version of importlib.import_module."""
|
|
483
|
+
if _is_import_in_progress(name):
|
|
484
|
+
return _original_import_module(name, package)
|
|
485
|
+
|
|
486
|
+
_mark_import_started(name)
|
|
487
|
+
try:
|
|
488
|
+
return _original_import_module(name, package)
|
|
489
|
+
finally:
|
|
490
|
+
_mark_import_finished(name)
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _patch_import_module() -> None:
|
|
494
|
+
"""Patch importlib.import_module to be lazy-aware."""
|
|
495
|
+
importlib.import_module = _lazy_aware_import_module
|
|
496
|
+
logger.debug("Patched importlib.import_module to be lazy-aware")
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _unpatch_import_module() -> None:
|
|
500
|
+
"""Restore original importlib.import_module."""
|
|
501
|
+
importlib.import_module = _original_import_module
|
|
502
|
+
logger.debug("Restored original importlib.import_module")
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
# =============================================================================
|
|
506
|
+
# ARCHIVE IMPORTS (from archive_imports.py)
|
|
507
|
+
# =============================================================================
|
|
508
|
+
|
|
509
|
+
_archive_path = None
|
|
510
|
+
_archive_added = False
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def get_archive_path() -> Path:
|
|
514
|
+
"""Get the path to the _archive folder."""
|
|
515
|
+
global _archive_path
|
|
516
|
+
if _archive_path is None:
|
|
517
|
+
current_file = Path(__file__)
|
|
518
|
+
_archive_path = current_file.parent.parent.parent.parent.parent.parent / "_archive"
|
|
519
|
+
return _archive_path
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def ensure_archive_in_path() -> None:
|
|
523
|
+
"""Ensure the archive folder is in sys.path for imports."""
|
|
524
|
+
global _archive_added
|
|
525
|
+
if not _archive_added:
|
|
526
|
+
archive_path = get_archive_path()
|
|
527
|
+
archive_str = str(archive_path)
|
|
528
|
+
if archive_str not in sys.path:
|
|
529
|
+
sys.path.insert(0, archive_str)
|
|
530
|
+
_archive_added = True
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def import_from_archive(module_name: str):
|
|
534
|
+
"""Import a module from the archived lazy code."""
|
|
535
|
+
ensure_archive_in_path()
|
|
536
|
+
return __import__(module_name, fromlist=[''])
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
# =============================================================================
|
|
540
|
+
# BOOTSTRAP (from bootstrap.py)
|
|
541
|
+
# =============================================================================
|
|
542
|
+
|
|
543
|
+
def _env_enabled(env_value: Optional[str]) -> Optional[bool]:
|
|
544
|
+
if not env_value:
|
|
545
|
+
return None
|
|
546
|
+
normalized = env_value.strip().lower()
|
|
547
|
+
if normalized in ('true', '1', 'yes', 'on'):
|
|
548
|
+
return True
|
|
549
|
+
if normalized in ('false', '0', 'no', 'off'):
|
|
550
|
+
return False
|
|
551
|
+
return None
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def bootstrap_lazy_mode(package_name: str) -> None:
|
|
555
|
+
"""Detect whether lazy mode should be enabled for ``package_name`` and bootstrap hooks."""
|
|
556
|
+
package_name = package_name.lower()
|
|
557
|
+
env_value = os.environ.get(f"{package_name.upper()}_LAZY_INSTALL")
|
|
558
|
+
env_enabled = _env_enabled(env_value)
|
|
559
|
+
enabled = env_enabled
|
|
560
|
+
|
|
561
|
+
if enabled is None:
|
|
562
|
+
from ...common.services import _detect_lazy_installation
|
|
563
|
+
enabled = _detect_lazy_installation(package_name)
|
|
564
|
+
|
|
565
|
+
if not enabled:
|
|
566
|
+
return
|
|
567
|
+
|
|
568
|
+
from ..facade import config_package_lazy_install_enabled
|
|
569
|
+
|
|
570
|
+
config_package_lazy_install_enabled(
|
|
571
|
+
package_name,
|
|
572
|
+
enabled=True,
|
|
573
|
+
install_hook=True,
|
|
574
|
+
)
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def bootstrap_lazy_mode_deferred(package_name: str) -> None:
|
|
578
|
+
"""Schedule lazy mode bootstrap to run AFTER the calling package finishes importing."""
|
|
579
|
+
package_name_lower = package_name.lower()
|
|
580
|
+
package_module_name = f"exonware.{package_name_lower}"
|
|
581
|
+
|
|
582
|
+
original_import = __builtins__.__import__ if hasattr(__builtins__, '__import__') else __import__
|
|
583
|
+
|
|
584
|
+
def _import_hook(name, *args, **kwargs):
|
|
585
|
+
result = original_import(name, *args, **kwargs)
|
|
586
|
+
|
|
587
|
+
if name == package_module_name or name.startswith(f"{package_module_name}."):
|
|
588
|
+
if package_module_name in sys.modules:
|
|
589
|
+
import threading
|
|
590
|
+
def _install_hook():
|
|
591
|
+
if hasattr(__builtins__, '__import__'):
|
|
592
|
+
__builtins__.__import__ = original_import
|
|
593
|
+
else:
|
|
594
|
+
import builtins
|
|
595
|
+
builtins.__import__ = original_import
|
|
596
|
+
bootstrap_lazy_mode(package_name_lower)
|
|
597
|
+
|
|
598
|
+
threading.Timer(0.0, _install_hook).start()
|
|
599
|
+
|
|
600
|
+
return result
|
|
601
|
+
|
|
602
|
+
if hasattr(__builtins__, '__import__'):
|
|
603
|
+
__builtins__.__import__ = _import_hook
|
|
604
|
+
else:
|
|
605
|
+
import builtins
|
|
606
|
+
builtins.__import__ = _import_hook
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
# =============================================================================
|
|
610
|
+
# LAZY LOADER (from loader.py)
|
|
611
|
+
# =============================================================================
|
|
612
|
+
|
|
613
|
+
class LazyLoader(AModuleHelper):
|
|
614
|
+
"""Thread-safe lazy loader for modules with caching."""
|
|
615
|
+
|
|
616
|
+
def load_module(self, module_path: str = None) -> ModuleType:
|
|
617
|
+
"""Thread-safe module loading with caching."""
|
|
618
|
+
if module_path is None:
|
|
619
|
+
module_path = self._module_path
|
|
620
|
+
|
|
621
|
+
if self._cached_module is not None:
|
|
622
|
+
return self._cached_module
|
|
623
|
+
|
|
624
|
+
with self._lock:
|
|
625
|
+
if self._cached_module is not None:
|
|
626
|
+
return self._cached_module
|
|
627
|
+
|
|
628
|
+
if self._loading:
|
|
629
|
+
raise ImportError(f"Circular import detected for {module_path}")
|
|
630
|
+
|
|
631
|
+
try:
|
|
632
|
+
self._loading = True
|
|
633
|
+
logger.debug(f"Lazy loading module: {module_path}")
|
|
634
|
+
|
|
635
|
+
self._cached_module = importlib.import_module(module_path)
|
|
636
|
+
|
|
637
|
+
logger.debug(f"Successfully loaded: {module_path}")
|
|
638
|
+
return self._cached_module
|
|
639
|
+
|
|
640
|
+
except Exception as e:
|
|
641
|
+
logger.error(f"Failed to load module {module_path}: {e}")
|
|
642
|
+
raise ImportError(f"Failed to load {module_path}: {e}") from e
|
|
643
|
+
finally:
|
|
644
|
+
self._loading = False
|
|
645
|
+
|
|
646
|
+
def unload_module(self, module_path: str) -> None:
|
|
647
|
+
"""Unload a module from cache."""
|
|
648
|
+
with self._lock:
|
|
649
|
+
if module_path == self._module_path:
|
|
650
|
+
self._cached_module = None
|
|
651
|
+
|
|
652
|
+
def is_loaded(self) -> bool:
|
|
653
|
+
"""Check if module is currently loaded."""
|
|
654
|
+
return self._cached_module is not None
|
|
655
|
+
|
|
656
|
+
def __getattr__(self, name: str) -> Any:
|
|
657
|
+
"""Get attribute from lazily loaded module."""
|
|
658
|
+
module = self.load_module()
|
|
659
|
+
try:
|
|
660
|
+
return getattr(module, name)
|
|
661
|
+
except AttributeError:
|
|
662
|
+
raise AttributeError(
|
|
663
|
+
f"module '{self._module_path}' has no attribute '{name}'"
|
|
664
|
+
)
|
|
665
|
+
|
|
666
|
+
def __dir__(self) -> list:
|
|
667
|
+
"""Return available attributes from loaded module."""
|
|
668
|
+
module = self.load_module()
|
|
669
|
+
return dir(module)
|
|
670
|
+
|
|
671
|
+
|
|
672
|
+
# =============================================================================
|
|
673
|
+
# LAZY MODULE REGISTRY (from registry.py)
|
|
674
|
+
# =============================================================================
|
|
675
|
+
|
|
676
|
+
class LazyModuleRegistry:
|
|
677
|
+
"""Registry for managing lazy-loaded modules with performance tracking."""
|
|
678
|
+
|
|
679
|
+
__slots__ = ('_modules', '_load_times', '_lock', '_access_counts')
|
|
680
|
+
|
|
681
|
+
def __init__(self):
|
|
682
|
+
self._modules: Dict[str, LazyLoader] = {}
|
|
683
|
+
self._load_times: Dict[str, float] = {}
|
|
684
|
+
self._access_counts: Dict[str, int] = {}
|
|
685
|
+
self._lock = threading.RLock()
|
|
686
|
+
|
|
687
|
+
def register_module(self, name: str, module_path: str) -> None:
|
|
688
|
+
"""Register a module for lazy loading."""
|
|
689
|
+
with self._lock:
|
|
690
|
+
if name in self._modules:
|
|
691
|
+
logger.warning(f"Module '{name}' already registered, overwriting")
|
|
692
|
+
|
|
693
|
+
self._modules[name] = LazyLoader(module_path)
|
|
694
|
+
self._access_counts[name] = 0
|
|
695
|
+
logger.debug(f"Registered lazy module: {name} -> {module_path}")
|
|
696
|
+
|
|
697
|
+
def get_module(self, name: str) -> LazyLoader:
|
|
698
|
+
"""Get a lazy-loaded module."""
|
|
699
|
+
with self._lock:
|
|
700
|
+
if name not in self._modules:
|
|
701
|
+
raise KeyError(f"Module '{name}' not registered")
|
|
702
|
+
|
|
703
|
+
self._access_counts[name] += 1
|
|
704
|
+
return self._modules[name]
|
|
705
|
+
|
|
706
|
+
def preload_frequently_used(self, threshold: int = 5) -> None:
|
|
707
|
+
"""Preload modules that are accessed frequently."""
|
|
708
|
+
with self._lock:
|
|
709
|
+
for name, count in self._access_counts.items():
|
|
710
|
+
if count >= threshold:
|
|
711
|
+
try:
|
|
712
|
+
start_time = time.time()
|
|
713
|
+
_ = self._modules[name].load_module()
|
|
714
|
+
self._load_times[name] = time.time() - start_time
|
|
715
|
+
log_event("hook", logger.info, f"Preloaded frequently used module: {name}")
|
|
716
|
+
except Exception as e:
|
|
717
|
+
logger.warning(f"Failed to preload {name}: {e}")
|
|
718
|
+
|
|
719
|
+
def get_stats(self) -> Dict[str, Any]:
|
|
720
|
+
"""Get loading statistics."""
|
|
721
|
+
with self._lock:
|
|
722
|
+
loaded_count = sum(
|
|
723
|
+
1 for loader in self._modules.values()
|
|
724
|
+
if loader.is_loaded()
|
|
725
|
+
)
|
|
726
|
+
|
|
727
|
+
return {
|
|
728
|
+
'total_registered': len(self._modules),
|
|
729
|
+
'loaded_count': loaded_count,
|
|
730
|
+
'unloaded_count': len(self._modules) - loaded_count,
|
|
731
|
+
'access_counts': self._access_counts.copy(),
|
|
732
|
+
'load_times': self._load_times.copy(),
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
def clear_cache(self) -> None:
|
|
736
|
+
"""Clear all cached modules."""
|
|
737
|
+
with self._lock:
|
|
738
|
+
for name, loader in self._modules.items():
|
|
739
|
+
loader.unload_module(loader._module_path)
|
|
740
|
+
log_event("config", logger.info, "Cleared all cached modules")
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
# =============================================================================
|
|
744
|
+
# LAZY IMPORTER (from importer.py)
|
|
745
|
+
# =============================================================================
|
|
746
|
+
|
|
747
|
+
class LazyImporter:
|
|
748
|
+
"""
|
|
749
|
+
Lazy importer that defers heavy module imports until first access.
|
|
750
|
+
Supports multiple load modes: NONE, AUTO, PRELOAD, BACKGROUND, CACHED,
|
|
751
|
+
TURBO, ADAPTIVE, HYPERPARALLEL, STREAMING, ULTRA, INTELLIGENT.
|
|
752
|
+
"""
|
|
753
|
+
|
|
754
|
+
__slots__ = (
|
|
755
|
+
'_enabled', '_load_mode', '_lazy_modules', '_loaded_modules', '_lock',
|
|
756
|
+
'_access_counts', '_background_tasks', '_async_loop',
|
|
757
|
+
'_multi_tier_cache', '_bytecode_cache', '_adaptive_learner',
|
|
758
|
+
'_parallel_loader', '_dependency_graph', '_load_times',
|
|
759
|
+
'_intelligent_selector', '_effective_mode', '_effective_install_mode'
|
|
760
|
+
)
|
|
761
|
+
|
|
762
|
+
def __init__(self):
|
|
763
|
+
"""Initialize lazy importer."""
|
|
764
|
+
self._enabled = False
|
|
765
|
+
self._load_mode = LazyLoadMode.NONE
|
|
766
|
+
self._lazy_modules: Dict[str, str] = {}
|
|
767
|
+
self._loaded_modules: Dict[str, ModuleType] = {}
|
|
768
|
+
self._access_counts: Dict[str, int] = {}
|
|
769
|
+
self._load_times: Dict[str, float] = {}
|
|
770
|
+
self._background_tasks: Dict[str, asyncio.Task] = {}
|
|
771
|
+
self._async_loop: Optional[asyncio.AbstractEventLoop] = None
|
|
772
|
+
|
|
773
|
+
# Superior mode components
|
|
774
|
+
self._multi_tier_cache: Optional[MultiTierCache] = None
|
|
775
|
+
self._bytecode_cache: Optional[BytecodeCache] = None
|
|
776
|
+
self._adaptive_learner: Optional[AdaptiveLearner] = None
|
|
777
|
+
self._parallel_loader: Optional[ParallelLoader] = None
|
|
778
|
+
self._dependency_graph: Optional[DependencyGraph] = None
|
|
779
|
+
self._intelligent_selector: Optional[IntelligentModeSelector] = None
|
|
780
|
+
|
|
781
|
+
# Effective modes (for INTELLIGENT mode)
|
|
782
|
+
self._effective_mode: Optional[LazyLoadMode] = None
|
|
783
|
+
self._effective_install_mode = None
|
|
784
|
+
|
|
785
|
+
self._lock = threading.RLock()
|
|
786
|
+
|
|
787
|
+
def _ensure_async_loop(self) -> asyncio.AbstractEventLoop:
|
|
788
|
+
"""Ensure async event loop is running for background loading."""
|
|
789
|
+
if self._async_loop is not None and self._async_loop.is_running():
|
|
790
|
+
return self._async_loop
|
|
791
|
+
|
|
792
|
+
with self._lock:
|
|
793
|
+
if self._async_loop is None or not self._async_loop.is_running():
|
|
794
|
+
loop_ready = threading.Event()
|
|
795
|
+
|
|
796
|
+
def _run_loop():
|
|
797
|
+
loop = asyncio.new_event_loop()
|
|
798
|
+
asyncio.set_event_loop(loop)
|
|
799
|
+
self._async_loop = loop
|
|
800
|
+
loop_ready.set()
|
|
801
|
+
loop.run_forever()
|
|
802
|
+
|
|
803
|
+
thread = threading.Thread(target=_run_loop, daemon=True, name="xwlazy-loader-async")
|
|
804
|
+
thread.start()
|
|
805
|
+
|
|
806
|
+
if not loop_ready.wait(timeout=5.0):
|
|
807
|
+
raise RuntimeError("Failed to start async loop for lazy loader")
|
|
808
|
+
|
|
809
|
+
return self._async_loop
|
|
810
|
+
|
|
811
|
+
def enable(self, load_mode: LazyLoadMode = LazyLoadMode.AUTO) -> None:
|
|
812
|
+
"""Enable lazy imports with specified load mode."""
|
|
813
|
+
with self._lock:
|
|
814
|
+
self._enabled = True
|
|
815
|
+
self._load_mode = load_mode
|
|
816
|
+
|
|
817
|
+
# Initialize superior mode components
|
|
818
|
+
if load_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA):
|
|
819
|
+
self._multi_tier_cache = MultiTierCache(l1_size=1000, enable_l3=True)
|
|
820
|
+
self._bytecode_cache = BytecodeCache()
|
|
821
|
+
|
|
822
|
+
if load_mode == LazyLoadMode.ADAPTIVE:
|
|
823
|
+
self._adaptive_learner = AdaptiveLearner()
|
|
824
|
+
self._multi_tier_cache = MultiTierCache(l1_size=1000, enable_l3=True)
|
|
825
|
+
|
|
826
|
+
if load_mode == LazyLoadMode.HYPERPARALLEL:
|
|
827
|
+
max_workers = min(os.cpu_count() or 4, 8)
|
|
828
|
+
self._parallel_loader = ParallelLoader(max_workers=max_workers)
|
|
829
|
+
self._dependency_graph = DependencyGraph()
|
|
830
|
+
|
|
831
|
+
if load_mode == LazyLoadMode.STREAMING:
|
|
832
|
+
self._ensure_async_loop()
|
|
833
|
+
|
|
834
|
+
if load_mode == LazyLoadMode.ULTRA:
|
|
835
|
+
# ULTRA combines all optimizations
|
|
836
|
+
if self._multi_tier_cache is None:
|
|
837
|
+
self._multi_tier_cache = MultiTierCache(l1_size=2000, enable_l3=True)
|
|
838
|
+
if self._bytecode_cache is None:
|
|
839
|
+
self._bytecode_cache = BytecodeCache()
|
|
840
|
+
if self._adaptive_learner is None:
|
|
841
|
+
self._adaptive_learner = AdaptiveLearner()
|
|
842
|
+
if self._parallel_loader is None:
|
|
843
|
+
self._parallel_loader = ParallelLoader(max_workers=min(os.cpu_count() or 4, 8))
|
|
844
|
+
if self._dependency_graph is None:
|
|
845
|
+
self._dependency_graph = DependencyGraph()
|
|
846
|
+
self._ensure_async_loop()
|
|
847
|
+
|
|
848
|
+
# INTELLIGENT mode: Initialize selector and determine initial mode
|
|
849
|
+
if load_mode == LazyLoadMode.INTELLIGENT:
|
|
850
|
+
self._intelligent_selector = IntelligentModeSelector()
|
|
851
|
+
# Detect initial load level and get optimal mode
|
|
852
|
+
initial_level = self._intelligent_selector.detect_load_level()
|
|
853
|
+
self._effective_mode, self._effective_install_mode = self._intelligent_selector.get_optimal_mode(initial_level)
|
|
854
|
+
logger.info(f"INTELLIGENT mode initialized: {initial_level.value} -> {self._effective_mode.value} + {self._effective_install_mode.value}")
|
|
855
|
+
# Enable the effective mode recursively
|
|
856
|
+
self.enable(self._effective_mode)
|
|
857
|
+
return # Early return, effective mode is already enabled
|
|
858
|
+
|
|
859
|
+
# For PRELOAD/TURBO/ULTRA modes, preload modules
|
|
860
|
+
if load_mode in (LazyLoadMode.PRELOAD, LazyLoadMode.TURBO, LazyLoadMode.ULTRA):
|
|
861
|
+
self._preload_all_modules()
|
|
862
|
+
# For BACKGROUND/STREAMING modes, ensure async loop is ready
|
|
863
|
+
elif load_mode in (LazyLoadMode.BACKGROUND, LazyLoadMode.STREAMING):
|
|
864
|
+
self._ensure_async_loop()
|
|
865
|
+
|
|
866
|
+
log_event("config", logger.info, f"Lazy imports enabled (mode: {load_mode.value})")
|
|
867
|
+
|
|
868
|
+
def disable(self) -> None:
|
|
869
|
+
"""Disable lazy imports."""
|
|
870
|
+
with self._lock:
|
|
871
|
+
self._enabled = False
|
|
872
|
+
|
|
873
|
+
# Cleanup cache resources
|
|
874
|
+
if self._multi_tier_cache:
|
|
875
|
+
self._multi_tier_cache.shutdown()
|
|
876
|
+
|
|
877
|
+
log_event("config", logger.info, "Lazy imports disabled")
|
|
878
|
+
|
|
879
|
+
def is_enabled(self) -> bool:
|
|
880
|
+
"""Check if lazy imports are enabled."""
|
|
881
|
+
return self._enabled
|
|
882
|
+
|
|
883
|
+
def register_lazy_module(self, module_name: str, module_path: str = None) -> None:
|
|
884
|
+
"""Register a module for lazy loading."""
|
|
885
|
+
with self._lock:
|
|
886
|
+
if module_path is None:
|
|
887
|
+
module_path = module_name
|
|
888
|
+
|
|
889
|
+
self._lazy_modules[module_name] = module_path
|
|
890
|
+
self._access_counts[module_name] = 0
|
|
891
|
+
logger.debug(f"Registered lazy module: {module_name} -> {module_path}")
|
|
892
|
+
|
|
893
|
+
async def _background_load_module(self, module_name: str, module_path: str) -> ModuleType:
|
|
894
|
+
"""Load module in background thread."""
|
|
895
|
+
try:
|
|
896
|
+
actual_module = importlib.import_module(module_path)
|
|
897
|
+
with self._lock:
|
|
898
|
+
self._loaded_modules[module_name] = actual_module
|
|
899
|
+
self._access_counts[module_name] += 1
|
|
900
|
+
logger.debug(f"Background loaded module: {module_name}")
|
|
901
|
+
return actual_module
|
|
902
|
+
except ImportError as e:
|
|
903
|
+
logger.error(f"Failed to background load {module_name}: {e}")
|
|
904
|
+
raise
|
|
905
|
+
|
|
906
|
+
def _preload_all_modules(self) -> None:
|
|
907
|
+
"""Preload all registered modules using appropriate strategy based on mode."""
|
|
908
|
+
if not self._lazy_modules:
|
|
909
|
+
return
|
|
910
|
+
|
|
911
|
+
with self._lock:
|
|
912
|
+
modules_to_load = [
|
|
913
|
+
(name, path) for name, path in self._lazy_modules.items()
|
|
914
|
+
if name not in self._loaded_modules
|
|
915
|
+
]
|
|
916
|
+
|
|
917
|
+
if not modules_to_load:
|
|
918
|
+
return
|
|
919
|
+
|
|
920
|
+
# HYPERPARALLEL/ULTRA: Use thread pool executor
|
|
921
|
+
if self._load_mode in (LazyLoadMode.HYPERPARALLEL, LazyLoadMode.ULTRA) and self._parallel_loader:
|
|
922
|
+
module_paths = [path for _, path in modules_to_load]
|
|
923
|
+
results = self._parallel_loader.load_modules_parallel(module_paths)
|
|
924
|
+
|
|
925
|
+
with self._lock:
|
|
926
|
+
for (name, path), (module, error) in zip(modules_to_load, results.items()):
|
|
927
|
+
if module is not None:
|
|
928
|
+
self._loaded_modules[name] = module
|
|
929
|
+
self._access_counts[name] = 0
|
|
930
|
+
if self._adaptive_learner:
|
|
931
|
+
self._adaptive_learner.record_import(name, 0.0)
|
|
932
|
+
|
|
933
|
+
log_event("hook", logger.info, f"Parallel preloaded {len([r for r in results.values() if r[0] is not None])} modules")
|
|
934
|
+
return
|
|
935
|
+
|
|
936
|
+
# TURBO/ULTRA: Preload with predictive caching
|
|
937
|
+
if self._load_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._multi_tier_cache:
|
|
938
|
+
# Get predictive modules to prioritize
|
|
939
|
+
predictive_keys = self._multi_tier_cache.get_predictive_keys(limit=10)
|
|
940
|
+
priority_modules = [(name, path) for name, path in modules_to_load if name in predictive_keys]
|
|
941
|
+
normal_modules = [(name, path) for name, path in modules_to_load if name not in predictive_keys]
|
|
942
|
+
modules_to_load = priority_modules + normal_modules
|
|
943
|
+
|
|
944
|
+
# ADAPTIVE: Preload based on learned patterns
|
|
945
|
+
if self._load_mode == LazyLoadMode.ADAPTIVE and self._adaptive_learner:
|
|
946
|
+
priority_modules = self._adaptive_learner.get_priority_modules(limit=10)
|
|
947
|
+
priority_list = [(name, path) for name, path in modules_to_load if name in priority_modules]
|
|
948
|
+
normal_list = [(name, path) for name, path in modules_to_load if name not in priority_modules]
|
|
949
|
+
modules_to_load = priority_list + normal_list
|
|
950
|
+
|
|
951
|
+
# Default: Use asyncio for parallel loading
|
|
952
|
+
loop = self._ensure_async_loop()
|
|
953
|
+
|
|
954
|
+
async def _preload_all():
|
|
955
|
+
tasks = [
|
|
956
|
+
self._background_load_module(name, path)
|
|
957
|
+
for name, path in modules_to_load
|
|
958
|
+
]
|
|
959
|
+
if tasks:
|
|
960
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
961
|
+
log_event("hook", logger.info, f"Preloaded {len(tasks)} modules")
|
|
962
|
+
|
|
963
|
+
asyncio.run_coroutine_threadsafe(_preload_all(), loop)
|
|
964
|
+
|
|
965
|
+
def import_module(self, module_name: str, package_name: str = None) -> Any:
|
|
966
|
+
"""Import a module with lazy loading."""
|
|
967
|
+
start_time = time.time()
|
|
968
|
+
|
|
969
|
+
# Fast path: Check if already in sys.modules (lock-free read)
|
|
970
|
+
if module_name in sys.modules:
|
|
971
|
+
# Lock-free check first
|
|
972
|
+
if module_name not in self._loaded_modules:
|
|
973
|
+
with self._lock:
|
|
974
|
+
# Double-check after acquiring lock
|
|
975
|
+
if module_name not in self._loaded_modules:
|
|
976
|
+
self._loaded_modules[module_name] = sys.modules[module_name]
|
|
977
|
+
# Update access count (requires lock)
|
|
978
|
+
with self._lock:
|
|
979
|
+
self._access_counts[module_name] = self._access_counts.get(module_name, 0) + 1
|
|
980
|
+
load_time = time.time() - start_time
|
|
981
|
+
if self._adaptive_learner:
|
|
982
|
+
self._adaptive_learner.record_import(module_name, load_time)
|
|
983
|
+
if self._load_mode == LazyLoadMode.INTELLIGENT:
|
|
984
|
+
self._total_import_time = getattr(self, '_total_import_time', 0.0) + load_time
|
|
985
|
+
return sys.modules[module_name]
|
|
986
|
+
|
|
987
|
+
# Fast path: Check if already loaded (lock-free read)
|
|
988
|
+
if module_name in self._loaded_modules:
|
|
989
|
+
with self._lock:
|
|
990
|
+
# Double-check and update
|
|
991
|
+
if module_name in self._loaded_modules:
|
|
992
|
+
self._access_counts[module_name] = self._access_counts.get(module_name, 0) + 1
|
|
993
|
+
if self._adaptive_learner:
|
|
994
|
+
self._adaptive_learner.record_import(module_name, 0.0)
|
|
995
|
+
return self._loaded_modules[module_name]
|
|
996
|
+
|
|
997
|
+
# Check enabled state and get module path (requires lock)
|
|
998
|
+
with self._lock:
|
|
999
|
+
if not self._enabled or self._load_mode == LazyLoadMode.NONE:
|
|
1000
|
+
return importlib.import_module(module_name)
|
|
1001
|
+
|
|
1002
|
+
if module_name in self._lazy_modules:
|
|
1003
|
+
module_path = self._lazy_modules[module_name]
|
|
1004
|
+
else:
|
|
1005
|
+
return importlib.import_module(module_name)
|
|
1006
|
+
|
|
1007
|
+
# Update total import time for intelligent mode (initialization)
|
|
1008
|
+
if self._load_mode == LazyLoadMode.INTELLIGENT:
|
|
1009
|
+
if not hasattr(self, '_total_import_time'):
|
|
1010
|
+
self._total_import_time = 0.0
|
|
1011
|
+
|
|
1012
|
+
# INTELLIGENT mode: Check if mode switch is needed and determine effective mode
|
|
1013
|
+
effective_load_mode = self._load_mode
|
|
1014
|
+
if self._load_mode == LazyLoadMode.INTELLIGENT and self._intelligent_selector:
|
|
1015
|
+
# Throttle load level detection (cache for 0.1s to avoid excessive checks)
|
|
1016
|
+
current_time = time.time()
|
|
1017
|
+
last_check = getattr(self, '_last_load_level_check', 0.0)
|
|
1018
|
+
check_interval = 0.1 # 100ms throttle
|
|
1019
|
+
|
|
1020
|
+
if current_time - last_check >= check_interval:
|
|
1021
|
+
# Fast path: lock-free reads for stats
|
|
1022
|
+
module_count = len(self._loaded_modules) # Dict read is thread-safe
|
|
1023
|
+
total_import_time = getattr(self, '_total_import_time', 0.0)
|
|
1024
|
+
import_count = sum(self._access_counts.values()) # Dict read is thread-safe
|
|
1025
|
+
|
|
1026
|
+
# Cache psutil import and memory check (only check every 0.5s)
|
|
1027
|
+
last_memory_check = getattr(self, '_last_memory_check', 0.0)
|
|
1028
|
+
memory_mb = getattr(self, '_cached_memory_mb', 0.0)
|
|
1029
|
+
|
|
1030
|
+
if current_time - last_memory_check >= 0.5:
|
|
1031
|
+
try:
|
|
1032
|
+
import psutil
|
|
1033
|
+
process = psutil.Process()
|
|
1034
|
+
memory_mb = process.memory_info().rss / 1024 / 1024
|
|
1035
|
+
self._cached_memory_mb = memory_mb
|
|
1036
|
+
self._last_memory_check = current_time
|
|
1037
|
+
except Exception:
|
|
1038
|
+
memory_mb = 0.0
|
|
1039
|
+
|
|
1040
|
+
# Detect current load level (lock-free)
|
|
1041
|
+
detected_level = self._intelligent_selector.detect_load_level(
|
|
1042
|
+
module_count=module_count,
|
|
1043
|
+
total_import_time=total_import_time,
|
|
1044
|
+
import_count=import_count,
|
|
1045
|
+
memory_usage_mb=memory_mb
|
|
1046
|
+
)
|
|
1047
|
+
|
|
1048
|
+
# Check if mode switch is needed (requires lock for write)
|
|
1049
|
+
current_mode_tuple = (self._effective_mode or self._load_mode, self._effective_install_mode)
|
|
1050
|
+
if self._intelligent_selector.should_switch_mode(current_mode_tuple, detected_level):
|
|
1051
|
+
optimal_load, optimal_install = self._intelligent_selector.get_optimal_mode(detected_level)
|
|
1052
|
+
if optimal_load != self._effective_mode or optimal_install != self._effective_install_mode:
|
|
1053
|
+
with self._lock: # Only lock for mode switch
|
|
1054
|
+
if optimal_load != self._effective_mode or optimal_install != self._effective_install_mode:
|
|
1055
|
+
logger.info(f"INTELLIGENT mode switching: {detected_level.value} -> {optimal_load.value} + {optimal_install.value}")
|
|
1056
|
+
self._effective_mode = optimal_load
|
|
1057
|
+
self._effective_install_mode = optimal_install
|
|
1058
|
+
# Switch to optimal mode (re-enable with new mode)
|
|
1059
|
+
self.enable(optimal_load)
|
|
1060
|
+
|
|
1061
|
+
self._last_load_level_check = current_time
|
|
1062
|
+
|
|
1063
|
+
# Use effective mode for processing
|
|
1064
|
+
effective_load_mode = self._effective_mode or self._load_mode
|
|
1065
|
+
|
|
1066
|
+
# Use effective mode for all checks
|
|
1067
|
+
check_mode = effective_load_mode
|
|
1068
|
+
|
|
1069
|
+
# TURBO/ULTRA: Check multi-tier cache first
|
|
1070
|
+
if check_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._multi_tier_cache:
|
|
1071
|
+
cached_module = self._multi_tier_cache.get(module_name)
|
|
1072
|
+
if cached_module is not None:
|
|
1073
|
+
with self._lock:
|
|
1074
|
+
self._loaded_modules[module_name] = cached_module
|
|
1075
|
+
self._access_counts[module_name] += 1
|
|
1076
|
+
self._load_times[module_name] = time.time() - start_time
|
|
1077
|
+
if self._adaptive_learner:
|
|
1078
|
+
self._adaptive_learner.record_import(module_name, self._load_times[module_name])
|
|
1079
|
+
return cached_module
|
|
1080
|
+
|
|
1081
|
+
# ADAPTIVE: Check cache and predict next imports
|
|
1082
|
+
if check_mode == LazyLoadMode.ADAPTIVE:
|
|
1083
|
+
if self._multi_tier_cache:
|
|
1084
|
+
cached_module = self._multi_tier_cache.get(module_name)
|
|
1085
|
+
if cached_module is not None:
|
|
1086
|
+
with self._lock:
|
|
1087
|
+
self._loaded_modules[module_name] = cached_module
|
|
1088
|
+
self._access_counts[module_name] += 1
|
|
1089
|
+
load_time = time.time() - start_time
|
|
1090
|
+
self._load_times[module_name] = load_time
|
|
1091
|
+
if self._adaptive_learner:
|
|
1092
|
+
self._adaptive_learner.record_import(module_name, load_time)
|
|
1093
|
+
|
|
1094
|
+
# Predict and preload next likely imports
|
|
1095
|
+
if self._adaptive_learner:
|
|
1096
|
+
next_imports = self._adaptive_learner.predict_next_imports(module_name, limit=3)
|
|
1097
|
+
self._preload_predictive_modules(next_imports)
|
|
1098
|
+
|
|
1099
|
+
return cached_module
|
|
1100
|
+
|
|
1101
|
+
# Record import for learning
|
|
1102
|
+
with self._lock:
|
|
1103
|
+
if self._adaptive_learner:
|
|
1104
|
+
# Will be updated after load
|
|
1105
|
+
pass
|
|
1106
|
+
|
|
1107
|
+
# HYPERPARALLEL: Use parallel loading
|
|
1108
|
+
if check_mode == LazyLoadMode.HYPERPARALLEL and self._parallel_loader:
|
|
1109
|
+
results = self._parallel_loader.load_modules_parallel([module_path])
|
|
1110
|
+
module, error = results.get(module_path, (None, None))
|
|
1111
|
+
if module is not None:
|
|
1112
|
+
with self._lock:
|
|
1113
|
+
self._loaded_modules[module_name] = module
|
|
1114
|
+
self._access_counts[module_name] += 1
|
|
1115
|
+
self._load_times[module_name] = time.time() - start_time
|
|
1116
|
+
return module
|
|
1117
|
+
elif error:
|
|
1118
|
+
raise error
|
|
1119
|
+
|
|
1120
|
+
# STREAMING: Load asynchronously in background
|
|
1121
|
+
if check_mode == LazyLoadMode.STREAMING:
|
|
1122
|
+
return self._streaming_load(module_name, module_path)
|
|
1123
|
+
|
|
1124
|
+
# BACKGROUND mode: Load in background, return placeholder
|
|
1125
|
+
if check_mode == LazyLoadMode.BACKGROUND:
|
|
1126
|
+
return self._background_placeholder_load(module_name, module_path)
|
|
1127
|
+
|
|
1128
|
+
# TURBO/ULTRA: Load with bytecode cache
|
|
1129
|
+
actual_module = None
|
|
1130
|
+
if check_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._bytecode_cache:
|
|
1131
|
+
# Try to load from bytecode cache first
|
|
1132
|
+
bytecode = self._bytecode_cache.get_cached_bytecode(module_path)
|
|
1133
|
+
if bytecode is not None:
|
|
1134
|
+
try:
|
|
1135
|
+
# Load from bytecode
|
|
1136
|
+
code = compile(bytecode, f"<cached {module_path}>", "exec")
|
|
1137
|
+
actual_module = importlib.import_module(module_path)
|
|
1138
|
+
except Exception as e:
|
|
1139
|
+
logger.debug(f"Failed to load from bytecode cache: {e}")
|
|
1140
|
+
|
|
1141
|
+
# Load module (standard or cached)
|
|
1142
|
+
if actual_module is None:
|
|
1143
|
+
try:
|
|
1144
|
+
actual_module = importlib.import_module(module_path)
|
|
1145
|
+
|
|
1146
|
+
# Cache bytecode for TURBO/ULTRA
|
|
1147
|
+
if check_mode in (LazyLoadMode.TURBO, LazyLoadMode.ULTRA) and self._bytecode_cache:
|
|
1148
|
+
try:
|
|
1149
|
+
# Get compiled bytecode from module
|
|
1150
|
+
if hasattr(actual_module, '__file__') and actual_module.__file__:
|
|
1151
|
+
pyc_path = actual_module.__file__.replace('.py', '.pyc')
|
|
1152
|
+
if os.path.exists(pyc_path):
|
|
1153
|
+
with open(pyc_path, 'rb') as f:
|
|
1154
|
+
f.seek(16) # Skip header
|
|
1155
|
+
bytecode = f.read()
|
|
1156
|
+
self._bytecode_cache.cache_bytecode(module_path, bytecode)
|
|
1157
|
+
except Exception as e:
|
|
1158
|
+
logger.debug(f"Failed to cache bytecode: {e}")
|
|
1159
|
+
except ImportError as e:
|
|
1160
|
+
logger.error(f"Failed to lazy load {module_name}: {e}")
|
|
1161
|
+
raise
|
|
1162
|
+
|
|
1163
|
+
load_time = time.time() - start_time
|
|
1164
|
+
|
|
1165
|
+
with self._lock:
|
|
1166
|
+
self._loaded_modules[module_name] = actual_module
|
|
1167
|
+
self._access_counts[module_name] += 1
|
|
1168
|
+
self._load_times[module_name] = load_time
|
|
1169
|
+
|
|
1170
|
+
# Update total import time for intelligent mode
|
|
1171
|
+
if self._load_mode == LazyLoadMode.INTELLIGENT:
|
|
1172
|
+
self._total_import_time = getattr(self, '_total_import_time', 0.0) + load_time
|
|
1173
|
+
|
|
1174
|
+
# Cache in multi-tier cache for TURBO/ULTRA/ADAPTIVE
|
|
1175
|
+
if self._multi_tier_cache:
|
|
1176
|
+
self._multi_tier_cache.set(module_name, actual_module)
|
|
1177
|
+
|
|
1178
|
+
# Record for adaptive learning
|
|
1179
|
+
if self._adaptive_learner:
|
|
1180
|
+
self._adaptive_learner.record_import(module_name, load_time)
|
|
1181
|
+
|
|
1182
|
+
logger.debug(f"Lazy loaded module: {module_name} ({load_time*1000:.2f}ms)")
|
|
1183
|
+
|
|
1184
|
+
return actual_module
|
|
1185
|
+
|
|
1186
|
+
def _streaming_load(self, module_name: str, module_path: str) -> ModuleType:
|
|
1187
|
+
"""Load module asynchronously with streaming."""
|
|
1188
|
+
if module_name not in self._background_tasks or self._background_tasks[module_name].done():
|
|
1189
|
+
loop = self._ensure_async_loop()
|
|
1190
|
+
task = asyncio.run_coroutine_threadsafe(
|
|
1191
|
+
self._background_load_module(module_name, module_path),
|
|
1192
|
+
loop
|
|
1193
|
+
)
|
|
1194
|
+
self._background_tasks[module_name] = task
|
|
1195
|
+
|
|
1196
|
+
# Return placeholder that streams
|
|
1197
|
+
placeholder = ModuleType(module_name)
|
|
1198
|
+
placeholder.__path__ = []
|
|
1199
|
+
placeholder.__package__ = module_name
|
|
1200
|
+
|
|
1201
|
+
def _streaming_getattr(name):
|
|
1202
|
+
task = self._background_tasks.get(module_name)
|
|
1203
|
+
if task and not task.done():
|
|
1204
|
+
# Non-blocking check with short timeout
|
|
1205
|
+
try:
|
|
1206
|
+
task.result(timeout=0.01) # Very short timeout for streaming
|
|
1207
|
+
except Exception:
|
|
1208
|
+
pass # Still loading, continue
|
|
1209
|
+
|
|
1210
|
+
# Check if loaded now
|
|
1211
|
+
with self._lock:
|
|
1212
|
+
if module_name in self._loaded_modules:
|
|
1213
|
+
return getattr(self._loaded_modules[module_name], name)
|
|
1214
|
+
|
|
1215
|
+
# Still loading, wait for completion
|
|
1216
|
+
if task and not task.done():
|
|
1217
|
+
task.result(timeout=10.0)
|
|
1218
|
+
|
|
1219
|
+
with self._lock:
|
|
1220
|
+
if module_name in self._loaded_modules:
|
|
1221
|
+
return getattr(self._loaded_modules[module_name], name)
|
|
1222
|
+
raise AttributeError(f"module '{module_name}' has no attribute '{name}'")
|
|
1223
|
+
|
|
1224
|
+
placeholder.__getattr__ = _streaming_getattr # type: ignore[attr-defined]
|
|
1225
|
+
return placeholder
|
|
1226
|
+
|
|
1227
|
+
def _background_placeholder_load(self, module_name: str, module_path: str) -> ModuleType:
|
|
1228
|
+
"""Load module in background, return placeholder."""
|
|
1229
|
+
if module_name not in self._background_tasks or self._background_tasks[module_name].done():
|
|
1230
|
+
loop = self._ensure_async_loop()
|
|
1231
|
+
task = asyncio.run_coroutine_threadsafe(
|
|
1232
|
+
self._background_load_module(module_name, module_path),
|
|
1233
|
+
loop
|
|
1234
|
+
)
|
|
1235
|
+
self._background_tasks[module_name] = task
|
|
1236
|
+
|
|
1237
|
+
# Return placeholder module that will be replaced when loaded
|
|
1238
|
+
placeholder = ModuleType(module_name)
|
|
1239
|
+
placeholder.__path__ = []
|
|
1240
|
+
placeholder.__package__ = module_name
|
|
1241
|
+
|
|
1242
|
+
def _getattr(name):
|
|
1243
|
+
# Wait for background load to complete
|
|
1244
|
+
task = self._background_tasks.get(module_name)
|
|
1245
|
+
if task and not task.done():
|
|
1246
|
+
task.result(timeout=10.0) # Wait up to 10 seconds
|
|
1247
|
+
with self._lock:
|
|
1248
|
+
if module_name in self._loaded_modules:
|
|
1249
|
+
return getattr(self._loaded_modules[module_name], name)
|
|
1250
|
+
raise AttributeError(f"module '{module_name}' has no attribute '{name}'")
|
|
1251
|
+
|
|
1252
|
+
placeholder.__getattr__ = _getattr # type: ignore[attr-defined]
|
|
1253
|
+
return placeholder
|
|
1254
|
+
|
|
1255
|
+
def _preload_predictive_modules(self, module_names: list) -> None:
|
|
1256
|
+
"""Preload modules predicted to be needed soon."""
|
|
1257
|
+
if not module_names:
|
|
1258
|
+
return
|
|
1259
|
+
|
|
1260
|
+
with self._lock:
|
|
1261
|
+
modules_to_preload = [
|
|
1262
|
+
(name, self._lazy_modules[name])
|
|
1263
|
+
for name in module_names
|
|
1264
|
+
if name in self._lazy_modules and name not in self._loaded_modules
|
|
1265
|
+
]
|
|
1266
|
+
|
|
1267
|
+
if not modules_to_preload:
|
|
1268
|
+
return
|
|
1269
|
+
|
|
1270
|
+
# Preload in background
|
|
1271
|
+
loop = self._ensure_async_loop()
|
|
1272
|
+
|
|
1273
|
+
async def _preload_predictive():
|
|
1274
|
+
tasks = [
|
|
1275
|
+
self._background_load_module(name, path)
|
|
1276
|
+
for name, path in modules_to_preload
|
|
1277
|
+
]
|
|
1278
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
1279
|
+
|
|
1280
|
+
asyncio.run_coroutine_threadsafe(_preload_predictive(), loop)
|
|
1281
|
+
|
|
1282
|
+
def preload_module(self, module_name: str) -> bool:
|
|
1283
|
+
"""Preload a registered lazy module."""
|
|
1284
|
+
with self._lock:
|
|
1285
|
+
if module_name not in self._lazy_modules:
|
|
1286
|
+
logger.warning(f"Module {module_name} not registered for lazy loading")
|
|
1287
|
+
return False
|
|
1288
|
+
|
|
1289
|
+
try:
|
|
1290
|
+
self.import_module(module_name)
|
|
1291
|
+
log_event("hook", logger.info, f"Preloaded module: {module_name}")
|
|
1292
|
+
return True
|
|
1293
|
+
except Exception as e:
|
|
1294
|
+
logger.error(f"Failed to preload {module_name}: {e}")
|
|
1295
|
+
return False
|
|
1296
|
+
|
|
1297
|
+
def get_stats(self) -> Dict[str, Any]:
|
|
1298
|
+
"""Get lazy import statistics."""
|
|
1299
|
+
with self._lock:
|
|
1300
|
+
return {
|
|
1301
|
+
'enabled': self._enabled,
|
|
1302
|
+
'registered_modules': list(self._lazy_modules.keys()),
|
|
1303
|
+
'loaded_modules': list(self._loaded_modules.keys()),
|
|
1304
|
+
'access_counts': self._access_counts.copy(),
|
|
1305
|
+
'total_registered': len(self._lazy_modules),
|
|
1306
|
+
'total_loaded': len(self._loaded_modules)
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
|
|
1310
|
+
# =============================================================================
|
|
1311
|
+
# IMPORT HOOK (from import_hook.py)
|
|
1312
|
+
# =============================================================================
|
|
1313
|
+
|
|
1314
|
+
class LazyImportHook(AModuleHelper):
|
|
1315
|
+
"""
|
|
1316
|
+
Import hook that intercepts ImportError and auto-installs packages.
|
|
1317
|
+
Performance optimized with zero overhead for successful imports.
|
|
1318
|
+
"""
|
|
1319
|
+
|
|
1320
|
+
__slots__ = AModuleHelper.__slots__
|
|
1321
|
+
|
|
1322
|
+
def handle_import_error(self, module_name: str) -> Optional[Any]:
|
|
1323
|
+
"""Handle ImportError by attempting to install and re-import."""
|
|
1324
|
+
if not self._enabled:
|
|
1325
|
+
return None
|
|
1326
|
+
|
|
1327
|
+
try:
|
|
1328
|
+
# Deferred import to avoid circular dependency
|
|
1329
|
+
from ..facade import lazy_import_with_install
|
|
1330
|
+
module, success = lazy_import_with_install(
|
|
1331
|
+
module_name,
|
|
1332
|
+
installer_package=self._package_name
|
|
1333
|
+
)
|
|
1334
|
+
return module if success else None
|
|
1335
|
+
except Exception:
|
|
1336
|
+
return None
|
|
1337
|
+
|
|
1338
|
+
def install_hook(self) -> None:
|
|
1339
|
+
"""Install the import hook into sys.meta_path."""
|
|
1340
|
+
install_import_hook(self._package_name)
|
|
1341
|
+
|
|
1342
|
+
def uninstall_hook(self) -> None:
|
|
1343
|
+
"""Uninstall the import hook from sys.meta_path."""
|
|
1344
|
+
uninstall_import_hook(self._package_name)
|
|
1345
|
+
|
|
1346
|
+
def is_installed(self) -> bool:
|
|
1347
|
+
"""Check if hook is installed."""
|
|
1348
|
+
return is_import_hook_installed(self._package_name)
|
|
1349
|
+
|
|
1350
|
+
|
|
1351
|
+
# =============================================================================
|
|
1352
|
+
# META PATH FINDER (from meta_path_finder.py)
|
|
1353
|
+
# =============================================================================
|
|
1354
|
+
|
|
1355
|
+
# Wrapped class cache
|
|
1356
|
+
_WRAPPED_CLASS_CACHE: Dict[str, Set[str]] = defaultdict(set)
|
|
1357
|
+
_wrapped_cache_lock = threading.RLock()
|
|
1358
|
+
|
|
1359
|
+
# Default lazy methods
|
|
1360
|
+
_DEFAULT_LAZY_METHODS = tuple(
|
|
1361
|
+
filter(
|
|
1362
|
+
None,
|
|
1363
|
+
os.environ.get("XWLAZY_LAZY_METHODS", "").split(","),
|
|
1364
|
+
)
|
|
1365
|
+
)
|
|
1366
|
+
|
|
1367
|
+
# Lazy prefix method registry
|
|
1368
|
+
_lazy_prefix_method_registry: Dict[str, Tuple[str, ...]] = {}
|
|
1369
|
+
|
|
1370
|
+
# Package class hints
|
|
1371
|
+
_package_class_hints: Dict[str, Tuple[str, ...]] = {}
|
|
1372
|
+
_class_hint_lock = threading.RLock()
|
|
1373
|
+
|
|
1374
|
+
|
|
1375
|
+
def _set_package_class_hints(package_name: str, hints: Iterable[str]) -> None:
|
|
1376
|
+
"""Set class hints for a package."""
|
|
1377
|
+
normalized: Tuple[str, ...] = tuple(
|
|
1378
|
+
OrderedDict((hint.lower(), None) for hint in hints if hint).keys() # type: ignore[arg-type]
|
|
1379
|
+
)
|
|
1380
|
+
with _class_hint_lock:
|
|
1381
|
+
if normalized:
|
|
1382
|
+
_package_class_hints[package_name] = normalized
|
|
1383
|
+
else:
|
|
1384
|
+
_package_class_hints.pop(package_name, None)
|
|
1385
|
+
|
|
1386
|
+
|
|
1387
|
+
def _get_package_class_hints(package_name: str) -> Tuple[str, ...]:
|
|
1388
|
+
"""Get class hints for a package."""
|
|
1389
|
+
with _class_hint_lock:
|
|
1390
|
+
return _package_class_hints.get(package_name, ())
|
|
1391
|
+
|
|
1392
|
+
|
|
1393
|
+
def _clear_all_package_class_hints() -> None:
|
|
1394
|
+
"""Clear all package class hints."""
|
|
1395
|
+
with _class_hint_lock:
|
|
1396
|
+
_package_class_hints.clear()
|
|
1397
|
+
|
|
1398
|
+
|
|
1399
|
+
def register_lazy_module_methods(prefix: str, methods: Tuple[str, ...]) -> None:
|
|
1400
|
+
"""Register method names to enhance for all classes under a module prefix."""
|
|
1401
|
+
prefix = prefix.strip()
|
|
1402
|
+
if not prefix:
|
|
1403
|
+
return
|
|
1404
|
+
|
|
1405
|
+
if not prefix.endswith("."):
|
|
1406
|
+
prefix += "."
|
|
1407
|
+
|
|
1408
|
+
_lazy_prefix_method_registry[prefix] = methods
|
|
1409
|
+
log_event("config", logger.info, f"Registered lazy module methods for prefix {prefix}: {methods}")
|
|
1410
|
+
|
|
1411
|
+
|
|
1412
|
+
def _spec_for_existing_module(
|
|
1413
|
+
fullname: str,
|
|
1414
|
+
module: ModuleType,
|
|
1415
|
+
original_spec: Optional[importlib.machinery.ModuleSpec] = None,
|
|
1416
|
+
) -> importlib.machinery.ModuleSpec:
|
|
1417
|
+
"""Build a ModuleSpec whose loader simply returns an already-initialized module."""
|
|
1418
|
+
loader = _DeferredModuleLoader(module)
|
|
1419
|
+
spec = importlib.machinery.ModuleSpec(fullname, loader)
|
|
1420
|
+
if original_spec and original_spec.submodule_search_locations is not None:
|
|
1421
|
+
locations = list(original_spec.submodule_search_locations)
|
|
1422
|
+
spec.submodule_search_locations = locations
|
|
1423
|
+
if hasattr(module, "__path__"):
|
|
1424
|
+
module.__path__ = locations
|
|
1425
|
+
module.__loader__ = loader
|
|
1426
|
+
module.__spec__ = spec
|
|
1427
|
+
return spec
|
|
1428
|
+
|
|
1429
|
+
|
|
1430
|
+
class LazyMetaPathFinder:
|
|
1431
|
+
"""
|
|
1432
|
+
Custom meta path finder that intercepts failed imports.
|
|
1433
|
+
Performance optimized - only triggers when import would fail anyway.
|
|
1434
|
+
"""
|
|
1435
|
+
|
|
1436
|
+
__slots__ = ('_package_name', '_enabled')
|
|
1437
|
+
|
|
1438
|
+
def __init__(self, package_name: str = 'default'):
|
|
1439
|
+
"""Initialize meta path finder."""
|
|
1440
|
+
self._package_name = package_name
|
|
1441
|
+
self._enabled = True
|
|
1442
|
+
|
|
1443
|
+
def _build_async_placeholder(
|
|
1444
|
+
self,
|
|
1445
|
+
fullname: str,
|
|
1446
|
+
installer: LazyInstaller,
|
|
1447
|
+
) -> Optional[importlib.machinery.ModuleSpec]:
|
|
1448
|
+
"""Create and register a deferred module placeholder for async installs."""
|
|
1449
|
+
handle = installer.ensure_async_install(fullname)
|
|
1450
|
+
if handle is None:
|
|
1451
|
+
return None
|
|
1452
|
+
|
|
1453
|
+
missing = ModuleNotFoundError(f"No module named '{fullname}'")
|
|
1454
|
+
deferred = DeferredImportError(fullname, missing, self._package_name, async_handle=handle)
|
|
1455
|
+
|
|
1456
|
+
module = ModuleType(fullname)
|
|
1457
|
+
loader = _DeferredModuleLoader(module)
|
|
1458
|
+
|
|
1459
|
+
def _resolve_real_module():
|
|
1460
|
+
real_module = deferred._try_install_and_import()
|
|
1461
|
+
sys.modules[fullname] = real_module
|
|
1462
|
+
module.__dict__.clear()
|
|
1463
|
+
module.__dict__.update(real_module.__dict__)
|
|
1464
|
+
module.__loader__ = getattr(real_module, "__loader__", loader)
|
|
1465
|
+
module.__spec__ = getattr(real_module, "__spec__", None)
|
|
1466
|
+
module.__path__ = getattr(real_module, "__path__", getattr(module, "__path__", []))
|
|
1467
|
+
module.__class__ = real_module.__class__
|
|
1468
|
+
try:
|
|
1469
|
+
spec_obj = getattr(real_module, "__spec__", None) or importlib.util.find_spec(fullname)
|
|
1470
|
+
if spec_obj is not None:
|
|
1471
|
+
_spec_cache_put(fullname, spec_obj)
|
|
1472
|
+
except (ValueError, AttributeError, ImportError):
|
|
1473
|
+
pass
|
|
1474
|
+
return real_module
|
|
1475
|
+
|
|
1476
|
+
def _module_getattr(name):
|
|
1477
|
+
real = _resolve_real_module()
|
|
1478
|
+
if name in module.__dict__:
|
|
1479
|
+
return module.__dict__[name]
|
|
1480
|
+
return getattr(real, name)
|
|
1481
|
+
|
|
1482
|
+
def _module_dir():
|
|
1483
|
+
try:
|
|
1484
|
+
real = _resolve_real_module()
|
|
1485
|
+
return dir(real)
|
|
1486
|
+
except Exception:
|
|
1487
|
+
return []
|
|
1488
|
+
|
|
1489
|
+
module.__getattr__ = _module_getattr # type: ignore[attr-defined]
|
|
1490
|
+
module.__dir__ = _module_dir # type: ignore[attr-defined]
|
|
1491
|
+
module.__loader__ = loader
|
|
1492
|
+
module.__package__ = fullname
|
|
1493
|
+
module.__path__ = []
|
|
1494
|
+
|
|
1495
|
+
spec = importlib.machinery.ModuleSpec(fullname, loader)
|
|
1496
|
+
spec.submodule_search_locations = []
|
|
1497
|
+
module.__spec__ = spec
|
|
1498
|
+
|
|
1499
|
+
sys.modules[fullname] = module
|
|
1500
|
+
log_event("hook", logger.info, f"⏳ [HOOK] Deferred import placeholder created for '{fullname}'")
|
|
1501
|
+
return spec
|
|
1502
|
+
|
|
1503
|
+
def find_module(self, fullname: str, path: Optional[str] = None):
|
|
1504
|
+
"""Find module - returns None to let standard import continue."""
|
|
1505
|
+
return None
|
|
1506
|
+
|
|
1507
|
+
def find_spec(self, fullname: str, path: Optional[str] = None, target=None):
|
|
1508
|
+
"""
|
|
1509
|
+
Find module spec - intercepts imports to enable two-stage lazy loading.
|
|
1510
|
+
|
|
1511
|
+
PERFORMANCE: Optimized for zero overhead on successful imports.
|
|
1512
|
+
"""
|
|
1513
|
+
# Debug logging for msgpack to trace why it's not being intercepted
|
|
1514
|
+
if fullname == 'msgpack':
|
|
1515
|
+
logger.info(f"[HOOK] find_spec called for msgpack, enabled={self._enabled}, in_sys_modules={fullname in sys.modules}, installing={getattr(_installing_state, 'active', False)}, importing={getattr(_importing_state, 'active', False)}")
|
|
1516
|
+
|
|
1517
|
+
# CRITICAL: Check installing state FIRST to prevent recursion during installation
|
|
1518
|
+
if getattr(_installing_state, 'active', False):
|
|
1519
|
+
if fullname == 'msgpack':
|
|
1520
|
+
logger.info(f"[HOOK] Installation in progress, skipping msgpack")
|
|
1521
|
+
logger.debug(f"[HOOK] Installation in progress, skipping {fullname} to prevent recursion")
|
|
1522
|
+
return None
|
|
1523
|
+
|
|
1524
|
+
# Fast path 1: Hook disabled
|
|
1525
|
+
if not self._enabled:
|
|
1526
|
+
if fullname == 'msgpack':
|
|
1527
|
+
logger.info(f"[HOOK] Hook disabled, skipping msgpack")
|
|
1528
|
+
return None
|
|
1529
|
+
|
|
1530
|
+
# Fast path 2: Module already loaded or partially initialized
|
|
1531
|
+
if fullname in sys.modules:
|
|
1532
|
+
if fullname == 'msgpack':
|
|
1533
|
+
logger.info(f"[HOOK] msgpack already in sys.modules, skipping")
|
|
1534
|
+
return None
|
|
1535
|
+
|
|
1536
|
+
# Fast path 3: Skip C extension modules and internal modules
|
|
1537
|
+
if fullname.startswith('_'):
|
|
1538
|
+
logger.debug(f"[HOOK] Skipping C extension/internal module {fullname}")
|
|
1539
|
+
return None
|
|
1540
|
+
|
|
1541
|
+
# Fast path 4: Check if parent package is partially initialized
|
|
1542
|
+
if '.' in fullname:
|
|
1543
|
+
parent_package = fullname.split('.', 1)[0]
|
|
1544
|
+
if parent_package in sys.modules:
|
|
1545
|
+
logger.debug(f"[HOOK] Skipping {fullname} - parent {parent_package} is partially initialized")
|
|
1546
|
+
return None
|
|
1547
|
+
if _is_import_in_progress(parent_package):
|
|
1548
|
+
logger.debug(f"[HOOK] Skipping {fullname} - parent {parent_package} import in progress")
|
|
1549
|
+
return None
|
|
1550
|
+
|
|
1551
|
+
# ROOT CAUSE FIX: Check lazy install status FIRST
|
|
1552
|
+
root_name = fullname.split('.', 1)[0]
|
|
1553
|
+
_watched_registry = get_watched_registry()
|
|
1554
|
+
|
|
1555
|
+
# Check if lazy install is enabled
|
|
1556
|
+
lazy_install_enabled = LazyInstallConfig.is_enabled(self._package_name)
|
|
1557
|
+
install_mode = LazyInstallConfig.get_install_mode(self._package_name)
|
|
1558
|
+
|
|
1559
|
+
# If lazy install is disabled, only intercept watched modules
|
|
1560
|
+
if not lazy_install_enabled or install_mode == LazyInstallMode.NONE:
|
|
1561
|
+
if not _watched_registry.has_root(root_name):
|
|
1562
|
+
logger.debug(f"[HOOK] Module {fullname} not in watched registry and lazy install disabled, skipping interception")
|
|
1563
|
+
return None
|
|
1564
|
+
|
|
1565
|
+
# Check persistent installation cache FIRST
|
|
1566
|
+
try:
|
|
1567
|
+
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
1568
|
+
package_name = installer._dependency_mapper.get_package_name(root_name)
|
|
1569
|
+
if package_name and installer.is_package_installed(package_name):
|
|
1570
|
+
logger.debug(f"[HOOK] Package {package_name} is installed (cache check), skipping interception of {fullname}")
|
|
1571
|
+
return None
|
|
1572
|
+
except Exception:
|
|
1573
|
+
pass
|
|
1574
|
+
|
|
1575
|
+
# Fast path 4: Cached spec
|
|
1576
|
+
cached_spec = _spec_cache_get(fullname)
|
|
1577
|
+
if cached_spec is not None:
|
|
1578
|
+
return cached_spec
|
|
1579
|
+
|
|
1580
|
+
# Fast path 5: Stdlib/builtin check
|
|
1581
|
+
if fullname.startswith('importlib') or fullname.startswith('_frozen_importlib'):
|
|
1582
|
+
return None
|
|
1583
|
+
|
|
1584
|
+
if '.' not in fullname:
|
|
1585
|
+
if DependencyMapper._is_stdlib_or_builtin(fullname):
|
|
1586
|
+
return None
|
|
1587
|
+
if fullname in DependencyMapper.DENY_LIST:
|
|
1588
|
+
return None
|
|
1589
|
+
|
|
1590
|
+
# Fast path 6: Import in progress
|
|
1591
|
+
if _is_import_in_progress(fullname):
|
|
1592
|
+
if fullname == 'msgpack':
|
|
1593
|
+
logger.info(f"[HOOK] msgpack import in progress, skipping")
|
|
1594
|
+
return None
|
|
1595
|
+
|
|
1596
|
+
if getattr(_importing_state, 'active', False):
|
|
1597
|
+
if fullname == 'msgpack':
|
|
1598
|
+
logger.info(f"[HOOK] Global importing active, skipping msgpack")
|
|
1599
|
+
return None
|
|
1600
|
+
|
|
1601
|
+
# Install mode check already done above
|
|
1602
|
+
matching_prefixes: Tuple[str, ...] = ()
|
|
1603
|
+
if _watched_registry.has_root(root_name):
|
|
1604
|
+
matching_prefixes = _watched_registry.get_matching_prefixes(fullname)
|
|
1605
|
+
|
|
1606
|
+
# Debug: Check if msgpack reaches this point
|
|
1607
|
+
if fullname == 'msgpack':
|
|
1608
|
+
logger.debug(f"[HOOK] msgpack: matching_prefixes={matching_prefixes}, has_root={_watched_registry.has_root(root_name)}")
|
|
1609
|
+
|
|
1610
|
+
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
1611
|
+
|
|
1612
|
+
# Two-stage lazy loading for serialization and archive modules
|
|
1613
|
+
if matching_prefixes:
|
|
1614
|
+
for prefix in matching_prefixes:
|
|
1615
|
+
if not _watched_registry.is_prefix_owned_by(self._package_name, prefix):
|
|
1616
|
+
continue
|
|
1617
|
+
if fullname.startswith(prefix):
|
|
1618
|
+
module_suffix = fullname[len(prefix):]
|
|
1619
|
+
|
|
1620
|
+
if module_suffix:
|
|
1621
|
+
log_event("hook", logger.info, f"[HOOK] Candidate for wrapping: {fullname}")
|
|
1622
|
+
|
|
1623
|
+
_mark_import_started(fullname)
|
|
1624
|
+
try:
|
|
1625
|
+
if getattr(_importing_state, 'active', False):
|
|
1626
|
+
logger.debug(f"[HOOK] Recursion guard active, skipping {fullname}")
|
|
1627
|
+
return None
|
|
1628
|
+
|
|
1629
|
+
try:
|
|
1630
|
+
logger.debug(f"[HOOK] Looking for spec: {fullname}")
|
|
1631
|
+
spec = _spec_cache_get(fullname)
|
|
1632
|
+
if spec is None:
|
|
1633
|
+
try:
|
|
1634
|
+
spec = importlib.util.find_spec(fullname)
|
|
1635
|
+
except (ValueError, AttributeError, ImportError):
|
|
1636
|
+
pass
|
|
1637
|
+
if spec is not None:
|
|
1638
|
+
_spec_cache_put(fullname, spec)
|
|
1639
|
+
if spec is not None:
|
|
1640
|
+
logger.debug(f"[HOOK] Spec found, trying normal import: {fullname}")
|
|
1641
|
+
_importing_state.active = True
|
|
1642
|
+
try:
|
|
1643
|
+
__import__(fullname)
|
|
1644
|
+
|
|
1645
|
+
module = sys.modules.get(fullname)
|
|
1646
|
+
if module:
|
|
1647
|
+
try:
|
|
1648
|
+
self._enhance_classes_with_class_methods(module)
|
|
1649
|
+
except Exception as enhance_exc:
|
|
1650
|
+
logger.debug(f"[HOOK] Could not enhance classes in {fullname}: {enhance_exc}")
|
|
1651
|
+
spec = _spec_for_existing_module(fullname, module, spec)
|
|
1652
|
+
log_event("hook", logger.info, f"✓ [HOOK] Module {fullname} imported successfully, no wrapping needed")
|
|
1653
|
+
if spec is not None:
|
|
1654
|
+
_spec_cache_put(fullname, spec)
|
|
1655
|
+
return spec
|
|
1656
|
+
return None
|
|
1657
|
+
finally:
|
|
1658
|
+
_importing_state.active = False
|
|
1659
|
+
except ImportError as e:
|
|
1660
|
+
if '.' not in module_suffix:
|
|
1661
|
+
log_event("hook", logger.info, f"⚠ [HOOK] Module {fullname} has missing dependencies, wrapping: {e}")
|
|
1662
|
+
wrapped_spec = self._wrap_serialization_module(fullname)
|
|
1663
|
+
if wrapped_spec is not None:
|
|
1664
|
+
log_event("hook", logger.info, f"✓ [HOOK] Successfully wrapped: {fullname}")
|
|
1665
|
+
return wrapped_spec
|
|
1666
|
+
logger.warning(f"✗ [HOOK] Failed to wrap: {fullname}")
|
|
1667
|
+
else:
|
|
1668
|
+
logger.debug(f"[HOOK] Import failed for nested module {fullname}: {e}")
|
|
1669
|
+
except (ModuleNotFoundError,) as e:
|
|
1670
|
+
logger.debug(f"[HOOK] Module {fullname} not found, skipping wrap: {e}")
|
|
1671
|
+
pass
|
|
1672
|
+
except Exception as e:
|
|
1673
|
+
logger.warning(f"[HOOK] Error checking module {fullname}: {e}")
|
|
1674
|
+
finally:
|
|
1675
|
+
_mark_import_finished(fullname)
|
|
1676
|
+
|
|
1677
|
+
return None
|
|
1678
|
+
|
|
1679
|
+
# If we had matching prefixes but didn't match any, continue to lazy install logic
|
|
1680
|
+
if matching_prefixes:
|
|
1681
|
+
logger.debug(f"[HOOK] {fullname} had matching prefixes but didn't match any, continuing to lazy install")
|
|
1682
|
+
|
|
1683
|
+
# For lazy installation, handle submodules by checking if parent package is installed
|
|
1684
|
+
if '.' in fullname:
|
|
1685
|
+
parent_package = fullname.split('.', 1)[0]
|
|
1686
|
+
if lazy_install_enabled:
|
|
1687
|
+
try:
|
|
1688
|
+
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
1689
|
+
package_name = installer._dependency_mapper.get_package_name(parent_package)
|
|
1690
|
+
if package_name and not installer.is_package_installed(package_name):
|
|
1691
|
+
logger.debug(f"[HOOK] Parent package {parent_package} not installed, intercepting parent")
|
|
1692
|
+
return self.find_spec(parent_package, path, target)
|
|
1693
|
+
except Exception:
|
|
1694
|
+
pass
|
|
1695
|
+
return None
|
|
1696
|
+
if DependencyMapper._is_stdlib_or_builtin(fullname):
|
|
1697
|
+
return None
|
|
1698
|
+
if fullname in DependencyMapper.DENY_LIST:
|
|
1699
|
+
return None
|
|
1700
|
+
|
|
1701
|
+
# ROOT CAUSE FIX: For lazy installation, intercept missing imports and install them
|
|
1702
|
+
logger.debug(f"[HOOK] Checking lazy install for {fullname}: enabled={lazy_install_enabled}, install_mode={install_mode}")
|
|
1703
|
+
if lazy_install_enabled:
|
|
1704
|
+
if _is_import_in_progress(fullname):
|
|
1705
|
+
return None
|
|
1706
|
+
|
|
1707
|
+
_mark_import_started(fullname)
|
|
1708
|
+
try:
|
|
1709
|
+
# Guard against recursion when importing facade
|
|
1710
|
+
if 'exonware.xwlazy.facade' in sys.modules or 'xwlazy.facade' in sys.modules:
|
|
1711
|
+
from ..facade import lazy_import_with_install
|
|
1712
|
+
else:
|
|
1713
|
+
if _is_import_in_progress('exonware.xwlazy.facade') or _is_import_in_progress('xwlazy.facade'):
|
|
1714
|
+
logger.debug(f"[HOOK] Facade import in progress, skipping {fullname} to prevent recursion")
|
|
1715
|
+
return None
|
|
1716
|
+
from ..facade import lazy_import_with_install
|
|
1717
|
+
|
|
1718
|
+
_importing_state.active = True
|
|
1719
|
+
try:
|
|
1720
|
+
module, success = lazy_import_with_install(
|
|
1721
|
+
fullname,
|
|
1722
|
+
installer_package=self._package_name
|
|
1723
|
+
)
|
|
1724
|
+
finally:
|
|
1725
|
+
_importing_state.active = False
|
|
1726
|
+
|
|
1727
|
+
if success and module:
|
|
1728
|
+
# Module was successfully installed and imported
|
|
1729
|
+
xwlazy_finder_names = {'LazyMetaPathFinder', 'LazyPathFinder', 'LazyLoader'}
|
|
1730
|
+
xwlazy_finders = [f for f in sys.meta_path if type(f).__name__ in xwlazy_finder_names]
|
|
1731
|
+
for finder in xwlazy_finders:
|
|
1732
|
+
try:
|
|
1733
|
+
sys.meta_path.remove(finder)
|
|
1734
|
+
except ValueError:
|
|
1735
|
+
pass
|
|
1736
|
+
|
|
1737
|
+
try:
|
|
1738
|
+
if fullname in sys.modules:
|
|
1739
|
+
del sys.modules[fullname]
|
|
1740
|
+
importlib.invalidate_caches()
|
|
1741
|
+
sys.path_importer_cache.clear()
|
|
1742
|
+
real_module = importlib.import_module(fullname)
|
|
1743
|
+
sys.modules[fullname] = real_module
|
|
1744
|
+
logger.debug(f"[HOOK] Successfully installed {fullname}, replaced module in sys.modules with real module")
|
|
1745
|
+
finally:
|
|
1746
|
+
for finder in reversed(xwlazy_finders):
|
|
1747
|
+
if finder not in sys.meta_path:
|
|
1748
|
+
sys.meta_path.insert(0, finder)
|
|
1749
|
+
return None
|
|
1750
|
+
else:
|
|
1751
|
+
logger.debug(f"[HOOK] Failed to install/import {fullname}")
|
|
1752
|
+
try:
|
|
1753
|
+
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
1754
|
+
if installer.is_async_enabled():
|
|
1755
|
+
placeholder = self._build_async_placeholder(fullname, installer)
|
|
1756
|
+
if placeholder is not None:
|
|
1757
|
+
return placeholder
|
|
1758
|
+
except Exception:
|
|
1759
|
+
pass
|
|
1760
|
+
return None
|
|
1761
|
+
except Exception as e:
|
|
1762
|
+
logger.debug(f"Lazy import hook failed for {fullname}: {e}")
|
|
1763
|
+
return None
|
|
1764
|
+
finally:
|
|
1765
|
+
_mark_import_finished(fullname)
|
|
1766
|
+
|
|
1767
|
+
def _wrap_serialization_module(self, fullname: str):
|
|
1768
|
+
"""Wrap serialization module loading to defer missing dependencies."""
|
|
1769
|
+
log_event("hook", logger.info, f"[STAGE 1] Starting wrap of module: {fullname}")
|
|
1770
|
+
|
|
1771
|
+
try:
|
|
1772
|
+
logger.debug(f"[STAGE 1] Getting spec for: {fullname}")
|
|
1773
|
+
try:
|
|
1774
|
+
sys.meta_path.remove(self)
|
|
1775
|
+
except ValueError:
|
|
1776
|
+
pass
|
|
1777
|
+
try:
|
|
1778
|
+
spec = importlib.util.find_spec(fullname)
|
|
1779
|
+
finally:
|
|
1780
|
+
if self not in sys.meta_path:
|
|
1781
|
+
sys.meta_path.insert(0, self)
|
|
1782
|
+
if not spec or not spec.loader:
|
|
1783
|
+
logger.warning(f"[STAGE 1] No spec or loader for: {fullname}")
|
|
1784
|
+
return None
|
|
1785
|
+
|
|
1786
|
+
logger.debug(f"[STAGE 1] Creating module from spec: {fullname}")
|
|
1787
|
+
module = importlib.util.module_from_spec(spec)
|
|
1788
|
+
|
|
1789
|
+
deferred_imports = {}
|
|
1790
|
+
|
|
1791
|
+
logger.debug(f"[STAGE 1] Setting up import wrapper for: {fullname}")
|
|
1792
|
+
original_import = builtins.__import__
|
|
1793
|
+
|
|
1794
|
+
def capture_import_errors(name, *args, **kwargs):
|
|
1795
|
+
"""Intercept imports and defer ONLY external missing packages."""
|
|
1796
|
+
logger.debug(f"[STAGE 1] capture_import_errors: Trying to import '{name}' in {fullname}")
|
|
1797
|
+
|
|
1798
|
+
if _is_import_in_progress(name):
|
|
1799
|
+
logger.debug(f"[STAGE 1] Import '{name}' already in progress, using original_import")
|
|
1800
|
+
return original_import(name, *args, **kwargs)
|
|
1801
|
+
|
|
1802
|
+
_mark_import_started(name)
|
|
1803
|
+
try:
|
|
1804
|
+
result = original_import(name, *args, **kwargs)
|
|
1805
|
+
logger.debug(f"[STAGE 1] ✓ Successfully imported '{name}'")
|
|
1806
|
+
return result
|
|
1807
|
+
except ImportError as e:
|
|
1808
|
+
logger.debug(f"[STAGE 1] ✗ Import failed for '{name}': {e}")
|
|
1809
|
+
|
|
1810
|
+
host_alias = self._package_name or ""
|
|
1811
|
+
if name.startswith('exonware.') or (host_alias and name.startswith(f"{host_alias}.")):
|
|
1812
|
+
log_event("hook", logger.info, f"[STAGE 1] Letting internal import '{name}' fail normally (internal package)")
|
|
1813
|
+
raise
|
|
1814
|
+
|
|
1815
|
+
if '.' in name:
|
|
1816
|
+
log_event("hook", logger.info, f"[STAGE 1] Letting submodule '{name}' fail normally (has dots)")
|
|
1817
|
+
raise
|
|
1818
|
+
|
|
1819
|
+
log_event("hook", logger.info, f"⏳ [STAGE 1] DEFERRING missing external package '{name}' in {fullname}")
|
|
1820
|
+
async_handle = None
|
|
1821
|
+
try:
|
|
1822
|
+
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
1823
|
+
async_handle = installer.schedule_async_install(name)
|
|
1824
|
+
except Exception as schedule_exc:
|
|
1825
|
+
logger.debug(f"[STAGE 1] Async install scheduling failed for '{name}': {schedule_exc}")
|
|
1826
|
+
deferred = DeferredImportError(name, e, self._package_name, async_handle=async_handle)
|
|
1827
|
+
deferred_imports[name] = deferred
|
|
1828
|
+
return deferred
|
|
1829
|
+
finally:
|
|
1830
|
+
_mark_import_finished(name)
|
|
1831
|
+
|
|
1832
|
+
logger.debug(f"[STAGE 1] Executing module with import wrapper: {fullname}")
|
|
1833
|
+
builtins.__import__ = capture_import_errors
|
|
1834
|
+
try:
|
|
1835
|
+
spec.loader.exec_module(module)
|
|
1836
|
+
logger.debug(f"[STAGE 1] Module execution completed: {fullname}")
|
|
1837
|
+
|
|
1838
|
+
if deferred_imports:
|
|
1839
|
+
log_event("hook", logger.info, f"✓ [STAGE 1] Module {fullname} loaded with {len(deferred_imports)} deferred imports: {list(deferred_imports.keys())}")
|
|
1840
|
+
self._replace_none_with_deferred(module, deferred_imports)
|
|
1841
|
+
self._wrap_module_classes(module, deferred_imports)
|
|
1842
|
+
else:
|
|
1843
|
+
log_event("hook", logger.info, f"✓ [STAGE 1] Module {fullname} loaded with NO deferred imports (all dependencies available)")
|
|
1844
|
+
|
|
1845
|
+
self._enhance_classes_with_class_methods(module)
|
|
1846
|
+
|
|
1847
|
+
finally:
|
|
1848
|
+
logger.debug(f"[STAGE 1] Restoring original __import__")
|
|
1849
|
+
builtins.__import__ = original_import
|
|
1850
|
+
|
|
1851
|
+
logger.debug(f"[STAGE 1] Registering module in sys.modules: {fullname}")
|
|
1852
|
+
sys.modules[fullname] = module
|
|
1853
|
+
final_spec = _spec_for_existing_module(fullname, module, spec)
|
|
1854
|
+
_spec_cache_put(fullname, final_spec)
|
|
1855
|
+
log_event("hook", logger.info, f"✓ [STAGE 1] Successfully wrapped and registered: {fullname}")
|
|
1856
|
+
return final_spec
|
|
1857
|
+
|
|
1858
|
+
except Exception as e:
|
|
1859
|
+
logger.debug(f"Could not wrap {fullname}: {e}")
|
|
1860
|
+
return None
|
|
1861
|
+
|
|
1862
|
+
def _replace_none_with_deferred(self, module, deferred_imports: Dict):
|
|
1863
|
+
"""Replace None values in module namespace with deferred import proxies."""
|
|
1864
|
+
logger.debug(f"[STAGE 1] Replacing None with deferred imports in {module.__name__}")
|
|
1865
|
+
replaced_count = 0
|
|
1866
|
+
|
|
1867
|
+
for dep_name, deferred_import in deferred_imports.items():
|
|
1868
|
+
if hasattr(module, dep_name):
|
|
1869
|
+
current_value = getattr(module, dep_name)
|
|
1870
|
+
if current_value is None:
|
|
1871
|
+
log_event("hook", logger.info, f"[STAGE 1] Replacing {dep_name}=None with deferred import proxy in {module.__name__}")
|
|
1872
|
+
setattr(module, dep_name, deferred_import)
|
|
1873
|
+
replaced_count += 1
|
|
1874
|
+
|
|
1875
|
+
if replaced_count > 0:
|
|
1876
|
+
log_event("hook", logger.info, f"✓ [STAGE 1] Replaced {replaced_count} None values with deferred imports in {module.__name__}")
|
|
1877
|
+
|
|
1878
|
+
def _wrap_module_classes(self, module, deferred_imports: Dict):
|
|
1879
|
+
"""Wrap classes in a module that depend on deferred imports."""
|
|
1880
|
+
module_name = getattr(module, '__name__', '<unknown>')
|
|
1881
|
+
logger.debug(f"[STAGE 1] Wrapping classes in {module_name} (deferred: {list(deferred_imports.keys())})")
|
|
1882
|
+
module_file = (getattr(module, '__file__', '') or '').lower()
|
|
1883
|
+
lower_map = {dep_name.lower(): dep_name for dep_name in deferred_imports.keys()}
|
|
1884
|
+
class_hints = _get_package_class_hints(self._package_name)
|
|
1885
|
+
with _wrapped_cache_lock:
|
|
1886
|
+
already_wrapped = _WRAPPED_CLASS_CACHE.setdefault(module_name, set()).copy()
|
|
1887
|
+
pending_lower = {lower for lower in lower_map.keys() if lower_map[lower] not in already_wrapped}
|
|
1888
|
+
if not pending_lower:
|
|
1889
|
+
logger.debug(f"[STAGE 1] All deferred imports already wrapped for {module_name}")
|
|
1890
|
+
return
|
|
1891
|
+
dep_entries = [(lower, deferred_imports[lower_map[lower]]) for lower in pending_lower]
|
|
1892
|
+
wrapped_count = 0
|
|
1893
|
+
newly_wrapped: Set[str] = set()
|
|
1894
|
+
|
|
1895
|
+
for name, obj in list(module.__dict__.items()):
|
|
1896
|
+
if not pending_lower:
|
|
1897
|
+
break
|
|
1898
|
+
if not isinstance(obj, type):
|
|
1899
|
+
continue
|
|
1900
|
+
lower_name = name.lower()
|
|
1901
|
+
if class_hints and not any(hint in lower_name for hint in class_hints):
|
|
1902
|
+
continue
|
|
1903
|
+
target_lower = None
|
|
1904
|
+
target_deferred = None
|
|
1905
|
+
for dep_lower, deferred in dep_entries:
|
|
1906
|
+
if dep_lower not in pending_lower:
|
|
1907
|
+
continue
|
|
1908
|
+
if dep_lower in lower_name or dep_lower in module_file:
|
|
1909
|
+
target_lower = dep_lower
|
|
1910
|
+
target_deferred = deferred
|
|
1911
|
+
break
|
|
1912
|
+
if target_deferred is None or target_lower is None:
|
|
1913
|
+
continue
|
|
1914
|
+
|
|
1915
|
+
logger.debug(f"[STAGE 1] Class '{name}' depends on deferred import, wrapping...")
|
|
1916
|
+
wrapped = self._create_lazy_class_wrapper(obj, target_deferred)
|
|
1917
|
+
module.__dict__[name] = wrapped
|
|
1918
|
+
wrapped_count += 1
|
|
1919
|
+
origin_name = lower_map.get(target_lower, target_lower)
|
|
1920
|
+
newly_wrapped.add(origin_name)
|
|
1921
|
+
pending_lower.discard(target_lower)
|
|
1922
|
+
log_event("hook", logger.info, f"✓ [STAGE 1] Wrapped class '{name}' in {module_name}")
|
|
1923
|
+
|
|
1924
|
+
if newly_wrapped:
|
|
1925
|
+
with _wrapped_cache_lock:
|
|
1926
|
+
cache = _WRAPPED_CLASS_CACHE.setdefault(module_name, set())
|
|
1927
|
+
cache.update(newly_wrapped)
|
|
1928
|
+
|
|
1929
|
+
log_event("hook", logger.info, f"[STAGE 1] Wrapped {wrapped_count} classes in {module_name}")
|
|
1930
|
+
|
|
1931
|
+
def _enhance_classes_with_class_methods(self, module):
|
|
1932
|
+
"""Enhance classes that registered lazy class methods."""
|
|
1933
|
+
if module is None:
|
|
1934
|
+
return
|
|
1935
|
+
|
|
1936
|
+
methods_to_apply: Tuple[str, ...] = ()
|
|
1937
|
+
for prefix, methods in _lazy_prefix_method_registry.items():
|
|
1938
|
+
if module.__name__.startswith(prefix.rstrip('.')):
|
|
1939
|
+
methods_to_apply = methods
|
|
1940
|
+
break
|
|
1941
|
+
|
|
1942
|
+
if not methods_to_apply:
|
|
1943
|
+
methods_to_apply = _DEFAULT_LAZY_METHODS
|
|
1944
|
+
|
|
1945
|
+
if not methods_to_apply:
|
|
1946
|
+
return
|
|
1947
|
+
|
|
1948
|
+
enhanced = 0
|
|
1949
|
+
for name, obj in list(module.__dict__.items()):
|
|
1950
|
+
if not isinstance(obj, type):
|
|
1951
|
+
continue
|
|
1952
|
+
for method_name in methods_to_apply:
|
|
1953
|
+
attr = obj.__dict__.get(method_name)
|
|
1954
|
+
if attr is None:
|
|
1955
|
+
continue
|
|
1956
|
+
if getattr(attr, "__lazy_wrapped__", False):
|
|
1957
|
+
continue
|
|
1958
|
+
if not callable(attr):
|
|
1959
|
+
continue
|
|
1960
|
+
|
|
1961
|
+
if isinstance(attr, (classmethod, staticmethod)):
|
|
1962
|
+
continue
|
|
1963
|
+
|
|
1964
|
+
import inspect
|
|
1965
|
+
try:
|
|
1966
|
+
sig = inspect.signature(attr)
|
|
1967
|
+
params = list(sig.parameters.keys())
|
|
1968
|
+
if params and params[0] == 'self':
|
|
1969
|
+
logger.debug(
|
|
1970
|
+
"[LAZY ENHANCE] Wrapping instance method %s.%s.%s for class-level access",
|
|
1971
|
+
module.__name__,
|
|
1972
|
+
name,
|
|
1973
|
+
method_name,
|
|
1974
|
+
)
|
|
1975
|
+
except Exception:
|
|
1976
|
+
pass
|
|
1977
|
+
|
|
1978
|
+
try:
|
|
1979
|
+
original_func = attr
|
|
1980
|
+
|
|
1981
|
+
def class_method_wrapper(func):
|
|
1982
|
+
def _class_call(cls, *args, **kwargs):
|
|
1983
|
+
instance = cls()
|
|
1984
|
+
return func(instance, *args, **kwargs)
|
|
1985
|
+
_class_call.__name__ = getattr(func, '__name__', 'lazy_method')
|
|
1986
|
+
_class_call.__doc__ = func.__doc__
|
|
1987
|
+
_class_call.__lazy_wrapped__ = True
|
|
1988
|
+
return _class_call
|
|
1989
|
+
|
|
1990
|
+
setattr(
|
|
1991
|
+
obj,
|
|
1992
|
+
method_name,
|
|
1993
|
+
classmethod(class_method_wrapper(original_func)),
|
|
1994
|
+
)
|
|
1995
|
+
enhanced += 1
|
|
1996
|
+
logger.debug(
|
|
1997
|
+
"[LAZY ENHANCE] Added class-level %s() to %s.%s",
|
|
1998
|
+
method_name,
|
|
1999
|
+
module.__name__,
|
|
2000
|
+
name,
|
|
2001
|
+
)
|
|
2002
|
+
except Exception as exc:
|
|
2003
|
+
logger.debug(
|
|
2004
|
+
"[LAZY ENHANCE] Skipped %s.%s.%s: %s",
|
|
2005
|
+
module.__name__,
|
|
2006
|
+
name,
|
|
2007
|
+
method_name,
|
|
2008
|
+
exc,
|
|
2009
|
+
)
|
|
2010
|
+
|
|
2011
|
+
if enhanced:
|
|
2012
|
+
log_event("enhance", logger.info, "✓ [LAZY ENHANCE] Added %s convenience methods in %s", enhanced, module.__name__)
|
|
2013
|
+
|
|
2014
|
+
def _create_lazy_class_wrapper(self, original_class, deferred_import: DeferredImportError):
|
|
2015
|
+
"""Create a wrapper class that installs dependencies when instantiated."""
|
|
2016
|
+
class LazyClassWrapper:
|
|
2017
|
+
"""Lazy wrapper that installs dependencies on first instantiation."""
|
|
2018
|
+
|
|
2019
|
+
def __init__(self, *args, **kwargs):
|
|
2020
|
+
"""Install dependency and create real instance."""
|
|
2021
|
+
deferred_import._try_install_and_import()
|
|
2022
|
+
|
|
2023
|
+
real_module = importlib.reload(sys.modules[original_class.__module__])
|
|
2024
|
+
real_class = getattr(real_module, original_class.__name__)
|
|
2025
|
+
|
|
2026
|
+
real_instance = real_class(*args, **kwargs)
|
|
2027
|
+
self.__class__ = real_class
|
|
2028
|
+
self.__dict__ = real_instance.__dict__
|
|
2029
|
+
|
|
2030
|
+
def __repr__(self):
|
|
2031
|
+
return f"<Lazy{original_class.__name__}: will install dependencies on init>"
|
|
2032
|
+
|
|
2033
|
+
LazyClassWrapper.__name__ = f"Lazy{original_class.__name__}"
|
|
2034
|
+
LazyClassWrapper.__qualname__ = f"Lazy{original_class.__qualname__}"
|
|
2035
|
+
LazyClassWrapper.__module__ = original_class.__module__
|
|
2036
|
+
LazyClassWrapper.__doc__ = original_class.__doc__
|
|
2037
|
+
|
|
2038
|
+
return LazyClassWrapper
|
|
2039
|
+
|
|
2040
|
+
|
|
2041
|
+
# Registry of installed hooks per package
|
|
2042
|
+
_installed_hooks: Dict[str, LazyMetaPathFinder] = {}
|
|
2043
|
+
_hook_lock = threading.RLock()
|
|
2044
|
+
|
|
2045
|
+
|
|
2046
|
+
def install_import_hook(package_name: str = 'default') -> None:
|
|
2047
|
+
"""Install performant import hook for automatic lazy installation."""
|
|
2048
|
+
global _installed_hooks
|
|
2049
|
+
|
|
2050
|
+
log_event("hook", logger.info, f"[HOOK INSTALL] Installing import hook for package: {package_name}")
|
|
2051
|
+
|
|
2052
|
+
with _hook_lock:
|
|
2053
|
+
if package_name in _installed_hooks:
|
|
2054
|
+
log_event("hook", logger.info, f"[HOOK INSTALL] Import hook already installed for {package_name}")
|
|
2055
|
+
return
|
|
2056
|
+
|
|
2057
|
+
logger.debug(f"[HOOK INSTALL] Creating LazyMetaPathFinder for {package_name}")
|
|
2058
|
+
hook = LazyMetaPathFinder(package_name)
|
|
2059
|
+
|
|
2060
|
+
logger.debug(f"[HOOK INSTALL] Current sys.meta_path has {len(sys.meta_path)} entries")
|
|
2061
|
+
sys.meta_path.insert(0, hook)
|
|
2062
|
+
_installed_hooks[package_name] = hook
|
|
2063
|
+
|
|
2064
|
+
log_event("hook", logger.info, f"✅ [HOOK INSTALL] Lazy import hook installed for {package_name} (now {len(sys.meta_path)} meta_path entries)")
|
|
2065
|
+
|
|
2066
|
+
|
|
2067
|
+
def uninstall_import_hook(package_name: str = 'default') -> None:
|
|
2068
|
+
"""Uninstall import hook for a package."""
|
|
2069
|
+
global _installed_hooks
|
|
2070
|
+
|
|
2071
|
+
with _hook_lock:
|
|
2072
|
+
if package_name in _installed_hooks:
|
|
2073
|
+
hook = _installed_hooks[package_name]
|
|
2074
|
+
try:
|
|
2075
|
+
sys.meta_path.remove(hook)
|
|
2076
|
+
except ValueError:
|
|
2077
|
+
pass
|
|
2078
|
+
del _installed_hooks[package_name]
|
|
2079
|
+
log_event("hook", logger.info, f"Lazy import hook uninstalled for {package_name}")
|
|
2080
|
+
|
|
2081
|
+
|
|
2082
|
+
def is_import_hook_installed(package_name: str = 'default') -> bool:
|
|
2083
|
+
"""Check if import hook is installed for a package."""
|
|
2084
|
+
return package_name in _installed_hooks
|
|
2085
|
+
|
|
2086
|
+
|
|
2087
|
+
def register_lazy_module_prefix(prefix: str) -> None:
|
|
2088
|
+
"""Register an import prefix for lazy wrapping."""
|
|
2089
|
+
_watched_registry = get_watched_registry()
|
|
2090
|
+
_watched_registry.add(prefix)
|
|
2091
|
+
normalized = _normalize_prefix(prefix)
|
|
2092
|
+
if normalized:
|
|
2093
|
+
log_event("config", logger.info, "Registered lazy module prefix: %s", normalized)
|
|
2094
|
+
|
|
2095
|
+
|
|
2096
|
+
# =============================================================================
|
|
2097
|
+
# EXPORTS
|
|
2098
|
+
# =============================================================================
|
|
2099
|
+
|
|
2100
|
+
__all__ = [
|
|
2101
|
+
# Logging utilities
|
|
2102
|
+
'get_logger',
|
|
2103
|
+
'log_event',
|
|
2104
|
+
'print_formatted',
|
|
2105
|
+
'format_message',
|
|
2106
|
+
'is_log_category_enabled',
|
|
2107
|
+
'set_log_category',
|
|
2108
|
+
'set_log_categories',
|
|
2109
|
+
'get_log_categories',
|
|
2110
|
+
'XWLazyFormatter',
|
|
2111
|
+
# Import tracking
|
|
2112
|
+
'_is_import_in_progress',
|
|
2113
|
+
'_mark_import_started',
|
|
2114
|
+
'_mark_import_finished',
|
|
2115
|
+
'get_importing_state',
|
|
2116
|
+
'get_installing_state',
|
|
2117
|
+
'_installation_depth',
|
|
2118
|
+
'_installation_depth_lock',
|
|
2119
|
+
# Prefix trie
|
|
2120
|
+
'_PrefixTrie',
|
|
2121
|
+
# Watched registry
|
|
2122
|
+
'WatchedPrefixRegistry',
|
|
2123
|
+
'get_watched_registry',
|
|
2124
|
+
# Deferred loader
|
|
2125
|
+
'_DeferredModuleLoader',
|
|
2126
|
+
# Cache utilities
|
|
2127
|
+
# Parallel utilities
|
|
2128
|
+
'ParallelLoader',
|
|
2129
|
+
'DependencyGraph',
|
|
2130
|
+
# Module patching
|
|
2131
|
+
'_lazy_aware_import_module',
|
|
2132
|
+
'_patch_import_module',
|
|
2133
|
+
'_unpatch_import_module',
|
|
2134
|
+
# Archive imports
|
|
2135
|
+
'get_archive_path',
|
|
2136
|
+
'ensure_archive_in_path',
|
|
2137
|
+
'import_from_archive',
|
|
2138
|
+
# Bootstrap
|
|
2139
|
+
'bootstrap_lazy_mode',
|
|
2140
|
+
'bootstrap_lazy_mode_deferred',
|
|
2141
|
+
# Lazy loader
|
|
2142
|
+
'LazyLoader',
|
|
2143
|
+
# Lazy module registry
|
|
2144
|
+
'LazyModuleRegistry',
|
|
2145
|
+
# Lazy importer
|
|
2146
|
+
'LazyImporter',
|
|
2147
|
+
# Import hook
|
|
2148
|
+
'LazyImportHook',
|
|
2149
|
+
# Meta path finder
|
|
2150
|
+
'LazyMetaPathFinder',
|
|
2151
|
+
'install_import_hook',
|
|
2152
|
+
'uninstall_import_hook',
|
|
2153
|
+
'is_import_hook_installed',
|
|
2154
|
+
'register_lazy_module_prefix',
|
|
2155
|
+
'register_lazy_module_methods',
|
|
2156
|
+
'_set_package_class_hints',
|
|
2157
|
+
'_get_package_class_hints',
|
|
2158
|
+
'_clear_all_package_class_hints',
|
|
2159
|
+
'_spec_for_existing_module',
|
|
2160
|
+
]
|
|
2161
|
+
|