exonware-xwlazy 0.1.0.11__py3-none-any.whl → 0.1.0.19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- exonware/__init__.py +22 -0
- exonware/xwlazy/__init__.py +0 -0
- exonware/xwlazy/common/__init__.py +47 -0
- exonware/xwlazy/common/base.py +58 -0
- exonware/xwlazy/common/cache.py +506 -0
- exonware/xwlazy/common/logger.py +268 -0
- exonware/xwlazy/common/services/__init__.py +72 -0
- exonware/xwlazy/common/services/dependency_mapper.py +234 -0
- exonware/xwlazy/common/services/install_async_utils.py +169 -0
- exonware/xwlazy/common/services/install_cache_utils.py +257 -0
- exonware/xwlazy/common/services/keyword_detection.py +292 -0
- exonware/xwlazy/common/services/spec_cache.py +173 -0
- exonware/xwlazy/common/strategies/__init__.py +28 -0
- exonware/xwlazy/common/strategies/caching_dict.py +45 -0
- exonware/xwlazy/common/strategies/caching_installation.py +89 -0
- exonware/xwlazy/common/strategies/caching_lfu.py +67 -0
- exonware/xwlazy/common/strategies/caching_lru.py +64 -0
- exonware/xwlazy/common/strategies/caching_multitier.py +60 -0
- exonware/xwlazy/common/strategies/caching_ttl.py +60 -0
- {xwlazy/lazy → exonware/xwlazy}/config.py +52 -20
- exonware/xwlazy/contracts.py +1410 -0
- exonware/xwlazy/defs.py +397 -0
- xwlazy/lazy/lazy_errors.py → exonware/xwlazy/errors.py +21 -8
- exonware/xwlazy/facade.py +1049 -0
- exonware/xwlazy/module/__init__.py +18 -0
- exonware/xwlazy/module/base.py +569 -0
- exonware/xwlazy/module/data.py +17 -0
- exonware/xwlazy/module/facade.py +247 -0
- exonware/xwlazy/module/importer_engine.py +2161 -0
- exonware/xwlazy/module/strategies/__init__.py +22 -0
- exonware/xwlazy/module/strategies/module_helper_lazy.py +94 -0
- exonware/xwlazy/module/strategies/module_helper_simple.py +66 -0
- exonware/xwlazy/module/strategies/module_manager_advanced.py +112 -0
- exonware/xwlazy/module/strategies/module_manager_simple.py +96 -0
- exonware/xwlazy/package/__init__.py +18 -0
- exonware/xwlazy/package/base.py +807 -0
- xwlazy/lazy/host_conf.py → exonware/xwlazy/package/conf.py +62 -10
- exonware/xwlazy/package/data.py +17 -0
- exonware/xwlazy/package/facade.py +481 -0
- exonware/xwlazy/package/services/__init__.py +84 -0
- exonware/xwlazy/package/services/async_install_handle.py +89 -0
- exonware/xwlazy/package/services/config_manager.py +246 -0
- exonware/xwlazy/package/services/discovery.py +374 -0
- {xwlazy/lazy → exonware/xwlazy/package/services}/host_packages.py +43 -16
- exonware/xwlazy/package/services/install_async.py +278 -0
- exonware/xwlazy/package/services/install_cache.py +146 -0
- exonware/xwlazy/package/services/install_interactive.py +60 -0
- exonware/xwlazy/package/services/install_policy.py +158 -0
- exonware/xwlazy/package/services/install_registry.py +56 -0
- exonware/xwlazy/package/services/install_result.py +17 -0
- exonware/xwlazy/package/services/install_sbom.py +154 -0
- exonware/xwlazy/package/services/install_utils.py +83 -0
- exonware/xwlazy/package/services/installer_engine.py +408 -0
- exonware/xwlazy/package/services/lazy_installer.py +720 -0
- {xwlazy/lazy → exonware/xwlazy/package/services}/manifest.py +42 -25
- exonware/xwlazy/package/services/strategy_registry.py +188 -0
- exonware/xwlazy/package/strategies/__init__.py +57 -0
- exonware/xwlazy/package/strategies/package_discovery_file.py +130 -0
- exonware/xwlazy/package/strategies/package_discovery_hybrid.py +85 -0
- exonware/xwlazy/package/strategies/package_discovery_manifest.py +102 -0
- exonware/xwlazy/package/strategies/package_execution_async.py +114 -0
- exonware/xwlazy/package/strategies/package_execution_cached.py +91 -0
- exonware/xwlazy/package/strategies/package_execution_pip.py +100 -0
- exonware/xwlazy/package/strategies/package_execution_wheel.py +107 -0
- exonware/xwlazy/package/strategies/package_mapping_discovery_first.py +101 -0
- exonware/xwlazy/package/strategies/package_mapping_hybrid.py +106 -0
- exonware/xwlazy/package/strategies/package_mapping_manifest_first.py +101 -0
- exonware/xwlazy/package/strategies/package_policy_allow_list.py +58 -0
- exonware/xwlazy/package/strategies/package_policy_deny_list.py +58 -0
- exonware/xwlazy/package/strategies/package_policy_permissive.py +47 -0
- exonware/xwlazy/package/strategies/package_timing_clean.py +68 -0
- exonware/xwlazy/package/strategies/package_timing_full.py +67 -0
- exonware/xwlazy/package/strategies/package_timing_smart.py +69 -0
- exonware/xwlazy/package/strategies/package_timing_temporary.py +67 -0
- exonware/xwlazy/runtime/__init__.py +18 -0
- exonware/xwlazy/runtime/adaptive_learner.py +131 -0
- exonware/xwlazy/runtime/base.py +276 -0
- exonware/xwlazy/runtime/facade.py +95 -0
- exonware/xwlazy/runtime/intelligent_selector.py +173 -0
- exonware/xwlazy/runtime/metrics.py +64 -0
- exonware/xwlazy/runtime/performance.py +39 -0
- exonware/xwlazy/version.py +2 -2
- {exonware_xwlazy-0.1.0.11.dist-info → exonware_xwlazy-0.1.0.19.dist-info}/METADATA +86 -10
- exonware_xwlazy-0.1.0.19.dist-info/RECORD +87 -0
- exonware_xwlazy-0.1.0.11.dist-info/RECORD +0 -20
- xwlazy/__init__.py +0 -34
- xwlazy/lazy/__init__.py +0 -301
- xwlazy/lazy/bootstrap.py +0 -106
- xwlazy/lazy/lazy_base.py +0 -465
- xwlazy/lazy/lazy_contracts.py +0 -290
- xwlazy/lazy/lazy_core.py +0 -3727
- xwlazy/lazy/logging_utils.py +0 -194
- xwlazy/version.py +0 -77
- /xwlazy/lazy/lazy_state.py → /exonware/xwlazy/common/services/state_manager.py +0 -0
- {exonware_xwlazy-0.1.0.11.dist-info → exonware_xwlazy-0.1.0.19.dist-info}/WHEEL +0 -0
- {exonware_xwlazy-0.1.0.11.dist-info → exonware_xwlazy-0.1.0.19.dist-info}/licenses/LICENSE +0 -0
xwlazy/lazy/lazy_core.py
DELETED
|
@@ -1,3727 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
#exonware/xwlazy/src/exonware/xwlazy/lazy/lazy_core.py
|
|
3
|
-
|
|
4
|
-
Company: eXonware.com
|
|
5
|
-
Author: Eng. Muhammad AlShehri
|
|
6
|
-
Email: connect@exonware.com
|
|
7
|
-
Version: 0.1.0.16
|
|
8
|
-
Generation Date: 10-Oct-2025
|
|
9
|
-
|
|
10
|
-
Lazy Loading System - Core Implementation
|
|
11
|
-
|
|
12
|
-
This module consolidates all lazy loading functionality into a single implementation
|
|
13
|
-
following DEV_GUIDELINES.md structure. It provides per-package lazy loading with:
|
|
14
|
-
- Automatic dependency discovery
|
|
15
|
-
- Secure package installation
|
|
16
|
-
- Import hooks for two-stage loading
|
|
17
|
-
- Performance monitoring and caching
|
|
18
|
-
- SBOM generation and lockfile management
|
|
19
|
-
|
|
20
|
-
Design Patterns Applied:
|
|
21
|
-
- Facade: LazySystemFacade provides unified API
|
|
22
|
-
- Strategy: Pluggable discovery/installation strategies
|
|
23
|
-
- Template Method: Base classes define workflows
|
|
24
|
-
- Singleton: Global instances for system-wide state
|
|
25
|
-
- Registry: Per-package isolation
|
|
26
|
-
- Observer: Performance monitoring
|
|
27
|
-
- Proxy: Deferred loading
|
|
28
|
-
|
|
29
|
-
Core Goal: Per-Package Lazy Loading
|
|
30
|
-
- Each package (xwsystem, xwnode, xwdata) can independently enable lazy mode
|
|
31
|
-
- Only packages installed with [lazy] extra get auto-installation
|
|
32
|
-
- Logs missing imports per package
|
|
33
|
-
- Installs on first actual usage (two-stage loading)
|
|
34
|
-
"""
|
|
35
|
-
|
|
36
|
-
import os
|
|
37
|
-
import re
|
|
38
|
-
import json
|
|
39
|
-
import sys
|
|
40
|
-
import subprocess
|
|
41
|
-
import importlib
|
|
42
|
-
import importlib.abc
|
|
43
|
-
import importlib.machinery
|
|
44
|
-
import importlib.util
|
|
45
|
-
import builtins
|
|
46
|
-
import threading
|
|
47
|
-
import time
|
|
48
|
-
import shutil
|
|
49
|
-
import sysconfig
|
|
50
|
-
import tempfile
|
|
51
|
-
import zipfile
|
|
52
|
-
import inspect
|
|
53
|
-
from collections import Counter, OrderedDict, defaultdict
|
|
54
|
-
from contextlib import suppress
|
|
55
|
-
from functools import lru_cache
|
|
56
|
-
from concurrent.futures import Future, ThreadPoolExecutor
|
|
57
|
-
from datetime import datetime
|
|
58
|
-
from pathlib import Path
|
|
59
|
-
from typing import Dict, Iterable, List, Optional, Set, Tuple, Any, Callable
|
|
60
|
-
from types import ModuleType
|
|
61
|
-
|
|
62
|
-
from .lazy_contracts import DependencyInfo, LazyInstallMode
|
|
63
|
-
from .lazy_errors import (
|
|
64
|
-
LazySystemError,
|
|
65
|
-
LazyInstallError,
|
|
66
|
-
LazyDiscoveryError,
|
|
67
|
-
ExternallyManagedError,
|
|
68
|
-
DeferredImportError,
|
|
69
|
-
)
|
|
70
|
-
from .lazy_base import (
|
|
71
|
-
APackageDiscovery,
|
|
72
|
-
APackageInstaller,
|
|
73
|
-
AImportHook,
|
|
74
|
-
ALazyLoader,
|
|
75
|
-
)
|
|
76
|
-
from .logging_utils import get_logger, log_event, print_formatted, format_message
|
|
77
|
-
from .manifest import PackageManifest, get_manifest_loader
|
|
78
|
-
from .lazy_state import LazyStateManager
|
|
79
|
-
|
|
80
|
-
try:
|
|
81
|
-
_STDLIB_MODULE_SET: Set[str] = set(sys.stdlib_module_names) # type: ignore[attr-defined]
|
|
82
|
-
except AttributeError:
|
|
83
|
-
_STDLIB_MODULE_SET = set()
|
|
84
|
-
_STDLIB_MODULE_SET.update(sys.builtin_module_names)
|
|
85
|
-
|
|
86
|
-
logger = get_logger("xwlazy.lazy")
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def _log(category: str, message: str, *args) -> None:
|
|
90
|
-
log_event(category, logger.info, message, *args)
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
def _get_trigger_file() -> Optional[str]:
|
|
94
|
-
"""Get the file that triggered the import (from call stack)."""
|
|
95
|
-
try:
|
|
96
|
-
# Walk up the call stack to find the first non-xwlazy file
|
|
97
|
-
# Look for files in xwsystem, xwnode, xwdata, or user code
|
|
98
|
-
for frame_info in inspect.stack():
|
|
99
|
-
filename = frame_info.filename
|
|
100
|
-
# Skip xwlazy internal files and importlib
|
|
101
|
-
if ('xwlazy' not in filename and
|
|
102
|
-
'importlib' not in filename and
|
|
103
|
-
'<frozen' not in filename and
|
|
104
|
-
filename.endswith('.py')):
|
|
105
|
-
# Return just the filename, not full path
|
|
106
|
-
basename = os.path.basename(filename)
|
|
107
|
-
# If it's a serialization file, use that
|
|
108
|
-
if 'serialization' in filename or 'formats' in filename:
|
|
109
|
-
# Extract the format name (e.g., bson.py -> BsonSerializer)
|
|
110
|
-
if basename.endswith('.py'):
|
|
111
|
-
basename = basename[:-3] # Remove .py
|
|
112
|
-
return f"{basename.capitalize()}Serializer" if basename else None
|
|
113
|
-
return basename
|
|
114
|
-
except Exception:
|
|
115
|
-
pass
|
|
116
|
-
return None
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
@lru_cache(maxsize=1024)
|
|
120
|
-
def _cached_stdlib_check(module_name: str) -> bool:
|
|
121
|
-
try:
|
|
122
|
-
spec = importlib.util.find_spec(module_name)
|
|
123
|
-
if spec is None:
|
|
124
|
-
return False
|
|
125
|
-
if spec.origin in ("built-in", None):
|
|
126
|
-
return True
|
|
127
|
-
origin = spec.origin or ""
|
|
128
|
-
return (
|
|
129
|
-
"python" in origin.lower()
|
|
130
|
-
and "site-packages" not in origin.lower()
|
|
131
|
-
and "dist-packages" not in origin.lower()
|
|
132
|
-
)
|
|
133
|
-
except Exception:
|
|
134
|
-
return False
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
# =============================================================================
|
|
138
|
-
# SECTION 1: PACKAGE DISCOVERY (~350 lines)
|
|
139
|
-
# =============================================================================
|
|
140
|
-
|
|
141
|
-
class DependencyMapper:
|
|
142
|
-
"""
|
|
143
|
-
Maps import names to package names using dynamic discovery.
|
|
144
|
-
Optimized with caching to avoid repeated file I/O.
|
|
145
|
-
"""
|
|
146
|
-
|
|
147
|
-
__slots__ = (
|
|
148
|
-
'_discovery',
|
|
149
|
-
'_package_import_mapping',
|
|
150
|
-
'_import_package_mapping',
|
|
151
|
-
'_cached',
|
|
152
|
-
'_lock',
|
|
153
|
-
'_package_name',
|
|
154
|
-
'_manifest_generation',
|
|
155
|
-
'_manifest_dependencies',
|
|
156
|
-
'_manifest_signature',
|
|
157
|
-
'_manifest_empty',
|
|
158
|
-
)
|
|
159
|
-
|
|
160
|
-
def __init__(self, package_name: str = 'default'):
|
|
161
|
-
"""Initialize dependency mapper."""
|
|
162
|
-
self._discovery = None # Lazy init to avoid circular imports
|
|
163
|
-
self._package_import_mapping = {}
|
|
164
|
-
self._import_package_mapping = {}
|
|
165
|
-
self._cached = False
|
|
166
|
-
self._lock = threading.RLock()
|
|
167
|
-
self._package_name = package_name
|
|
168
|
-
self._manifest_generation = -1
|
|
169
|
-
self._manifest_dependencies: Dict[str, str] = {}
|
|
170
|
-
self._manifest_signature: Optional[Tuple[str, float, float]] = None
|
|
171
|
-
self._manifest_empty = False
|
|
172
|
-
|
|
173
|
-
def set_package_name(self, package_name: str) -> None:
|
|
174
|
-
"""Update the owning package name (affects manifest lookups)."""
|
|
175
|
-
normalized = (package_name or 'default').strip().lower() or 'default'
|
|
176
|
-
if normalized != self._package_name:
|
|
177
|
-
self._package_name = normalized
|
|
178
|
-
self._manifest_generation = -1
|
|
179
|
-
self._manifest_dependencies = {}
|
|
180
|
-
|
|
181
|
-
def _get_discovery(self):
|
|
182
|
-
"""Get discovery instance (lazy init)."""
|
|
183
|
-
if self._discovery is None:
|
|
184
|
-
self._discovery = get_lazy_discovery()
|
|
185
|
-
return self._discovery
|
|
186
|
-
|
|
187
|
-
def _ensure_mappings_cached(self) -> None:
|
|
188
|
-
"""Ensure mappings are cached (lazy initialization)."""
|
|
189
|
-
if self._cached:
|
|
190
|
-
return
|
|
191
|
-
|
|
192
|
-
with self._lock:
|
|
193
|
-
if self._cached:
|
|
194
|
-
return
|
|
195
|
-
|
|
196
|
-
discovery = self._get_discovery()
|
|
197
|
-
self._package_import_mapping = discovery.get_package_import_mapping()
|
|
198
|
-
self._import_package_mapping = discovery.get_import_package_mapping()
|
|
199
|
-
self._cached = True
|
|
200
|
-
|
|
201
|
-
def _ensure_manifest_cached(self, loader=None) -> None:
|
|
202
|
-
if loader is None:
|
|
203
|
-
loader = get_manifest_loader()
|
|
204
|
-
signature = loader.get_manifest_signature(self._package_name)
|
|
205
|
-
if signature == self._manifest_signature and (self._manifest_dependencies or self._manifest_empty):
|
|
206
|
-
return
|
|
207
|
-
|
|
208
|
-
shared = loader.get_shared_dependencies(self._package_name, signature)
|
|
209
|
-
if shared is not None:
|
|
210
|
-
self._manifest_generation = loader.generation
|
|
211
|
-
self._manifest_signature = signature
|
|
212
|
-
self._manifest_dependencies = shared
|
|
213
|
-
self._manifest_empty = len(shared) == 0
|
|
214
|
-
return
|
|
215
|
-
|
|
216
|
-
manifest = loader.get_manifest(self._package_name)
|
|
217
|
-
current_generation = loader.generation
|
|
218
|
-
|
|
219
|
-
dependencies: Dict[str, str] = {}
|
|
220
|
-
manifest_empty = True
|
|
221
|
-
if manifest and manifest.dependencies:
|
|
222
|
-
dependencies = {
|
|
223
|
-
key.lower(): value
|
|
224
|
-
for key, value in manifest.dependencies.items()
|
|
225
|
-
if key and value
|
|
226
|
-
}
|
|
227
|
-
manifest_empty = False
|
|
228
|
-
|
|
229
|
-
self._manifest_generation = current_generation
|
|
230
|
-
self._manifest_signature = signature
|
|
231
|
-
self._manifest_dependencies = dependencies
|
|
232
|
-
self._manifest_empty = manifest_empty
|
|
233
|
-
|
|
234
|
-
@staticmethod
|
|
235
|
-
def _is_stdlib_or_builtin(module_name: str) -> bool:
|
|
236
|
-
"""Return True if the module is built-in or part of the stdlib."""
|
|
237
|
-
root = module_name.split('.', 1)[0]
|
|
238
|
-
needs_cache = False
|
|
239
|
-
if module_name in _STDLIB_MODULE_SET or root in _STDLIB_MODULE_SET:
|
|
240
|
-
return True
|
|
241
|
-
if _cached_stdlib_check(module_name):
|
|
242
|
-
needs_cache = True
|
|
243
|
-
if needs_cache:
|
|
244
|
-
_cache_spec_if_missing(module_name)
|
|
245
|
-
return needs_cache
|
|
246
|
-
|
|
247
|
-
DENY_LIST: Set[str] = {
|
|
248
|
-
# POSIX-only modules that don't exist on Windows but try to auto-install
|
|
249
|
-
"pwd",
|
|
250
|
-
"grp",
|
|
251
|
-
"spwd",
|
|
252
|
-
"nis",
|
|
253
|
-
"termios",
|
|
254
|
-
"tty",
|
|
255
|
-
"pty",
|
|
256
|
-
"fcntl",
|
|
257
|
-
# Windows-only internals
|
|
258
|
-
"winreg",
|
|
259
|
-
"winsound",
|
|
260
|
-
"_winapi",
|
|
261
|
-
"_dbm",
|
|
262
|
-
# Internal optional modules that must never trigger auto-install
|
|
263
|
-
"compression",
|
|
264
|
-
"socks",
|
|
265
|
-
"wimlib",
|
|
266
|
-
# Optional dependencies with Python 2 compatibility shims (Python 3.8+ only)
|
|
267
|
-
"inspect2", # Python 2 compatibility shim, not needed on Python 3.8+
|
|
268
|
-
"rich", # Optional CLI enhancement for httpx, not required for core functionality
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
def _should_skip_auto_install(self, import_name: str) -> bool:
|
|
272
|
-
"""Determine whether an import should bypass lazy installation."""
|
|
273
|
-
if self._is_stdlib_or_builtin(import_name):
|
|
274
|
-
logger.debug("Skipping lazy install for stdlib module '%s'", import_name)
|
|
275
|
-
return True
|
|
276
|
-
|
|
277
|
-
if import_name in self.DENY_LIST:
|
|
278
|
-
logger.debug("Skipping lazy install for denied module '%s'", import_name)
|
|
279
|
-
return True
|
|
280
|
-
|
|
281
|
-
return False
|
|
282
|
-
|
|
283
|
-
def get_package_name(self, import_name: str) -> Optional[str]:
|
|
284
|
-
"""Get package name from import name."""
|
|
285
|
-
if self._should_skip_auto_install(import_name):
|
|
286
|
-
return None
|
|
287
|
-
|
|
288
|
-
if _spec_cache_get(import_name):
|
|
289
|
-
return None
|
|
290
|
-
|
|
291
|
-
loader = get_manifest_loader()
|
|
292
|
-
generation_changed = self._manifest_generation != loader.generation
|
|
293
|
-
manifest_uninitialized = not self._manifest_dependencies and not self._manifest_empty
|
|
294
|
-
if generation_changed or manifest_uninitialized:
|
|
295
|
-
self._ensure_manifest_cached(loader)
|
|
296
|
-
manifest_hit = self._manifest_dependencies.get(import_name.lower())
|
|
297
|
-
if manifest_hit:
|
|
298
|
-
return manifest_hit
|
|
299
|
-
|
|
300
|
-
self._ensure_mappings_cached()
|
|
301
|
-
return self._import_package_mapping.get(import_name, import_name)
|
|
302
|
-
|
|
303
|
-
def get_import_names(self, package_name: str) -> List[str]:
|
|
304
|
-
"""Get all possible import names for a package."""
|
|
305
|
-
self._ensure_mappings_cached()
|
|
306
|
-
return self._package_import_mapping.get(package_name, [package_name])
|
|
307
|
-
|
|
308
|
-
def get_package_import_mapping(self) -> Dict[str, List[str]]:
|
|
309
|
-
"""Get complete package to import names mapping."""
|
|
310
|
-
self._ensure_mappings_cached()
|
|
311
|
-
return self._package_import_mapping.copy()
|
|
312
|
-
|
|
313
|
-
def get_import_package_mapping(self) -> Dict[str, str]:
|
|
314
|
-
"""Get complete import to package name mapping."""
|
|
315
|
-
self._ensure_mappings_cached()
|
|
316
|
-
return self._import_package_mapping.copy()
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
class LazyDiscovery(APackageDiscovery):
|
|
320
|
-
"""
|
|
321
|
-
Discovers dependencies from project configuration sources.
|
|
322
|
-
Implements caching with file modification time checks.
|
|
323
|
-
"""
|
|
324
|
-
|
|
325
|
-
# System/built-in modules that should NEVER be auto-installed
|
|
326
|
-
SYSTEM_MODULES_BLACKLIST = {
|
|
327
|
-
'pwd', 'grp', 'spwd', 'crypt', 'nis', 'syslog', 'termios', 'tty', 'pty',
|
|
328
|
-
'fcntl', 'resource', 'msvcrt', 'winreg', 'winsound', '_winapi',
|
|
329
|
-
'rpython', 'rply', 'rnc2rng', '_dbm',
|
|
330
|
-
'sys', 'os', 'io', 'time', 'datetime', 'json', 'csv', 'math',
|
|
331
|
-
'random', 're', 'collections', 'itertools', 'functools', 'operator',
|
|
332
|
-
'pathlib', 'shutil', 'glob', 'tempfile', 'pickle', 'copy', 'types',
|
|
333
|
-
'typing', 'abc', 'enum', 'dataclasses', 'contextlib', 'warnings',
|
|
334
|
-
'logging', 'threading', 'multiprocessing', 'subprocess', 'queue',
|
|
335
|
-
'socket', 'select', 'signal', 'asyncio', 'concurrent', 'email',
|
|
336
|
-
'http', 'urllib', 'xml', 'html', 'sqlite3', 'base64', 'hashlib',
|
|
337
|
-
'hmac', 'secrets', 'ssl', 'binascii', 'struct', 'array', 'weakref',
|
|
338
|
-
'gc', 'inspect', 'traceback', 'atexit', 'codecs', 'locale', 'gettext',
|
|
339
|
-
'argparse', 'optparse', 'configparser', 'fileinput', 'stat', 'platform',
|
|
340
|
-
'unittest', 'doctest', 'pdb', 'profile', 'cProfile', 'timeit', 'trace',
|
|
341
|
-
# Internal / optional modules that must never trigger auto-install
|
|
342
|
-
'compression', 'socks', 'wimlib',
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
# Common import name to package name mappings
|
|
346
|
-
COMMON_MAPPINGS = {
|
|
347
|
-
'cv2': 'opencv-python',
|
|
348
|
-
'PIL': 'Pillow',
|
|
349
|
-
'Pillow': 'Pillow',
|
|
350
|
-
'yaml': 'PyYAML',
|
|
351
|
-
'sklearn': 'scikit-learn',
|
|
352
|
-
'bs4': 'beautifulsoup4',
|
|
353
|
-
'dateutil': 'python-dateutil',
|
|
354
|
-
'requests_oauthlib': 'requests-oauthlib',
|
|
355
|
-
'google': 'google-api-python-client',
|
|
356
|
-
'jwt': 'PyJWT',
|
|
357
|
-
'crypto': 'pycrypto',
|
|
358
|
-
'Crypto': 'pycrypto',
|
|
359
|
-
'MySQLdb': 'mysqlclient',
|
|
360
|
-
'psycopg2': 'psycopg2-binary',
|
|
361
|
-
'bson': 'pymongo',
|
|
362
|
-
'lxml': 'lxml',
|
|
363
|
-
'numpy': 'numpy',
|
|
364
|
-
'pandas': 'pandas',
|
|
365
|
-
'matplotlib': 'matplotlib',
|
|
366
|
-
'seaborn': 'seaborn',
|
|
367
|
-
'plotly': 'plotly',
|
|
368
|
-
'django': 'Django',
|
|
369
|
-
'flask': 'Flask',
|
|
370
|
-
'fastapi': 'fastapi',
|
|
371
|
-
'uvicorn': 'uvicorn',
|
|
372
|
-
'pytest': 'pytest',
|
|
373
|
-
'black': 'black',
|
|
374
|
-
'isort': 'isort',
|
|
375
|
-
'mypy': 'mypy',
|
|
376
|
-
'psutil': 'psutil',
|
|
377
|
-
'colorama': 'colorama',
|
|
378
|
-
'pytz': 'pytz',
|
|
379
|
-
'aiofiles': 'aiofiles',
|
|
380
|
-
'watchdog': 'watchdog',
|
|
381
|
-
'wand': 'Wand',
|
|
382
|
-
'exifread': 'ExifRead',
|
|
383
|
-
'piexif': 'piexif',
|
|
384
|
-
'rawpy': 'rawpy',
|
|
385
|
-
'imageio': 'imageio',
|
|
386
|
-
'scipy': 'scipy',
|
|
387
|
-
'scikit-image': 'scikit-image',
|
|
388
|
-
'opencv-python': 'opencv-python',
|
|
389
|
-
'opencv-contrib-python': 'opencv-contrib-python',
|
|
390
|
-
'opentelemetry': 'opentelemetry-api',
|
|
391
|
-
'opentelemetry.trace': 'opentelemetry-api',
|
|
392
|
-
'opentelemetry.sdk': 'opentelemetry-sdk',
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
def _discover_from_sources(self) -> None:
|
|
396
|
-
"""Discover dependencies from all sources."""
|
|
397
|
-
self._discover_from_pyproject_toml()
|
|
398
|
-
self._discover_from_requirements_txt()
|
|
399
|
-
self._discover_from_setup_py()
|
|
400
|
-
self._discover_from_custom_config()
|
|
401
|
-
|
|
402
|
-
def _is_cache_valid(self) -> bool:
|
|
403
|
-
"""Check if cached dependencies are still valid."""
|
|
404
|
-
if not self._cache_valid or not self._cached_dependencies:
|
|
405
|
-
return False
|
|
406
|
-
|
|
407
|
-
config_files = [
|
|
408
|
-
self.project_root / 'pyproject.toml',
|
|
409
|
-
self.project_root / 'requirements.txt',
|
|
410
|
-
self.project_root / 'setup.py',
|
|
411
|
-
]
|
|
412
|
-
|
|
413
|
-
for config_file in config_files:
|
|
414
|
-
if config_file.exists():
|
|
415
|
-
try:
|
|
416
|
-
current_mtime = config_file.stat().st_mtime
|
|
417
|
-
cached_mtime = self._file_mtimes.get(str(config_file), 0)
|
|
418
|
-
if current_mtime > cached_mtime:
|
|
419
|
-
return False
|
|
420
|
-
except:
|
|
421
|
-
return False
|
|
422
|
-
|
|
423
|
-
return True
|
|
424
|
-
|
|
425
|
-
def _update_file_mtimes(self) -> None:
|
|
426
|
-
"""Update file modification times for cache validation."""
|
|
427
|
-
config_files = [
|
|
428
|
-
self.project_root / 'pyproject.toml',
|
|
429
|
-
self.project_root / 'requirements.txt',
|
|
430
|
-
self.project_root / 'setup.py',
|
|
431
|
-
]
|
|
432
|
-
for config_file in config_files:
|
|
433
|
-
if config_file.exists():
|
|
434
|
-
try:
|
|
435
|
-
self._file_mtimes[str(config_file)] = config_file.stat().st_mtime
|
|
436
|
-
except:
|
|
437
|
-
pass
|
|
438
|
-
|
|
439
|
-
def _discover_from_pyproject_toml(self) -> None:
|
|
440
|
-
"""Discover dependencies from pyproject.toml."""
|
|
441
|
-
pyproject_path = self.project_root / 'pyproject.toml'
|
|
442
|
-
if not pyproject_path.exists():
|
|
443
|
-
return
|
|
444
|
-
|
|
445
|
-
try:
|
|
446
|
-
try:
|
|
447
|
-
import tomllib # Python 3.11+
|
|
448
|
-
toml_parser = tomllib # type: ignore[assignment]
|
|
449
|
-
except ImportError:
|
|
450
|
-
try:
|
|
451
|
-
import tomli as tomllib # type: ignore[assignment]
|
|
452
|
-
toml_parser = tomllib
|
|
453
|
-
except ImportError:
|
|
454
|
-
_log(
|
|
455
|
-
"discovery",
|
|
456
|
-
"TOML parser not available; attempting to lazy-install 'tomli'...",
|
|
457
|
-
)
|
|
458
|
-
try:
|
|
459
|
-
subprocess.run(
|
|
460
|
-
[sys.executable, "-m", "pip", "install", "tomli"],
|
|
461
|
-
check=False,
|
|
462
|
-
capture_output=True,
|
|
463
|
-
)
|
|
464
|
-
import tomli as tomllib # type: ignore[assignment]
|
|
465
|
-
toml_parser = tomllib
|
|
466
|
-
except Exception as install_exc:
|
|
467
|
-
logger.warning(
|
|
468
|
-
"tomli installation failed; skipping pyproject.toml discovery "
|
|
469
|
-
f"({install_exc})"
|
|
470
|
-
)
|
|
471
|
-
return
|
|
472
|
-
|
|
473
|
-
with open(pyproject_path, 'rb') as f:
|
|
474
|
-
data = toml_parser.load(f)
|
|
475
|
-
|
|
476
|
-
dependencies = []
|
|
477
|
-
if 'project' in data and 'dependencies' in data['project']:
|
|
478
|
-
dependencies.extend(data['project']['dependencies'])
|
|
479
|
-
|
|
480
|
-
if 'project' in data and 'optional-dependencies' in data['project']:
|
|
481
|
-
for group_name, group_deps in data['project']['optional-dependencies'].items():
|
|
482
|
-
dependencies.extend(group_deps)
|
|
483
|
-
|
|
484
|
-
if 'build-system' in data and 'requires' in data['build-system']:
|
|
485
|
-
dependencies.extend(data['build-system']['requires'])
|
|
486
|
-
|
|
487
|
-
for dep in dependencies:
|
|
488
|
-
self._parse_dependency_string(dep, 'pyproject.toml')
|
|
489
|
-
|
|
490
|
-
self._discovery_sources.append('pyproject.toml')
|
|
491
|
-
except Exception as e:
|
|
492
|
-
logger.warning(f"Could not parse pyproject.toml: {e}")
|
|
493
|
-
|
|
494
|
-
def _discover_from_requirements_txt(self) -> None:
|
|
495
|
-
"""Discover dependencies from requirements.txt."""
|
|
496
|
-
requirements_path = self.project_root / 'requirements.txt'
|
|
497
|
-
if not requirements_path.exists():
|
|
498
|
-
return
|
|
499
|
-
|
|
500
|
-
try:
|
|
501
|
-
with open(requirements_path, 'r', encoding='utf-8') as f:
|
|
502
|
-
for line in f:
|
|
503
|
-
line = line.strip()
|
|
504
|
-
if line and not line.startswith('#'):
|
|
505
|
-
self._parse_dependency_string(line, 'requirements.txt')
|
|
506
|
-
|
|
507
|
-
self._discovery_sources.append('requirements.txt')
|
|
508
|
-
except Exception as e:
|
|
509
|
-
logger.warning(f"Could not parse requirements.txt: {e}")
|
|
510
|
-
|
|
511
|
-
def _discover_from_setup_py(self) -> None:
|
|
512
|
-
"""Discover dependencies from setup.py."""
|
|
513
|
-
setup_path = self.project_root / 'setup.py'
|
|
514
|
-
if not setup_path.exists():
|
|
515
|
-
return
|
|
516
|
-
|
|
517
|
-
try:
|
|
518
|
-
with open(setup_path, 'r', encoding='utf-8') as f:
|
|
519
|
-
content = f.read()
|
|
520
|
-
|
|
521
|
-
install_requires_match = re.search(
|
|
522
|
-
r'install_requires\s*=\s*\[(.*?)\]',
|
|
523
|
-
content,
|
|
524
|
-
re.DOTALL
|
|
525
|
-
)
|
|
526
|
-
if install_requires_match:
|
|
527
|
-
deps_str = install_requires_match.group(1)
|
|
528
|
-
deps = re.findall(r'["\']([^"\']+)["\']', deps_str)
|
|
529
|
-
for dep in deps:
|
|
530
|
-
self._parse_dependency_string(dep, 'setup.py')
|
|
531
|
-
|
|
532
|
-
self._discovery_sources.append('setup.py')
|
|
533
|
-
except Exception as e:
|
|
534
|
-
logger.warning(f"Could not parse setup.py: {e}")
|
|
535
|
-
|
|
536
|
-
def _discover_from_custom_config(self) -> None:
|
|
537
|
-
"""Discover dependencies from custom configuration files."""
|
|
538
|
-
config_files = [
|
|
539
|
-
'dependency-mappings.json',
|
|
540
|
-
'lazy-dependencies.json',
|
|
541
|
-
'dependencies.json'
|
|
542
|
-
]
|
|
543
|
-
|
|
544
|
-
for config_file in config_files:
|
|
545
|
-
config_path = self.project_root / config_file
|
|
546
|
-
if config_path.exists():
|
|
547
|
-
try:
|
|
548
|
-
with open(config_path, 'r', encoding='utf-8') as f:
|
|
549
|
-
data = json.load(f)
|
|
550
|
-
|
|
551
|
-
if isinstance(data, dict):
|
|
552
|
-
for import_name, package_name in data.items():
|
|
553
|
-
self.discovered_dependencies[import_name] = DependencyInfo(
|
|
554
|
-
import_name=import_name,
|
|
555
|
-
package_name=package_name,
|
|
556
|
-
source=config_file,
|
|
557
|
-
category='custom'
|
|
558
|
-
)
|
|
559
|
-
|
|
560
|
-
self._discovery_sources.append(config_file)
|
|
561
|
-
except Exception as e:
|
|
562
|
-
logger.warning(f"Could not parse {config_file}: {e}")
|
|
563
|
-
|
|
564
|
-
def _parse_dependency_string(self, dep_str: str, source: str) -> None:
|
|
565
|
-
"""Parse a dependency string and extract dependency information."""
|
|
566
|
-
dep_str = re.sub(r'[>=<!=~]+.*', '', dep_str)
|
|
567
|
-
dep_str = re.sub(r'\[.*\]', '', dep_str)
|
|
568
|
-
dep_str = dep_str.strip()
|
|
569
|
-
|
|
570
|
-
if not dep_str:
|
|
571
|
-
return
|
|
572
|
-
|
|
573
|
-
import_name = dep_str
|
|
574
|
-
package_name = dep_str
|
|
575
|
-
|
|
576
|
-
if dep_str in self.COMMON_MAPPINGS:
|
|
577
|
-
package_name = self.COMMON_MAPPINGS[dep_str]
|
|
578
|
-
elif dep_str in self.COMMON_MAPPINGS.values():
|
|
579
|
-
for imp_name, pkg_name in self.COMMON_MAPPINGS.items():
|
|
580
|
-
if pkg_name == dep_str:
|
|
581
|
-
import_name = imp_name
|
|
582
|
-
break
|
|
583
|
-
|
|
584
|
-
self.discovered_dependencies[import_name] = DependencyInfo(
|
|
585
|
-
import_name=import_name,
|
|
586
|
-
package_name=package_name,
|
|
587
|
-
source=source,
|
|
588
|
-
category='discovered'
|
|
589
|
-
)
|
|
590
|
-
|
|
591
|
-
def _add_common_mappings(self) -> None:
|
|
592
|
-
"""Add common mappings that might not be in dependency files."""
|
|
593
|
-
for import_name, package_name in self.COMMON_MAPPINGS.items():
|
|
594
|
-
if import_name not in self.discovered_dependencies:
|
|
595
|
-
self.discovered_dependencies[import_name] = DependencyInfo(
|
|
596
|
-
import_name=import_name,
|
|
597
|
-
package_name=package_name,
|
|
598
|
-
source='common_mappings',
|
|
599
|
-
category='common'
|
|
600
|
-
)
|
|
601
|
-
|
|
602
|
-
def get_package_for_import(self, import_name: str) -> Optional[str]:
|
|
603
|
-
"""Get package name for a given import name."""
|
|
604
|
-
mapping = self.discover_all_dependencies()
|
|
605
|
-
return mapping.get(import_name)
|
|
606
|
-
|
|
607
|
-
def get_imports_for_package(self, package_name: str) -> List[str]:
|
|
608
|
-
"""Get all possible import names for a package."""
|
|
609
|
-
mapping = self.get_package_import_mapping()
|
|
610
|
-
return mapping.get(package_name, [package_name])
|
|
611
|
-
|
|
612
|
-
def get_package_import_mapping(self) -> Dict[str, List[str]]:
|
|
613
|
-
"""Get mapping of package names to their possible import names."""
|
|
614
|
-
self.discover_all_dependencies()
|
|
615
|
-
|
|
616
|
-
package_to_imports = {}
|
|
617
|
-
for import_name, dep_info in self.discovered_dependencies.items():
|
|
618
|
-
package_name = dep_info.package_name
|
|
619
|
-
|
|
620
|
-
if package_name not in package_to_imports:
|
|
621
|
-
package_to_imports[package_name] = [package_name]
|
|
622
|
-
|
|
623
|
-
if import_name != package_name:
|
|
624
|
-
if import_name not in package_to_imports[package_name]:
|
|
625
|
-
package_to_imports[package_name].append(import_name)
|
|
626
|
-
|
|
627
|
-
return package_to_imports
|
|
628
|
-
|
|
629
|
-
def get_import_package_mapping(self) -> Dict[str, str]:
|
|
630
|
-
"""Get mapping of import names to package names."""
|
|
631
|
-
self.discover_all_dependencies()
|
|
632
|
-
return {import_name: dep_info.package_name for import_name, dep_info in self.discovered_dependencies.items()}
|
|
633
|
-
|
|
634
|
-
def export_to_json(self, file_path: str) -> None:
|
|
635
|
-
"""Export discovered dependencies to JSON file."""
|
|
636
|
-
data = {
|
|
637
|
-
'dependencies': {name: info.package_name for name, info in self.discovered_dependencies.items()},
|
|
638
|
-
'sources': self.get_discovery_sources(),
|
|
639
|
-
'total_count': len(self.discovered_dependencies)
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
with open(file_path, 'w', encoding='utf-8') as f:
|
|
643
|
-
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
# Global discovery instance
|
|
647
|
-
_discovery = None
|
|
648
|
-
_discovery_lock = threading.RLock()
|
|
649
|
-
|
|
650
|
-
def get_lazy_discovery(project_root: Optional[str] = None) -> LazyDiscovery:
|
|
651
|
-
"""Get the global lazy discovery instance."""
|
|
652
|
-
global _discovery
|
|
653
|
-
if _discovery is None:
|
|
654
|
-
with _discovery_lock:
|
|
655
|
-
if _discovery is None:
|
|
656
|
-
_discovery = LazyDiscovery(project_root)
|
|
657
|
-
return _discovery
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
def discover_dependencies(project_root: Optional[str] = None) -> Dict[str, str]:
|
|
661
|
-
"""Discover all dependencies for the current project."""
|
|
662
|
-
discovery = get_lazy_discovery(project_root)
|
|
663
|
-
return discovery.discover_all_dependencies()
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
def export_dependency_mappings(file_path: str, project_root: Optional[str] = None) -> None:
|
|
667
|
-
"""Export discovered dependency mappings to a JSON file."""
|
|
668
|
-
discovery = get_lazy_discovery(project_root)
|
|
669
|
-
discovery.export_to_json(file_path)
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
# =============================================================================
|
|
673
|
-
# SECTION 2: PACKAGE INSTALLATION (~550 lines)
|
|
674
|
-
# =============================================================================
|
|
675
|
-
|
|
676
|
-
def _is_externally_managed() -> bool:
|
|
677
|
-
"""Check if Python environment is externally managed (PEP 668)."""
|
|
678
|
-
marker_file = Path(sys.prefix) / "EXTERNALLY-MANAGED"
|
|
679
|
-
return marker_file.exists()
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
def _check_pip_audit_available() -> bool:
|
|
683
|
-
"""Check if pip-audit is available for vulnerability scanning."""
|
|
684
|
-
try:
|
|
685
|
-
result = subprocess.run(
|
|
686
|
-
[sys.executable, '-m', 'pip', 'list'],
|
|
687
|
-
capture_output=True,
|
|
688
|
-
text=True,
|
|
689
|
-
timeout=5
|
|
690
|
-
)
|
|
691
|
-
return 'pip-audit' in result.stdout
|
|
692
|
-
except Exception:
|
|
693
|
-
return False
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
class LazyInstallPolicy:
|
|
697
|
-
"""
|
|
698
|
-
Security and policy configuration for lazy installation.
|
|
699
|
-
Per-package allow/deny lists, index URLs, and security settings.
|
|
700
|
-
"""
|
|
701
|
-
__slots__ = ()
|
|
702
|
-
|
|
703
|
-
_allow_lists: Dict[str, Set[str]] = {}
|
|
704
|
-
_deny_lists: Dict[str, Set[str]] = {}
|
|
705
|
-
_index_urls: Dict[str, str] = {}
|
|
706
|
-
_extra_index_urls: Dict[str, List[str]] = {}
|
|
707
|
-
_trusted_hosts: Dict[str, List[str]] = {}
|
|
708
|
-
_require_hashes: Dict[str, bool] = {}
|
|
709
|
-
_verify_ssl: Dict[str, bool] = {}
|
|
710
|
-
_lockfile_paths: Dict[str, str] = {}
|
|
711
|
-
_lock = threading.RLock()
|
|
712
|
-
|
|
713
|
-
@classmethod
|
|
714
|
-
def set_allow_list(cls, package_name: str, allowed_packages: List[str]) -> None:
|
|
715
|
-
"""Set allow list for a package (only these can be installed)."""
|
|
716
|
-
with cls._lock:
|
|
717
|
-
cls._allow_lists[package_name] = set(allowed_packages)
|
|
718
|
-
_log("config", f"Set allow list for {package_name}: {len(allowed_packages)} packages")
|
|
719
|
-
|
|
720
|
-
@classmethod
|
|
721
|
-
def set_deny_list(cls, package_name: str, denied_packages: List[str]) -> None:
|
|
722
|
-
"""Set deny list for a package (these cannot be installed)."""
|
|
723
|
-
with cls._lock:
|
|
724
|
-
cls._deny_lists[package_name] = set(denied_packages)
|
|
725
|
-
_log("config", f"Set deny list for {package_name}: {len(denied_packages)} packages")
|
|
726
|
-
|
|
727
|
-
@classmethod
|
|
728
|
-
def add_to_allow_list(cls, package_name: str, allowed_package: str) -> None:
|
|
729
|
-
"""Add single package to allow list."""
|
|
730
|
-
with cls._lock:
|
|
731
|
-
if package_name not in cls._allow_lists:
|
|
732
|
-
cls._allow_lists[package_name] = set()
|
|
733
|
-
cls._allow_lists[package_name].add(allowed_package)
|
|
734
|
-
|
|
735
|
-
@classmethod
|
|
736
|
-
def add_to_deny_list(cls, package_name: str, denied_package: str) -> None:
|
|
737
|
-
"""Add single package to deny list."""
|
|
738
|
-
with cls._lock:
|
|
739
|
-
if package_name not in cls._deny_lists:
|
|
740
|
-
cls._deny_lists[package_name] = set()
|
|
741
|
-
cls._deny_lists[package_name].add(denied_package)
|
|
742
|
-
|
|
743
|
-
@classmethod
|
|
744
|
-
def is_package_allowed(cls, installer_package: str, target_package: str) -> Tuple[bool, str]:
|
|
745
|
-
"""Check if target_package can be installed by installer_package."""
|
|
746
|
-
with cls._lock:
|
|
747
|
-
if installer_package in cls._deny_lists:
|
|
748
|
-
if target_package in cls._deny_lists[installer_package]:
|
|
749
|
-
return False, f"Package '{target_package}' is in deny list"
|
|
750
|
-
|
|
751
|
-
if installer_package in cls._allow_lists:
|
|
752
|
-
if target_package not in cls._allow_lists[installer_package]:
|
|
753
|
-
return False, f"Package '{target_package}' not in allow list"
|
|
754
|
-
|
|
755
|
-
return True, "OK"
|
|
756
|
-
|
|
757
|
-
@classmethod
|
|
758
|
-
def set_index_url(cls, package_name: str, index_url: str) -> None:
|
|
759
|
-
"""Set PyPI index URL for a package."""
|
|
760
|
-
with cls._lock:
|
|
761
|
-
cls._index_urls[package_name] = index_url
|
|
762
|
-
_log("config", f"Set index URL for {package_name}: {index_url}")
|
|
763
|
-
|
|
764
|
-
@classmethod
|
|
765
|
-
def set_extra_index_urls(cls, package_name: str, urls: List[str]) -> None:
|
|
766
|
-
"""Set extra index URLs for a package."""
|
|
767
|
-
with cls._lock:
|
|
768
|
-
cls._extra_index_urls[package_name] = urls
|
|
769
|
-
_log("config", f"Set {len(urls)} extra index URLs for {package_name}")
|
|
770
|
-
|
|
771
|
-
@classmethod
|
|
772
|
-
def add_trusted_host(cls, package_name: str, host: str) -> None:
|
|
773
|
-
"""Add trusted host for a package."""
|
|
774
|
-
with cls._lock:
|
|
775
|
-
if package_name not in cls._trusted_hosts:
|
|
776
|
-
cls._trusted_hosts[package_name] = []
|
|
777
|
-
cls._trusted_hosts[package_name].append(host)
|
|
778
|
-
|
|
779
|
-
@classmethod
|
|
780
|
-
def get_pip_args(cls, package_name: str) -> List[str]:
|
|
781
|
-
"""Get pip install arguments for a package based on policy."""
|
|
782
|
-
args = []
|
|
783
|
-
|
|
784
|
-
with cls._lock:
|
|
785
|
-
if package_name in cls._index_urls:
|
|
786
|
-
args.extend(['--index-url', cls._index_urls[package_name]])
|
|
787
|
-
|
|
788
|
-
if package_name in cls._extra_index_urls:
|
|
789
|
-
for url in cls._extra_index_urls[package_name]:
|
|
790
|
-
args.extend(['--extra-index-url', url])
|
|
791
|
-
|
|
792
|
-
if package_name in cls._trusted_hosts:
|
|
793
|
-
for host in cls._trusted_hosts[package_name]:
|
|
794
|
-
args.extend(['--trusted-host', host])
|
|
795
|
-
|
|
796
|
-
if cls._require_hashes.get(package_name, False):
|
|
797
|
-
args.append('--require-hashes')
|
|
798
|
-
|
|
799
|
-
if not cls._verify_ssl.get(package_name, True):
|
|
800
|
-
args.append('--no-verify-ssl')
|
|
801
|
-
|
|
802
|
-
return args
|
|
803
|
-
|
|
804
|
-
@classmethod
|
|
805
|
-
def set_lockfile_path(cls, package_name: str, path: str) -> None:
|
|
806
|
-
"""Set lockfile path for a package."""
|
|
807
|
-
with cls._lock:
|
|
808
|
-
cls._lockfile_paths[package_name] = path
|
|
809
|
-
|
|
810
|
-
@classmethod
|
|
811
|
-
def get_lockfile_path(cls, package_name: str) -> Optional[str]:
|
|
812
|
-
"""Get lockfile path for a package."""
|
|
813
|
-
with cls._lock:
|
|
814
|
-
return cls._lockfile_paths.get(package_name)
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
_ENV_ASYNC_INSTALL = os.environ.get("XWLAZY_ASYNC_INSTALL", "").strip().lower() in {"1", "true", "yes", "on"}
|
|
818
|
-
_ENV_ASYNC_WORKERS = int(os.environ.get("XWLAZY_ASYNC_WORKERS", "0") or 0)
|
|
819
|
-
|
|
820
|
-
_SPEC_CACHE_MAX = int(os.environ.get("XWLAZY_SPEC_CACHE_MAX", "512") or 512)
|
|
821
|
-
_SPEC_CACHE_TTL = float(os.environ.get("XWLAZY_SPEC_CACHE_TTL", "60") or 60.0)
|
|
822
|
-
_spec_cache_lock = threading.RLock()
|
|
823
|
-
_spec_cache: "OrderedDict[str, Tuple[importlib.machinery.ModuleSpec, float]]" = OrderedDict()
|
|
824
|
-
|
|
825
|
-
_DEFAULT_ASYNC_CACHE_DIR = Path(
|
|
826
|
-
os.environ.get(
|
|
827
|
-
"XWLAZY_ASYNC_CACHE_DIR",
|
|
828
|
-
os.path.join(os.path.expanduser("~"), ".xwlazy", "wheel-cache"),
|
|
829
|
-
)
|
|
830
|
-
)
|
|
831
|
-
_KNOWN_MISSING_CACHE_LIMIT = int(os.environ.get("XWLAZY_MISSING_CACHE_MAX", "128") or 128)
|
|
832
|
-
_KNOWN_MISSING_CACHE_TTL = float(os.environ.get("XWLAZY_MISSING_CACHE_TTL", "120") or 120.0)
|
|
833
|
-
_WRAPPED_CLASS_CACHE: Dict[str, Set[str]] = defaultdict(set)
|
|
834
|
-
_wrapped_cache_lock = threading.RLock()
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
class LazyInstaller(APackageInstaller):
|
|
838
|
-
"""
|
|
839
|
-
Lazy installer that automatically installs missing packages on import failure.
|
|
840
|
-
Each instance is isolated per package to prevent interference.
|
|
841
|
-
"""
|
|
842
|
-
|
|
843
|
-
__slots__ = APackageInstaller.__slots__ + (
|
|
844
|
-
'_dependency_mapper',
|
|
845
|
-
'_auto_approve_all',
|
|
846
|
-
'_async_enabled',
|
|
847
|
-
'_async_workers',
|
|
848
|
-
'_async_executor',
|
|
849
|
-
'_async_pending',
|
|
850
|
-
'_known_missing',
|
|
851
|
-
'_async_cache_dir',
|
|
852
|
-
)
|
|
853
|
-
|
|
854
|
-
def __init__(self, package_name: str = 'default'):
|
|
855
|
-
"""Initialize lazy installer for a specific package."""
|
|
856
|
-
super().__init__(package_name)
|
|
857
|
-
self._dependency_mapper = DependencyMapper(package_name)
|
|
858
|
-
self._auto_approve_all = False
|
|
859
|
-
self._async_enabled = False
|
|
860
|
-
self._async_workers = 1
|
|
861
|
-
self._async_executor: Optional[ThreadPoolExecutor] = None
|
|
862
|
-
self._async_pending: Dict[str, Future] = {}
|
|
863
|
-
self._known_missing: "OrderedDict[str, float]" = OrderedDict()
|
|
864
|
-
self._async_cache_dir = _DEFAULT_ASYNC_CACHE_DIR
|
|
865
|
-
|
|
866
|
-
def _ask_user_permission(self, package_name: str, module_name: str) -> bool:
|
|
867
|
-
"""Ask user for permission to install a package."""
|
|
868
|
-
if self._auto_approve_all:
|
|
869
|
-
return True
|
|
870
|
-
|
|
871
|
-
print(f"\n{'='*60}")
|
|
872
|
-
print(f"Lazy Installation Active - {self._package_name}")
|
|
873
|
-
print(f"{'='*60}")
|
|
874
|
-
print(f"Package: {package_name}")
|
|
875
|
-
print(f"Module: {module_name}")
|
|
876
|
-
print(f"{'='*60}")
|
|
877
|
-
print(f"\nThe module '{module_name}' is not installed.")
|
|
878
|
-
print(f"Would you like to install '{package_name}'?")
|
|
879
|
-
print(f"\nOptions:")
|
|
880
|
-
print(f" [Y] Yes - Install this package")
|
|
881
|
-
print(f" [N] No - Skip this package")
|
|
882
|
-
print(f" [A] All - Install this and all future packages without asking")
|
|
883
|
-
print(f" [Q] Quit - Cancel and raise ImportError")
|
|
884
|
-
print(f"{'='*60}")
|
|
885
|
-
|
|
886
|
-
while True:
|
|
887
|
-
try:
|
|
888
|
-
choice = input("Your choice [Y/N/A/Q]: ").strip().upper()
|
|
889
|
-
|
|
890
|
-
if choice in ('Y', 'YES', ''):
|
|
891
|
-
return True
|
|
892
|
-
elif choice in ('N', 'NO'):
|
|
893
|
-
return False
|
|
894
|
-
elif choice in ('A', 'ALL'):
|
|
895
|
-
self._auto_approve_all = True
|
|
896
|
-
return True
|
|
897
|
-
elif choice in ('Q', 'QUIT'):
|
|
898
|
-
raise KeyboardInterrupt("User cancelled installation")
|
|
899
|
-
else:
|
|
900
|
-
print(f"Invalid choice '{choice}'. Please enter Y, N, A, or Q.")
|
|
901
|
-
except (EOFError, KeyboardInterrupt):
|
|
902
|
-
print("\n❌ Installation cancelled by user")
|
|
903
|
-
return False
|
|
904
|
-
|
|
905
|
-
def install_package(self, package_name: str, module_name: str = None) -> bool:
|
|
906
|
-
"""Install a package using pip."""
|
|
907
|
-
with self._lock:
|
|
908
|
-
if package_name in self._installed_packages:
|
|
909
|
-
return True
|
|
910
|
-
|
|
911
|
-
if package_name in self._failed_packages:
|
|
912
|
-
return False
|
|
913
|
-
|
|
914
|
-
if self._mode == LazyInstallMode.DISABLED:
|
|
915
|
-
_log("install", f"Lazy installation disabled for {self._package_name}, skipping {package_name}")
|
|
916
|
-
return False
|
|
917
|
-
|
|
918
|
-
if self._mode == LazyInstallMode.WARN:
|
|
919
|
-
logger.warning(f"[WARN] Package '{package_name}' is missing but WARN mode is active - not installing")
|
|
920
|
-
print(f"[WARN] ({self._package_name}): Package '{package_name}' is missing (not installed in WARN mode)")
|
|
921
|
-
return False
|
|
922
|
-
|
|
923
|
-
if self._mode == LazyInstallMode.DRY_RUN:
|
|
924
|
-
print(f"[DRY RUN] ({self._package_name}): Would install package '{package_name}'")
|
|
925
|
-
return False
|
|
926
|
-
|
|
927
|
-
if self._mode == LazyInstallMode.INTERACTIVE:
|
|
928
|
-
if not self._ask_user_permission(package_name, module_name or package_name):
|
|
929
|
-
_log("install", f"User declined installation of {package_name}")
|
|
930
|
-
self._failed_packages.add(package_name)
|
|
931
|
-
return False
|
|
932
|
-
|
|
933
|
-
# Security checks
|
|
934
|
-
if _is_externally_managed():
|
|
935
|
-
logger.error(f"Cannot install {package_name}: Environment is externally managed (PEP 668)")
|
|
936
|
-
print(f"\n[ERROR] This Python environment is externally managed (PEP 668)")
|
|
937
|
-
print(f"Package '{package_name}' cannot be installed in this environment.")
|
|
938
|
-
print(f"\nSuggested solutions:")
|
|
939
|
-
print(f" 1. Create a virtual environment:")
|
|
940
|
-
print(f" python -m venv .venv")
|
|
941
|
-
print(f" .venv\\Scripts\\activate # Windows")
|
|
942
|
-
print(f" source .venv/bin/activate # Linux/macOS")
|
|
943
|
-
print(f" 2. Use pipx for isolated installs:")
|
|
944
|
-
print(f" pipx install {package_name}")
|
|
945
|
-
print(f" 3. Override with --break-system-packages (NOT RECOMMENDED)\n")
|
|
946
|
-
self._failed_packages.add(package_name)
|
|
947
|
-
return False
|
|
948
|
-
|
|
949
|
-
allowed, reason = LazyInstallPolicy.is_package_allowed(self._package_name, package_name)
|
|
950
|
-
if not allowed:
|
|
951
|
-
logger.error(f"Cannot install {package_name}: {reason}")
|
|
952
|
-
print(f"\n[SECURITY] Package '{package_name}' blocked: {reason}\n")
|
|
953
|
-
self._failed_packages.add(package_name)
|
|
954
|
-
return False
|
|
955
|
-
|
|
956
|
-
# Show warning about missing library with trigger file
|
|
957
|
-
trigger_file = _get_trigger_file()
|
|
958
|
-
module_display = module_name or package_name
|
|
959
|
-
if trigger_file:
|
|
960
|
-
# Get the module name used (e.g., 'bson' from 'pymongo')
|
|
961
|
-
used_for = module_display if module_display != package_name else package_name
|
|
962
|
-
print_formatted("WARN", f"Missing library {package_name} used for ({used_for}) triggered by {trigger_file}", same_line=True)
|
|
963
|
-
else:
|
|
964
|
-
print_formatted("WARN", f"Missing library {package_name} used for ({module_display})", same_line=True)
|
|
965
|
-
|
|
966
|
-
# Proceed with installation
|
|
967
|
-
try:
|
|
968
|
-
print_formatted("INFO", f"Installing package: {package_name}", same_line=True)
|
|
969
|
-
policy_args = LazyInstallPolicy.get_pip_args(self._package_name) or []
|
|
970
|
-
|
|
971
|
-
cache_args = list(policy_args)
|
|
972
|
-
if self._install_from_cached_tree(package_name):
|
|
973
|
-
print_formatted("ACTION", f"Installing {package_name} via pip...", same_line=True)
|
|
974
|
-
time.sleep(0.1) # Brief pause for visual effect
|
|
975
|
-
self._finalize_install_success(package_name, "cache-tree")
|
|
976
|
-
return True
|
|
977
|
-
|
|
978
|
-
if self._install_from_cached_wheel(package_name, cache_args):
|
|
979
|
-
print_formatted("ACTION", f"Installing {package_name} via pip...", same_line=True)
|
|
980
|
-
wheel_path = self._cached_wheel_name(package_name)
|
|
981
|
-
self._materialize_cached_tree(package_name, wheel_path)
|
|
982
|
-
time.sleep(0.1) # Brief pause for visual effect
|
|
983
|
-
self._finalize_install_success(package_name, "cache")
|
|
984
|
-
return True
|
|
985
|
-
|
|
986
|
-
wheel_path = self._ensure_cached_wheel(package_name, cache_args)
|
|
987
|
-
if wheel_path and self._pip_install_from_path(wheel_path, cache_args):
|
|
988
|
-
print_formatted("ACTION", f"Installing {package_name} via pip...", same_line=True)
|
|
989
|
-
self._materialize_cached_tree(package_name, wheel_path)
|
|
990
|
-
time.sleep(0.1) # Brief pause for visual effect
|
|
991
|
-
self._finalize_install_success(package_name, "wheel")
|
|
992
|
-
return True
|
|
993
|
-
|
|
994
|
-
# Show installation message with animated dots
|
|
995
|
-
print_formatted("ACTION", f"Installing {package_name} via pip...", same_line=True)
|
|
996
|
-
|
|
997
|
-
# Animate dots while installing
|
|
998
|
-
stop_animation = threading.Event()
|
|
999
|
-
|
|
1000
|
-
def animate_dots():
|
|
1001
|
-
dots = ["", ".", "..", "..."]
|
|
1002
|
-
i = 0
|
|
1003
|
-
while not stop_animation.is_set():
|
|
1004
|
-
msg = f"Installing {package_name} via pip{dots[i % len(dots)]}"
|
|
1005
|
-
print_formatted("ACTION", msg, same_line=True)
|
|
1006
|
-
i += 1
|
|
1007
|
-
time.sleep(0.3)
|
|
1008
|
-
|
|
1009
|
-
animator = threading.Thread(target=animate_dots, daemon=True)
|
|
1010
|
-
animator.start()
|
|
1011
|
-
|
|
1012
|
-
try:
|
|
1013
|
-
pip_args = [sys.executable, '-m', 'pip', 'install']
|
|
1014
|
-
if policy_args:
|
|
1015
|
-
pip_args.extend(policy_args)
|
|
1016
|
-
logger.debug(f"Using policy args: {policy_args}")
|
|
1017
|
-
|
|
1018
|
-
pip_args.append(package_name)
|
|
1019
|
-
|
|
1020
|
-
result = subprocess.run(
|
|
1021
|
-
pip_args,
|
|
1022
|
-
capture_output=True,
|
|
1023
|
-
text=True,
|
|
1024
|
-
check=True
|
|
1025
|
-
)
|
|
1026
|
-
finally:
|
|
1027
|
-
stop_animation.set()
|
|
1028
|
-
animator.join(timeout=0.5)
|
|
1029
|
-
|
|
1030
|
-
self._finalize_install_success(package_name, "pip")
|
|
1031
|
-
wheel_path = self._ensure_cached_wheel(package_name, cache_args)
|
|
1032
|
-
if wheel_path:
|
|
1033
|
-
self._materialize_cached_tree(package_name, wheel_path)
|
|
1034
|
-
return True
|
|
1035
|
-
|
|
1036
|
-
except subprocess.CalledProcessError as e:
|
|
1037
|
-
logger.error(f"Failed to install {package_name}: {e.stderr}")
|
|
1038
|
-
print(f"[FAIL] Failed to install {package_name}\n")
|
|
1039
|
-
self._failed_packages.add(package_name)
|
|
1040
|
-
return False
|
|
1041
|
-
except Exception as e:
|
|
1042
|
-
logger.error(f"Unexpected error installing {package_name}: {e}")
|
|
1043
|
-
print(f"[ERROR] Unexpected error: {e}\n")
|
|
1044
|
-
self._failed_packages.add(package_name)
|
|
1045
|
-
return False
|
|
1046
|
-
|
|
1047
|
-
def _finalize_install_success(self, package_name: str, source: str) -> None:
|
|
1048
|
-
self._installed_packages.add(package_name)
|
|
1049
|
-
# Show final success message (this will replace all previous same-line messages)
|
|
1050
|
-
print_formatted("SUCCESS", f"Successfully installed via {source}: {package_name}", same_line=True)
|
|
1051
|
-
# Add newline after final message so cursor moves to next line
|
|
1052
|
-
print()
|
|
1053
|
-
if _check_pip_audit_available():
|
|
1054
|
-
self._run_vulnerability_audit(package_name)
|
|
1055
|
-
self._update_lockfile(package_name)
|
|
1056
|
-
|
|
1057
|
-
def _run_vulnerability_audit(self, package_name: str) -> None:
|
|
1058
|
-
"""Run vulnerability audit on installed package using pip-audit."""
|
|
1059
|
-
try:
|
|
1060
|
-
result = subprocess.run(
|
|
1061
|
-
[sys.executable, '-m', 'pip_audit', '-r', '-', '--format', 'json'],
|
|
1062
|
-
input=package_name,
|
|
1063
|
-
capture_output=True,
|
|
1064
|
-
text=True,
|
|
1065
|
-
timeout=30
|
|
1066
|
-
)
|
|
1067
|
-
|
|
1068
|
-
if result.returncode == 0:
|
|
1069
|
-
_log("audit", f"Vulnerability audit passed for {package_name}")
|
|
1070
|
-
else:
|
|
1071
|
-
try:
|
|
1072
|
-
audit_data = json.loads(result.stdout)
|
|
1073
|
-
if audit_data.get('vulnerabilities'):
|
|
1074
|
-
logger.warning(f"[SECURITY] Vulnerabilities found in {package_name}: {audit_data}")
|
|
1075
|
-
print(f"[SECURITY WARNING] Package '{package_name}' has known vulnerabilities")
|
|
1076
|
-
print(f"Run 'pip-audit' for details")
|
|
1077
|
-
except json.JSONDecodeError:
|
|
1078
|
-
logger.warning(f"Could not parse audit results for {package_name}")
|
|
1079
|
-
except subprocess.TimeoutExpired:
|
|
1080
|
-
logger.warning(f"Vulnerability audit timed out for {package_name}")
|
|
1081
|
-
except Exception as e:
|
|
1082
|
-
logger.debug(f"Vulnerability audit skipped for {package_name}: {e}")
|
|
1083
|
-
|
|
1084
|
-
def _update_lockfile(self, package_name: str) -> None:
|
|
1085
|
-
"""Update lockfile with newly installed package."""
|
|
1086
|
-
lockfile_path = LazyInstallPolicy.get_lockfile_path(self._package_name)
|
|
1087
|
-
if not lockfile_path:
|
|
1088
|
-
return
|
|
1089
|
-
|
|
1090
|
-
try:
|
|
1091
|
-
version = self._get_installed_version(package_name)
|
|
1092
|
-
if not version:
|
|
1093
|
-
return
|
|
1094
|
-
|
|
1095
|
-
lockfile_path = Path(lockfile_path)
|
|
1096
|
-
if lockfile_path.exists():
|
|
1097
|
-
with open(lockfile_path, 'r', encoding='utf-8') as f:
|
|
1098
|
-
lockdata = json.load(f)
|
|
1099
|
-
else:
|
|
1100
|
-
lockdata = {
|
|
1101
|
-
"metadata": {
|
|
1102
|
-
"generated_by": f"xwlazy-{self._package_name}",
|
|
1103
|
-
"version": "1.0"
|
|
1104
|
-
},
|
|
1105
|
-
"packages": {}
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
|
-
lockdata["packages"][package_name] = {
|
|
1109
|
-
"version": version,
|
|
1110
|
-
"installed_at": datetime.now().isoformat(),
|
|
1111
|
-
"installer": self._package_name
|
|
1112
|
-
}
|
|
1113
|
-
|
|
1114
|
-
lockfile_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1115
|
-
with open(lockfile_path, 'w', encoding='utf-8') as f:
|
|
1116
|
-
json.dump(lockdata, f, indent=2)
|
|
1117
|
-
|
|
1118
|
-
_log("sbom", f"Updated lockfile: {lockfile_path}")
|
|
1119
|
-
except Exception as e:
|
|
1120
|
-
logger.warning(f"Failed to update lockfile: {e}")
|
|
1121
|
-
|
|
1122
|
-
def _get_installed_version(self, package_name: str) -> Optional[str]:
|
|
1123
|
-
"""Get installed version of a package."""
|
|
1124
|
-
try:
|
|
1125
|
-
result = subprocess.run(
|
|
1126
|
-
[sys.executable, '-m', 'pip', 'show', package_name],
|
|
1127
|
-
capture_output=True,
|
|
1128
|
-
text=True,
|
|
1129
|
-
timeout=5
|
|
1130
|
-
)
|
|
1131
|
-
|
|
1132
|
-
if result.returncode == 0:
|
|
1133
|
-
for line in result.stdout.split('\n'):
|
|
1134
|
-
if line.startswith('Version:'):
|
|
1135
|
-
return line.split(':', 1)[1].strip()
|
|
1136
|
-
except Exception as e:
|
|
1137
|
-
logger.debug(f"Could not get version for {package_name}: {e}")
|
|
1138
|
-
return None
|
|
1139
|
-
|
|
1140
|
-
def apply_manifest(self, manifest: Optional[PackageManifest]) -> None:
|
|
1141
|
-
"""Apply manifest-driven configuration such as async installs."""
|
|
1142
|
-
env_override = _ENV_ASYNC_INSTALL
|
|
1143
|
-
desired_async = bool(env_override or (manifest and manifest.async_installs))
|
|
1144
|
-
desired_workers = _ENV_ASYNC_WORKERS or (manifest.async_workers if manifest else 1)
|
|
1145
|
-
desired_workers = max(1, desired_workers)
|
|
1146
|
-
|
|
1147
|
-
with self._lock:
|
|
1148
|
-
if desired_workers != self._async_workers and self._async_executor:
|
|
1149
|
-
self._async_executor.shutdown(wait=False)
|
|
1150
|
-
self._async_executor = None
|
|
1151
|
-
self._async_workers = desired_workers
|
|
1152
|
-
|
|
1153
|
-
if not desired_async and self._async_executor:
|
|
1154
|
-
self._async_executor.shutdown(wait=False)
|
|
1155
|
-
self._async_executor = None
|
|
1156
|
-
self._async_pending.clear()
|
|
1157
|
-
|
|
1158
|
-
self._async_enabled = desired_async
|
|
1159
|
-
|
|
1160
|
-
def _prune_known_missing(self) -> None:
|
|
1161
|
-
"""Remove stale entries from the known-missing cache."""
|
|
1162
|
-
if not self._known_missing:
|
|
1163
|
-
return
|
|
1164
|
-
now = time.monotonic()
|
|
1165
|
-
with self._lock:
|
|
1166
|
-
while self._known_missing:
|
|
1167
|
-
_, ts = next(iter(self._known_missing.items()))
|
|
1168
|
-
if now - ts <= _KNOWN_MISSING_CACHE_TTL:
|
|
1169
|
-
break
|
|
1170
|
-
self._known_missing.popitem(last=False)
|
|
1171
|
-
|
|
1172
|
-
def _mark_module_missing(self, module_name: str) -> None:
|
|
1173
|
-
"""Remember modules that failed to import recently."""
|
|
1174
|
-
with self._lock:
|
|
1175
|
-
self._prune_known_missing()
|
|
1176
|
-
_spec_cache_clear(module_name)
|
|
1177
|
-
self._known_missing[module_name] = time.monotonic()
|
|
1178
|
-
while len(self._known_missing) > _KNOWN_MISSING_CACHE_LIMIT:
|
|
1179
|
-
self._known_missing.popitem(last=False)
|
|
1180
|
-
|
|
1181
|
-
def _clear_module_missing(self, module_name: str) -> None:
|
|
1182
|
-
"""Remove a module from the known-missing cache."""
|
|
1183
|
-
with self._lock:
|
|
1184
|
-
self._known_missing.pop(module_name, None)
|
|
1185
|
-
|
|
1186
|
-
def _get_async_cache_dir(self) -> Path:
|
|
1187
|
-
path = Path(self._async_cache_dir).expanduser()
|
|
1188
|
-
path.mkdir(parents=True, exist_ok=True)
|
|
1189
|
-
return path
|
|
1190
|
-
|
|
1191
|
-
def _cached_wheel_name(self, package_name: str) -> Path:
|
|
1192
|
-
safe = package_name.replace("/", "_").replace("\\", "_").replace(":", "_")
|
|
1193
|
-
return self._get_async_cache_dir() / f"{safe}.whl"
|
|
1194
|
-
|
|
1195
|
-
def _install_from_cached_wheel(self, package_name: str, policy_args: Optional[List[str]] = None) -> bool:
|
|
1196
|
-
wheel_path = self._cached_wheel_name(package_name)
|
|
1197
|
-
if not wheel_path.exists():
|
|
1198
|
-
return False
|
|
1199
|
-
return self._pip_install_from_path(wheel_path, policy_args)
|
|
1200
|
-
|
|
1201
|
-
def _pip_install_from_path(self, wheel_path: Path, policy_args: Optional[List[str]] = None) -> bool:
|
|
1202
|
-
try:
|
|
1203
|
-
pip_args = [
|
|
1204
|
-
sys.executable,
|
|
1205
|
-
'-m',
|
|
1206
|
-
'pip',
|
|
1207
|
-
'install',
|
|
1208
|
-
'--no-deps',
|
|
1209
|
-
'--no-input',
|
|
1210
|
-
'--disable-pip-version-check',
|
|
1211
|
-
]
|
|
1212
|
-
if policy_args:
|
|
1213
|
-
pip_args.extend(policy_args)
|
|
1214
|
-
pip_args.append(str(wheel_path))
|
|
1215
|
-
result = subprocess.run(
|
|
1216
|
-
pip_args,
|
|
1217
|
-
capture_output=True,
|
|
1218
|
-
text=True,
|
|
1219
|
-
check=True,
|
|
1220
|
-
)
|
|
1221
|
-
return result.returncode == 0
|
|
1222
|
-
except subprocess.CalledProcessError:
|
|
1223
|
-
return False
|
|
1224
|
-
|
|
1225
|
-
def _ensure_cached_wheel(self, package_name: str, policy_args: Optional[List[str]] = None) -> Optional[Path]:
|
|
1226
|
-
wheel_path = self._cached_wheel_name(package_name)
|
|
1227
|
-
if wheel_path.exists():
|
|
1228
|
-
return wheel_path
|
|
1229
|
-
cache_dir = self._get_async_cache_dir()
|
|
1230
|
-
try:
|
|
1231
|
-
pip_args = [
|
|
1232
|
-
sys.executable,
|
|
1233
|
-
'-m',
|
|
1234
|
-
'pip',
|
|
1235
|
-
'wheel',
|
|
1236
|
-
'--no-deps',
|
|
1237
|
-
'--disable-pip-version-check',
|
|
1238
|
-
]
|
|
1239
|
-
if policy_args:
|
|
1240
|
-
pip_args.extend(policy_args)
|
|
1241
|
-
pip_args.extend(['--wheel-dir', str(cache_dir), package_name])
|
|
1242
|
-
result = subprocess.run(
|
|
1243
|
-
pip_args,
|
|
1244
|
-
capture_output=True,
|
|
1245
|
-
text=True,
|
|
1246
|
-
check=True,
|
|
1247
|
-
)
|
|
1248
|
-
if result.returncode != 0:
|
|
1249
|
-
return None
|
|
1250
|
-
candidates = sorted(cache_dir.glob("*.whl"), key=lambda p: p.stat().st_mtime, reverse=True)
|
|
1251
|
-
if not candidates:
|
|
1252
|
-
return None
|
|
1253
|
-
primary = candidates[0]
|
|
1254
|
-
if wheel_path.exists():
|
|
1255
|
-
with suppress(Exception):
|
|
1256
|
-
wheel_path.unlink()
|
|
1257
|
-
primary.rename(wheel_path)
|
|
1258
|
-
for leftover in candidates[1:]:
|
|
1259
|
-
with suppress(Exception):
|
|
1260
|
-
leftover.unlink()
|
|
1261
|
-
return wheel_path
|
|
1262
|
-
except subprocess.CalledProcessError:
|
|
1263
|
-
return None
|
|
1264
|
-
|
|
1265
|
-
def _cached_install_dir(self, package_name: str) -> Path:
|
|
1266
|
-
safe = package_name.replace("/", "_").replace("\\", "_").replace(":", "_")
|
|
1267
|
-
return self._get_async_cache_dir() / "installs" / safe
|
|
1268
|
-
|
|
1269
|
-
def _has_cached_install_tree(self, package_name: str) -> bool:
|
|
1270
|
-
target = self._cached_install_dir(package_name)
|
|
1271
|
-
return target.exists() and any(target.iterdir())
|
|
1272
|
-
|
|
1273
|
-
def _site_packages_dir(self) -> Path:
|
|
1274
|
-
purelib = sysconfig.get_paths().get("purelib")
|
|
1275
|
-
if not purelib:
|
|
1276
|
-
purelib = sysconfig.get_paths().get("platlib", sys.prefix)
|
|
1277
|
-
path = Path(purelib)
|
|
1278
|
-
path.mkdir(parents=True, exist_ok=True)
|
|
1279
|
-
return path
|
|
1280
|
-
|
|
1281
|
-
def _install_from_cached_tree(self, package_name: str) -> bool:
|
|
1282
|
-
src = self._cached_install_dir(package_name)
|
|
1283
|
-
if not src.exists() or not any(src.iterdir()):
|
|
1284
|
-
return False
|
|
1285
|
-
target_root = self._site_packages_dir()
|
|
1286
|
-
try:
|
|
1287
|
-
for item in src.iterdir():
|
|
1288
|
-
dest = target_root / item.name
|
|
1289
|
-
if dest.exists():
|
|
1290
|
-
if dest.is_dir():
|
|
1291
|
-
shutil.rmtree(dest, ignore_errors=True)
|
|
1292
|
-
else:
|
|
1293
|
-
with suppress(FileNotFoundError):
|
|
1294
|
-
dest.unlink()
|
|
1295
|
-
if item.is_dir():
|
|
1296
|
-
shutil.copytree(item, dest)
|
|
1297
|
-
else:
|
|
1298
|
-
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
1299
|
-
shutil.copy2(item, dest)
|
|
1300
|
-
return True
|
|
1301
|
-
except Exception as exc:
|
|
1302
|
-
logger.debug("Cached tree install failed for %s: %s", package_name, exc)
|
|
1303
|
-
return False
|
|
1304
|
-
|
|
1305
|
-
def _materialize_cached_tree(self, package_name: str, wheel_path: Path) -> None:
|
|
1306
|
-
if not wheel_path or not wheel_path.exists():
|
|
1307
|
-
return
|
|
1308
|
-
target_dir = self._cached_install_dir(package_name)
|
|
1309
|
-
if target_dir.exists() and any(target_dir.iterdir()):
|
|
1310
|
-
return
|
|
1311
|
-
parent = target_dir.parent
|
|
1312
|
-
parent.mkdir(parents=True, exist_ok=True)
|
|
1313
|
-
temp_dir = Path(
|
|
1314
|
-
tempfile.mkdtemp(prefix="xwlazy-cache-", dir=str(parent))
|
|
1315
|
-
)
|
|
1316
|
-
try:
|
|
1317
|
-
with zipfile.ZipFile(wheel_path, "r") as archive:
|
|
1318
|
-
archive.extractall(temp_dir)
|
|
1319
|
-
if target_dir.exists():
|
|
1320
|
-
shutil.rmtree(target_dir, ignore_errors=True)
|
|
1321
|
-
shutil.move(str(temp_dir), str(target_dir))
|
|
1322
|
-
except Exception as exc:
|
|
1323
|
-
logger.debug("Failed to materialize cached tree for %s: %s", package_name, exc)
|
|
1324
|
-
with suppress(Exception):
|
|
1325
|
-
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
1326
|
-
else:
|
|
1327
|
-
# temp_dir was moved; nothing to clean.
|
|
1328
|
-
return
|
|
1329
|
-
finally:
|
|
1330
|
-
if temp_dir.exists():
|
|
1331
|
-
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
1332
|
-
|
|
1333
|
-
def is_module_known_missing(self, module_name: str) -> bool:
|
|
1334
|
-
"""Return True if module recently failed to import."""
|
|
1335
|
-
self._prune_known_missing()
|
|
1336
|
-
with self._lock:
|
|
1337
|
-
return module_name in self._known_missing
|
|
1338
|
-
|
|
1339
|
-
def is_async_enabled(self) -> bool:
|
|
1340
|
-
"""Return True if async installers are enabled for this package."""
|
|
1341
|
-
return self._async_enabled
|
|
1342
|
-
|
|
1343
|
-
def ensure_async_install(self, module_name: str) -> Optional["AsyncInstallHandle"]:
|
|
1344
|
-
"""
|
|
1345
|
-
Schedule (or reuse) an async install job for module_name if async is enabled.
|
|
1346
|
-
"""
|
|
1347
|
-
if not self._async_enabled:
|
|
1348
|
-
return None
|
|
1349
|
-
return self.schedule_async_install(module_name)
|
|
1350
|
-
|
|
1351
|
-
def schedule_async_install(self, module_name: str) -> Optional["AsyncInstallHandle"]:
|
|
1352
|
-
"""Schedule installation of a dependency in the background."""
|
|
1353
|
-
if not self._async_enabled:
|
|
1354
|
-
return None
|
|
1355
|
-
|
|
1356
|
-
package_name = self._dependency_mapper.get_package_name(module_name) or module_name
|
|
1357
|
-
if not package_name:
|
|
1358
|
-
return None
|
|
1359
|
-
|
|
1360
|
-
with self._lock:
|
|
1361
|
-
future = self._async_pending.get(module_name)
|
|
1362
|
-
if future is None:
|
|
1363
|
-
self._mark_module_missing(module_name)
|
|
1364
|
-
if self._async_executor is None:
|
|
1365
|
-
self._async_executor = ThreadPoolExecutor(
|
|
1366
|
-
max_workers=self._async_workers,
|
|
1367
|
-
thread_name_prefix=f"xwlazy-{self._package_name}-install",
|
|
1368
|
-
)
|
|
1369
|
-
def _run_install():
|
|
1370
|
-
if self._install_from_cached_tree(package_name):
|
|
1371
|
-
self._finalize_install_success(package_name, "cache-tree")
|
|
1372
|
-
return True
|
|
1373
|
-
return self.install_package(package_name, module_name)
|
|
1374
|
-
|
|
1375
|
-
future = self._async_executor.submit(_run_install)
|
|
1376
|
-
self._async_pending[module_name] = future
|
|
1377
|
-
|
|
1378
|
-
def _cleanup(_future: Future, name: str = module_name, pkg: str = package_name) -> None:
|
|
1379
|
-
with self._lock:
|
|
1380
|
-
self._async_pending.pop(name, None)
|
|
1381
|
-
try:
|
|
1382
|
-
result = bool(_future.result())
|
|
1383
|
-
except Exception:
|
|
1384
|
-
result = False
|
|
1385
|
-
if result:
|
|
1386
|
-
self._clear_module_missing(name)
|
|
1387
|
-
try:
|
|
1388
|
-
importlib.import_module(name)
|
|
1389
|
-
except Exception:
|
|
1390
|
-
pass
|
|
1391
|
-
|
|
1392
|
-
future.add_done_callback(_cleanup)
|
|
1393
|
-
|
|
1394
|
-
return AsyncInstallHandle(future, module_name, package_name, self._package_name)
|
|
1395
|
-
|
|
1396
|
-
def generate_sbom(self) -> Dict:
|
|
1397
|
-
"""Generate Software Bill of Materials (SBOM) for installed packages."""
|
|
1398
|
-
sbom = {
|
|
1399
|
-
"metadata": {
|
|
1400
|
-
"format": "xwlazy-sbom",
|
|
1401
|
-
"version": "1.0",
|
|
1402
|
-
"generated_at": datetime.now().isoformat(),
|
|
1403
|
-
"installer_package": self._package_name
|
|
1404
|
-
},
|
|
1405
|
-
"packages": []
|
|
1406
|
-
}
|
|
1407
|
-
|
|
1408
|
-
for pkg in self._installed_packages:
|
|
1409
|
-
version = self._get_installed_version(pkg)
|
|
1410
|
-
sbom["packages"].append({
|
|
1411
|
-
"name": pkg,
|
|
1412
|
-
"version": version or "unknown",
|
|
1413
|
-
"installed_by": self._package_name,
|
|
1414
|
-
"source": "pypi"
|
|
1415
|
-
})
|
|
1416
|
-
|
|
1417
|
-
return sbom
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
def export_sbom(self, output_path: str) -> bool:
|
|
1422
|
-
"""Export SBOM to file."""
|
|
1423
|
-
try:
|
|
1424
|
-
sbom = self.generate_sbom()
|
|
1425
|
-
output_path = Path(output_path)
|
|
1426
|
-
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1427
|
-
|
|
1428
|
-
with open(output_path, 'w', encoding='utf-8') as f:
|
|
1429
|
-
json.dump(sbom, f, indent=2)
|
|
1430
|
-
|
|
1431
|
-
_log("sbom", f"Exported SBOM to: {output_path}")
|
|
1432
|
-
return True
|
|
1433
|
-
except Exception as e:
|
|
1434
|
-
logger.error(f"Failed to export SBOM: {e}")
|
|
1435
|
-
return False
|
|
1436
|
-
|
|
1437
|
-
def is_package_installed(self, package_name: str) -> bool:
|
|
1438
|
-
"""Check if a package is already installed."""
|
|
1439
|
-
return package_name in self._installed_packages
|
|
1440
|
-
|
|
1441
|
-
def install_and_import(self, module_name: str, package_name: str = None) -> Tuple[Optional[ModuleType], bool]:
|
|
1442
|
-
"""Install package and import module."""
|
|
1443
|
-
if not self.is_enabled():
|
|
1444
|
-
return None, False
|
|
1445
|
-
|
|
1446
|
-
if package_name is None:
|
|
1447
|
-
package_name = self._dependency_mapper.get_package_name(module_name)
|
|
1448
|
-
if package_name is None:
|
|
1449
|
-
logger.debug(f"Module '{module_name}' is a system/built-in module, not installing")
|
|
1450
|
-
return None, False
|
|
1451
|
-
|
|
1452
|
-
try:
|
|
1453
|
-
module = importlib.import_module(module_name)
|
|
1454
|
-
self._clear_module_missing(module_name)
|
|
1455
|
-
_spec_cache_put(module_name, importlib.util.find_spec(module_name))
|
|
1456
|
-
return module, True
|
|
1457
|
-
except ImportError:
|
|
1458
|
-
pass
|
|
1459
|
-
|
|
1460
|
-
if self._async_enabled:
|
|
1461
|
-
handle = self.schedule_async_install(module_name)
|
|
1462
|
-
if handle is not None:
|
|
1463
|
-
return None, False
|
|
1464
|
-
|
|
1465
|
-
if self.install_package(package_name, module_name):
|
|
1466
|
-
try:
|
|
1467
|
-
module = importlib.import_module(module_name)
|
|
1468
|
-
self._clear_module_missing(module_name)
|
|
1469
|
-
_spec_cache_put(module_name, importlib.util.find_spec(module_name))
|
|
1470
|
-
return module, True
|
|
1471
|
-
except ImportError as e:
|
|
1472
|
-
logger.error(f"Still cannot import {module_name} after installing {package_name}: {e}")
|
|
1473
|
-
return None, False
|
|
1474
|
-
|
|
1475
|
-
self._mark_module_missing(module_name)
|
|
1476
|
-
return None, False
|
|
1477
|
-
|
|
1478
|
-
def _check_security_policy(self, package_name: str) -> Tuple[bool, str]:
|
|
1479
|
-
"""Check security policy for package."""
|
|
1480
|
-
return LazyInstallPolicy.is_package_allowed(self._package_name, package_name)
|
|
1481
|
-
|
|
1482
|
-
def _run_pip_install(self, package_name: str, args: List[str]) -> bool:
|
|
1483
|
-
"""Run pip install with arguments."""
|
|
1484
|
-
if self._install_from_cached_wheel(package_name):
|
|
1485
|
-
return True
|
|
1486
|
-
try:
|
|
1487
|
-
pip_args = [
|
|
1488
|
-
sys.executable,
|
|
1489
|
-
'-m',
|
|
1490
|
-
'pip',
|
|
1491
|
-
'install',
|
|
1492
|
-
'--disable-pip-version-check',
|
|
1493
|
-
'--no-input',
|
|
1494
|
-
] + args + [package_name]
|
|
1495
|
-
result = subprocess.run(
|
|
1496
|
-
pip_args,
|
|
1497
|
-
capture_output=True,
|
|
1498
|
-
text=True,
|
|
1499
|
-
check=True,
|
|
1500
|
-
)
|
|
1501
|
-
if result.returncode == 0:
|
|
1502
|
-
self._ensure_cached_wheel(package_name)
|
|
1503
|
-
return True
|
|
1504
|
-
return False
|
|
1505
|
-
except subprocess.CalledProcessError:
|
|
1506
|
-
return False
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
class AsyncInstallHandle:
|
|
1510
|
-
"""Lightweight handle for background installation jobs."""
|
|
1511
|
-
|
|
1512
|
-
__slots__ = ("future", "module_name", "package_name", "installer_package")
|
|
1513
|
-
|
|
1514
|
-
def __init__(
|
|
1515
|
-
self,
|
|
1516
|
-
future: Future,
|
|
1517
|
-
module_name: str,
|
|
1518
|
-
package_name: str,
|
|
1519
|
-
installer_package: str,
|
|
1520
|
-
) -> None:
|
|
1521
|
-
self.future = future
|
|
1522
|
-
self.module_name = module_name
|
|
1523
|
-
self.package_name = package_name
|
|
1524
|
-
self.installer_package = installer_package
|
|
1525
|
-
|
|
1526
|
-
def wait(self, timeout: Optional[float] = None) -> bool:
|
|
1527
|
-
try:
|
|
1528
|
-
result = self.future.result(timeout=timeout)
|
|
1529
|
-
return bool(result)
|
|
1530
|
-
except Exception:
|
|
1531
|
-
return False
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
class LazyInstallerRegistry:
|
|
1535
|
-
"""Registry to manage separate lazy installer instances per package."""
|
|
1536
|
-
_instances: Dict[str, LazyInstaller] = {}
|
|
1537
|
-
_lock = threading.RLock()
|
|
1538
|
-
|
|
1539
|
-
@classmethod
|
|
1540
|
-
def get_instance(cls, package_name: str = 'default') -> LazyInstaller:
|
|
1541
|
-
"""Get or create a lazy installer instance for a package."""
|
|
1542
|
-
with cls._lock:
|
|
1543
|
-
if package_name not in cls._instances:
|
|
1544
|
-
cls._instances[package_name] = LazyInstaller(package_name)
|
|
1545
|
-
return cls._instances[package_name]
|
|
1546
|
-
|
|
1547
|
-
@classmethod
|
|
1548
|
-
def get_all_instances(cls) -> Dict[str, LazyInstaller]:
|
|
1549
|
-
"""Get all lazy installer instances."""
|
|
1550
|
-
with cls._lock:
|
|
1551
|
-
return cls._instances.copy()
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
def sync_manifest_configuration(package_name: str) -> Optional[PackageManifest]:
|
|
1555
|
-
"""
|
|
1556
|
-
Load manifest data for a package and propagate configuration hooks.
|
|
1557
|
-
"""
|
|
1558
|
-
loader = get_manifest_loader()
|
|
1559
|
-
manifest = loader.get_manifest(package_name)
|
|
1560
|
-
prefixes = manifest.watched_prefixes if manifest else ()
|
|
1561
|
-
_watched_registry.register_package(package_name, prefixes)
|
|
1562
|
-
if manifest:
|
|
1563
|
-
_set_package_class_hints(package_name, manifest.class_wrap_prefixes)
|
|
1564
|
-
else:
|
|
1565
|
-
_set_package_class_hints(package_name, ())
|
|
1566
|
-
|
|
1567
|
-
installer = LazyInstallerRegistry.get_instance(package_name)
|
|
1568
|
-
installer.apply_manifest(manifest)
|
|
1569
|
-
return manifest
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
def refresh_lazy_manifests(package_name: Optional[str] = None) -> None:
|
|
1573
|
-
"""
|
|
1574
|
-
Clear manifest caches and re-apply configuration for one or all packages.
|
|
1575
|
-
"""
|
|
1576
|
-
loader = get_manifest_loader()
|
|
1577
|
-
loader.clear_cache()
|
|
1578
|
-
_spec_cache_clear()
|
|
1579
|
-
|
|
1580
|
-
if package_name:
|
|
1581
|
-
_set_package_class_hints(package_name, ())
|
|
1582
|
-
sync_manifest_configuration(package_name)
|
|
1583
|
-
return
|
|
1584
|
-
|
|
1585
|
-
_clear_all_package_class_hints()
|
|
1586
|
-
for pkg_name in LazyInstallerRegistry.get_all_instances().keys():
|
|
1587
|
-
sync_manifest_configuration(pkg_name)
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
# =============================================================================
|
|
1591
|
-
# SECTION 3: IMPORT HOOKS & TWO-STAGE LOADING (~450 lines)
|
|
1592
|
-
# =============================================================================
|
|
1593
|
-
|
|
1594
|
-
# Global import tracking cache - Prevents infinite loops
|
|
1595
|
-
_import_in_progress: Dict[int, Set[str]] = defaultdict(set)
|
|
1596
|
-
_import_cache_lock = threading.RLock()
|
|
1597
|
-
_importing = threading.local()
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
def _normalize_prefix(prefix: str) -> str:
|
|
1601
|
-
prefix = prefix.strip()
|
|
1602
|
-
if not prefix:
|
|
1603
|
-
return ""
|
|
1604
|
-
if not prefix.endswith("."):
|
|
1605
|
-
prefix += "."
|
|
1606
|
-
return prefix
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
def _spec_cache_prune_locked(now: Optional[float] = None) -> None:
|
|
1610
|
-
if not _spec_cache:
|
|
1611
|
-
return
|
|
1612
|
-
current = now or time.monotonic()
|
|
1613
|
-
while _spec_cache:
|
|
1614
|
-
fullname, (_, ts) = next(iter(_spec_cache.items()))
|
|
1615
|
-
if current - ts <= _SPEC_CACHE_TTL and len(_spec_cache) <= _SPEC_CACHE_MAX:
|
|
1616
|
-
break
|
|
1617
|
-
_spec_cache.popitem(last=False)
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
def _spec_cache_get(fullname: str) -> Optional[importlib.machinery.ModuleSpec]:
|
|
1621
|
-
with _spec_cache_lock:
|
|
1622
|
-
_spec_cache_prune_locked()
|
|
1623
|
-
entry = _spec_cache.get(fullname)
|
|
1624
|
-
if entry is None:
|
|
1625
|
-
return None
|
|
1626
|
-
spec, _ = entry
|
|
1627
|
-
_spec_cache.move_to_end(fullname)
|
|
1628
|
-
return spec
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
def _spec_cache_put(fullname: str, spec: Optional[importlib.machinery.ModuleSpec]) -> None:
|
|
1632
|
-
if spec is None:
|
|
1633
|
-
return
|
|
1634
|
-
with _spec_cache_lock:
|
|
1635
|
-
_spec_cache[fullname] = (spec, time.monotonic())
|
|
1636
|
-
_spec_cache.move_to_end(fullname)
|
|
1637
|
-
_spec_cache_prune_locked()
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
def _spec_cache_clear(fullname: Optional[str] = None) -> None:
|
|
1641
|
-
with _spec_cache_lock:
|
|
1642
|
-
if fullname is None:
|
|
1643
|
-
_spec_cache.clear()
|
|
1644
|
-
else:
|
|
1645
|
-
_spec_cache.pop(fullname, None)
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
def _cache_spec_if_missing(fullname: str) -> None:
|
|
1649
|
-
"""Ensure a ModuleSpec is cached for a known-good module."""
|
|
1650
|
-
if _spec_cache_get(fullname):
|
|
1651
|
-
return
|
|
1652
|
-
try:
|
|
1653
|
-
spec = importlib.util.find_spec(fullname)
|
|
1654
|
-
except Exception:
|
|
1655
|
-
spec = None
|
|
1656
|
-
if spec is not None:
|
|
1657
|
-
_spec_cache_put(fullname, spec)
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
class _DeferredModuleLoader(importlib.abc.Loader):
|
|
1661
|
-
"""Loader that simply returns a preconstructed module placeholder."""
|
|
1662
|
-
|
|
1663
|
-
def __init__(self, module: ModuleType) -> None:
|
|
1664
|
-
self._module = module
|
|
1665
|
-
|
|
1666
|
-
def create_module(self, spec): # noqa: D401 - standard loader hook
|
|
1667
|
-
return self._module
|
|
1668
|
-
|
|
1669
|
-
def exec_module(self, module): # noqa: D401 - nothing to execute
|
|
1670
|
-
return None
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
class _PrefixTrie:
|
|
1674
|
-
__slots__ = ("_root",)
|
|
1675
|
-
|
|
1676
|
-
def __init__(self) -> None:
|
|
1677
|
-
self._root: Dict[str, Dict[str, Any]] = {}
|
|
1678
|
-
|
|
1679
|
-
def add(self, prefix: str) -> None:
|
|
1680
|
-
node = self._root
|
|
1681
|
-
for char in prefix:
|
|
1682
|
-
node = node.setdefault(char, {})
|
|
1683
|
-
node["_end"] = prefix
|
|
1684
|
-
|
|
1685
|
-
def iter_matches(self, value: str) -> Tuple[str, ...]:
|
|
1686
|
-
node = self._root
|
|
1687
|
-
matches: List[str] = []
|
|
1688
|
-
for char in value:
|
|
1689
|
-
end_value = node.get("_end")
|
|
1690
|
-
if end_value:
|
|
1691
|
-
matches.append(end_value)
|
|
1692
|
-
node = node.get(char)
|
|
1693
|
-
if node is None:
|
|
1694
|
-
break
|
|
1695
|
-
else:
|
|
1696
|
-
end_value = node.get("_end")
|
|
1697
|
-
if end_value:
|
|
1698
|
-
matches.append(end_value)
|
|
1699
|
-
return tuple(matches)
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
class WatchedPrefixRegistry:
|
|
1703
|
-
"""
|
|
1704
|
-
Maintain watched prefixes and provide fast trie-based membership checks.
|
|
1705
|
-
"""
|
|
1706
|
-
|
|
1707
|
-
__slots__ = (
|
|
1708
|
-
"_lock",
|
|
1709
|
-
"_prefix_refcounts",
|
|
1710
|
-
"_owner_map",
|
|
1711
|
-
"_prefixes",
|
|
1712
|
-
"_trie",
|
|
1713
|
-
"_dirty",
|
|
1714
|
-
"_root_refcounts",
|
|
1715
|
-
"_root_snapshot",
|
|
1716
|
-
"_root_snapshot_dirty",
|
|
1717
|
-
)
|
|
1718
|
-
|
|
1719
|
-
def __init__(self, initial: Optional[List[str]] = None) -> None:
|
|
1720
|
-
self._lock = threading.RLock()
|
|
1721
|
-
self._prefix_refcounts: Counter[str] = Counter()
|
|
1722
|
-
self._owner_map: Dict[str, Set[str]] = {}
|
|
1723
|
-
self._prefixes: Set[str] = set()
|
|
1724
|
-
self._trie = _PrefixTrie()
|
|
1725
|
-
self._dirty = False
|
|
1726
|
-
self._root_refcounts: Counter[str] = Counter()
|
|
1727
|
-
self._root_snapshot: Set[str] = set()
|
|
1728
|
-
self._root_snapshot_dirty = False
|
|
1729
|
-
if initial:
|
|
1730
|
-
for prefix in initial:
|
|
1731
|
-
self._register_manual(prefix)
|
|
1732
|
-
|
|
1733
|
-
def _register_manual(self, prefix: str) -> None:
|
|
1734
|
-
normalized = _normalize_prefix(prefix)
|
|
1735
|
-
if not normalized:
|
|
1736
|
-
return
|
|
1737
|
-
owner = "__manual__"
|
|
1738
|
-
owners = self._owner_map.setdefault(owner, set())
|
|
1739
|
-
if normalized in owners:
|
|
1740
|
-
return
|
|
1741
|
-
owners.add(normalized)
|
|
1742
|
-
self._add_prefix(normalized)
|
|
1743
|
-
|
|
1744
|
-
def _add_prefix(self, prefix: str) -> None:
|
|
1745
|
-
if not prefix:
|
|
1746
|
-
return
|
|
1747
|
-
self._prefix_refcounts[prefix] += 1
|
|
1748
|
-
if self._prefix_refcounts[prefix] == 1:
|
|
1749
|
-
self._prefixes.add(prefix)
|
|
1750
|
-
self._dirty = True
|
|
1751
|
-
root = prefix.split('.', 1)[0]
|
|
1752
|
-
self._root_refcounts[root] += 1
|
|
1753
|
-
self._root_snapshot_dirty = True
|
|
1754
|
-
|
|
1755
|
-
def _remove_prefix(self, prefix: str) -> None:
|
|
1756
|
-
if prefix not in self._prefix_refcounts:
|
|
1757
|
-
return
|
|
1758
|
-
self._prefix_refcounts[prefix] -= 1
|
|
1759
|
-
if self._prefix_refcounts[prefix] <= 0:
|
|
1760
|
-
self._prefix_refcounts.pop(prefix, None)
|
|
1761
|
-
self._prefixes.discard(prefix)
|
|
1762
|
-
self._dirty = True
|
|
1763
|
-
root = prefix.split('.', 1)[0]
|
|
1764
|
-
self._root_refcounts[root] -= 1
|
|
1765
|
-
if self._root_refcounts[root] <= 0:
|
|
1766
|
-
self._root_refcounts.pop(root, None)
|
|
1767
|
-
self._root_snapshot_dirty = True
|
|
1768
|
-
|
|
1769
|
-
def _ensure_trie(self) -> None:
|
|
1770
|
-
if not self._dirty:
|
|
1771
|
-
return
|
|
1772
|
-
self._trie = _PrefixTrie()
|
|
1773
|
-
for prefix in self._prefixes:
|
|
1774
|
-
self._trie.add(prefix)
|
|
1775
|
-
self._dirty = False
|
|
1776
|
-
|
|
1777
|
-
def add(self, prefix: str) -> None:
|
|
1778
|
-
normalized = _normalize_prefix(prefix)
|
|
1779
|
-
if not normalized:
|
|
1780
|
-
return
|
|
1781
|
-
with self._lock:
|
|
1782
|
-
self._register_manual(normalized)
|
|
1783
|
-
|
|
1784
|
-
def is_empty(self) -> bool:
|
|
1785
|
-
with self._lock:
|
|
1786
|
-
return not self._prefixes
|
|
1787
|
-
|
|
1788
|
-
def register_package(self, package_name: str, prefixes: Iterable[str]) -> None:
|
|
1789
|
-
owner_key = f"pkg::{package_name.lower()}"
|
|
1790
|
-
normalized = {_normalize_prefix(p) for p in prefixes if _normalize_prefix(p)}
|
|
1791
|
-
with self._lock:
|
|
1792
|
-
current = self._owner_map.get(owner_key, set())
|
|
1793
|
-
to_remove = current - normalized
|
|
1794
|
-
to_add = normalized - current
|
|
1795
|
-
|
|
1796
|
-
for prefix in to_remove:
|
|
1797
|
-
self._remove_prefix(prefix)
|
|
1798
|
-
for prefix in to_add:
|
|
1799
|
-
self._add_prefix(prefix)
|
|
1800
|
-
|
|
1801
|
-
if normalized:
|
|
1802
|
-
self._owner_map[owner_key] = normalized
|
|
1803
|
-
elif owner_key in self._owner_map:
|
|
1804
|
-
self._owner_map.pop(owner_key, None)
|
|
1805
|
-
|
|
1806
|
-
def is_prefix_owned_by(self, package_name: str, prefix: str) -> bool:
|
|
1807
|
-
normalized = _normalize_prefix(prefix)
|
|
1808
|
-
owner_key = f"pkg::{package_name.lower()}"
|
|
1809
|
-
with self._lock:
|
|
1810
|
-
if normalized in self._owner_map.get("__manual__", set()):
|
|
1811
|
-
return True
|
|
1812
|
-
return normalized in self._owner_map.get(owner_key, set())
|
|
1813
|
-
|
|
1814
|
-
def get_matching_prefixes(self, module_name: str) -> Tuple[str, ...]:
|
|
1815
|
-
with self._lock:
|
|
1816
|
-
if not self._prefixes:
|
|
1817
|
-
return ()
|
|
1818
|
-
self._ensure_trie()
|
|
1819
|
-
return self._trie.iter_matches(module_name)
|
|
1820
|
-
|
|
1821
|
-
def has_root(self, root_name: str) -> bool:
|
|
1822
|
-
snapshot = self._root_snapshot
|
|
1823
|
-
if not self._root_snapshot_dirty:
|
|
1824
|
-
return root_name in snapshot
|
|
1825
|
-
with self._lock:
|
|
1826
|
-
if self._root_snapshot_dirty:
|
|
1827
|
-
self._root_snapshot = set(self._root_refcounts.keys())
|
|
1828
|
-
self._root_snapshot_dirty = False
|
|
1829
|
-
return root_name in self._root_snapshot
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
_DEFAULT_WATCHED_PREFIXES = tuple(
|
|
1833
|
-
filter(
|
|
1834
|
-
None,
|
|
1835
|
-
os.environ.get(
|
|
1836
|
-
"XWLAZY_LAZY_PREFIXES",
|
|
1837
|
-
"",
|
|
1838
|
-
).split(";"),
|
|
1839
|
-
)
|
|
1840
|
-
)
|
|
1841
|
-
_watched_registry = WatchedPrefixRegistry(list(_DEFAULT_WATCHED_PREFIXES))
|
|
1842
|
-
|
|
1843
|
-
_DEFAULT_LAZY_METHODS = tuple(
|
|
1844
|
-
filter(
|
|
1845
|
-
None,
|
|
1846
|
-
os.environ.get("XWLAZY_LAZY_METHODS", "").split(","),
|
|
1847
|
-
)
|
|
1848
|
-
)
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
def register_lazy_module_prefix(prefix: str) -> None:
|
|
1852
|
-
"""Register an import prefix for lazy wrapping."""
|
|
1853
|
-
_watched_registry.add(prefix)
|
|
1854
|
-
normalized = _normalize_prefix(prefix)
|
|
1855
|
-
if normalized:
|
|
1856
|
-
_log("config", "Registered lazy module prefix: %s", normalized)
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
_lazy_prefix_method_registry: Dict[str, Tuple[str, ...]] = {}
|
|
1860
|
-
|
|
1861
|
-
_package_class_hints: Dict[str, Tuple[str, ...]] = {}
|
|
1862
|
-
_class_hint_lock = threading.RLock()
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
def _set_package_class_hints(package_name: str, hints: Iterable[str]) -> None:
|
|
1866
|
-
normalized: Tuple[str, ...] = tuple(
|
|
1867
|
-
OrderedDict((hint.lower(), None) for hint in hints if hint).keys() # type: ignore[arg-type]
|
|
1868
|
-
)
|
|
1869
|
-
with _class_hint_lock:
|
|
1870
|
-
if normalized:
|
|
1871
|
-
_package_class_hints[package_name] = normalized
|
|
1872
|
-
else:
|
|
1873
|
-
_package_class_hints.pop(package_name, None)
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
def _get_package_class_hints(package_name: str) -> Tuple[str, ...]:
|
|
1877
|
-
with _class_hint_lock:
|
|
1878
|
-
return _package_class_hints.get(package_name, ())
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
def _clear_all_package_class_hints() -> None:
|
|
1882
|
-
with _class_hint_lock:
|
|
1883
|
-
_package_class_hints.clear()
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
def register_lazy_module_methods(prefix: str, methods: Tuple[str, ...]) -> None:
|
|
1887
|
-
"""Register method names to enhance for all classes under a module prefix."""
|
|
1888
|
-
prefix = prefix.strip()
|
|
1889
|
-
if not prefix:
|
|
1890
|
-
return
|
|
1891
|
-
|
|
1892
|
-
if not prefix.endswith("."):
|
|
1893
|
-
prefix += "."
|
|
1894
|
-
|
|
1895
|
-
normalized_methods = tuple(m for m in methods if m)
|
|
1896
|
-
if not normalized_methods:
|
|
1897
|
-
return
|
|
1898
|
-
|
|
1899
|
-
_lazy_prefix_method_registry[prefix] = normalized_methods
|
|
1900
|
-
|
|
1901
|
-
# Retroactively enhance modules that were imported before this registration.
|
|
1902
|
-
target_prefix = prefix.rstrip(".")
|
|
1903
|
-
finder = LazyMetaPathFinder()
|
|
1904
|
-
for name, module in list(sys.modules.items()):
|
|
1905
|
-
if not isinstance(module, ModuleType):
|
|
1906
|
-
continue
|
|
1907
|
-
if not name.startswith(target_prefix):
|
|
1908
|
-
continue
|
|
1909
|
-
finder._enhance_classes_with_class_methods(module)
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
def _is_import_in_progress(module_name: str) -> bool:
|
|
1913
|
-
"""Check if a module import is currently in progress for this thread."""
|
|
1914
|
-
thread_id = threading.get_ident()
|
|
1915
|
-
with _import_cache_lock:
|
|
1916
|
-
return module_name in _import_in_progress.get(thread_id, set())
|
|
1917
|
-
|
|
1918
|
-
def _mark_import_started(module_name: str) -> None:
|
|
1919
|
-
"""Mark a module import as started for this thread."""
|
|
1920
|
-
thread_id = threading.get_ident()
|
|
1921
|
-
with _import_cache_lock:
|
|
1922
|
-
_import_in_progress[thread_id].add(module_name)
|
|
1923
|
-
|
|
1924
|
-
def _mark_import_finished(module_name: str) -> None:
|
|
1925
|
-
"""Mark a module import as finished for this thread."""
|
|
1926
|
-
thread_id = threading.get_ident()
|
|
1927
|
-
with _import_cache_lock:
|
|
1928
|
-
stack = _import_in_progress.get(thread_id)
|
|
1929
|
-
if not stack:
|
|
1930
|
-
return
|
|
1931
|
-
stack.discard(module_name)
|
|
1932
|
-
if not stack:
|
|
1933
|
-
_import_in_progress.pop(thread_id, None)
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
class LazyImportHook(AImportHook):
|
|
1937
|
-
"""
|
|
1938
|
-
Import hook that intercepts ImportError and auto-installs packages.
|
|
1939
|
-
Performance optimized with zero overhead for successful imports.
|
|
1940
|
-
"""
|
|
1941
|
-
|
|
1942
|
-
__slots__ = AImportHook.__slots__
|
|
1943
|
-
|
|
1944
|
-
def handle_import_error(self, module_name: str) -> Optional[Any]:
|
|
1945
|
-
"""Handle ImportError by attempting to install and re-import."""
|
|
1946
|
-
if not self._enabled:
|
|
1947
|
-
return None
|
|
1948
|
-
|
|
1949
|
-
try:
|
|
1950
|
-
module, success = lazy_import_with_install(
|
|
1951
|
-
module_name,
|
|
1952
|
-
installer_package=self._package_name
|
|
1953
|
-
)
|
|
1954
|
-
return module if success else None
|
|
1955
|
-
except:
|
|
1956
|
-
return None
|
|
1957
|
-
|
|
1958
|
-
def install_hook(self) -> None:
|
|
1959
|
-
"""Install the import hook into sys.meta_path."""
|
|
1960
|
-
install_import_hook(self._package_name)
|
|
1961
|
-
|
|
1962
|
-
def uninstall_hook(self) -> None:
|
|
1963
|
-
"""Uninstall the import hook from sys.meta_path."""
|
|
1964
|
-
uninstall_import_hook(self._package_name)
|
|
1965
|
-
|
|
1966
|
-
def is_installed(self) -> bool:
|
|
1967
|
-
"""Check if hook is installed."""
|
|
1968
|
-
return is_import_hook_installed(self._package_name)
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
class LazyMetaPathFinder:
|
|
1972
|
-
"""
|
|
1973
|
-
Custom meta path finder that intercepts failed imports.
|
|
1974
|
-
Performance optimized - only triggers when import would fail anyway.
|
|
1975
|
-
"""
|
|
1976
|
-
|
|
1977
|
-
__slots__ = ('_package_name', '_enabled')
|
|
1978
|
-
|
|
1979
|
-
def __init__(self, package_name: str = 'default'):
|
|
1980
|
-
"""Initialize meta path finder."""
|
|
1981
|
-
self._package_name = package_name
|
|
1982
|
-
self._enabled = True
|
|
1983
|
-
|
|
1984
|
-
def _build_async_placeholder(
|
|
1985
|
-
self,
|
|
1986
|
-
fullname: str,
|
|
1987
|
-
installer: LazyInstaller,
|
|
1988
|
-
) -> Optional[importlib.machinery.ModuleSpec]:
|
|
1989
|
-
"""Create and register a deferred module placeholder for async installs."""
|
|
1990
|
-
handle = installer.ensure_async_install(fullname)
|
|
1991
|
-
if handle is None:
|
|
1992
|
-
return None
|
|
1993
|
-
|
|
1994
|
-
missing = ModuleNotFoundError(f"No module named '{fullname}'")
|
|
1995
|
-
deferred = DeferredImportError(fullname, missing, self._package_name, async_handle=handle)
|
|
1996
|
-
|
|
1997
|
-
module = ModuleType(fullname)
|
|
1998
|
-
loader = _DeferredModuleLoader(module)
|
|
1999
|
-
|
|
2000
|
-
def _resolve_real_module():
|
|
2001
|
-
real_module = deferred._try_install_and_import()
|
|
2002
|
-
sys.modules[fullname] = real_module
|
|
2003
|
-
module.__dict__.clear()
|
|
2004
|
-
module.__dict__.update(real_module.__dict__)
|
|
2005
|
-
module.__loader__ = getattr(real_module, "__loader__", loader)
|
|
2006
|
-
module.__spec__ = getattr(real_module, "__spec__", None)
|
|
2007
|
-
module.__path__ = getattr(real_module, "__path__", getattr(module, "__path__", []))
|
|
2008
|
-
module.__class__ = real_module.__class__
|
|
2009
|
-
spec_obj = getattr(real_module, "__spec__", None) or importlib.util.find_spec(fullname)
|
|
2010
|
-
if spec_obj is not None:
|
|
2011
|
-
_spec_cache_put(fullname, spec_obj)
|
|
2012
|
-
return real_module
|
|
2013
|
-
|
|
2014
|
-
def _module_getattr(name):
|
|
2015
|
-
real = _resolve_real_module()
|
|
2016
|
-
return getattr(real, name)
|
|
2017
|
-
|
|
2018
|
-
def _module_dir():
|
|
2019
|
-
try:
|
|
2020
|
-
real = _resolve_real_module()
|
|
2021
|
-
return dir(real)
|
|
2022
|
-
except Exception:
|
|
2023
|
-
return []
|
|
2024
|
-
|
|
2025
|
-
module.__getattr__ = _module_getattr # type: ignore[attr-defined]
|
|
2026
|
-
module.__dir__ = _module_dir # type: ignore[attr-defined]
|
|
2027
|
-
module.__loader__ = loader
|
|
2028
|
-
module.__package__ = fullname
|
|
2029
|
-
module.__path__ = []
|
|
2030
|
-
|
|
2031
|
-
spec = importlib.machinery.ModuleSpec(fullname, loader)
|
|
2032
|
-
spec.submodule_search_locations = []
|
|
2033
|
-
module.__spec__ = spec
|
|
2034
|
-
|
|
2035
|
-
sys.modules[fullname] = module
|
|
2036
|
-
_log("hook", f"⏳ [HOOK] Deferred import placeholder created for '{fullname}'")
|
|
2037
|
-
return spec
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
def _spec_for_existing_module(
|
|
2041
|
-
fullname: str,
|
|
2042
|
-
module: ModuleType,
|
|
2043
|
-
original_spec: Optional[importlib.machinery.ModuleSpec] = None,
|
|
2044
|
-
) -> importlib.machinery.ModuleSpec:
|
|
2045
|
-
"""
|
|
2046
|
-
Build a ModuleSpec whose loader simply returns an already-initialized module.
|
|
2047
|
-
|
|
2048
|
-
Used to hand control back to importlib without re-executing third-party code.
|
|
2049
|
-
"""
|
|
2050
|
-
loader = _DeferredModuleLoader(module)
|
|
2051
|
-
spec = importlib.machinery.ModuleSpec(fullname, loader)
|
|
2052
|
-
if original_spec and original_spec.submodule_search_locations is not None:
|
|
2053
|
-
locations = list(original_spec.submodule_search_locations)
|
|
2054
|
-
spec.submodule_search_locations = locations
|
|
2055
|
-
if hasattr(module, "__path__"):
|
|
2056
|
-
module.__path__ = locations
|
|
2057
|
-
module.__loader__ = loader
|
|
2058
|
-
module.__spec__ = spec
|
|
2059
|
-
return spec
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
class LazyMetaPathFinder:
|
|
2063
|
-
"""
|
|
2064
|
-
Custom meta path finder that intercepts failed imports.
|
|
2065
|
-
Performance optimized - only triggers when import would fail anyway.
|
|
2066
|
-
"""
|
|
2067
|
-
|
|
2068
|
-
__slots__ = ('_package_name', '_enabled')
|
|
2069
|
-
|
|
2070
|
-
def __init__(self, package_name: str = 'default'):
|
|
2071
|
-
"""Initialize meta path finder."""
|
|
2072
|
-
self._package_name = package_name
|
|
2073
|
-
self._enabled = True
|
|
2074
|
-
|
|
2075
|
-
def _build_async_placeholder(
|
|
2076
|
-
self,
|
|
2077
|
-
fullname: str,
|
|
2078
|
-
installer: LazyInstaller,
|
|
2079
|
-
) -> Optional[importlib.machinery.ModuleSpec]:
|
|
2080
|
-
"""Create and register a deferred module placeholder for async installs."""
|
|
2081
|
-
handle = installer.ensure_async_install(fullname)
|
|
2082
|
-
if handle is None:
|
|
2083
|
-
return None
|
|
2084
|
-
|
|
2085
|
-
missing = ModuleNotFoundError(f"No module named '{fullname}'")
|
|
2086
|
-
deferred = DeferredImportError(fullname, missing, self._package_name, async_handle=handle)
|
|
2087
|
-
|
|
2088
|
-
module = ModuleType(fullname)
|
|
2089
|
-
loader = _DeferredModuleLoader(module)
|
|
2090
|
-
|
|
2091
|
-
def _resolve_real_module():
|
|
2092
|
-
real_module = deferred._try_install_and_import()
|
|
2093
|
-
sys.modules[fullname] = real_module
|
|
2094
|
-
module.__dict__.clear()
|
|
2095
|
-
module.__dict__.update(real_module.__dict__)
|
|
2096
|
-
module.__loader__ = getattr(real_module, "__loader__", loader)
|
|
2097
|
-
module.__spec__ = getattr(real_module, "__spec__", None)
|
|
2098
|
-
module.__path__ = getattr(real_module, "__path__", getattr(module, "__path__", []))
|
|
2099
|
-
module.__class__ = real_module.__class__
|
|
2100
|
-
spec_obj = getattr(real_module, "__spec__", None) or importlib.util.find_spec(fullname)
|
|
2101
|
-
if spec_obj is not None:
|
|
2102
|
-
_spec_cache_put(fullname, spec_obj)
|
|
2103
|
-
return real_module
|
|
2104
|
-
|
|
2105
|
-
def _module_getattr(name):
|
|
2106
|
-
real = _resolve_real_module()
|
|
2107
|
-
return getattr(real, name)
|
|
2108
|
-
|
|
2109
|
-
def _module_dir():
|
|
2110
|
-
try:
|
|
2111
|
-
real = _resolve_real_module()
|
|
2112
|
-
return dir(real)
|
|
2113
|
-
except Exception:
|
|
2114
|
-
return []
|
|
2115
|
-
|
|
2116
|
-
module.__getattr__ = _module_getattr # type: ignore[attr-defined]
|
|
2117
|
-
module.__dir__ = _module_dir # type: ignore[attr-defined]
|
|
2118
|
-
module.__loader__ = loader
|
|
2119
|
-
module.__package__ = fullname
|
|
2120
|
-
module.__path__ = []
|
|
2121
|
-
|
|
2122
|
-
spec = importlib.machinery.ModuleSpec(fullname, loader)
|
|
2123
|
-
spec.submodule_search_locations = []
|
|
2124
|
-
module.__spec__ = spec
|
|
2125
|
-
|
|
2126
|
-
sys.modules[fullname] = module
|
|
2127
|
-
_log("hook", f"⏳ [HOOK] Deferred import placeholder created for '{fullname}'")
|
|
2128
|
-
return spec
|
|
2129
|
-
|
|
2130
|
-
def find_module(self, fullname: str, path: Optional[str] = None):
|
|
2131
|
-
"""Find module - returns None to let standard import continue."""
|
|
2132
|
-
return None
|
|
2133
|
-
|
|
2134
|
-
def find_spec(self, fullname: str, path: Optional[str] = None, target=None):
|
|
2135
|
-
"""Find module spec - intercepts imports to enable two-stage lazy loading."""
|
|
2136
|
-
if not self._enabled:
|
|
2137
|
-
return None
|
|
2138
|
-
|
|
2139
|
-
# CRITICAL: Bail out immediately for stdlib/builtin and importlib internals
|
|
2140
|
-
# to avoid interfering with importlib.resources (Microsoft Store Python bug)
|
|
2141
|
-
if fullname.startswith('importlib') or fullname.startswith('_frozen_importlib'):
|
|
2142
|
-
return None
|
|
2143
|
-
|
|
2144
|
-
if '.' not in fullname:
|
|
2145
|
-
if DependencyMapper._is_stdlib_or_builtin(fullname):
|
|
2146
|
-
return None
|
|
2147
|
-
if fullname in DependencyMapper.DENY_LIST:
|
|
2148
|
-
return None
|
|
2149
|
-
|
|
2150
|
-
if _is_import_in_progress(fullname):
|
|
2151
|
-
logger.debug(f"[RECURSION GUARD] Import '{fullname}' already in progress, skipping hook")
|
|
2152
|
-
return None
|
|
2153
|
-
|
|
2154
|
-
if getattr(_importing, 'active', False):
|
|
2155
|
-
logger.debug(f"[RECURSION GUARD] Lazy wrapping suspended while importing '{fullname}'")
|
|
2156
|
-
return None
|
|
2157
|
-
|
|
2158
|
-
cached_spec = _spec_cache_get(fullname)
|
|
2159
|
-
if cached_spec is not None:
|
|
2160
|
-
return cached_spec
|
|
2161
|
-
|
|
2162
|
-
lazy_enabled = is_lazy_install_enabled(self._package_name)
|
|
2163
|
-
if not lazy_enabled and _watched_registry.is_empty():
|
|
2164
|
-
return None
|
|
2165
|
-
|
|
2166
|
-
root_name = fullname.split('.', 1)[0]
|
|
2167
|
-
matching_prefixes: Tuple[str, ...] = ()
|
|
2168
|
-
if _watched_registry.has_root(root_name):
|
|
2169
|
-
matching_prefixes = _watched_registry.get_matching_prefixes(fullname)
|
|
2170
|
-
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
2171
|
-
|
|
2172
|
-
# Two-stage lazy loading for serialization and archive modules
|
|
2173
|
-
# Host packages register prefixes to monitor (serialization modules, archives, etc.)
|
|
2174
|
-
# Priority impact: Usability (#2) - Missing dependencies not auto-installed
|
|
2175
|
-
for prefix in matching_prefixes:
|
|
2176
|
-
if not _watched_registry.is_prefix_owned_by(self._package_name, prefix):
|
|
2177
|
-
continue
|
|
2178
|
-
if fullname.startswith(prefix):
|
|
2179
|
-
module_suffix = fullname[len(prefix):]
|
|
2180
|
-
|
|
2181
|
-
if module_suffix:
|
|
2182
|
-
_log("hook", f"[HOOK] Candidate for wrapping: {fullname}")
|
|
2183
|
-
|
|
2184
|
-
_mark_import_started(fullname)
|
|
2185
|
-
try:
|
|
2186
|
-
if getattr(_importing, 'active', False):
|
|
2187
|
-
logger.debug(f"[HOOK] Recursion guard active, skipping {fullname}")
|
|
2188
|
-
return None
|
|
2189
|
-
|
|
2190
|
-
try:
|
|
2191
|
-
logger.debug(f"[HOOK] Looking for spec: {fullname}")
|
|
2192
|
-
spec = _spec_cache_get(fullname)
|
|
2193
|
-
if spec is None:
|
|
2194
|
-
# Temporarily remove hook to avoid interfering with nested imports
|
|
2195
|
-
try:
|
|
2196
|
-
sys.meta_path.remove(self)
|
|
2197
|
-
except ValueError:
|
|
2198
|
-
pass
|
|
2199
|
-
try:
|
|
2200
|
-
spec = importlib.util.find_spec(fullname)
|
|
2201
|
-
finally:
|
|
2202
|
-
# Restore hook
|
|
2203
|
-
if self not in sys.meta_path:
|
|
2204
|
-
sys.meta_path.insert(0, self)
|
|
2205
|
-
if spec is not None:
|
|
2206
|
-
_spec_cache_put(fullname, spec)
|
|
2207
|
-
if spec is not None:
|
|
2208
|
-
logger.debug(f"[HOOK] Spec found, trying normal import: {fullname}")
|
|
2209
|
-
_importing.active = True
|
|
2210
|
-
try:
|
|
2211
|
-
# Temporarily remove hook during __import__ to avoid interfering with nested imports
|
|
2212
|
-
try:
|
|
2213
|
-
sys.meta_path.remove(self)
|
|
2214
|
-
except ValueError:
|
|
2215
|
-
pass
|
|
2216
|
-
try:
|
|
2217
|
-
__import__(fullname)
|
|
2218
|
-
finally:
|
|
2219
|
-
if self not in sys.meta_path:
|
|
2220
|
-
sys.meta_path.insert(0, self)
|
|
2221
|
-
|
|
2222
|
-
module = sys.modules.get(fullname)
|
|
2223
|
-
if module:
|
|
2224
|
-
try:
|
|
2225
|
-
self._enhance_classes_with_class_methods(module)
|
|
2226
|
-
except Exception as enhance_exc:
|
|
2227
|
-
logger.debug(f"[HOOK] Could not enhance classes in {fullname}: {enhance_exc}")
|
|
2228
|
-
spec = _spec_for_existing_module(fullname, module, spec)
|
|
2229
|
-
_log("hook", f"✓ [HOOK] Module {fullname} imported successfully, no wrapping needed")
|
|
2230
|
-
if spec is not None:
|
|
2231
|
-
_spec_cache_put(fullname, spec)
|
|
2232
|
-
return spec
|
|
2233
|
-
return None
|
|
2234
|
-
finally:
|
|
2235
|
-
_importing.active = False
|
|
2236
|
-
except ImportError as e:
|
|
2237
|
-
if '.' not in module_suffix:
|
|
2238
|
-
_log("hook", f"⚠ [HOOK] Module {fullname} has missing dependencies, wrapping: {e}")
|
|
2239
|
-
wrapped_spec = self._wrap_serialization_module(fullname)
|
|
2240
|
-
if wrapped_spec is not None:
|
|
2241
|
-
_log("hook", f"✓ [HOOK] Successfully wrapped: {fullname}")
|
|
2242
|
-
return wrapped_spec
|
|
2243
|
-
logger.warning(f"✗ [HOOK] Failed to wrap: {fullname}")
|
|
2244
|
-
else:
|
|
2245
|
-
logger.debug(f"[HOOK] Import failed for nested module {fullname}: {e}")
|
|
2246
|
-
except (ModuleNotFoundError,) as e:
|
|
2247
|
-
logger.debug(f"[HOOK] Module {fullname} not found, skipping wrap: {e}")
|
|
2248
|
-
pass
|
|
2249
|
-
except Exception as e:
|
|
2250
|
-
logger.warning(f"[HOOK] Error checking module {fullname}: {e}")
|
|
2251
|
-
finally:
|
|
2252
|
-
_mark_import_finished(fullname)
|
|
2253
|
-
|
|
2254
|
-
return None
|
|
2255
|
-
|
|
2256
|
-
# Only handle top-level packages
|
|
2257
|
-
if '.' in fullname:
|
|
2258
|
-
return None
|
|
2259
|
-
if DependencyMapper._is_stdlib_or_builtin(fullname):
|
|
2260
|
-
return None
|
|
2261
|
-
if fullname in DependencyMapper.DENY_LIST:
|
|
2262
|
-
return None
|
|
2263
|
-
|
|
2264
|
-
_mark_import_started(fullname)
|
|
2265
|
-
try:
|
|
2266
|
-
try:
|
|
2267
|
-
if not lazy_enabled:
|
|
2268
|
-
return None
|
|
2269
|
-
|
|
2270
|
-
module, success = lazy_import_with_install(
|
|
2271
|
-
fullname,
|
|
2272
|
-
installer_package=self._package_name
|
|
2273
|
-
)
|
|
2274
|
-
|
|
2275
|
-
if success and module:
|
|
2276
|
-
spec = getattr(module, "__spec__", None)
|
|
2277
|
-
if spec is None:
|
|
2278
|
-
try:
|
|
2279
|
-
sys.meta_path.remove(self)
|
|
2280
|
-
except ValueError:
|
|
2281
|
-
pass
|
|
2282
|
-
try:
|
|
2283
|
-
spec = importlib.util.find_spec(fullname)
|
|
2284
|
-
finally:
|
|
2285
|
-
if self not in sys.meta_path:
|
|
2286
|
-
sys.meta_path.insert(0, self)
|
|
2287
|
-
if spec is not None and spec.loader:
|
|
2288
|
-
module.__spec__ = spec
|
|
2289
|
-
module.__loader__ = spec.loader
|
|
2290
|
-
_spec_cache_put(fullname, spec)
|
|
2291
|
-
return spec
|
|
2292
|
-
return None
|
|
2293
|
-
if not success and installer.is_async_enabled():
|
|
2294
|
-
placeholder = self._build_async_placeholder(fullname, installer)
|
|
2295
|
-
if placeholder is not None:
|
|
2296
|
-
return placeholder
|
|
2297
|
-
|
|
2298
|
-
except Exception as e:
|
|
2299
|
-
logger.debug(f"Lazy import hook failed for {fullname}: {e}")
|
|
2300
|
-
|
|
2301
|
-
return None
|
|
2302
|
-
finally:
|
|
2303
|
-
_mark_import_finished(fullname)
|
|
2304
|
-
|
|
2305
|
-
def _wrap_serialization_module(self, fullname: str):
|
|
2306
|
-
"""Wrap serialization module loading to defer missing dependencies."""
|
|
2307
|
-
_log("hook", f"[STAGE 1] Starting wrap of module: {fullname}")
|
|
2308
|
-
|
|
2309
|
-
try:
|
|
2310
|
-
logger.debug(f"[STAGE 1] Getting spec for: {fullname}")
|
|
2311
|
-
# Temporarily remove hook to avoid interfering with nested imports
|
|
2312
|
-
try:
|
|
2313
|
-
sys.meta_path.remove(self)
|
|
2314
|
-
except ValueError:
|
|
2315
|
-
pass
|
|
2316
|
-
try:
|
|
2317
|
-
spec = importlib.util.find_spec(fullname)
|
|
2318
|
-
finally:
|
|
2319
|
-
if self not in sys.meta_path:
|
|
2320
|
-
sys.meta_path.insert(0, self)
|
|
2321
|
-
if not spec or not spec.loader:
|
|
2322
|
-
logger.warning(f"[STAGE 1] No spec or loader for: {fullname}")
|
|
2323
|
-
return None
|
|
2324
|
-
|
|
2325
|
-
logger.debug(f"[STAGE 1] Creating module from spec: {fullname}")
|
|
2326
|
-
module = importlib.util.module_from_spec(spec)
|
|
2327
|
-
|
|
2328
|
-
deferred_imports = {}
|
|
2329
|
-
|
|
2330
|
-
logger.debug(f"[STAGE 1] Setting up import wrapper for: {fullname}")
|
|
2331
|
-
original_import = builtins.__import__
|
|
2332
|
-
|
|
2333
|
-
def capture_import_errors(name, *args, **kwargs):
|
|
2334
|
-
"""Intercept imports and defer ONLY external missing packages."""
|
|
2335
|
-
logger.debug(f"[STAGE 1] capture_import_errors: Trying to import '{name}' in {fullname}")
|
|
2336
|
-
|
|
2337
|
-
if _is_import_in_progress(name):
|
|
2338
|
-
logger.debug(f"[STAGE 1] Import '{name}' already in progress, using original_import")
|
|
2339
|
-
return original_import(name, *args, **kwargs)
|
|
2340
|
-
|
|
2341
|
-
_mark_import_started(name)
|
|
2342
|
-
try:
|
|
2343
|
-
result = original_import(name, *args, **kwargs)
|
|
2344
|
-
logger.debug(f"[STAGE 1] ✓ Successfully imported '{name}'")
|
|
2345
|
-
return result
|
|
2346
|
-
except ImportError as e:
|
|
2347
|
-
logger.debug(f"[STAGE 1] ✗ Import failed for '{name}': {e}")
|
|
2348
|
-
|
|
2349
|
-
host_alias = self._package_name or ""
|
|
2350
|
-
if name.startswith('exonware.') or (host_alias and name.startswith(f"{host_alias}.")):
|
|
2351
|
-
_log("hook", f"[STAGE 1] Letting internal import '{name}' fail normally (internal package)")
|
|
2352
|
-
raise
|
|
2353
|
-
|
|
2354
|
-
if '.' in name:
|
|
2355
|
-
_log("hook", f"[STAGE 1] Letting submodule '{name}' fail normally (has dots)")
|
|
2356
|
-
raise
|
|
2357
|
-
|
|
2358
|
-
_log("hook", f"⏳ [STAGE 1] DEFERRING missing external package '{name}' in {fullname}")
|
|
2359
|
-
async_handle = None
|
|
2360
|
-
try:
|
|
2361
|
-
installer = LazyInstallerRegistry.get_instance(self._package_name)
|
|
2362
|
-
async_handle = installer.schedule_async_install(name)
|
|
2363
|
-
except Exception as schedule_exc:
|
|
2364
|
-
logger.debug(f"[STAGE 1] Async install scheduling failed for '{name}': {schedule_exc}")
|
|
2365
|
-
deferred = DeferredImportError(name, e, self._package_name, async_handle=async_handle)
|
|
2366
|
-
deferred_imports[name] = deferred
|
|
2367
|
-
return deferred
|
|
2368
|
-
finally:
|
|
2369
|
-
_mark_import_finished(name)
|
|
2370
|
-
|
|
2371
|
-
logger.debug(f"[STAGE 1] Executing module with import wrapper: {fullname}")
|
|
2372
|
-
builtins.__import__ = capture_import_errors
|
|
2373
|
-
try:
|
|
2374
|
-
spec.loader.exec_module(module)
|
|
2375
|
-
logger.debug(f"[STAGE 1] Module execution completed: {fullname}")
|
|
2376
|
-
|
|
2377
|
-
if deferred_imports:
|
|
2378
|
-
_log("hook", f"✓ [STAGE 1] Module {fullname} loaded with {len(deferred_imports)} deferred imports: {list(deferred_imports.keys())}")
|
|
2379
|
-
# Replace None values with deferred import proxies (for modules that catch ImportError and set to None)
|
|
2380
|
-
self._replace_none_with_deferred(module, deferred_imports)
|
|
2381
|
-
self._wrap_module_classes(module, deferred_imports)
|
|
2382
|
-
else:
|
|
2383
|
-
_log("hook", f"✓ [STAGE 1] Module {fullname} loaded with NO deferred imports (all dependencies available)")
|
|
2384
|
-
|
|
2385
|
-
# Always enhance serializers with class-level convenience methods
|
|
2386
|
-
self._enhance_classes_with_class_methods(module)
|
|
2387
|
-
|
|
2388
|
-
finally:
|
|
2389
|
-
logger.debug(f"[STAGE 1] Restoring original __import__")
|
|
2390
|
-
builtins.__import__ = original_import
|
|
2391
|
-
|
|
2392
|
-
logger.debug(f"[STAGE 1] Registering module in sys.modules: {fullname}")
|
|
2393
|
-
sys.modules[fullname] = module
|
|
2394
|
-
final_spec = _spec_for_existing_module(fullname, module, spec)
|
|
2395
|
-
_spec_cache_put(fullname, final_spec)
|
|
2396
|
-
_log("hook", f"✓ [STAGE 1] Successfully wrapped and registered: {fullname}")
|
|
2397
|
-
return final_spec
|
|
2398
|
-
|
|
2399
|
-
except Exception as e:
|
|
2400
|
-
logger.debug(f"Could not wrap {fullname}: {e}")
|
|
2401
|
-
return None
|
|
2402
|
-
|
|
2403
|
-
def _replace_none_with_deferred(self, module, deferred_imports: Dict):
|
|
2404
|
-
"""
|
|
2405
|
-
Replace None values in module namespace with deferred import proxies.
|
|
2406
|
-
|
|
2407
|
-
Some modules catch ImportError and set the variable to None (e.g., yaml = None).
|
|
2408
|
-
This method replaces those None values with DeferredImportError proxies so the
|
|
2409
|
-
hook can install missing packages when the variable is accessed.
|
|
2410
|
-
"""
|
|
2411
|
-
logger.debug(f"[STAGE 1] Replacing None with deferred imports in {module.__name__}")
|
|
2412
|
-
replaced_count = 0
|
|
2413
|
-
|
|
2414
|
-
for dep_name, deferred_import in deferred_imports.items():
|
|
2415
|
-
# Check if module has this variable set to None
|
|
2416
|
-
if hasattr(module, dep_name):
|
|
2417
|
-
current_value = getattr(module, dep_name)
|
|
2418
|
-
if current_value is None:
|
|
2419
|
-
_log("hook", f"[STAGE 1] Replacing {dep_name}=None with deferred import proxy in {module.__name__}")
|
|
2420
|
-
setattr(module, dep_name, deferred_import)
|
|
2421
|
-
replaced_count += 1
|
|
2422
|
-
|
|
2423
|
-
if replaced_count > 0:
|
|
2424
|
-
_log("hook", f"✓ [STAGE 1] Replaced {replaced_count} None values with deferred imports in {module.__name__}")
|
|
2425
|
-
|
|
2426
|
-
def _wrap_module_classes(self, module, deferred_imports: Dict):
|
|
2427
|
-
"""Wrap classes in a module that depend on deferred imports."""
|
|
2428
|
-
module_name = getattr(module, '__name__', '<unknown>')
|
|
2429
|
-
logger.debug(f"[STAGE 1] Wrapping classes in {module_name} (deferred: {list(deferred_imports.keys())})")
|
|
2430
|
-
module_file = (getattr(module, '__file__', '') or '').lower()
|
|
2431
|
-
lower_map = {dep_name.lower(): dep_name for dep_name in deferred_imports.keys()}
|
|
2432
|
-
class_hints = _get_package_class_hints(self._package_name)
|
|
2433
|
-
with _wrapped_cache_lock:
|
|
2434
|
-
already_wrapped = _WRAPPED_CLASS_CACHE.setdefault(module_name, set()).copy()
|
|
2435
|
-
pending_lower = {lower for lower in lower_map.keys() if lower_map[lower] not in already_wrapped}
|
|
2436
|
-
if not pending_lower:
|
|
2437
|
-
logger.debug(f"[STAGE 1] All deferred imports already wrapped for {module_name}")
|
|
2438
|
-
return
|
|
2439
|
-
dep_entries = [(lower, deferred_imports[lower_map[lower]]) for lower in pending_lower]
|
|
2440
|
-
wrapped_count = 0
|
|
2441
|
-
newly_wrapped: Set[str] = set()
|
|
2442
|
-
|
|
2443
|
-
for name, obj in list(module.__dict__.items()):
|
|
2444
|
-
if not pending_lower:
|
|
2445
|
-
break
|
|
2446
|
-
if not isinstance(obj, type):
|
|
2447
|
-
continue
|
|
2448
|
-
lower_name = name.lower()
|
|
2449
|
-
if class_hints and not any(hint in lower_name for hint in class_hints):
|
|
2450
|
-
continue
|
|
2451
|
-
target_lower = None
|
|
2452
|
-
target_deferred = None
|
|
2453
|
-
for dep_lower, deferred in dep_entries:
|
|
2454
|
-
if dep_lower not in pending_lower:
|
|
2455
|
-
continue
|
|
2456
|
-
if dep_lower in lower_name or dep_lower in module_file:
|
|
2457
|
-
target_lower = dep_lower
|
|
2458
|
-
target_deferred = deferred
|
|
2459
|
-
break
|
|
2460
|
-
if target_deferred is None or target_lower is None:
|
|
2461
|
-
continue
|
|
2462
|
-
|
|
2463
|
-
logger.debug(f"[STAGE 1] Class '{name}' depends on deferred import, wrapping...")
|
|
2464
|
-
wrapped = self._create_lazy_class_wrapper(obj, target_deferred)
|
|
2465
|
-
module.__dict__[name] = wrapped
|
|
2466
|
-
wrapped_count += 1
|
|
2467
|
-
origin_name = lower_map.get(target_lower, target_lower)
|
|
2468
|
-
newly_wrapped.add(origin_name)
|
|
2469
|
-
pending_lower.discard(target_lower)
|
|
2470
|
-
_log("hook", f"✓ [STAGE 1] Wrapped class '{name}' in {module_name}")
|
|
2471
|
-
|
|
2472
|
-
if newly_wrapped:
|
|
2473
|
-
with _wrapped_cache_lock:
|
|
2474
|
-
cache = _WRAPPED_CLASS_CACHE.setdefault(module_name, set())
|
|
2475
|
-
cache.update(newly_wrapped)
|
|
2476
|
-
|
|
2477
|
-
_log("hook", f"[STAGE 1] Wrapped {wrapped_count} classes in {module_name}")
|
|
2478
|
-
|
|
2479
|
-
def _enhance_classes_with_class_methods(self, module):
|
|
2480
|
-
"""
|
|
2481
|
-
Enhance classes that registered lazy class methods.
|
|
2482
|
-
|
|
2483
|
-
Root cause: Original implementation wrapped instance methods as classmethods,
|
|
2484
|
-
breaking normal usage (e.g., serializer.encode(data) failed with missing 'value').
|
|
2485
|
-
|
|
2486
|
-
Fix: Only wrap if method is NOT already an instance method. Instance methods
|
|
2487
|
-
should remain as-is; we only add classmethod wrappers for static/class methods.
|
|
2488
|
-
|
|
2489
|
-
Priority: Usability (#2) - Preserve normal API usage patterns
|
|
2490
|
-
"""
|
|
2491
|
-
if module is None:
|
|
2492
|
-
return
|
|
2493
|
-
|
|
2494
|
-
methods_to_apply: Tuple[str, ...] = ()
|
|
2495
|
-
for prefix, methods in _lazy_prefix_method_registry.items():
|
|
2496
|
-
if module.__name__.startswith(prefix.rstrip('.')):
|
|
2497
|
-
methods_to_apply = methods
|
|
2498
|
-
break
|
|
2499
|
-
|
|
2500
|
-
if not methods_to_apply:
|
|
2501
|
-
methods_to_apply = _DEFAULT_LAZY_METHODS
|
|
2502
|
-
|
|
2503
|
-
if not methods_to_apply:
|
|
2504
|
-
return
|
|
2505
|
-
|
|
2506
|
-
enhanced = 0
|
|
2507
|
-
for name, obj in list(module.__dict__.items()):
|
|
2508
|
-
if not isinstance(obj, type):
|
|
2509
|
-
continue
|
|
2510
|
-
for method_name in methods_to_apply:
|
|
2511
|
-
attr = obj.__dict__.get(method_name)
|
|
2512
|
-
if attr is None:
|
|
2513
|
-
continue
|
|
2514
|
-
if getattr(attr, "__lazy_wrapped__", False):
|
|
2515
|
-
continue
|
|
2516
|
-
if not callable(attr):
|
|
2517
|
-
continue
|
|
2518
|
-
|
|
2519
|
-
# Skip if already a classmethod or staticmethod descriptor
|
|
2520
|
-
if isinstance(attr, (classmethod, staticmethod)):
|
|
2521
|
-
continue
|
|
2522
|
-
|
|
2523
|
-
# Wrap instance methods to auto-instantiate when called class-level
|
|
2524
|
-
# Root cause: json_run.py uses BsonSerializer.encode(data) without instantiation
|
|
2525
|
-
# Solution: Wrapper auto-instantiates and delegates to instance method
|
|
2526
|
-
# Priority: Usability (#2) - Enable convenient class-level API
|
|
2527
|
-
import inspect
|
|
2528
|
-
try:
|
|
2529
|
-
sig = inspect.signature(attr)
|
|
2530
|
-
params = list(sig.parameters.keys())
|
|
2531
|
-
# Instance methods (first param is 'self') get wrapped for class-level convenience
|
|
2532
|
-
if params and params[0] == 'self':
|
|
2533
|
-
logger.debug(
|
|
2534
|
-
"[LAZY ENHANCE] Wrapping instance method %s.%s.%s for class-level access",
|
|
2535
|
-
module.__name__,
|
|
2536
|
-
name,
|
|
2537
|
-
method_name,
|
|
2538
|
-
)
|
|
2539
|
-
except Exception:
|
|
2540
|
-
# If we can't inspect, try wrapping anyway
|
|
2541
|
-
pass
|
|
2542
|
-
|
|
2543
|
-
try:
|
|
2544
|
-
original_func = attr
|
|
2545
|
-
|
|
2546
|
-
def class_method_wrapper(func):
|
|
2547
|
-
def _class_call(cls, *args, **kwargs):
|
|
2548
|
-
instance = cls()
|
|
2549
|
-
return func(instance, *args, **kwargs)
|
|
2550
|
-
_class_call.__name__ = getattr(func, '__name__', 'lazy_method')
|
|
2551
|
-
_class_call.__doc__ = func.__doc__
|
|
2552
|
-
_class_call.__lazy_wrapped__ = True
|
|
2553
|
-
return _class_call
|
|
2554
|
-
|
|
2555
|
-
setattr(
|
|
2556
|
-
obj,
|
|
2557
|
-
method_name,
|
|
2558
|
-
classmethod(class_method_wrapper(original_func)),
|
|
2559
|
-
)
|
|
2560
|
-
enhanced += 1
|
|
2561
|
-
logger.debug(
|
|
2562
|
-
"[LAZY ENHANCE] Added class-level %s() to %s.%s",
|
|
2563
|
-
method_name,
|
|
2564
|
-
module.__name__,
|
|
2565
|
-
name,
|
|
2566
|
-
)
|
|
2567
|
-
except Exception as exc:
|
|
2568
|
-
logger.debug(
|
|
2569
|
-
"[LAZY ENHANCE] Skipped %s.%s.%s: %s",
|
|
2570
|
-
module.__name__,
|
|
2571
|
-
name,
|
|
2572
|
-
method_name,
|
|
2573
|
-
exc,
|
|
2574
|
-
)
|
|
2575
|
-
|
|
2576
|
-
if enhanced:
|
|
2577
|
-
_log("enhance", "✓ [LAZY ENHANCE] Added %s convenience methods in %s", enhanced, module.__name__)
|
|
2578
|
-
|
|
2579
|
-
def _create_lazy_class_wrapper(self, original_class, deferred_import: DeferredImportError):
|
|
2580
|
-
"""Create a wrapper class that installs dependencies when instantiated."""
|
|
2581
|
-
class LazyClassWrapper:
|
|
2582
|
-
"""Lazy wrapper that installs dependencies on first instantiation."""
|
|
2583
|
-
|
|
2584
|
-
def __init__(self, *args, **kwargs):
|
|
2585
|
-
"""Install dependency and create real instance."""
|
|
2586
|
-
deferred_import._try_install_and_import()
|
|
2587
|
-
|
|
2588
|
-
real_module = importlib.reload(sys.modules[original_class.__module__])
|
|
2589
|
-
real_class = getattr(real_module, original_class.__name__)
|
|
2590
|
-
|
|
2591
|
-
real_instance = real_class(*args, **kwargs)
|
|
2592
|
-
self.__class__ = real_class
|
|
2593
|
-
self.__dict__ = real_instance.__dict__
|
|
2594
|
-
|
|
2595
|
-
def __repr__(self):
|
|
2596
|
-
return f"<Lazy{original_class.__name__}: will install dependencies on init>"
|
|
2597
|
-
|
|
2598
|
-
LazyClassWrapper.__name__ = f"Lazy{original_class.__name__}"
|
|
2599
|
-
LazyClassWrapper.__qualname__ = f"Lazy{original_class.__qualname__}"
|
|
2600
|
-
LazyClassWrapper.__module__ = original_class.__module__
|
|
2601
|
-
LazyClassWrapper.__doc__ = original_class.__doc__
|
|
2602
|
-
|
|
2603
|
-
return LazyClassWrapper
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
# Registry of installed hooks per package
|
|
2607
|
-
_installed_hooks: Dict[str, LazyMetaPathFinder] = {}
|
|
2608
|
-
_hook_lock = threading.RLock()
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
def install_import_hook(package_name: str = 'default') -> None:
|
|
2612
|
-
"""Install performant import hook for automatic lazy installation."""
|
|
2613
|
-
global _installed_hooks
|
|
2614
|
-
|
|
2615
|
-
_log("hook", f"[HOOK INSTALL] Installing import hook for package: {package_name}")
|
|
2616
|
-
|
|
2617
|
-
with _hook_lock:
|
|
2618
|
-
if package_name in _installed_hooks:
|
|
2619
|
-
_log("hook", f"[HOOK INSTALL] Import hook already installed for {package_name}")
|
|
2620
|
-
return
|
|
2621
|
-
|
|
2622
|
-
logger.debug(f"[HOOK INSTALL] Creating LazyMetaPathFinder for {package_name}")
|
|
2623
|
-
hook = LazyMetaPathFinder(package_name)
|
|
2624
|
-
|
|
2625
|
-
logger.debug(f"[HOOK INSTALL] Current sys.meta_path has {len(sys.meta_path)} entries")
|
|
2626
|
-
sys.meta_path.insert(0, hook)
|
|
2627
|
-
_installed_hooks[package_name] = hook
|
|
2628
|
-
|
|
2629
|
-
_log("hook", f"✅ [HOOK INSTALL] Lazy import hook installed for {package_name} (now {len(sys.meta_path)} meta_path entries)")
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
def uninstall_import_hook(package_name: str = 'default') -> None:
|
|
2633
|
-
"""Uninstall import hook for a package."""
|
|
2634
|
-
global _installed_hooks
|
|
2635
|
-
|
|
2636
|
-
with _hook_lock:
|
|
2637
|
-
if package_name in _installed_hooks:
|
|
2638
|
-
hook = _installed_hooks[package_name]
|
|
2639
|
-
try:
|
|
2640
|
-
sys.meta_path.remove(hook)
|
|
2641
|
-
except ValueError:
|
|
2642
|
-
pass
|
|
2643
|
-
del _installed_hooks[package_name]
|
|
2644
|
-
_log("hook", f"Lazy import hook uninstalled for {package_name}")
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
def is_import_hook_installed(package_name: str = 'default') -> bool:
|
|
2648
|
-
"""Check if import hook is installed for a package."""
|
|
2649
|
-
return package_name in _installed_hooks
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
# =============================================================================
|
|
2653
|
-
# SECTION 4: LAZY LOADING & CACHING (~300 lines)
|
|
2654
|
-
# =============================================================================
|
|
2655
|
-
|
|
2656
|
-
class LazyLoader(ALazyLoader):
|
|
2657
|
-
"""
|
|
2658
|
-
Thread-safe lazy loader for modules with caching.
|
|
2659
|
-
Implements Proxy pattern for deferred module loading.
|
|
2660
|
-
"""
|
|
2661
|
-
|
|
2662
|
-
def load_module(self, module_path: str = None) -> ModuleType:
|
|
2663
|
-
"""Thread-safe module loading with caching."""
|
|
2664
|
-
if module_path is None:
|
|
2665
|
-
module_path = self._module_path
|
|
2666
|
-
|
|
2667
|
-
if self._cached_module is not None:
|
|
2668
|
-
return self._cached_module
|
|
2669
|
-
|
|
2670
|
-
with self._lock:
|
|
2671
|
-
if self._cached_module is not None:
|
|
2672
|
-
return self._cached_module
|
|
2673
|
-
|
|
2674
|
-
if self._loading:
|
|
2675
|
-
raise ImportError(f"Circular import detected for {module_path}")
|
|
2676
|
-
|
|
2677
|
-
try:
|
|
2678
|
-
self._loading = True
|
|
2679
|
-
logger.debug(f"Lazy loading module: {module_path}")
|
|
2680
|
-
|
|
2681
|
-
self._cached_module = importlib.import_module(module_path)
|
|
2682
|
-
|
|
2683
|
-
logger.debug(f"Successfully loaded: {module_path}")
|
|
2684
|
-
return self._cached_module
|
|
2685
|
-
|
|
2686
|
-
except Exception as e:
|
|
2687
|
-
logger.error(f"Failed to load module {module_path}: {e}")
|
|
2688
|
-
raise ImportError(f"Failed to load {module_path}: {e}") from e
|
|
2689
|
-
finally:
|
|
2690
|
-
self._loading = False
|
|
2691
|
-
|
|
2692
|
-
def unload_module(self, module_path: str) -> None:
|
|
2693
|
-
"""Unload a module from cache."""
|
|
2694
|
-
with self._lock:
|
|
2695
|
-
if module_path == self._module_path:
|
|
2696
|
-
self._cached_module = None
|
|
2697
|
-
|
|
2698
|
-
def __getattr__(self, name: str) -> Any:
|
|
2699
|
-
"""Get attribute from lazily loaded module."""
|
|
2700
|
-
module = self.load_module()
|
|
2701
|
-
try:
|
|
2702
|
-
return getattr(module, name)
|
|
2703
|
-
except AttributeError:
|
|
2704
|
-
raise AttributeError(
|
|
2705
|
-
f"module '{self._module_path}' has no attribute '{name}'"
|
|
2706
|
-
)
|
|
2707
|
-
|
|
2708
|
-
def __dir__(self) -> list:
|
|
2709
|
-
"""Return available attributes from loaded module."""
|
|
2710
|
-
module = self.load_module()
|
|
2711
|
-
return dir(module)
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
class LazyImporter:
|
|
2715
|
-
"""
|
|
2716
|
-
Lazy importer that defers heavy module imports until first access.
|
|
2717
|
-
"""
|
|
2718
|
-
|
|
2719
|
-
__slots__ = ('_enabled', '_lazy_modules', '_loaded_modules', '_lock', '_access_counts')
|
|
2720
|
-
|
|
2721
|
-
def __init__(self):
|
|
2722
|
-
"""Initialize lazy importer."""
|
|
2723
|
-
self._enabled = False
|
|
2724
|
-
self._lazy_modules: Dict[str, str] = {}
|
|
2725
|
-
self._loaded_modules: Dict[str, ModuleType] = {}
|
|
2726
|
-
self._access_counts: Dict[str, int] = {}
|
|
2727
|
-
self._lock = threading.RLock()
|
|
2728
|
-
|
|
2729
|
-
def enable(self) -> None:
|
|
2730
|
-
"""Enable lazy imports."""
|
|
2731
|
-
with self._lock:
|
|
2732
|
-
self._enabled = True
|
|
2733
|
-
_log("config", "Lazy imports enabled")
|
|
2734
|
-
|
|
2735
|
-
def disable(self) -> None:
|
|
2736
|
-
"""Disable lazy imports."""
|
|
2737
|
-
with self._lock:
|
|
2738
|
-
self._enabled = False
|
|
2739
|
-
_log("config", "Lazy imports disabled")
|
|
2740
|
-
|
|
2741
|
-
def is_enabled(self) -> bool:
|
|
2742
|
-
"""Check if lazy imports are enabled."""
|
|
2743
|
-
return self._enabled
|
|
2744
|
-
|
|
2745
|
-
def register_lazy_module(self, module_name: str, module_path: str = None) -> None:
|
|
2746
|
-
"""Register a module for lazy loading."""
|
|
2747
|
-
with self._lock:
|
|
2748
|
-
if module_path is None:
|
|
2749
|
-
module_path = module_name
|
|
2750
|
-
|
|
2751
|
-
self._lazy_modules[module_name] = module_path
|
|
2752
|
-
self._access_counts[module_name] = 0
|
|
2753
|
-
logger.debug(f"Registered lazy module: {module_name} -> {module_path}")
|
|
2754
|
-
|
|
2755
|
-
def import_module(self, module_name: str, package_name: str = None) -> Any:
|
|
2756
|
-
"""Import a module with lazy loading."""
|
|
2757
|
-
with self._lock:
|
|
2758
|
-
if not self._enabled:
|
|
2759
|
-
return importlib.import_module(module_name)
|
|
2760
|
-
|
|
2761
|
-
if module_name in self._loaded_modules:
|
|
2762
|
-
self._access_counts[module_name] += 1
|
|
2763
|
-
return self._loaded_modules[module_name]
|
|
2764
|
-
|
|
2765
|
-
if module_name in self._lazy_modules:
|
|
2766
|
-
module_path = self._lazy_modules[module_name]
|
|
2767
|
-
|
|
2768
|
-
try:
|
|
2769
|
-
actual_module = importlib.import_module(module_path)
|
|
2770
|
-
self._loaded_modules[module_name] = actual_module
|
|
2771
|
-
self._access_counts[module_name] += 1
|
|
2772
|
-
|
|
2773
|
-
logger.debug(f"Lazy loaded module: {module_name}")
|
|
2774
|
-
return actual_module
|
|
2775
|
-
|
|
2776
|
-
except ImportError as e:
|
|
2777
|
-
logger.error(f"Failed to lazy load {module_name}: {e}")
|
|
2778
|
-
raise
|
|
2779
|
-
else:
|
|
2780
|
-
return importlib.import_module(module_name)
|
|
2781
|
-
|
|
2782
|
-
def preload_module(self, module_name: str) -> bool:
|
|
2783
|
-
"""Preload a registered lazy module."""
|
|
2784
|
-
with self._lock:
|
|
2785
|
-
if module_name not in self._lazy_modules:
|
|
2786
|
-
logger.warning(f"Module {module_name} not registered for lazy loading")
|
|
2787
|
-
return False
|
|
2788
|
-
|
|
2789
|
-
try:
|
|
2790
|
-
self.import_module(module_name)
|
|
2791
|
-
_log("hook", f"Preloaded module: {module_name}")
|
|
2792
|
-
return True
|
|
2793
|
-
except Exception as e:
|
|
2794
|
-
logger.error(f"Failed to preload {module_name}: {e}")
|
|
2795
|
-
return False
|
|
2796
|
-
|
|
2797
|
-
def get_stats(self) -> Dict[str, Any]:
|
|
2798
|
-
"""Get lazy import statistics."""
|
|
2799
|
-
with self._lock:
|
|
2800
|
-
return {
|
|
2801
|
-
'enabled': self._enabled,
|
|
2802
|
-
'registered_modules': list(self._lazy_modules.keys()),
|
|
2803
|
-
'loaded_modules': list(self._loaded_modules.keys()),
|
|
2804
|
-
'access_counts': self._access_counts.copy(),
|
|
2805
|
-
'total_registered': len(self._lazy_modules),
|
|
2806
|
-
'total_loaded': len(self._loaded_modules)
|
|
2807
|
-
}
|
|
2808
|
-
|
|
2809
|
-
|
|
2810
|
-
class LazyModuleRegistry:
|
|
2811
|
-
"""
|
|
2812
|
-
Registry for managing lazy-loaded modules with performance tracking.
|
|
2813
|
-
"""
|
|
2814
|
-
|
|
2815
|
-
__slots__ = ('_modules', '_load_times', '_lock', '_access_counts')
|
|
2816
|
-
|
|
2817
|
-
def __init__(self):
|
|
2818
|
-
"""Initialize the registry."""
|
|
2819
|
-
self._modules: Dict[str, LazyLoader] = {}
|
|
2820
|
-
self._load_times: Dict[str, float] = {}
|
|
2821
|
-
self._access_counts: Dict[str, int] = {}
|
|
2822
|
-
self._lock = threading.RLock()
|
|
2823
|
-
|
|
2824
|
-
def register_module(self, name: str, module_path: str) -> None:
|
|
2825
|
-
"""Register a module for lazy loading."""
|
|
2826
|
-
with self._lock:
|
|
2827
|
-
if name in self._modules:
|
|
2828
|
-
logger.warning(f"Module '{name}' already registered, overwriting")
|
|
2829
|
-
|
|
2830
|
-
self._modules[name] = LazyLoader(module_path)
|
|
2831
|
-
self._access_counts[name] = 0
|
|
2832
|
-
logger.debug(f"Registered lazy module: {name} -> {module_path}")
|
|
2833
|
-
|
|
2834
|
-
def get_module(self, name: str) -> LazyLoader:
|
|
2835
|
-
"""Get a lazy-loaded module."""
|
|
2836
|
-
with self._lock:
|
|
2837
|
-
if name not in self._modules:
|
|
2838
|
-
raise KeyError(f"Module '{name}' not registered")
|
|
2839
|
-
|
|
2840
|
-
self._access_counts[name] += 1
|
|
2841
|
-
return self._modules[name]
|
|
2842
|
-
|
|
2843
|
-
def preload_frequently_used(self, threshold: int = 5) -> None:
|
|
2844
|
-
"""Preload modules that are accessed frequently."""
|
|
2845
|
-
with self._lock:
|
|
2846
|
-
for name, count in self._access_counts.items():
|
|
2847
|
-
if count >= threshold:
|
|
2848
|
-
try:
|
|
2849
|
-
_ = self._modules[name].load_module()
|
|
2850
|
-
_log("hook", f"Preloaded frequently used module: {name}")
|
|
2851
|
-
except Exception as e:
|
|
2852
|
-
logger.warning(f"Failed to preload {name}: {e}")
|
|
2853
|
-
|
|
2854
|
-
def get_stats(self) -> Dict[str, Any]:
|
|
2855
|
-
"""Get loading statistics."""
|
|
2856
|
-
with self._lock:
|
|
2857
|
-
loaded_count = sum(
|
|
2858
|
-
1 for loader in self._modules.values()
|
|
2859
|
-
if loader.is_loaded()
|
|
2860
|
-
)
|
|
2861
|
-
|
|
2862
|
-
return {
|
|
2863
|
-
'total_registered': len(self._modules),
|
|
2864
|
-
'loaded_count': loaded_count,
|
|
2865
|
-
'unloaded_count': len(self._modules) - loaded_count,
|
|
2866
|
-
'access_counts': self._access_counts.copy(),
|
|
2867
|
-
'load_times': self._load_times.copy(),
|
|
2868
|
-
}
|
|
2869
|
-
|
|
2870
|
-
def clear_cache(self) -> None:
|
|
2871
|
-
"""Clear all cached modules."""
|
|
2872
|
-
with self._lock:
|
|
2873
|
-
for name, loader in self._modules.items():
|
|
2874
|
-
loader.unload_module(loader._module_path)
|
|
2875
|
-
_log("config", "Cleared all cached modules")
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
class LazyPerformanceMonitor:
|
|
2879
|
-
"""Performance monitor for lazy loading operations."""
|
|
2880
|
-
|
|
2881
|
-
__slots__ = ('_load_times', '_access_counts', '_memory_usage')
|
|
2882
|
-
|
|
2883
|
-
def __init__(self):
|
|
2884
|
-
"""Initialize performance monitor."""
|
|
2885
|
-
self._load_times = {}
|
|
2886
|
-
self._access_counts = {}
|
|
2887
|
-
self._memory_usage = {}
|
|
2888
|
-
|
|
2889
|
-
def record_load_time(self, module: str, load_time: float) -> None:
|
|
2890
|
-
"""Record module load time."""
|
|
2891
|
-
self._load_times[module] = load_time
|
|
2892
|
-
|
|
2893
|
-
def record_access(self, module: str) -> None:
|
|
2894
|
-
"""Record module access."""
|
|
2895
|
-
self._access_counts[module] = self._access_counts.get(module, 0) + 1
|
|
2896
|
-
|
|
2897
|
-
def get_stats(self) -> Dict[str, Any]:
|
|
2898
|
-
"""Get performance statistics."""
|
|
2899
|
-
return {
|
|
2900
|
-
'load_times': self._load_times.copy(),
|
|
2901
|
-
'access_counts': self._access_counts.copy(),
|
|
2902
|
-
'memory_usage': self._memory_usage.copy()
|
|
2903
|
-
}
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
# Global instances
|
|
2907
|
-
_lazy_importer = LazyImporter()
|
|
2908
|
-
_global_registry = LazyModuleRegistry()
|
|
2909
|
-
|
|
2910
|
-
|
|
2911
|
-
_lazy_importer = LazyImporter()
|
|
2912
|
-
_global_registry = LazyModuleRegistry()
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
def enable_lazy_imports() -> None:
|
|
2916
|
-
"""Enable lazy imports (loader only)."""
|
|
2917
|
-
_lazy_importer.enable()
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
def disable_lazy_imports() -> None:
|
|
2921
|
-
"""Disable lazy imports (loader only)."""
|
|
2922
|
-
_lazy_importer.disable()
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
def is_lazy_import_enabled() -> bool:
|
|
2926
|
-
"""Check if lazy imports are enabled."""
|
|
2927
|
-
return _lazy_importer.is_enabled()
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
def lazy_import(module_name: str, package_name: str = None) -> Any:
|
|
2931
|
-
"""Import a module with lazy loading."""
|
|
2932
|
-
return _lazy_importer.import_module(module_name, package_name)
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
def register_lazy_module(module_name: str, module_path: str = None) -> None:
|
|
2936
|
-
"""Register a module for lazy loading."""
|
|
2937
|
-
_lazy_importer.register_lazy_module(module_name, module_path)
|
|
2938
|
-
_global_registry.register_module(module_name, module_path or module_name)
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
def preload_module(module_name: str) -> bool:
|
|
2942
|
-
"""Preload a registered lazy module."""
|
|
2943
|
-
return _lazy_importer.preload_module(module_name)
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
def get_lazy_module(name: str) -> LazyLoader:
|
|
2947
|
-
"""Get a lazy-loaded module from the global registry."""
|
|
2948
|
-
return _global_registry.get_module(name)
|
|
2949
|
-
|
|
2950
|
-
|
|
2951
|
-
def get_loading_stats() -> Dict[str, Any]:
|
|
2952
|
-
"""Get loading statistics from the global registry."""
|
|
2953
|
-
return _global_registry.get_stats()
|
|
2954
|
-
|
|
2955
|
-
|
|
2956
|
-
def preload_frequently_used(threshold: int = 5) -> None:
|
|
2957
|
-
"""Preload frequently used modules from the global registry."""
|
|
2958
|
-
_global_registry.preload_frequently_used(threshold)
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
def get_lazy_import_stats() -> Dict[str, Any]:
|
|
2962
|
-
"""Get lazy import statistics."""
|
|
2963
|
-
return _lazy_importer.get_stats()
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
# =============================================================================
|
|
2967
|
-
# SECTION 5: CONFIGURATION & REGISTRY (~200 lines)
|
|
2968
|
-
# =============================================================================
|
|
2969
|
-
|
|
2970
|
-
# Performance optimization: Cache detection results per package
|
|
2971
|
-
_lazy_detection_cache: Dict[str, bool] = {}
|
|
2972
|
-
_lazy_detection_lock = threading.RLock()
|
|
2973
|
-
|
|
2974
|
-
# Keyword-based detection configuration
|
|
2975
|
-
_KEYWORD_DETECTION_ENABLED: bool = True
|
|
2976
|
-
_KEYWORD_TO_CHECK: str = "xwlazy-enabled"
|
|
2977
|
-
|
|
2978
|
-
# Performance optimization: Module-level constant for mode enum conversion
|
|
2979
|
-
_MODE_ENUM_MAP = {
|
|
2980
|
-
"auto": LazyInstallMode.AUTO,
|
|
2981
|
-
"interactive": LazyInstallMode.INTERACTIVE,
|
|
2982
|
-
"warn": LazyInstallMode.WARN,
|
|
2983
|
-
"disabled": LazyInstallMode.DISABLED,
|
|
2984
|
-
"dry_run": LazyInstallMode.DRY_RUN,
|
|
2985
|
-
}
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
def _lazy_env_override(package_name: str) -> Optional[bool]:
|
|
2989
|
-
env_var = f"{package_name.upper()}_LAZY_INSTALL"
|
|
2990
|
-
raw_value = os.environ.get(env_var)
|
|
2991
|
-
if raw_value is None:
|
|
2992
|
-
return None
|
|
2993
|
-
|
|
2994
|
-
normalized = raw_value.strip().lower()
|
|
2995
|
-
if normalized in ("true", "1", "yes", "on"):
|
|
2996
|
-
return True
|
|
2997
|
-
if normalized in ("false", "0", "no", "off"):
|
|
2998
|
-
return False
|
|
2999
|
-
return None
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
def _lazy_marker_installed() -> bool:
|
|
3003
|
-
if sys.version_info < (3, 8):
|
|
3004
|
-
return False
|
|
3005
|
-
|
|
3006
|
-
try:
|
|
3007
|
-
from importlib import metadata
|
|
3008
|
-
except Exception as exc:
|
|
3009
|
-
logger.debug(f"importlib.metadata unavailable for lazy detection: {exc}")
|
|
3010
|
-
return False
|
|
3011
|
-
|
|
3012
|
-
try:
|
|
3013
|
-
metadata.distribution("exonware-xwlazy")
|
|
3014
|
-
_log("config", "✅ Detected exonware-xwlazy marker package")
|
|
3015
|
-
return True
|
|
3016
|
-
except metadata.PackageNotFoundError:
|
|
3017
|
-
_log("config", "❌ exonware-xwlazy marker package not installed")
|
|
3018
|
-
return False
|
|
3019
|
-
except Exception as exc:
|
|
3020
|
-
logger.debug(f"Failed to inspect marker package: {exc}")
|
|
3021
|
-
return False
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
def _check_package_keywords(package_name: Optional[str] = None, keyword: Optional[str] = None) -> bool:
|
|
3025
|
-
"""
|
|
3026
|
-
Check if any installed package has the specified keyword in its metadata.
|
|
3027
|
-
|
|
3028
|
-
This allows packages to opt-in to lazy loading by adding:
|
|
3029
|
-
[project]
|
|
3030
|
-
keywords = ["xwlazy-enabled"]
|
|
3031
|
-
|
|
3032
|
-
in their pyproject.toml file. The keyword is stored in the package's
|
|
3033
|
-
metadata when installed.
|
|
3034
|
-
|
|
3035
|
-
Args:
|
|
3036
|
-
package_name: The package name to check (or None to check all packages)
|
|
3037
|
-
keyword: The keyword to look for (default: uses _KEYWORD_TO_CHECK)
|
|
3038
|
-
|
|
3039
|
-
Returns:
|
|
3040
|
-
True if the keyword is found in any relevant package's metadata
|
|
3041
|
-
"""
|
|
3042
|
-
if not _KEYWORD_DETECTION_ENABLED:
|
|
3043
|
-
return False
|
|
3044
|
-
|
|
3045
|
-
if sys.version_info < (3, 8):
|
|
3046
|
-
return False
|
|
3047
|
-
|
|
3048
|
-
try:
|
|
3049
|
-
from importlib import metadata
|
|
3050
|
-
except Exception as exc:
|
|
3051
|
-
logger.debug(f"importlib.metadata unavailable for keyword detection: {exc}")
|
|
3052
|
-
return False
|
|
3053
|
-
|
|
3054
|
-
search_keyword = (keyword or _KEYWORD_TO_CHECK).lower()
|
|
3055
|
-
|
|
3056
|
-
try:
|
|
3057
|
-
if package_name:
|
|
3058
|
-
# Check specific package
|
|
3059
|
-
try:
|
|
3060
|
-
dist = metadata.distribution(package_name)
|
|
3061
|
-
keywords = dist.metadata.get_all('Keywords', [])
|
|
3062
|
-
if keywords:
|
|
3063
|
-
# Keywords can be a single string or list
|
|
3064
|
-
all_keywords = []
|
|
3065
|
-
for kw in keywords:
|
|
3066
|
-
if isinstance(kw, str):
|
|
3067
|
-
# Split comma-separated keywords
|
|
3068
|
-
all_keywords.extend(k.strip().lower() for k in kw.split(','))
|
|
3069
|
-
else:
|
|
3070
|
-
all_keywords.append(str(kw).lower())
|
|
3071
|
-
|
|
3072
|
-
if search_keyword in all_keywords:
|
|
3073
|
-
_log("config", f"✅ Detected '{search_keyword}' keyword in package: {package_name}")
|
|
3074
|
-
return True
|
|
3075
|
-
except metadata.PackageNotFoundError:
|
|
3076
|
-
return False
|
|
3077
|
-
else:
|
|
3078
|
-
# Check all installed packages
|
|
3079
|
-
for dist in metadata.distributions():
|
|
3080
|
-
try:
|
|
3081
|
-
keywords = dist.metadata.get_all('Keywords', [])
|
|
3082
|
-
if keywords:
|
|
3083
|
-
all_keywords = []
|
|
3084
|
-
for kw in keywords:
|
|
3085
|
-
if isinstance(kw, str):
|
|
3086
|
-
all_keywords.extend(k.strip().lower() for k in kw.split(','))
|
|
3087
|
-
else:
|
|
3088
|
-
all_keywords.append(str(kw).lower())
|
|
3089
|
-
|
|
3090
|
-
if search_keyword in all_keywords:
|
|
3091
|
-
package_found = dist.metadata.get('Name', 'unknown')
|
|
3092
|
-
_log("config", f"✅ Detected '{search_keyword}' keyword in package: {package_found}")
|
|
3093
|
-
return True
|
|
3094
|
-
except Exception:
|
|
3095
|
-
continue
|
|
3096
|
-
except Exception as exc:
|
|
3097
|
-
logger.debug(f"Failed to check package keywords: {exc}")
|
|
3098
|
-
|
|
3099
|
-
return False
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
def _detect_lazy_installation(package_name: str) -> bool:
|
|
3103
|
-
with _lazy_detection_lock:
|
|
3104
|
-
cached = _lazy_detection_cache.get(package_name)
|
|
3105
|
-
if cached is not None:
|
|
3106
|
-
return cached
|
|
3107
|
-
|
|
3108
|
-
env_override = _lazy_env_override(package_name)
|
|
3109
|
-
if env_override is not None:
|
|
3110
|
-
with _lazy_detection_lock:
|
|
3111
|
-
_lazy_detection_cache[package_name] = env_override
|
|
3112
|
-
return env_override
|
|
3113
|
-
|
|
3114
|
-
state_manager = LazyStateManager(package_name)
|
|
3115
|
-
manual_state = state_manager.get_manual_state()
|
|
3116
|
-
if manual_state is not None:
|
|
3117
|
-
with _lazy_detection_lock:
|
|
3118
|
-
_lazy_detection_cache[package_name] = manual_state
|
|
3119
|
-
return manual_state
|
|
3120
|
-
|
|
3121
|
-
cached_state = state_manager.get_cached_auto_state()
|
|
3122
|
-
if cached_state is not None:
|
|
3123
|
-
with _lazy_detection_lock:
|
|
3124
|
-
_lazy_detection_cache[package_name] = cached_state
|
|
3125
|
-
return cached_state
|
|
3126
|
-
|
|
3127
|
-
# Check marker package first (existing behavior)
|
|
3128
|
-
marker_detected = _lazy_marker_installed()
|
|
3129
|
-
|
|
3130
|
-
# Also check for keyword in package metadata (new feature)
|
|
3131
|
-
keyword_detected = _check_package_keywords(package_name)
|
|
3132
|
-
|
|
3133
|
-
# Enable if either marker package OR keyword is found
|
|
3134
|
-
detected = marker_detected or keyword_detected
|
|
3135
|
-
|
|
3136
|
-
state_manager.set_auto_state(detected)
|
|
3137
|
-
|
|
3138
|
-
with _lazy_detection_lock:
|
|
3139
|
-
_lazy_detection_cache[package_name] = detected
|
|
3140
|
-
|
|
3141
|
-
return detected
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
class LazyInstallConfig:
|
|
3145
|
-
"""Global configuration for lazy installation per package."""
|
|
3146
|
-
_configs: Dict[str, bool] = {}
|
|
3147
|
-
_modes: Dict[str, str] = {}
|
|
3148
|
-
_initialized: Dict[str, bool] = {}
|
|
3149
|
-
_manual_overrides: Dict[str, bool] = {}
|
|
3150
|
-
|
|
3151
|
-
@classmethod
|
|
3152
|
-
def set(
|
|
3153
|
-
cls,
|
|
3154
|
-
package_name: str,
|
|
3155
|
-
enabled: bool,
|
|
3156
|
-
mode: str = "auto",
|
|
3157
|
-
install_hook: bool = True,
|
|
3158
|
-
manual: bool = False,
|
|
3159
|
-
) -> None:
|
|
3160
|
-
"""Enable or disable lazy installation for a specific package."""
|
|
3161
|
-
package_key = package_name.lower()
|
|
3162
|
-
state_manager = LazyStateManager(package_name)
|
|
3163
|
-
|
|
3164
|
-
if manual:
|
|
3165
|
-
cls._manual_overrides[package_key] = True
|
|
3166
|
-
state_manager.set_manual_state(enabled)
|
|
3167
|
-
elif cls._manual_overrides.get(package_key):
|
|
3168
|
-
logger.debug(
|
|
3169
|
-
f"Lazy install config for {package_key} already overridden manually; skipping auto configuration."
|
|
3170
|
-
)
|
|
3171
|
-
return
|
|
3172
|
-
else:
|
|
3173
|
-
state_manager.set_manual_state(None)
|
|
3174
|
-
|
|
3175
|
-
cls._configs[package_key] = enabled
|
|
3176
|
-
cls._modes[package_key] = mode
|
|
3177
|
-
|
|
3178
|
-
cls._initialize_package(package_key, enabled, mode, install_hook=install_hook)
|
|
3179
|
-
|
|
3180
|
-
@classmethod
|
|
3181
|
-
def _initialize_package(cls, package_key: str, enabled: bool, mode: str, install_hook: bool = True) -> None:
|
|
3182
|
-
"""Initialize lazy installation for a specific package."""
|
|
3183
|
-
if enabled:
|
|
3184
|
-
try:
|
|
3185
|
-
enable_lazy_install(package_key)
|
|
3186
|
-
|
|
3187
|
-
mode_enum = _MODE_ENUM_MAP.get(mode.lower(), LazyInstallMode.AUTO)
|
|
3188
|
-
set_lazy_install_mode(package_key, mode_enum)
|
|
3189
|
-
|
|
3190
|
-
if install_hook:
|
|
3191
|
-
if not is_import_hook_installed(package_key):
|
|
3192
|
-
install_import_hook(package_key)
|
|
3193
|
-
_log("config", f"✅ Lazy installation initialized for {package_key} (mode: {mode}, hook: installed)")
|
|
3194
|
-
else:
|
|
3195
|
-
uninstall_import_hook(package_key)
|
|
3196
|
-
_log("config", f"✅ Lazy installation initialized for {package_key} (mode: {mode}, hook: disabled)")
|
|
3197
|
-
|
|
3198
|
-
cls._initialized[package_key] = True
|
|
3199
|
-
sync_manifest_configuration(package_key)
|
|
3200
|
-
except ImportError as e:
|
|
3201
|
-
logger.warning(f"⚠️ Could not enable lazy install for {package_key}: {e}")
|
|
3202
|
-
else:
|
|
3203
|
-
try:
|
|
3204
|
-
disable_lazy_install(package_key)
|
|
3205
|
-
except ImportError:
|
|
3206
|
-
pass
|
|
3207
|
-
uninstall_import_hook(package_key)
|
|
3208
|
-
cls._initialized[package_key] = False
|
|
3209
|
-
_log("config", f"❌ Lazy installation disabled for {package_key}")
|
|
3210
|
-
sync_manifest_configuration(package_key)
|
|
3211
|
-
|
|
3212
|
-
@classmethod
|
|
3213
|
-
def is_enabled(cls, package_name: str) -> bool:
|
|
3214
|
-
"""Check if lazy installation is enabled for a package."""
|
|
3215
|
-
return cls._configs.get(package_name.lower(), False)
|
|
3216
|
-
|
|
3217
|
-
@classmethod
|
|
3218
|
-
def get_mode(cls, package_name: str) -> str:
|
|
3219
|
-
"""Get the lazy installation mode for a package."""
|
|
3220
|
-
return cls._modes.get(package_name.lower(), "auto")
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
def config_package_lazy_install_enabled(
|
|
3224
|
-
package_name: str,
|
|
3225
|
-
enabled: bool = None,
|
|
3226
|
-
mode: str = "auto",
|
|
3227
|
-
install_hook: bool = True
|
|
3228
|
-
) -> None:
|
|
3229
|
-
"""
|
|
3230
|
-
Simple one-line configuration for package lazy installation.
|
|
3231
|
-
|
|
3232
|
-
Args:
|
|
3233
|
-
package_name: Package name (e.g., "xwsystem", "xwnode", "xwdata")
|
|
3234
|
-
enabled: True to enable, False to disable, None to auto-detect from pip installation
|
|
3235
|
-
mode: Installation mode - "auto", "interactive", "disabled", "dry_run"
|
|
3236
|
-
install_hook: Whether to install the import hook (default: True)
|
|
3237
|
-
|
|
3238
|
-
Examples:
|
|
3239
|
-
# Auto-detect from installation
|
|
3240
|
-
config_package_lazy_install_enabled("your_package_name")
|
|
3241
|
-
|
|
3242
|
-
# Force enable
|
|
3243
|
-
config_package_lazy_install_enabled("xwnode", True, "interactive")
|
|
3244
|
-
|
|
3245
|
-
# Force disable
|
|
3246
|
-
config_package_lazy_install_enabled("xwdata", False)
|
|
3247
|
-
"""
|
|
3248
|
-
manual_override = enabled is not None
|
|
3249
|
-
if enabled is None:
|
|
3250
|
-
enabled = _detect_lazy_installation(package_name)
|
|
3251
|
-
|
|
3252
|
-
LazyInstallConfig.set(
|
|
3253
|
-
package_name,
|
|
3254
|
-
enabled,
|
|
3255
|
-
mode,
|
|
3256
|
-
install_hook=install_hook,
|
|
3257
|
-
manual=manual_override,
|
|
3258
|
-
)
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
# =============================================================================
|
|
3262
|
-
# SECTION 6: FACADE - UNIFIED API (~150 lines)
|
|
3263
|
-
# =============================================================================
|
|
3264
|
-
|
|
3265
|
-
class LazyModeFacade:
|
|
3266
|
-
"""
|
|
3267
|
-
Main facade for lazy mode operations.
|
|
3268
|
-
Provides a unified interface for lazy loading functionality.
|
|
3269
|
-
"""
|
|
3270
|
-
|
|
3271
|
-
__slots__ = ('_enabled', '_strategy', '_config', '_performance_monitor')
|
|
3272
|
-
|
|
3273
|
-
def __init__(self):
|
|
3274
|
-
"""Initialize lazy mode facade."""
|
|
3275
|
-
self._enabled = False
|
|
3276
|
-
self._strategy = None
|
|
3277
|
-
self._config = {}
|
|
3278
|
-
self._performance_monitor = None
|
|
3279
|
-
|
|
3280
|
-
def enable(self, strategy: str = "on_demand", **kwargs) -> None:
|
|
3281
|
-
"""Enable lazy mode with specified strategy."""
|
|
3282
|
-
self._enabled = True
|
|
3283
|
-
self._strategy = strategy
|
|
3284
|
-
|
|
3285
|
-
package_name = kwargs.pop('package_name', 'default').lower()
|
|
3286
|
-
enable_lazy_import_flag = kwargs.pop('enable_lazy_imports', True)
|
|
3287
|
-
enable_lazy_install_flag = kwargs.pop('enable_lazy_install', True)
|
|
3288
|
-
lazy_install_mode = kwargs.pop('lazy_install_mode', "auto")
|
|
3289
|
-
install_hook = kwargs.pop('install_hook', True)
|
|
3290
|
-
|
|
3291
|
-
self._config.update({
|
|
3292
|
-
'package_name': package_name,
|
|
3293
|
-
'enable_lazy_imports': enable_lazy_import_flag,
|
|
3294
|
-
'enable_lazy_install': enable_lazy_install_flag,
|
|
3295
|
-
'lazy_install_mode': lazy_install_mode,
|
|
3296
|
-
'install_hook': install_hook,
|
|
3297
|
-
})
|
|
3298
|
-
self._config.update(kwargs)
|
|
3299
|
-
|
|
3300
|
-
_log("config", f"Lazy mode enabled with strategy: {strategy}")
|
|
3301
|
-
|
|
3302
|
-
if enable_lazy_import_flag:
|
|
3303
|
-
_lazy_importer.enable()
|
|
3304
|
-
else:
|
|
3305
|
-
_lazy_importer.disable()
|
|
3306
|
-
|
|
3307
|
-
if enable_lazy_install_flag:
|
|
3308
|
-
config_package_lazy_install_enabled(
|
|
3309
|
-
package_name,
|
|
3310
|
-
True,
|
|
3311
|
-
lazy_install_mode,
|
|
3312
|
-
install_hook=install_hook,
|
|
3313
|
-
)
|
|
3314
|
-
else:
|
|
3315
|
-
config_package_lazy_install_enabled(
|
|
3316
|
-
package_name,
|
|
3317
|
-
False,
|
|
3318
|
-
install_hook=install_hook,
|
|
3319
|
-
)
|
|
3320
|
-
uninstall_import_hook(package_name)
|
|
3321
|
-
|
|
3322
|
-
if self._config.get('enable_monitoring', True):
|
|
3323
|
-
self._performance_monitor = LazyPerformanceMonitor()
|
|
3324
|
-
|
|
3325
|
-
def disable(self) -> None:
|
|
3326
|
-
"""Disable lazy mode and cleanup resources."""
|
|
3327
|
-
self._enabled = False
|
|
3328
|
-
self._strategy = None
|
|
3329
|
-
|
|
3330
|
-
package_name = self._config.get('package_name', 'default')
|
|
3331
|
-
|
|
3332
|
-
if self._config.get('enable_lazy_imports', True):
|
|
3333
|
-
_lazy_importer.disable()
|
|
3334
|
-
|
|
3335
|
-
if self._config.get('enable_lazy_install', True):
|
|
3336
|
-
LazyInstallConfig.set(
|
|
3337
|
-
package_name,
|
|
3338
|
-
False,
|
|
3339
|
-
self._config.get('lazy_install_mode', 'auto'),
|
|
3340
|
-
install_hook=self._config.get('install_hook', True),
|
|
3341
|
-
)
|
|
3342
|
-
|
|
3343
|
-
if self._config.get('clear_cache_on_disable', True):
|
|
3344
|
-
_global_registry.clear_cache()
|
|
3345
|
-
|
|
3346
|
-
self._performance_monitor = None
|
|
3347
|
-
|
|
3348
|
-
_log("config", "Lazy mode disabled")
|
|
3349
|
-
|
|
3350
|
-
def is_enabled(self) -> bool:
|
|
3351
|
-
"""Check if lazy mode is currently enabled."""
|
|
3352
|
-
return self._enabled
|
|
3353
|
-
|
|
3354
|
-
def get_stats(self) -> Dict[str, Any]:
|
|
3355
|
-
"""Get lazy mode performance statistics."""
|
|
3356
|
-
stats = _global_registry.get_stats()
|
|
3357
|
-
stats.update({
|
|
3358
|
-
'enabled': self._enabled,
|
|
3359
|
-
'strategy': self._strategy,
|
|
3360
|
-
'config': self._config.copy()
|
|
3361
|
-
})
|
|
3362
|
-
|
|
3363
|
-
if self._performance_monitor:
|
|
3364
|
-
stats['performance'] = self._performance_monitor.get_stats()
|
|
3365
|
-
|
|
3366
|
-
return stats
|
|
3367
|
-
|
|
3368
|
-
def configure(self, **kwargs) -> None:
|
|
3369
|
-
"""Configure lazy mode settings."""
|
|
3370
|
-
self._config.update(kwargs)
|
|
3371
|
-
logger.debug(f"Lazy mode configuration updated: {kwargs}")
|
|
3372
|
-
|
|
3373
|
-
def preload(self, modules: List[str]) -> None:
|
|
3374
|
-
"""Preload specified modules."""
|
|
3375
|
-
for module_name in modules:
|
|
3376
|
-
try:
|
|
3377
|
-
loader = _global_registry.get_module(module_name)
|
|
3378
|
-
_ = loader.load_module()
|
|
3379
|
-
_log("hook", f"Preloaded module: {module_name}")
|
|
3380
|
-
except KeyError:
|
|
3381
|
-
logger.warning(f"Module not registered: {module_name}")
|
|
3382
|
-
except Exception as e:
|
|
3383
|
-
logger.error(f"Failed to preload {module_name}: {e}")
|
|
3384
|
-
|
|
3385
|
-
def optimize(self) -> None:
|
|
3386
|
-
"""Run optimization based on current usage patterns."""
|
|
3387
|
-
if not self._enabled:
|
|
3388
|
-
return
|
|
3389
|
-
|
|
3390
|
-
threshold = self._config.get('preload_threshold', 5)
|
|
3391
|
-
_global_registry.preload_frequently_used(threshold)
|
|
3392
|
-
|
|
3393
|
-
_log("config", "Lazy mode optimization completed")
|
|
3394
|
-
|
|
3395
|
-
|
|
3396
|
-
# Global lazy mode facade instance
|
|
3397
|
-
_lazy_facade = LazyModeFacade()
|
|
3398
|
-
|
|
3399
|
-
|
|
3400
|
-
def enable_lazy_mode(strategy: str = "on_demand", **kwargs) -> None:
|
|
3401
|
-
"""Enable lazy mode with specified strategy."""
|
|
3402
|
-
_lazy_facade.enable(strategy, **kwargs)
|
|
3403
|
-
|
|
3404
|
-
|
|
3405
|
-
def disable_lazy_mode() -> None:
|
|
3406
|
-
"""Disable lazy mode and cleanup resources."""
|
|
3407
|
-
_lazy_facade.disable()
|
|
3408
|
-
|
|
3409
|
-
|
|
3410
|
-
def is_lazy_mode_enabled() -> bool:
|
|
3411
|
-
"""Check if lazy mode is currently enabled."""
|
|
3412
|
-
return _lazy_facade.is_enabled()
|
|
3413
|
-
|
|
3414
|
-
|
|
3415
|
-
def get_lazy_mode_stats() -> Dict[str, Any]:
|
|
3416
|
-
"""Get lazy mode performance statistics."""
|
|
3417
|
-
return _lazy_facade.get_stats()
|
|
3418
|
-
|
|
3419
|
-
|
|
3420
|
-
def configure_lazy_mode(**kwargs) -> None:
|
|
3421
|
-
"""Configure lazy mode settings."""
|
|
3422
|
-
_lazy_facade.configure(**kwargs)
|
|
3423
|
-
|
|
3424
|
-
|
|
3425
|
-
def preload_modules(modules: List[str]) -> None:
|
|
3426
|
-
"""Preload specified modules."""
|
|
3427
|
-
_lazy_facade.preload(modules)
|
|
3428
|
-
|
|
3429
|
-
|
|
3430
|
-
def optimize_lazy_mode() -> None:
|
|
3431
|
-
"""Run optimization based on current usage patterns."""
|
|
3432
|
-
_lazy_facade.optimize()
|
|
3433
|
-
|
|
3434
|
-
|
|
3435
|
-
# =============================================================================
|
|
3436
|
-
# SECTION 7: PUBLIC API - SIMPLE FUNCTIONS (~200 lines)
|
|
3437
|
-
# =============================================================================
|
|
3438
|
-
|
|
3439
|
-
def enable_lazy_install(package_name: str = 'default') -> None:
|
|
3440
|
-
"""Enable lazy installation for a specific package."""
|
|
3441
|
-
installer = LazyInstallerRegistry.get_instance(package_name)
|
|
3442
|
-
installer.enable()
|
|
3443
|
-
|
|
3444
|
-
|
|
3445
|
-
def disable_lazy_install(package_name: str = 'default') -> None:
|
|
3446
|
-
"""Disable lazy installation for a specific package."""
|
|
3447
|
-
installer = LazyInstallerRegistry.get_instance(package_name)
|
|
3448
|
-
installer.disable()
|
|
3449
|
-
|
|
3450
|
-
|
|
3451
|
-
def is_lazy_install_enabled(package_name: str = 'default') -> bool:
|
|
3452
|
-
"""Check if lazy installation is enabled for a specific package."""
|
|
3453
|
-
installer = LazyInstallerRegistry.get_instance(package_name)
|
|
3454
|
-
return installer.is_enabled()
|
|
3455
|
-
|
|
3456
|
-
|
|
3457
|
-
def set_lazy_install_mode(package_name: str, mode: LazyInstallMode) -> None:
|
|
3458
|
-
"""Set the lazy installation mode for a specific package."""
|
|
3459
|
-
installer = LazyInstallerRegistry.get_instance(package_name)
|
|
3460
|
-
installer.set_mode(mode)
|
|
3461
|
-
|
|
3462
|
-
|
|
3463
|
-
def get_lazy_install_mode(package_name: str = 'default') -> LazyInstallMode:
|
|
3464
|
-
"""Get the lazy installation mode for a specific package."""
|
|
3465
|
-
installer = LazyInstallerRegistry.get_instance(package_name)
|
|
3466
|
-
return installer.get_mode()
|
|
3467
|
-
|
|
3468
|
-
|
|
3469
|
-
def install_missing_package(package_name: str, installer_package: str = 'default') -> bool:
|
|
3470
|
-
"""Install a missing package."""
|
|
3471
|
-
installer = LazyInstallerRegistry.get_instance(installer_package)
|
|
3472
|
-
return installer.install_package(package_name)
|
|
3473
|
-
|
|
3474
|
-
|
|
3475
|
-
def install_and_import(
|
|
3476
|
-
module_name: str,
|
|
3477
|
-
package_name: str = None,
|
|
3478
|
-
installer_package: str = 'default'
|
|
3479
|
-
) -> Tuple[Optional[ModuleType], bool]:
|
|
3480
|
-
"""Install package and import module."""
|
|
3481
|
-
installer = LazyInstallerRegistry.get_instance(installer_package)
|
|
3482
|
-
return installer.install_and_import(module_name, package_name)
|
|
3483
|
-
|
|
3484
|
-
|
|
3485
|
-
def get_lazy_install_stats(package_name: str = 'default') -> Dict[str, Any]:
|
|
3486
|
-
"""Get lazy installation statistics for a specific package."""
|
|
3487
|
-
installer = LazyInstallerRegistry.get_instance(package_name)
|
|
3488
|
-
return installer.get_stats()
|
|
3489
|
-
|
|
3490
|
-
|
|
3491
|
-
def get_all_lazy_install_stats() -> Dict[str, Dict[str, Any]]:
|
|
3492
|
-
"""Get lazy installation statistics for all packages."""
|
|
3493
|
-
all_instances = LazyInstallerRegistry.get_all_instances()
|
|
3494
|
-
return {name: inst.get_stats() for name, inst in all_instances.items()}
|
|
3495
|
-
|
|
3496
|
-
|
|
3497
|
-
def lazy_import_with_install(
|
|
3498
|
-
module_name: str,
|
|
3499
|
-
package_name: str = None,
|
|
3500
|
-
installer_package: str = 'default'
|
|
3501
|
-
) -> Tuple[Optional[ModuleType], bool]:
|
|
3502
|
-
"""
|
|
3503
|
-
Lazy import with automatic installation.
|
|
3504
|
-
|
|
3505
|
-
This function attempts to import a module, and if it fails due to ImportError,
|
|
3506
|
-
it automatically installs the corresponding package using pip before retrying.
|
|
3507
|
-
"""
|
|
3508
|
-
installer = LazyInstallerRegistry.get_instance(installer_package)
|
|
3509
|
-
return installer.install_and_import(module_name, package_name)
|
|
3510
|
-
|
|
3511
|
-
|
|
3512
|
-
def xwimport(
|
|
3513
|
-
module_name: str,
|
|
3514
|
-
package_name: str = None,
|
|
3515
|
-
installer_package: str = 'default'
|
|
3516
|
-
) -> Any:
|
|
3517
|
-
"""
|
|
3518
|
-
Simple lazy import with automatic installation.
|
|
3519
|
-
|
|
3520
|
-
This function either returns the imported module or raises an ImportError.
|
|
3521
|
-
"""
|
|
3522
|
-
module, available = lazy_import_with_install(module_name, package_name, installer_package)
|
|
3523
|
-
if not available:
|
|
3524
|
-
raise ImportError(f"Module {module_name} is not available and could not be installed")
|
|
3525
|
-
return module
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
# Security & Policy APIs
|
|
3529
|
-
def set_package_allow_list(package_name: str, allowed_packages: List[str]) -> None:
|
|
3530
|
-
"""Set allow list for a package (only these packages can be installed)."""
|
|
3531
|
-
LazyInstallPolicy.set_allow_list(package_name, allowed_packages)
|
|
3532
|
-
|
|
3533
|
-
|
|
3534
|
-
def set_package_deny_list(package_name: str, denied_packages: List[str]) -> None:
|
|
3535
|
-
"""Set deny list for a package (these packages cannot be installed)."""
|
|
3536
|
-
LazyInstallPolicy.set_deny_list(package_name, denied_packages)
|
|
3537
|
-
|
|
3538
|
-
|
|
3539
|
-
def add_to_package_allow_list(package_name: str, allowed_package: str) -> None:
|
|
3540
|
-
"""Add single package to allow list."""
|
|
3541
|
-
LazyInstallPolicy.add_to_allow_list(package_name, allowed_package)
|
|
3542
|
-
|
|
3543
|
-
|
|
3544
|
-
def add_to_package_deny_list(package_name: str, denied_package: str) -> None:
|
|
3545
|
-
"""Add single package to deny list."""
|
|
3546
|
-
LazyInstallPolicy.add_to_deny_list(package_name, denied_package)
|
|
3547
|
-
|
|
3548
|
-
|
|
3549
|
-
def set_package_index_url(package_name: str, index_url: str) -> None:
|
|
3550
|
-
"""Set PyPI index URL for a package."""
|
|
3551
|
-
LazyInstallPolicy.set_index_url(package_name, index_url)
|
|
3552
|
-
|
|
3553
|
-
|
|
3554
|
-
def set_package_extra_index_urls(package_name: str, urls: List[str]) -> None:
|
|
3555
|
-
"""Set extra index URLs for a package."""
|
|
3556
|
-
LazyInstallPolicy.set_extra_index_urls(package_name, urls)
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
def add_package_trusted_host(package_name: str, host: str) -> None:
|
|
3560
|
-
"""Add trusted host for a package."""
|
|
3561
|
-
LazyInstallPolicy.add_trusted_host(package_name, host)
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
def set_package_lockfile(package_name: str, lockfile_path: str) -> None:
|
|
3565
|
-
"""Set lockfile path for a package to track installed dependencies."""
|
|
3566
|
-
LazyInstallPolicy.set_lockfile_path(package_name, lockfile_path)
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
def generate_package_sbom(package_name: str = 'default', output_path: str = None) -> Dict:
|
|
3570
|
-
"""Generate Software Bill of Materials (SBOM) for installed packages."""
|
|
3571
|
-
installer = LazyInstallerRegistry.get_instance(package_name)
|
|
3572
|
-
sbom = installer.generate_sbom()
|
|
3573
|
-
|
|
3574
|
-
if output_path:
|
|
3575
|
-
installer.export_sbom(output_path)
|
|
3576
|
-
|
|
3577
|
-
return sbom
|
|
3578
|
-
|
|
3579
|
-
|
|
3580
|
-
def check_externally_managed_environment() -> bool:
|
|
3581
|
-
"""Check if current Python environment is externally managed (PEP 668)."""
|
|
3582
|
-
return _is_externally_managed()
|
|
3583
|
-
|
|
3584
|
-
|
|
3585
|
-
# Keyword-based detection API
|
|
3586
|
-
def enable_keyword_detection(enabled: bool = True, keyword: Optional[str] = None) -> None:
|
|
3587
|
-
"""
|
|
3588
|
-
Enable/disable keyword-based auto-detection of lazy loading.
|
|
3589
|
-
|
|
3590
|
-
When enabled, xwlazy will check installed packages for a keyword
|
|
3591
|
-
(default: "xwlazy-enabled") in their metadata. Packages can opt-in
|
|
3592
|
-
by adding the keyword to their pyproject.toml:
|
|
3593
|
-
|
|
3594
|
-
[project]
|
|
3595
|
-
keywords = ["xwlazy-enabled"]
|
|
3596
|
-
|
|
3597
|
-
Args:
|
|
3598
|
-
enabled: Whether to enable keyword detection (default: True)
|
|
3599
|
-
keyword: Custom keyword to check (default: "xwlazy-enabled")
|
|
3600
|
-
"""
|
|
3601
|
-
global _KEYWORD_DETECTION_ENABLED, _KEYWORD_TO_CHECK
|
|
3602
|
-
_KEYWORD_DETECTION_ENABLED = enabled
|
|
3603
|
-
if keyword is not None:
|
|
3604
|
-
_KEYWORD_TO_CHECK = keyword
|
|
3605
|
-
# Clear cache to force re-detection
|
|
3606
|
-
with _lazy_detection_lock:
|
|
3607
|
-
_lazy_detection_cache.clear()
|
|
3608
|
-
|
|
3609
|
-
|
|
3610
|
-
def is_keyword_detection_enabled() -> bool:
|
|
3611
|
-
"""Return whether keyword-based detection is enabled."""
|
|
3612
|
-
return _KEYWORD_DETECTION_ENABLED
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
def get_keyword_detection_keyword() -> str:
|
|
3616
|
-
"""Get the keyword currently being checked for auto-detection."""
|
|
3617
|
-
return _KEYWORD_TO_CHECK
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
def check_package_keywords(package_name: Optional[str] = None, keyword: Optional[str] = None) -> bool:
|
|
3621
|
-
"""
|
|
3622
|
-
Check if a package (or any package) has the specified keyword in its metadata.
|
|
3623
|
-
|
|
3624
|
-
This is the public API for the keyword detection functionality.
|
|
3625
|
-
|
|
3626
|
-
Args:
|
|
3627
|
-
package_name: The package name to check (or None to check all packages)
|
|
3628
|
-
keyword: The keyword to look for (default: uses configured keyword)
|
|
3629
|
-
|
|
3630
|
-
Returns:
|
|
3631
|
-
True if the keyword is found in the package's metadata
|
|
3632
|
-
"""
|
|
3633
|
-
return _check_package_keywords(package_name, keyword)
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
# =============================================================================
|
|
3637
|
-
# EXPORT ALL
|
|
3638
|
-
# =============================================================================
|
|
3639
|
-
|
|
3640
|
-
__all__ = [
|
|
3641
|
-
# Core classes
|
|
3642
|
-
'DependencyMapper',
|
|
3643
|
-
'LazyDiscovery',
|
|
3644
|
-
'LazyInstaller',
|
|
3645
|
-
'LazyInstallPolicy',
|
|
3646
|
-
'LazyInstallerRegistry',
|
|
3647
|
-
'LazyImportHook',
|
|
3648
|
-
'LazyMetaPathFinder',
|
|
3649
|
-
'LazyLoader',
|
|
3650
|
-
'LazyImporter',
|
|
3651
|
-
'LazyModuleRegistry',
|
|
3652
|
-
'LazyPerformanceMonitor',
|
|
3653
|
-
'LazyInstallConfig',
|
|
3654
|
-
'LazyModeFacade',
|
|
3655
|
-
'WatchedPrefixRegistry',
|
|
3656
|
-
'AsyncInstallHandle',
|
|
3657
|
-
|
|
3658
|
-
# Discovery functions
|
|
3659
|
-
'get_lazy_discovery',
|
|
3660
|
-
'discover_dependencies',
|
|
3661
|
-
'export_dependency_mappings',
|
|
3662
|
-
|
|
3663
|
-
# Install functions
|
|
3664
|
-
'enable_lazy_install',
|
|
3665
|
-
'disable_lazy_install',
|
|
3666
|
-
'is_lazy_install_enabled',
|
|
3667
|
-
'set_lazy_install_mode',
|
|
3668
|
-
'get_lazy_install_mode',
|
|
3669
|
-
'install_missing_package',
|
|
3670
|
-
'install_and_import',
|
|
3671
|
-
'get_lazy_install_stats',
|
|
3672
|
-
'get_all_lazy_install_stats',
|
|
3673
|
-
'lazy_import_with_install',
|
|
3674
|
-
'xwimport',
|
|
3675
|
-
|
|
3676
|
-
# Hook functions
|
|
3677
|
-
'install_import_hook',
|
|
3678
|
-
'uninstall_import_hook',
|
|
3679
|
-
'is_import_hook_installed',
|
|
3680
|
-
|
|
3681
|
-
# Lazy loading functions
|
|
3682
|
-
'enable_lazy_imports',
|
|
3683
|
-
'disable_lazy_imports',
|
|
3684
|
-
'is_lazy_import_enabled',
|
|
3685
|
-
'lazy_import',
|
|
3686
|
-
'register_lazy_module',
|
|
3687
|
-
'preload_module',
|
|
3688
|
-
'get_lazy_module',
|
|
3689
|
-
'get_loading_stats',
|
|
3690
|
-
'preload_frequently_used',
|
|
3691
|
-
'get_lazy_import_stats',
|
|
3692
|
-
|
|
3693
|
-
# Lazy mode facade functions
|
|
3694
|
-
'enable_lazy_mode',
|
|
3695
|
-
'disable_lazy_mode',
|
|
3696
|
-
'is_lazy_mode_enabled',
|
|
3697
|
-
'get_lazy_mode_stats',
|
|
3698
|
-
'configure_lazy_mode',
|
|
3699
|
-
'preload_modules',
|
|
3700
|
-
'optimize_lazy_mode',
|
|
3701
|
-
|
|
3702
|
-
# Configuration
|
|
3703
|
-
'config_package_lazy_install_enabled',
|
|
3704
|
-
'sync_manifest_configuration',
|
|
3705
|
-
'refresh_lazy_manifests',
|
|
3706
|
-
|
|
3707
|
-
# Security & Policy
|
|
3708
|
-
'set_package_allow_list',
|
|
3709
|
-
'set_package_deny_list',
|
|
3710
|
-
'add_to_package_allow_list',
|
|
3711
|
-
'add_to_package_deny_list',
|
|
3712
|
-
'set_package_index_url',
|
|
3713
|
-
'set_package_extra_index_urls',
|
|
3714
|
-
'add_package_trusted_host',
|
|
3715
|
-
'set_package_lockfile',
|
|
3716
|
-
'generate_package_sbom',
|
|
3717
|
-
'check_externally_managed_environment',
|
|
3718
|
-
'register_lazy_module_prefix',
|
|
3719
|
-
'register_lazy_module_methods',
|
|
3720
|
-
|
|
3721
|
-
# Keyword-based detection
|
|
3722
|
-
'enable_keyword_detection',
|
|
3723
|
-
'is_keyword_detection_enabled',
|
|
3724
|
-
'get_keyword_detection_keyword',
|
|
3725
|
-
'check_package_keywords',
|
|
3726
|
-
]
|
|
3727
|
-
|