exonware-xwsystem 0.0.1.407__py3-none-any.whl → 0.0.1.409__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. exonware/__init__.py +21 -12
  2. exonware/conf.py +89 -149
  3. exonware/xwsystem/__init__.py +18 -159
  4. exonware/xwsystem/caching/__init__.py +1 -1
  5. exonware/xwsystem/caching/base.py +1 -1
  6. exonware/xwsystem/caching/bloom_cache.py +1 -1
  7. exonware/xwsystem/caching/cache_manager.py +1 -1
  8. exonware/xwsystem/caching/conditional.py +1 -1
  9. exonware/xwsystem/caching/contracts.py +1 -1
  10. exonware/xwsystem/caching/decorators.py +1 -1
  11. exonware/xwsystem/caching/defs.py +1 -1
  12. exonware/xwsystem/caching/disk_cache.py +1 -1
  13. exonware/xwsystem/caching/distributed.py +1 -1
  14. exonware/xwsystem/caching/errors.py +1 -1
  15. exonware/xwsystem/caching/events.py +1 -1
  16. exonware/xwsystem/caching/eviction_strategies.py +1 -1
  17. exonware/xwsystem/caching/fluent.py +1 -1
  18. exonware/xwsystem/caching/integrity.py +1 -1
  19. exonware/xwsystem/caching/lfu_cache.py +1 -1
  20. exonware/xwsystem/caching/lfu_optimized.py +1 -1
  21. exonware/xwsystem/caching/lru_cache.py +1 -1
  22. exonware/xwsystem/caching/memory_bounded.py +1 -1
  23. exonware/xwsystem/caching/metrics_exporter.py +1 -1
  24. exonware/xwsystem/caching/observable_cache.py +1 -1
  25. exonware/xwsystem/caching/pluggable_cache.py +1 -1
  26. exonware/xwsystem/caching/rate_limiter.py +1 -1
  27. exonware/xwsystem/caching/read_through.py +1 -1
  28. exonware/xwsystem/caching/secure_cache.py +1 -1
  29. exonware/xwsystem/caching/serializable.py +1 -1
  30. exonware/xwsystem/caching/stats.py +1 -1
  31. exonware/xwsystem/caching/tagging.py +1 -1
  32. exonware/xwsystem/caching/ttl_cache.py +1 -1
  33. exonware/xwsystem/caching/two_tier_cache.py +1 -1
  34. exonware/xwsystem/caching/utils.py +1 -1
  35. exonware/xwsystem/caching/validation.py +1 -1
  36. exonware/xwsystem/caching/warming.py +1 -1
  37. exonware/xwsystem/caching/write_behind.py +1 -1
  38. exonware/xwsystem/cli/__init__.py +1 -1
  39. exonware/xwsystem/cli/args.py +1 -1
  40. exonware/xwsystem/cli/base.py +1 -1
  41. exonware/xwsystem/cli/colors.py +1 -1
  42. exonware/xwsystem/cli/console.py +1 -1
  43. exonware/xwsystem/cli/contracts.py +1 -1
  44. exonware/xwsystem/cli/defs.py +1 -1
  45. exonware/xwsystem/cli/errors.py +1 -1
  46. exonware/xwsystem/cli/progress.py +1 -1
  47. exonware/xwsystem/cli/prompts.py +1 -1
  48. exonware/xwsystem/cli/tables.py +1 -1
  49. exonware/xwsystem/conf.py +1 -21
  50. exonware/xwsystem/config/__init__.py +1 -1
  51. exonware/xwsystem/config/base.py +1 -1
  52. exonware/xwsystem/config/contracts.py +1 -1
  53. exonware/xwsystem/config/defaults.py +1 -1
  54. exonware/xwsystem/config/defs.py +1 -1
  55. exonware/xwsystem/config/errors.py +1 -1
  56. exonware/xwsystem/config/logging.py +1 -1
  57. exonware/xwsystem/config/logging_setup.py +1 -1
  58. exonware/xwsystem/config/performance.py +1 -1
  59. exonware/xwsystem/http/__init__.py +1 -1
  60. exonware/xwsystem/http/advanced_client.py +5 -1
  61. exonware/xwsystem/http/base.py +1 -1
  62. exonware/xwsystem/http/client.py +5 -1
  63. exonware/xwsystem/http/contracts.py +1 -1
  64. exonware/xwsystem/http/defs.py +1 -1
  65. exonware/xwsystem/http/errors.py +1 -1
  66. exonware/xwsystem/io/__init__.py +1 -1
  67. exonware/xwsystem/io/archive/__init__.py +1 -1
  68. exonware/xwsystem/io/archive/archive.py +1 -1
  69. exonware/xwsystem/io/archive/archive_files.py +1 -1
  70. exonware/xwsystem/io/archive/archivers.py +1 -1
  71. exonware/xwsystem/io/archive/base.py +1 -1
  72. exonware/xwsystem/io/archive/codec_integration.py +1 -1
  73. exonware/xwsystem/io/archive/compression.py +1 -1
  74. exonware/xwsystem/io/archive/formats/__init__.py +1 -1
  75. exonware/xwsystem/io/archive/formats/brotli_format.py +1 -1
  76. exonware/xwsystem/io/archive/formats/lz4_format.py +1 -1
  77. exonware/xwsystem/io/archive/formats/rar.py +1 -1
  78. exonware/xwsystem/io/archive/formats/sevenzip.py +1 -1
  79. exonware/xwsystem/io/archive/formats/squashfs_format.py +1 -1
  80. exonware/xwsystem/io/archive/formats/tar.py +1 -1
  81. exonware/xwsystem/io/archive/formats/wim_format.py +1 -1
  82. exonware/xwsystem/io/archive/formats/zip.py +1 -1
  83. exonware/xwsystem/io/archive/formats/zpaq_format.py +1 -1
  84. exonware/xwsystem/io/archive/formats/zstandard.py +1 -1
  85. exonware/xwsystem/io/base.py +1 -1
  86. exonware/xwsystem/io/codec/__init__.py +1 -1
  87. exonware/xwsystem/io/codec/base.py +1 -1
  88. exonware/xwsystem/io/codec/contracts.py +1 -1
  89. exonware/xwsystem/io/codec/registry.py +1 -1
  90. exonware/xwsystem/io/common/__init__.py +1 -1
  91. exonware/xwsystem/io/common/base.py +1 -1
  92. exonware/xwsystem/io/common/lock.py +1 -1
  93. exonware/xwsystem/io/common/watcher.py +1 -1
  94. exonware/xwsystem/io/contracts.py +1 -1
  95. exonware/xwsystem/io/defs.py +1 -1
  96. exonware/xwsystem/io/errors.py +1 -1
  97. exonware/xwsystem/io/facade.py +1 -1
  98. exonware/xwsystem/io/file/__init__.py +1 -1
  99. exonware/xwsystem/io/file/base.py +1 -1
  100. exonware/xwsystem/io/file/conversion.py +1 -1
  101. exonware/xwsystem/io/file/file.py +1 -1
  102. exonware/xwsystem/io/file/paged_source.py +1 -1
  103. exonware/xwsystem/io/file/paging/__init__.py +1 -1
  104. exonware/xwsystem/io/file/paging/byte_paging.py +1 -1
  105. exonware/xwsystem/io/file/paging/line_paging.py +1 -1
  106. exonware/xwsystem/io/file/paging/record_paging.py +1 -1
  107. exonware/xwsystem/io/file/paging/registry.py +1 -1
  108. exonware/xwsystem/io/file/source.py +1 -1
  109. exonware/xwsystem/io/filesystem/__init__.py +1 -1
  110. exonware/xwsystem/io/filesystem/base.py +1 -1
  111. exonware/xwsystem/io/filesystem/local.py +1 -1
  112. exonware/xwsystem/io/folder/__init__.py +1 -1
  113. exonware/xwsystem/io/folder/base.py +1 -1
  114. exonware/xwsystem/io/folder/folder.py +1 -1
  115. exonware/xwsystem/io/serialization/__init__.py +1 -1
  116. exonware/xwsystem/io/serialization/auto_serializer.py +1 -1
  117. exonware/xwsystem/io/serialization/base.py +2 -2
  118. exonware/xwsystem/io/serialization/contracts.py +1 -1
  119. exonware/xwsystem/io/serialization/defs.py +1 -1
  120. exonware/xwsystem/io/serialization/errors.py +1 -1
  121. exonware/xwsystem/io/serialization/flyweight.py +1 -1
  122. exonware/xwsystem/io/serialization/format_detector.py +1 -1
  123. exonware/xwsystem/io/serialization/formats/__init__.py +1 -1
  124. exonware/xwsystem/io/serialization/formats/binary/bson.py +1 -1
  125. exonware/xwsystem/io/serialization/formats/binary/cbor.py +1 -1
  126. exonware/xwsystem/io/serialization/formats/binary/marshal.py +1 -1
  127. exonware/xwsystem/io/serialization/formats/binary/msgpack.py +1 -1
  128. exonware/xwsystem/io/serialization/formats/binary/pickle.py +1 -1
  129. exonware/xwsystem/io/serialization/formats/binary/plistlib.py +1 -1
  130. exonware/xwsystem/io/serialization/formats/database/dbm.py +1 -1
  131. exonware/xwsystem/io/serialization/formats/database/shelve.py +1 -1
  132. exonware/xwsystem/io/serialization/formats/database/sqlite3.py +1 -1
  133. exonware/xwsystem/io/serialization/formats/text/configparser.py +1 -1
  134. exonware/xwsystem/io/serialization/formats/text/csv.py +1 -1
  135. exonware/xwsystem/io/serialization/formats/text/formdata.py +1 -1
  136. exonware/xwsystem/io/serialization/formats/text/json.py +1 -1
  137. exonware/xwsystem/io/serialization/formats/text/json5.py +1 -1
  138. exonware/xwsystem/io/serialization/formats/text/jsonlines.py +1 -1
  139. exonware/xwsystem/io/serialization/formats/text/multipart.py +1 -1
  140. exonware/xwsystem/io/serialization/formats/text/toml.py +1 -1
  141. exonware/xwsystem/io/serialization/formats/text/xml.py +1 -1
  142. exonware/xwsystem/io/serialization/formats/text/yaml.py +1 -1
  143. exonware/xwsystem/io/serialization/registry.py +1 -1
  144. exonware/xwsystem/io/serialization/serializer.py +1 -1
  145. exonware/xwsystem/io/serialization/utils/__init__.py +1 -1
  146. exonware/xwsystem/io/serialization/utils/path_ops.py +1 -1
  147. exonware/xwsystem/io/stream/__init__.py +1 -1
  148. exonware/xwsystem/io/stream/async_operations.py +1 -1
  149. exonware/xwsystem/io/stream/base.py +1 -1
  150. exonware/xwsystem/io/stream/codec_io.py +1 -1
  151. exonware/xwsystem/ipc/async_fabric.py +1 -1
  152. exonware/xwsystem/ipc/base.py +1 -1
  153. exonware/xwsystem/ipc/contracts.py +1 -1
  154. exonware/xwsystem/ipc/defs.py +1 -1
  155. exonware/xwsystem/ipc/errors.py +1 -1
  156. exonware/xwsystem/lazy_bootstrap.py +79 -0
  157. exonware/xwsystem/monitoring/base.py +1 -1
  158. exonware/xwsystem/monitoring/contracts.py +1 -1
  159. exonware/xwsystem/monitoring/defs.py +1 -1
  160. exonware/xwsystem/monitoring/errors.py +1 -1
  161. exonware/xwsystem/monitoring/performance_manager_generic.py +1 -1
  162. exonware/xwsystem/monitoring/system_monitor.py +1 -1
  163. exonware/xwsystem/monitoring/tracing.py +17 -15
  164. exonware/xwsystem/monitoring/tracker.py +1 -1
  165. exonware/xwsystem/operations/__init__.py +1 -1
  166. exonware/xwsystem/operations/base.py +1 -1
  167. exonware/xwsystem/operations/defs.py +1 -1
  168. exonware/xwsystem/operations/diff.py +1 -1
  169. exonware/xwsystem/operations/merge.py +1 -1
  170. exonware/xwsystem/operations/patch.py +1 -1
  171. exonware/xwsystem/patterns/base.py +1 -1
  172. exonware/xwsystem/patterns/contracts.py +1 -1
  173. exonware/xwsystem/patterns/defs.py +1 -1
  174. exonware/xwsystem/patterns/errors.py +1 -1
  175. exonware/xwsystem/patterns/registry.py +1 -1
  176. exonware/xwsystem/plugins/__init__.py +1 -1
  177. exonware/xwsystem/plugins/base.py +1 -1
  178. exonware/xwsystem/plugins/contracts.py +1 -1
  179. exonware/xwsystem/plugins/defs.py +1 -1
  180. exonware/xwsystem/plugins/errors.py +1 -1
  181. exonware/xwsystem/runtime/__init__.py +1 -1
  182. exonware/xwsystem/runtime/base.py +1 -1
  183. exonware/xwsystem/runtime/contracts.py +1 -1
  184. exonware/xwsystem/runtime/defs.py +1 -1
  185. exonware/xwsystem/runtime/env.py +1 -1
  186. exonware/xwsystem/runtime/errors.py +1 -1
  187. exonware/xwsystem/runtime/reflection.py +1 -1
  188. exonware/xwsystem/security/auth.py +1 -1
  189. exonware/xwsystem/security/base.py +1 -1
  190. exonware/xwsystem/security/contracts.py +1 -1
  191. exonware/xwsystem/security/crypto.py +1 -1
  192. exonware/xwsystem/security/defs.py +1 -1
  193. exonware/xwsystem/security/errors.py +1 -1
  194. exonware/xwsystem/security/hazmat.py +1 -1
  195. exonware/xwsystem/shared/__init__.py +1 -1
  196. exonware/xwsystem/shared/base.py +1 -1
  197. exonware/xwsystem/shared/contracts.py +1 -1
  198. exonware/xwsystem/shared/defs.py +1 -1
  199. exonware/xwsystem/shared/errors.py +1 -1
  200. exonware/xwsystem/structures/base.py +1 -1
  201. exonware/xwsystem/structures/contracts.py +1 -1
  202. exonware/xwsystem/structures/defs.py +1 -1
  203. exonware/xwsystem/structures/errors.py +1 -1
  204. exonware/xwsystem/threading/async_primitives.py +1 -1
  205. exonware/xwsystem/threading/base.py +1 -1
  206. exonware/xwsystem/threading/contracts.py +1 -1
  207. exonware/xwsystem/threading/defs.py +1 -1
  208. exonware/xwsystem/threading/errors.py +1 -1
  209. exonware/xwsystem/utils/base.py +1 -1
  210. exonware/xwsystem/utils/contracts.py +1 -1
  211. exonware/xwsystem/utils/dt/__init__.py +1 -1
  212. exonware/xwsystem/utils/dt/base.py +1 -1
  213. exonware/xwsystem/utils/dt/contracts.py +1 -1
  214. exonware/xwsystem/utils/dt/defs.py +1 -1
  215. exonware/xwsystem/utils/dt/errors.py +1 -1
  216. exonware/xwsystem/utils/dt/formatting.py +1 -1
  217. exonware/xwsystem/utils/dt/humanize.py +1 -1
  218. exonware/xwsystem/utils/dt/parsing.py +1 -1
  219. exonware/xwsystem/utils/dt/timezone_utils.py +1 -1
  220. exonware/xwsystem/utils/errors.py +1 -1
  221. exonware/xwsystem/utils/test_runner.py +1 -1
  222. exonware/xwsystem/utils/utils_contracts.py +1 -1
  223. exonware/xwsystem/validation/__init__.py +1 -1
  224. exonware/xwsystem/validation/base.py +1 -1
  225. exonware/xwsystem/validation/contracts.py +1 -1
  226. exonware/xwsystem/validation/declarative.py +1 -1
  227. exonware/xwsystem/validation/defs.py +1 -1
  228. exonware/xwsystem/validation/errors.py +1 -1
  229. exonware/xwsystem/validation/fluent_validator.py +1 -1
  230. exonware/xwsystem/version.py +2 -2
  231. {exonware_xwsystem-0.0.1.407.dist-info → exonware_xwsystem-0.0.1.409.dist-info}/METADATA +3 -3
  232. exonware_xwsystem-0.0.1.409.dist-info/RECORD +274 -0
  233. exonware/xwsystem/_lazy_bootstrap.py +0 -77
  234. exonware/xwsystem/utils/lazy_package/ARCHITECTURE.md +0 -820
  235. exonware/xwsystem/utils/lazy_package/__init__.py +0 -268
  236. exonware/xwsystem/utils/lazy_package/config.py +0 -163
  237. exonware/xwsystem/utils/lazy_package/lazy_base.py +0 -465
  238. exonware/xwsystem/utils/lazy_package/lazy_contracts.py +0 -290
  239. exonware/xwsystem/utils/lazy_package/lazy_core.py +0 -2248
  240. exonware/xwsystem/utils/lazy_package/lazy_errors.py +0 -253
  241. exonware/xwsystem/utils/lazy_package/lazy_state.py +0 -86
  242. exonware_xwsystem-0.0.1.407.dist-info/RECORD +0 -282
  243. {exonware_xwsystem-0.0.1.407.dist-info → exonware_xwsystem-0.0.1.409.dist-info}/WHEEL +0 -0
  244. {exonware_xwsystem-0.0.1.407.dist-info → exonware_xwsystem-0.0.1.409.dist-info}/licenses/LICENSE +0 -0
@@ -1,820 +0,0 @@
1
- # Lazy Package Architecture
2
-
3
- **Company:** eXonware.com
4
- **Author:** Eng. Muhammad AlShehri
5
- **Email:** connect@exonware.com
6
- **Version:** 0.0.1.382
7
- **Date:** 10-Oct-2025
8
-
9
- ## Overview
10
-
11
- The Lazy Package provides per-package lazy loading with automatic installation of missing dependencies when they are actually used. This system enables each eXonware package (xwsystem, xwnode, xwdata, etc.) to independently enable lazy mode, providing maximum flexibility and zero interference between packages.
12
-
13
- ## Core Goal: Per-Package Scoping
14
-
15
- ### The Problem
16
- Traditional dependency management requires all dependencies to be installed upfront, even if they're never used. This leads to:
17
- - Large installation sizes
18
- - Longer installation times
19
- - Potential conflicts between packages
20
- - Wasted disk space
21
-
22
- ### The Solution: Per-Package Lazy Loading
23
-
24
- Each package can independently enable lazy mode:
25
-
26
- ```python
27
- # Installation
28
- pip install xwsystem[lazy] ← xwsystem gets lazy mode
29
- pip install xwnode ← xwnode does NOT get lazy mode
30
- pip install xwdata[lazy] ← xwdata gets lazy mode
31
-
32
- # Result
33
- xwsystem: Missing imports → LOG + AUTO-INSTALL on usage ✅
34
- xwnode: Missing imports → ImportError (normal behavior) ❌
35
- xwdata: Missing imports → LOG + AUTO-INSTALL on usage ✅
36
- ```
37
-
38
- ### Key Benefits
39
-
40
- 1. **Independence**: Each package's lazy mode doesn't affect others
41
- 2. **Flexibility**: Users choose per-package which mode to use
42
- 3. **Performance**: Dependencies installed only when actually needed
43
- 4. **Transparency**: Clean Python code with standard imports
44
-
45
- ## Architecture
46
-
47
- ### 4-File Structure (DEV_GUIDELINES.md Compliant)
48
-
49
- ```
50
- lazy_package/
51
- ├── lazy_contracts.py (~250 lines) - Interfaces, enums, protocols
52
- ├── lazy_errors.py (~120 lines) - Exception hierarchy
53
- ├── lazy_base.py (~450 lines) - Abstract base classes
54
- ├── lazy_core.py (~2000 lines) - Complete implementation
55
- ├── __init__.py - Public API exports
56
- └── ARCHITECTURE.md - This file
57
- ```
58
-
59
- #### lazy_contracts.py - Interfaces & Contracts
60
- Defines all interfaces following Strategy pattern:
61
- - `IPackageDiscovery` - Strategy for dependency discovery
62
- - `IPackageInstaller` - Strategy for package installation
63
- - `IImportHook` - Strategy for import interception
64
- - `IPackageCache` - Strategy for caching
65
- - `ILazyLoader` - Strategy for lazy loading
66
-
67
- Also includes:
68
- - `LazyInstallMode` enum (AUTO, INTERACTIVE, WARN, DISABLED, DRY_RUN)
69
- - `DependencyInfo` dataclass
70
-
71
- #### lazy_errors.py - Exception Hierarchy
72
- All exception classes:
73
- - `LazySystemError` - Base exception
74
- - `LazyInstallError` - Installation failures
75
- - `LazyDiscoveryError` - Discovery failures
76
- - `LazyHookError` - Hook operation failures
77
- - `LazySecurityError` - Security policy violations
78
- - `ExternallyManagedError` - PEP 668 errors
79
- - `DeferredImportError` - Two-stage loading placeholder
80
-
81
- #### lazy_base.py - Abstract Base Classes
82
- Abstract classes implementing Template Method pattern:
83
- - `APackageDiscovery` - Discovery workflow template
84
- - `APackageInstaller` - Installation workflow template
85
- - `AImportHook` - Hook workflow template
86
- - `APackageCache` - Caching workflow template
87
- - `ALazyLoader` - Loading workflow template
88
-
89
- All abstract classes:
90
- - Start with 'A' (per DEV_GUIDELINES.md)
91
- - Extend interfaces from lazy_contracts.py
92
- - Define common workflows with abstract steps
93
-
94
- #### lazy_core.py - Complete Implementation
95
- Organized in 7 sections:
96
-
97
- 1. **SECTION 1: PACKAGE DISCOVERY** (~350 lines)
98
- - `DependencyMapper` - Maps import names to package names
99
- - `LazyDiscovery` - Discovers from config files
100
- - Discovery from pyproject.toml, requirements.txt, setup.py
101
- - Caching with file modification time checks
102
- - SYSTEM_MODULES_BLACKLIST and COMMON_MAPPINGS
103
-
104
- 2. **SECTION 2: PACKAGE INSTALLATION** (~550 lines)
105
- - `LazyInstaller` - Per-package installer with all modes
106
- - `LazyInstallPolicy` - Security policies (allow/deny lists)
107
- - `LazyInstallerRegistry` - Per-package isolation **[CRITICAL]**
108
- - Interactive user prompts
109
- - PEP 668 externally-managed checks
110
- - Vulnerability auditing (pip-audit)
111
- - Lockfile management
112
- - SBOM generation
113
-
114
- 3. **SECTION 3: IMPORT HOOKS & TWO-STAGE LOADING** (~450 lines)
115
- - Recursion guards to prevent infinite loops
116
- - `DeferredImportError` integration for two-stage loading
117
- - `LazyMetaPathFinder` - Import interception
118
- - Serialization module wrapping
119
- - Per-package hook registry
120
-
121
- 4. **SECTION 4: LAZY LOADING & CACHING** (~300 lines)
122
- - `LazyLoader` - Thread-safe lazy loading with caching
123
- - `LazyImporter` - Module registration and access counting
124
- - `LazyModuleRegistry` - Registry pattern for modules
125
- - `LazyPerformanceMonitor` - Performance tracking
126
-
127
- 5. **SECTION 5: CONFIGURATION & REGISTRY** (~200 lines)
128
- - `LazyInstallConfig` - Per-package configuration
129
- - `config_package_lazy_install_enabled()` **[CRITICAL]**
130
- - Auto-detection of [lazy] extra from pip
131
-
132
- 6. **SECTION 6: FACADE - UNIFIED API** (~150 lines)
133
- - `LazyModeFacade` - Unified interface to all subsystems
134
- - Simplified API combining all features
135
-
136
- 7. **SECTION 7: PUBLIC API** (~200 lines)
137
- - All public functions for external use
138
- - Discovery, installation, hooks, loading, configuration APIs
139
- - Security and policy APIs
140
-
141
- ### Design Patterns Applied
142
-
143
- #### 1. Facade Pattern (MANDATORY per DEV_GUIDELINES.md)
144
- **LazyModeFacade** provides a unified interface to complex subsystems:
145
- - Hides complexity of discovery, installation, hooks, caching
146
- - Provides simple enable/disable/configure methods
147
- - Combines multiple subsystems into coherent interface
148
-
149
- ```python
150
- # Instead of managing 5 different systems
151
- facade = LazyModeFacade()
152
- facade.enable("on_demand") # One call does it all
153
- ```
154
-
155
- #### 2. Strategy Pattern
156
- Pluggable strategies for different behaviors:
157
- - **IPackageDiscovery**: Different discovery strategies (pyproject.toml, requirements.txt)
158
- - **IPackageInstaller**: Different installation modes (AUTO, INTERACTIVE, WARN)
159
- - **IImportHook**: Different hook implementations
160
-
161
- ```python
162
- # Different strategies can be swapped
163
- installer.set_mode(LazyInstallMode.INTERACTIVE) # Change strategy
164
- ```
165
-
166
- #### 3. Template Method Pattern
167
- Base classes define common workflows with abstract steps:
168
- - **APackageDiscovery.discover_all_dependencies()**: Common workflow
169
- - Check cache validity
170
- - Discover from sources (abstract)
171
- - Add common mappings
172
- - Update cache
173
- - **APackageInstaller.install_package()**: Common installation flow
174
- - Check if already installed
175
- - Check security policy (abstract)
176
- - Run pip install (abstract)
177
- - Update records
178
-
179
- ```python
180
- class LazyDiscovery(APackageDiscovery):
181
- def _discover_from_sources(self): # Implement abstract step
182
- self._discover_from_pyproject_toml()
183
- self._discover_from_requirements_txt()
184
- ```
185
-
186
- #### 4. Singleton Pattern
187
- Global instances for system-wide state:
188
- - `_discovery`: Global LazyDiscovery instance
189
- - `_lazy_facade`: Global LazyModeFacade instance
190
- - `_lazy_importer`: Global LazyImporter instance
191
- - Hook registries: One per package
192
-
193
- ```python
194
- # Always returns the same instance
195
- discovery = get_lazy_discovery() # Singleton
196
- ```
197
-
198
- #### 5. Registry Pattern
199
- **LazyInstallerRegistry** manages per-package isolation:
200
- - Each package gets its own LazyInstaller instance
201
- - Prevents interference between packages
202
- - Enables independent configuration
203
-
204
- ```python
205
- # Each package gets isolated installer
206
- xwsystem_installer = LazyInstallerRegistry.get_instance("xwsystem")
207
- xwnode_installer = LazyInstallerRegistry.get_instance("xwnode")
208
- # They don't interfere with each other!
209
- ```
210
-
211
- #### 6. Observer Pattern
212
- **LazyPerformanceMonitor** observes and tracks:
213
- - Module load times
214
- - Access counts
215
- - Memory usage
216
- - Performance metrics
217
-
218
- ```python
219
- monitor.record_load_time("fastavro", 0.5) # Observer pattern
220
- monitor.record_access("fastavro")
221
- ```
222
-
223
- #### 7. Proxy Pattern
224
- **LazyLoader** and **DeferredImportError** act as proxies:
225
- - LazyLoader: Proxies module access, loads on first use
226
- - DeferredImportError: Placeholder for failed imports, installs on first use
227
-
228
- ```python
229
- # LazyLoader acts as proxy
230
- loader = LazyLoader("heavy.module")
231
- loader.some_func() # Only NOW does it load the module
232
- ```
233
-
234
- #### 8. Factory Pattern
235
- Creating appropriate handlers based on context:
236
- - LazyInstallerRegistry creates installers per package
237
- - Discovery creates appropriate handlers for different file types
238
-
239
- ## Two-Stage Lazy Loading
240
-
241
- ### The Problem
242
- Traditional lazy loading: Import fails → Error immediately
243
-
244
- ### The Solution: Two-Stage Loading
245
-
246
- **Stage 1: Import Time** - Log + Return Placeholder
247
- ```python
248
- import fastavro # Missing!
249
- # → System logs: "[xwsystem] Missing: fastavro"
250
- # → Returns DeferredImportError placeholder
251
- # → NO ERROR! Import succeeds
252
- ```
253
-
254
- **Stage 2: Usage Time** - Install + Continue
255
- ```python
256
- from xwsystem.serialization.avro import AvroSerializer
257
- serializer = AvroSerializer() # ← NOW it installs fastavro
258
- # → System logs: "[xwsystem] Installing fastavro on first use"
259
- # → Installs package
260
- # → Returns real object
261
- # → Code continues normally
262
- ```
263
-
264
- ### Implementation
265
-
266
- **DeferredImportError** (in lazy_errors.py):
267
- - Acts as both error class and proxy object
268
- - Stores original ImportError for later
269
- - Implements `__getattr__` and `__call__` to trigger installation
270
- - Caches real module once installed
271
-
272
- **LazyMetaPathFinder** (in lazy_core.py):
273
- - Intercepts serialization module imports
274
- - Wraps module execution with custom `__import__`
275
- - Catches ImportError and returns DeferredImportError
276
- - Only defers external missing packages (not internal ones)
277
-
278
- ### Benefits
279
- 1. **Modules load even with missing dependencies**
280
- 2. **Dependencies installed only when actually used**
281
- 3. **Zero overhead for unused features**
282
- 4. **Clean code - no try/except ImportError needed**
283
-
284
- ## Per-Package Isolation
285
-
286
- ### LazyInstallerRegistry: The Key to Isolation
287
-
288
- Each package gets its own isolated LazyInstaller instance:
289
-
290
- ```python
291
- # xwsystem package
292
- config_package_lazy_install_enabled("xwsystem") # Enables lazy for xwsystem
293
- → LazyInstallerRegistry.get_instance("xwsystem") # Creates isolated installer
294
- → Only xwsystem's imports trigger auto-install
295
-
296
- # xwnode package
297
- # (no config_package_lazy_install_enabled call)
298
- → No installer created
299
- → Standard Python ImportError behavior
300
-
301
- # xwdata package
302
- config_package_lazy_install_enabled("xwdata") # Enables lazy for xwdata
303
- → LazyInstallerRegistry.get_instance("xwdata") # Creates separate isolated installer
304
- → Only xwdata's imports trigger auto-install
305
- ```
306
-
307
- ### Registry Implementation
308
-
309
- ```python
310
- class LazyInstallerRegistry:
311
- _instances: Dict[str, LazyInstaller] = {} # Per-package storage
312
-
313
- @classmethod
314
- def get_instance(cls, package_name: str) -> LazyInstaller:
315
- if package_name not in cls._instances:
316
- cls._instances[package_name] = LazyInstaller(package_name)
317
- return cls._instances[package_name]
318
- ```
319
-
320
- ### Statistics Per Package
321
-
322
- ```python
323
- # Get stats for specific package
324
- xwsystem_stats = get_lazy_install_stats("xwsystem")
325
- # {
326
- # 'package': 'xwsystem',
327
- # 'installed': ['fastavro', 'protobuf'],
328
- # 'failed': []
329
- # }
330
-
331
- # Get stats for all packages
332
- all_stats = get_all_lazy_install_stats()
333
- # {
334
- # 'xwsystem': {...},
335
- # 'xwdata': {...}
336
- # }
337
- ```
338
-
339
- ## Features Preserved
340
-
341
- All features from the original 8 files are preserved:
342
-
343
- ### 1. All 5 Installation Modes
344
- - **AUTO**: Automatically install without asking
345
- - **INTERACTIVE**: Ask user before installing each package
346
- - **WARN**: Log warning but don't install (monitoring mode)
347
- - **DISABLED**: Don't install anything
348
- - **DRY_RUN**: Show what would be installed but don't install
349
-
350
- ```python
351
- set_lazy_install_mode("xwsystem", LazyInstallMode.INTERACTIVE)
352
- ```
353
-
354
- ### 2. Security Policies
355
- - Allow lists: Only specified packages can be installed
356
- - Deny lists: Specified packages cannot be installed
357
- - Index URLs: Custom PyPI mirrors
358
- - Trusted hosts: Bypass SSL verification for specific hosts
359
-
360
- ```python
361
- set_package_allow_list("xwsystem", ["fastavro", "protobuf"])
362
- set_package_deny_list("xwsystem", ["suspicious-package"])
363
- set_package_index_url("xwsystem", "https://custom-pypi.org/simple")
364
- ```
365
-
366
- ### 3. SBOM Generation
367
- Software Bill of Materials for compliance:
368
-
369
- ```python
370
- sbom = generate_package_sbom("xwsystem", "sbom.json")
371
- # {
372
- # "metadata": {...},
373
- # "packages": [
374
- # {"name": "fastavro", "version": "1.8.0", "installed_by": "xwsystem"},
375
- # ...
376
- # ]
377
- # }
378
- ```
379
-
380
- ### 4. Lockfile Management
381
- Track installed packages with versions:
382
-
383
- ```python
384
- set_package_lockfile("xwsystem", "lazy-lock.json")
385
- # Automatically updated when packages are installed
386
- ```
387
-
388
- ### 5. Vulnerability Auditing
389
- Automatic scanning with pip-audit (if installed):
390
- - Runs after each package installation
391
- - Logs security warnings
392
- - Non-blocking (continues even if vulnerabilities found)
393
-
394
- ### 6. Interactive Mode
395
- User-friendly prompts for approval:
396
-
397
- ```
398
- ============================================================
399
- Lazy Installation Active - xwsystem
400
- ============================================================
401
- Package: fastavro
402
- Module: fastavro
403
- ============================================================
404
-
405
- The module 'fastavro' is not installed.
406
- Would you like to install 'fastavro'?
407
-
408
- Options:
409
- [Y] Yes - Install this package
410
- [N] No - Skip this package
411
- [A] All - Install this and all future packages without asking
412
- [Q] Quit - Cancel and raise ImportError
413
- ============================================================
414
- Your choice [Y/N/A/Q]:
415
- ```
416
-
417
- ### 7. Performance Monitoring
418
- Track loading performance:
419
- - Module load times
420
- - Access counts
421
- - Memory usage
422
- - Cache hit/miss ratios
423
-
424
- ```python
425
- stats = get_lazy_mode_stats()
426
- # {
427
- # 'enabled': True,
428
- # 'loaded_count': 15,
429
- # 'access_counts': {...},
430
- # 'performance': {...}
431
- # }
432
- ```
433
-
434
- ### 8. Caching with Validation
435
- Intelligent caching:
436
- - File modification time checks
437
- - Automatic cache invalidation
438
- - Thread-safe operations
439
- - Zero overhead for valid cache
440
-
441
- ## API Documentation
442
-
443
- ### Configuration API
444
-
445
- #### `config_package_lazy_install_enabled(package_name, enabled=None, mode="auto", install_hook=True)`
446
- **ONE-LINE SETUP** for per-package lazy loading.
447
-
448
- ```python
449
- # Auto-detect from pip installation
450
- config_package_lazy_install_enabled("xwsystem")
451
-
452
- # Force enable with interactive mode
453
- config_package_lazy_install_enabled("xwnode", True, "interactive")
454
-
455
- # Force disable
456
- config_package_lazy_install_enabled("xwdata", False)
457
- ```
458
-
459
- **Parameters:**
460
- - `package_name`: Package name (e.g., "xwsystem")
461
- - `enabled`: True/False/None (None = auto-detect from [lazy] extra)
462
- - `mode`: Installation mode ("auto", "interactive", "warn", "disabled", "dry_run")
463
- - `install_hook`: Whether to install import hook (default: True)
464
-
465
- ### Discovery API
466
-
467
- #### `discover_dependencies(project_root=None) -> Dict[str, str]`
468
- Discover all dependencies for the current project.
469
-
470
- ```python
471
- deps = discover_dependencies()
472
- # {'cv2': 'opencv-python', 'PIL': 'Pillow', ...}
473
- ```
474
-
475
- #### `get_lazy_discovery(project_root=None) -> LazyDiscovery`
476
- Get the global lazy discovery instance.
477
-
478
- ```python
479
- discovery = get_lazy_discovery()
480
- package_name = discovery.get_package_for_import("cv2") # "opencv-python"
481
- ```
482
-
483
- ### Installation API
484
-
485
- #### `enable_lazy_install(package_name="default")`
486
- Enable lazy installation for a specific package.
487
-
488
- ```python
489
- enable_lazy_install("xwsystem")
490
- ```
491
-
492
- #### `set_lazy_install_mode(package_name, mode)`
493
- Set the installation mode for a package.
494
-
495
- ```python
496
- set_lazy_install_mode("xwsystem", LazyInstallMode.INTERACTIVE)
497
- ```
498
-
499
- #### `lazy_import_with_install(module_name, package_name=None, installer_package="default")`
500
- Import with automatic installation.
501
-
502
- ```python
503
- module, success = lazy_import_with_install("fastavro", installer_package="xwsystem")
504
- if success:
505
- # Use module
506
- pass
507
- ```
508
-
509
- #### `xwimport(module_name, package_name=None, installer_package="default")`
510
- Simple lazy import (raises ImportError if fails).
511
-
512
- ```python
513
- fastavro = xwimport("fastavro", installer_package="xwsystem")
514
- ```
515
-
516
- ### Hook API
517
-
518
- #### `install_import_hook(package_name="default")`
519
- Install import hook for automatic lazy installation.
520
-
521
- ```python
522
- install_import_hook("xwsystem")
523
- ```
524
-
525
- #### `uninstall_import_hook(package_name="default")`
526
- Uninstall import hook.
527
-
528
- ```python
529
- uninstall_import_hook("xwsystem")
530
- ```
531
-
532
- ### Security API
533
-
534
- #### `set_package_allow_list(package_name, allowed_packages)`
535
- Set allow list for a package.
536
-
537
- ```python
538
- set_package_allow_list("xwsystem", ["fastavro", "protobuf", "msgpack"])
539
- ```
540
-
541
- #### `set_package_deny_list(package_name, denied_packages)`
542
- Set deny list for a package.
543
-
544
- ```python
545
- set_package_deny_list("xwsystem", ["malicious-package"])
546
- ```
547
-
548
- #### `set_package_index_url(package_name, index_url)`
549
- Set custom PyPI index URL.
550
-
551
- ```python
552
- set_package_index_url("xwsystem", "https://custom-pypi.org/simple")
553
- ```
554
-
555
- ### Statistics API
556
-
557
- #### `get_lazy_install_stats(package_name="default") -> Dict`
558
- Get lazy installation statistics for a specific package.
559
-
560
- ```python
561
- stats = get_lazy_install_stats("xwsystem")
562
- # {
563
- # 'enabled': True,
564
- # 'mode': 'auto',
565
- # 'package_name': 'xwsystem',
566
- # 'installed_packages': ['fastavro', 'protobuf'],
567
- # 'failed_packages': [],
568
- # 'total_installed': 2,
569
- # 'total_failed': 0
570
- # }
571
- ```
572
-
573
- #### `get_all_lazy_install_stats() -> Dict[str, Dict]`
574
- Get statistics for all packages.
575
-
576
- ```python
577
- all_stats = get_all_lazy_install_stats()
578
- # {
579
- # 'xwsystem': {...},
580
- # 'xwdata': {...}
581
- # }
582
- ```
583
-
584
- ## Usage Examples
585
-
586
- ### Example 1: Basic Setup (xwsystem)
587
-
588
- ```python
589
- # xwsystem/__init__.py
590
- from exonware.xwsystem.utils.lazy_package import config_package_lazy_install_enabled
591
-
592
- # One-line setup - auto-detects from pip install xwsystem[lazy]
593
- config_package_lazy_install_enabled("xwsystem")
594
-
595
- # That's it! Now use standard imports:
596
- # xwsystem/serialization/avro.py
597
- import fastavro # Auto-installed if missing! ✨
598
-
599
- # User code
600
- from xwsystem.serialization.avro import AvroSerializer
601
- serializer = AvroSerializer() # Installs fastavro on first use
602
- ```
603
-
604
- ### Example 2: Interactive Mode (xwnode)
605
-
606
- ```python
607
- # xwnode/__init__.py
608
- from exonware.xwsystem.utils.lazy_package import config_package_lazy_install_enabled
609
-
610
- # Force enable with interactive mode
611
- config_package_lazy_install_enabled("xwnode", True, "interactive")
612
-
613
- # xwnode/graph.py
614
- import networkx # Will ask user before installing
615
-
616
- # User code
617
- from xwnode.graph import GraphNode
618
- node = GraphNode() # Prompts: "Install networkx? [Y/N/A/Q]"
619
- ```
620
-
621
- ### Example 3: Security Policies (xwdata)
622
-
623
- ```python
624
- # xwdata/__init__.py
625
- from exonware.xwsystem.utils.lazy_package import (
626
- config_package_lazy_install_enabled,
627
- set_package_allow_list,
628
- set_package_lockfile,
629
- )
630
-
631
- # Enable lazy mode
632
- config_package_lazy_install_enabled("xwdata")
633
-
634
- # Security: Only allow specific packages
635
- set_package_allow_list("xwdata", [
636
- "pandas",
637
- "numpy",
638
- "openpyxl",
639
- "xlrd"
640
- ])
641
-
642
- # Track installations
643
- set_package_lockfile("xwdata", "xwdata-lazy-lock.json")
644
-
645
- # xwdata/formats/excel.py
646
- import openpyxl # ✅ Allowed
647
- import some_random_package # ❌ Blocked (not in allow list)
648
- ```
649
-
650
- ### Example 4: Custom PyPI Mirror
651
-
652
- ```python
653
- from exonware.xwsystem.utils.lazy_package import (
654
- config_package_lazy_install_enabled,
655
- set_package_index_url,
656
- add_package_trusted_host,
657
- )
658
-
659
- # Enable lazy mode
660
- config_package_lazy_install_enabled("xwsystem")
661
-
662
- # Use custom PyPI mirror
663
- set_package_index_url("xwsystem", "https://pypi.internal.company.com/simple")
664
- add_package_trusted_host("xwsystem", "pypi.internal.company.com")
665
- ```
666
-
667
- ### Example 5: SBOM Generation for Compliance
668
-
669
- ```python
670
- from exonware.xwsystem.utils.lazy_package import generate_package_sbom
671
-
672
- # Generate SBOM for xwsystem
673
- sbom = generate_package_sbom("xwsystem", "xwsystem-sbom.json")
674
-
675
- # SBOM contains:
676
- # - All packages installed by xwsystem lazy loader
677
- # - Versions
678
- # - Installation timestamps
679
- # - Source (PyPI)
680
- ```
681
-
682
- ## Performance Considerations
683
-
684
- ### Zero Overhead for Successful Imports
685
- Import hooks only trigger when imports fail:
686
- - Successful imports run at full native Python speed
687
- - No performance penalty for normal operation
688
- - Hooks are passive observers
689
-
690
- ### Aggressive Caching
691
- Multiple levels of caching:
692
- 1. **Discovery cache**: File modification time checks
693
- 2. **Module cache**: Loaded modules cached
694
- 3. **Detection cache**: Per-package [lazy] detection cached
695
- 4. **Thread-safe**: All caches use RLock for safety
696
-
697
- ### Lazy Initialization
698
- Everything initializes only when first needed:
699
- - Discovery doesn't load until first import failure
700
- - Installers created per-package on demand
701
- - Hooks installed only when enabled
702
-
703
- ### Metrics
704
- - **Import overhead**: ~0.1ms for successful imports
705
- - **First failure**: ~50ms (discovery + policy check)
706
- - **Subsequent failures**: ~5ms (cached discovery)
707
- - **Installation**: Depends on package size and network
708
-
709
- ## Security Considerations
710
-
711
- ### 1. PEP 668 Compliance
712
- Respects externally-managed Python environments:
713
- - Detects EXTERNALLY-MANAGED marker file
714
- - Refuses to install in system Python
715
- - Suggests virtual environment or pipx
716
-
717
- ### 2. Allow/Deny Lists
718
- Per-package security policies:
719
- - Allow list: Only specified packages installable
720
- - Deny list: Specified packages blocked
721
- - Checked before every installation
722
-
723
- ### 3. System Module Blacklist
724
- Built-in modules never auto-installed:
725
- - OS-level modules (pwd, grp, msvcrt, winreg)
726
- - Python standard library
727
- - Prevents accidental system modifications
728
-
729
- ### 4. Vulnerability Auditing
730
- Optional pip-audit integration:
731
- - Scans packages after installation
732
- - Logs security warnings
733
- - Non-blocking (doesn't prevent installation)
734
-
735
- ### 5. Custom PyPI Mirrors
736
- Support for internal PyPI servers:
737
- - Custom index URLs
738
- - Trusted hosts
739
- - SSL verification control
740
-
741
- ## Migration from Old Structure
742
-
743
- ### Old Import Pattern
744
- ```python
745
- # Old way (8 separate files)
746
- from exonware.xwsystem.utils.lazy_loader import LazyLoader
747
- from exonware.xwsystem.utils.lazy_discovery import config_package_lazy_install_enabled
748
- from exonware.xwsystem.utils.lazy_install import enable_lazy_install
749
- from exonware.xwsystem.utils.lazy_import_hook import install_import_hook
750
- ```
751
-
752
- ### New Import Pattern
753
- ```python
754
- # New way (1 unified package)
755
- from exonware.xwsystem.utils.lazy_package import (
756
- LazyLoader,
757
- config_package_lazy_install_enabled,
758
- enable_lazy_install,
759
- install_import_hook,
760
- )
761
- ```
762
-
763
- ### Backward Compatibility
764
- The `__init__.py` exports all symbols, maintaining the same public API.
765
- Existing code continues to work with minimal changes.
766
-
767
- ## Testing Strategy
768
-
769
- ### Unit Tests
770
- Test individual components:
771
- - Discovery from different sources
772
- - Installation modes
773
- - Security policy checks
774
- - Cache invalidation
775
-
776
- ### Integration Tests
777
- Test full workflows:
778
- - Per-package isolation
779
- - Two-stage lazy loading
780
- - Import hook interception
781
- - SBOM generation
782
-
783
- ### Performance Tests
784
- Benchmark critical paths:
785
- - Import overhead (should be < 1ms)
786
- - Discovery cache effectiveness
787
- - Thread safety under load
788
-
789
- ## Future Enhancements
790
-
791
- ### Planned for Version 1.x
792
- 1. **Async installation**: Install packages in background
793
- 2. **Dependency resolution**: Resolve conflicts automatically
794
- 3. **Network caching**: Cache downloaded packages
795
- 4. **Offline mode**: Use cached packages when offline
796
- 5. **Multi-language support**: Messages in multiple languages
797
-
798
- ### Under Consideration
799
- - Integration with Poetry, PDM
800
- - Conda environment support
801
- - Docker-aware installation
802
- - Cloud package mirrors (S3, Azure Blob)
803
-
804
- ## Conclusion
805
-
806
- The Lazy Package provides a production-grade, per-package lazy loading system that:
807
- - ✅ Follows DEV_GUIDELINES.md structure (contracts, errors, base, core)
808
- - ✅ Implements 8 design patterns (Facade, Strategy, Template Method, Singleton, Registry, Observer, Proxy, Factory)
809
- - ✅ Preserves ALL features (security, SBOM, lockfiles, interactive, caching, etc.)
810
- - ✅ Enables per-package isolation (no interference between packages)
811
- - ✅ Provides two-stage lazy loading (log on import, install on usage)
812
- - ✅ Maintains clean Python code (standard imports, no defensive programming)
813
- - ✅ Achieves zero overhead for successful imports
814
-
815
- **The result**: A simple, powerful, and transparent lazy loading system that "just works" while maintaining production-grade quality and security.
816
-
817
- ---
818
-
819
- *For questions or support, contact: connect@exonware.com*
820
-