unrealon 1.0.9__py3-none-any.whl → 1.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (302) hide show
  1. unrealon/__init__.py +23 -21
  2. unrealon-1.1.0.dist-info/METADATA +164 -0
  3. unrealon-1.1.0.dist-info/RECORD +82 -0
  4. {unrealon-1.0.9.dist-info → unrealon-1.1.0.dist-info}/WHEEL +1 -1
  5. unrealon-1.1.0.dist-info/entry_points.txt +9 -0
  6. {unrealon-1.0.9.dist-info → unrealon-1.1.0.dist-info/licenses}/LICENSE +1 -1
  7. unrealon_bridge/__init__.py +114 -0
  8. unrealon_bridge/cli.py +316 -0
  9. unrealon_bridge/client/__init__.py +93 -0
  10. unrealon_bridge/client/base.py +78 -0
  11. unrealon_bridge/client/commands.py +89 -0
  12. unrealon_bridge/client/connection.py +90 -0
  13. unrealon_bridge/client/events.py +65 -0
  14. unrealon_bridge/client/health.py +38 -0
  15. unrealon_bridge/client/html_parser.py +146 -0
  16. unrealon_bridge/client/logging.py +139 -0
  17. unrealon_bridge/client/proxy.py +70 -0
  18. unrealon_bridge/client/scheduler.py +450 -0
  19. unrealon_bridge/client/session.py +70 -0
  20. unrealon_bridge/configs/__init__.py +14 -0
  21. unrealon_bridge/configs/bridge_config.py +212 -0
  22. unrealon_bridge/configs/bridge_config.yaml +39 -0
  23. unrealon_bridge/models/__init__.py +138 -0
  24. unrealon_bridge/models/base.py +28 -0
  25. unrealon_bridge/models/command.py +41 -0
  26. unrealon_bridge/models/events.py +40 -0
  27. unrealon_bridge/models/html_parser.py +79 -0
  28. unrealon_bridge/models/logging.py +55 -0
  29. unrealon_bridge/models/parser.py +63 -0
  30. unrealon_bridge/models/proxy.py +41 -0
  31. unrealon_bridge/models/requests.py +95 -0
  32. unrealon_bridge/models/responses.py +88 -0
  33. unrealon_bridge/models/scheduler.py +592 -0
  34. unrealon_bridge/models/session.py +28 -0
  35. unrealon_bridge/server/__init__.py +91 -0
  36. unrealon_bridge/server/base.py +171 -0
  37. unrealon_bridge/server/handlers/__init__.py +23 -0
  38. unrealon_bridge/server/handlers/command.py +110 -0
  39. unrealon_bridge/server/handlers/html_parser.py +139 -0
  40. unrealon_bridge/server/handlers/logging.py +95 -0
  41. unrealon_bridge/server/handlers/parser.py +95 -0
  42. unrealon_bridge/server/handlers/proxy.py +75 -0
  43. unrealon_bridge/server/handlers/scheduler.py +545 -0
  44. unrealon_bridge/server/handlers/session.py +66 -0
  45. unrealon_browser/__init__.py +61 -18
  46. unrealon_browser/{src/cli → cli}/browser_cli.py +6 -13
  47. unrealon_browser/{src/cli → cli}/cookies_cli.py +5 -1
  48. unrealon_browser/{src/core → core}/browser_manager.py +2 -2
  49. unrealon_browser/{src/managers → managers}/captcha.py +1 -1
  50. unrealon_browser/{src/managers → managers}/cookies.py +1 -1
  51. unrealon_browser/managers/logger_bridge.py +231 -0
  52. unrealon_browser/{src/managers → managers}/profile.py +1 -1
  53. unrealon_driver/__init__.py +73 -19
  54. unrealon_driver/browser/__init__.py +8 -0
  55. unrealon_driver/browser/config.py +74 -0
  56. unrealon_driver/browser/manager.py +416 -0
  57. unrealon_driver/exceptions.py +28 -0
  58. unrealon_driver/parser/__init__.py +55 -0
  59. unrealon_driver/parser/cli_manager.py +141 -0
  60. unrealon_driver/parser/daemon_manager.py +227 -0
  61. unrealon_driver/parser/managers/__init__.py +46 -0
  62. unrealon_driver/parser/managers/browser.py +51 -0
  63. unrealon_driver/parser/managers/config.py +281 -0
  64. unrealon_driver/parser/managers/error.py +412 -0
  65. unrealon_driver/parser/managers/html.py +732 -0
  66. unrealon_driver/parser/managers/logging.py +609 -0
  67. unrealon_driver/parser/managers/result.py +321 -0
  68. unrealon_driver/parser/parser_manager.py +628 -0
  69. unrealon/sdk_config.py +0 -88
  70. unrealon-1.0.9.dist-info/METADATA +0 -810
  71. unrealon-1.0.9.dist-info/RECORD +0 -246
  72. unrealon_browser/pyproject.toml +0 -182
  73. unrealon_browser/src/__init__.py +0 -62
  74. unrealon_browser/src/managers/logger_bridge.py +0 -395
  75. unrealon_driver/README.md +0 -204
  76. unrealon_driver/pyproject.toml +0 -187
  77. unrealon_driver/src/__init__.py +0 -90
  78. unrealon_driver/src/cli/__init__.py +0 -10
  79. unrealon_driver/src/cli/main.py +0 -66
  80. unrealon_driver/src/cli/simple.py +0 -510
  81. unrealon_driver/src/config/__init__.py +0 -11
  82. unrealon_driver/src/config/auto_config.py +0 -478
  83. unrealon_driver/src/core/__init__.py +0 -18
  84. unrealon_driver/src/core/exceptions.py +0 -289
  85. unrealon_driver/src/core/parser.py +0 -638
  86. unrealon_driver/src/dto/__init__.py +0 -66
  87. unrealon_driver/src/dto/cli.py +0 -119
  88. unrealon_driver/src/dto/config.py +0 -18
  89. unrealon_driver/src/dto/events.py +0 -237
  90. unrealon_driver/src/dto/execution.py +0 -313
  91. unrealon_driver/src/dto/services.py +0 -311
  92. unrealon_driver/src/execution/__init__.py +0 -23
  93. unrealon_driver/src/execution/daemon_mode.py +0 -317
  94. unrealon_driver/src/execution/interactive_mode.py +0 -88
  95. unrealon_driver/src/execution/modes.py +0 -45
  96. unrealon_driver/src/execution/scheduled_mode.py +0 -209
  97. unrealon_driver/src/execution/test_mode.py +0 -250
  98. unrealon_driver/src/logging/__init__.py +0 -24
  99. unrealon_driver/src/logging/driver_logger.py +0 -512
  100. unrealon_driver/src/services/__init__.py +0 -24
  101. unrealon_driver/src/services/browser_service.py +0 -726
  102. unrealon_driver/src/services/llm/__init__.py +0 -15
  103. unrealon_driver/src/services/llm/browser_llm_service.py +0 -363
  104. unrealon_driver/src/services/llm/llm.py +0 -195
  105. unrealon_driver/src/services/logger_service.py +0 -232
  106. unrealon_driver/src/services/metrics_service.py +0 -185
  107. unrealon_driver/src/services/scheduler_service.py +0 -489
  108. unrealon_driver/src/services/websocket_service.py +0 -362
  109. unrealon_driver/src/utils/__init__.py +0 -16
  110. unrealon_driver/src/utils/service_factory.py +0 -317
  111. unrealon_driver/src/utils/time_formatter.py +0 -338
  112. unrealon_llm/README.md +0 -44
  113. unrealon_llm/__init__.py +0 -26
  114. unrealon_llm/pyproject.toml +0 -154
  115. unrealon_llm/src/__init__.py +0 -228
  116. unrealon_llm/src/cli/__init__.py +0 -0
  117. unrealon_llm/src/core/__init__.py +0 -11
  118. unrealon_llm/src/core/smart_client.py +0 -438
  119. unrealon_llm/src/dto/__init__.py +0 -155
  120. unrealon_llm/src/dto/models/__init__.py +0 -0
  121. unrealon_llm/src/dto/models/config.py +0 -343
  122. unrealon_llm/src/dto/models/core.py +0 -328
  123. unrealon_llm/src/dto/models/enums.py +0 -123
  124. unrealon_llm/src/dto/models/html_analysis.py +0 -345
  125. unrealon_llm/src/dto/models/statistics.py +0 -473
  126. unrealon_llm/src/dto/models/translation.py +0 -383
  127. unrealon_llm/src/dto/models/type_conversion.py +0 -462
  128. unrealon_llm/src/dto/schemas/__init__.py +0 -0
  129. unrealon_llm/src/exceptions.py +0 -392
  130. unrealon_llm/src/llm_config/__init__.py +0 -20
  131. unrealon_llm/src/llm_config/logging_config.py +0 -178
  132. unrealon_llm/src/llm_logging/__init__.py +0 -42
  133. unrealon_llm/src/llm_logging/llm_events.py +0 -107
  134. unrealon_llm/src/llm_logging/llm_logger.py +0 -466
  135. unrealon_llm/src/managers/__init__.py +0 -15
  136. unrealon_llm/src/managers/cache_manager.py +0 -67
  137. unrealon_llm/src/managers/cost_manager.py +0 -107
  138. unrealon_llm/src/managers/request_manager.py +0 -298
  139. unrealon_llm/src/modules/__init__.py +0 -0
  140. unrealon_llm/src/modules/html_processor/__init__.py +0 -25
  141. unrealon_llm/src/modules/html_processor/base_processor.py +0 -415
  142. unrealon_llm/src/modules/html_processor/details_processor.py +0 -85
  143. unrealon_llm/src/modules/html_processor/listing_processor.py +0 -91
  144. unrealon_llm/src/modules/html_processor/models/__init__.py +0 -20
  145. unrealon_llm/src/modules/html_processor/models/processing_models.py +0 -40
  146. unrealon_llm/src/modules/html_processor/models/universal_model.py +0 -56
  147. unrealon_llm/src/modules/html_processor/processor.py +0 -102
  148. unrealon_llm/src/modules/llm/__init__.py +0 -0
  149. unrealon_llm/src/modules/translator/__init__.py +0 -0
  150. unrealon_llm/src/provider.py +0 -116
  151. unrealon_llm/src/utils/__init__.py +0 -95
  152. unrealon_llm/src/utils/common.py +0 -64
  153. unrealon_llm/src/utils/data_extractor.py +0 -188
  154. unrealon_llm/src/utils/html_cleaner.py +0 -767
  155. unrealon_llm/src/utils/language_detector.py +0 -308
  156. unrealon_llm/src/utils/models_cache.py +0 -592
  157. unrealon_llm/src/utils/smart_counter.py +0 -229
  158. unrealon_llm/src/utils/token_counter.py +0 -189
  159. unrealon_sdk/README.md +0 -25
  160. unrealon_sdk/__init__.py +0 -30
  161. unrealon_sdk/pyproject.toml +0 -231
  162. unrealon_sdk/src/__init__.py +0 -150
  163. unrealon_sdk/src/cli/__init__.py +0 -12
  164. unrealon_sdk/src/cli/commands/__init__.py +0 -22
  165. unrealon_sdk/src/cli/commands/benchmark.py +0 -42
  166. unrealon_sdk/src/cli/commands/diagnostics.py +0 -573
  167. unrealon_sdk/src/cli/commands/health.py +0 -46
  168. unrealon_sdk/src/cli/commands/integration.py +0 -498
  169. unrealon_sdk/src/cli/commands/reports.py +0 -43
  170. unrealon_sdk/src/cli/commands/security.py +0 -36
  171. unrealon_sdk/src/cli/commands/server.py +0 -483
  172. unrealon_sdk/src/cli/commands/servers.py +0 -56
  173. unrealon_sdk/src/cli/commands/tests.py +0 -55
  174. unrealon_sdk/src/cli/main.py +0 -126
  175. unrealon_sdk/src/cli/utils/reporter.py +0 -519
  176. unrealon_sdk/src/clients/openapi.yaml +0 -3347
  177. unrealon_sdk/src/clients/python_http/__init__.py +0 -3
  178. unrealon_sdk/src/clients/python_http/api_config.py +0 -228
  179. unrealon_sdk/src/clients/python_http/models/BaseModel.py +0 -12
  180. unrealon_sdk/src/clients/python_http/models/BroadcastDeliveryStats.py +0 -33
  181. unrealon_sdk/src/clients/python_http/models/BroadcastMessage.py +0 -17
  182. unrealon_sdk/src/clients/python_http/models/BroadcastMessageRequest.py +0 -35
  183. unrealon_sdk/src/clients/python_http/models/BroadcastPriority.py +0 -10
  184. unrealon_sdk/src/clients/python_http/models/BroadcastResponse.py +0 -21
  185. unrealon_sdk/src/clients/python_http/models/BroadcastResultResponse.py +0 -33
  186. unrealon_sdk/src/clients/python_http/models/BroadcastTarget.py +0 -11
  187. unrealon_sdk/src/clients/python_http/models/ConnectionStats.py +0 -27
  188. unrealon_sdk/src/clients/python_http/models/ConnectionsResponse.py +0 -21
  189. unrealon_sdk/src/clients/python_http/models/DeveloperMessageResponse.py +0 -23
  190. unrealon_sdk/src/clients/python_http/models/ErrorResponse.py +0 -25
  191. unrealon_sdk/src/clients/python_http/models/HTTPValidationError.py +0 -16
  192. unrealon_sdk/src/clients/python_http/models/HealthResponse.py +0 -23
  193. unrealon_sdk/src/clients/python_http/models/HealthStatus.py +0 -33
  194. unrealon_sdk/src/clients/python_http/models/LogLevel.py +0 -10
  195. unrealon_sdk/src/clients/python_http/models/LoggingRequest.py +0 -27
  196. unrealon_sdk/src/clients/python_http/models/LoggingResponse.py +0 -23
  197. unrealon_sdk/src/clients/python_http/models/MaintenanceMode.py +0 -9
  198. unrealon_sdk/src/clients/python_http/models/MaintenanceModeRequest.py +0 -33
  199. unrealon_sdk/src/clients/python_http/models/MaintenanceStatusResponse.py +0 -39
  200. unrealon_sdk/src/clients/python_http/models/ParserCommandRequest.py +0 -25
  201. unrealon_sdk/src/clients/python_http/models/ParserMessageResponse.py +0 -21
  202. unrealon_sdk/src/clients/python_http/models/ParserRegistrationRequest.py +0 -28
  203. unrealon_sdk/src/clients/python_http/models/ParserRegistrationResponse.py +0 -25
  204. unrealon_sdk/src/clients/python_http/models/ParserType.py +0 -10
  205. unrealon_sdk/src/clients/python_http/models/ProxyBlockRequest.py +0 -19
  206. unrealon_sdk/src/clients/python_http/models/ProxyEndpointResponse.py +0 -20
  207. unrealon_sdk/src/clients/python_http/models/ProxyListResponse.py +0 -19
  208. unrealon_sdk/src/clients/python_http/models/ProxyProvider.py +0 -10
  209. unrealon_sdk/src/clients/python_http/models/ProxyPurchaseRequest.py +0 -25
  210. unrealon_sdk/src/clients/python_http/models/ProxyResponse.py +0 -47
  211. unrealon_sdk/src/clients/python_http/models/ProxyRotationRequest.py +0 -23
  212. unrealon_sdk/src/clients/python_http/models/ProxyStatus.py +0 -10
  213. unrealon_sdk/src/clients/python_http/models/ProxyUsageRequest.py +0 -19
  214. unrealon_sdk/src/clients/python_http/models/ProxyUsageStatsResponse.py +0 -26
  215. unrealon_sdk/src/clients/python_http/models/ServiceRegistrationDto.py +0 -23
  216. unrealon_sdk/src/clients/python_http/models/ServiceStatsResponse.py +0 -31
  217. unrealon_sdk/src/clients/python_http/models/SessionStartRequest.py +0 -23
  218. unrealon_sdk/src/clients/python_http/models/SuccessResponse.py +0 -25
  219. unrealon_sdk/src/clients/python_http/models/SystemNotificationResponse.py +0 -23
  220. unrealon_sdk/src/clients/python_http/models/ValidationError.py +0 -18
  221. unrealon_sdk/src/clients/python_http/models/ValidationErrorResponse.py +0 -21
  222. unrealon_sdk/src/clients/python_http/models/WebSocketMetrics.py +0 -21
  223. unrealon_sdk/src/clients/python_http/models/__init__.py +0 -44
  224. unrealon_sdk/src/clients/python_http/services/None_service.py +0 -35
  225. unrealon_sdk/src/clients/python_http/services/ParserManagement_service.py +0 -190
  226. unrealon_sdk/src/clients/python_http/services/ProxyManagement_service.py +0 -289
  227. unrealon_sdk/src/clients/python_http/services/SocketLogging_service.py +0 -187
  228. unrealon_sdk/src/clients/python_http/services/SystemHealth_service.py +0 -119
  229. unrealon_sdk/src/clients/python_http/services/WebSocketAPI_service.py +0 -198
  230. unrealon_sdk/src/clients/python_http/services/__init__.py +0 -0
  231. unrealon_sdk/src/clients/python_http/services/admin_service.py +0 -125
  232. unrealon_sdk/src/clients/python_http/services/async_None_service.py +0 -35
  233. unrealon_sdk/src/clients/python_http/services/async_ParserManagement_service.py +0 -190
  234. unrealon_sdk/src/clients/python_http/services/async_ProxyManagement_service.py +0 -289
  235. unrealon_sdk/src/clients/python_http/services/async_SocketLogging_service.py +0 -189
  236. unrealon_sdk/src/clients/python_http/services/async_SystemHealth_service.py +0 -123
  237. unrealon_sdk/src/clients/python_http/services/async_WebSocketAPI_service.py +0 -200
  238. unrealon_sdk/src/clients/python_http/services/async_admin_service.py +0 -125
  239. unrealon_sdk/src/clients/python_websocket/__init__.py +0 -28
  240. unrealon_sdk/src/clients/python_websocket/client.py +0 -490
  241. unrealon_sdk/src/clients/python_websocket/events.py +0 -732
  242. unrealon_sdk/src/clients/python_websocket/example.py +0 -136
  243. unrealon_sdk/src/clients/python_websocket/types.py +0 -871
  244. unrealon_sdk/src/core/__init__.py +0 -64
  245. unrealon_sdk/src/core/client.py +0 -556
  246. unrealon_sdk/src/core/config.py +0 -465
  247. unrealon_sdk/src/core/exceptions.py +0 -239
  248. unrealon_sdk/src/core/metadata.py +0 -191
  249. unrealon_sdk/src/core/models.py +0 -142
  250. unrealon_sdk/src/core/types.py +0 -68
  251. unrealon_sdk/src/dto/__init__.py +0 -268
  252. unrealon_sdk/src/dto/authentication.py +0 -108
  253. unrealon_sdk/src/dto/cache.py +0 -208
  254. unrealon_sdk/src/dto/common.py +0 -19
  255. unrealon_sdk/src/dto/concurrency.py +0 -393
  256. unrealon_sdk/src/dto/events.py +0 -108
  257. unrealon_sdk/src/dto/health.py +0 -339
  258. unrealon_sdk/src/dto/load_balancing.py +0 -336
  259. unrealon_sdk/src/dto/logging.py +0 -230
  260. unrealon_sdk/src/dto/performance.py +0 -165
  261. unrealon_sdk/src/dto/rate_limiting.py +0 -295
  262. unrealon_sdk/src/dto/resource_pooling.py +0 -128
  263. unrealon_sdk/src/dto/structured_logging.py +0 -112
  264. unrealon_sdk/src/dto/task_scheduling.py +0 -121
  265. unrealon_sdk/src/dto/websocket.py +0 -55
  266. unrealon_sdk/src/enterprise/__init__.py +0 -59
  267. unrealon_sdk/src/enterprise/authentication.py +0 -401
  268. unrealon_sdk/src/enterprise/cache_manager.py +0 -578
  269. unrealon_sdk/src/enterprise/error_recovery.py +0 -494
  270. unrealon_sdk/src/enterprise/event_system.py +0 -549
  271. unrealon_sdk/src/enterprise/health_monitor.py +0 -747
  272. unrealon_sdk/src/enterprise/load_balancer.py +0 -964
  273. unrealon_sdk/src/enterprise/logging/__init__.py +0 -68
  274. unrealon_sdk/src/enterprise/logging/cleanup.py +0 -156
  275. unrealon_sdk/src/enterprise/logging/development.py +0 -744
  276. unrealon_sdk/src/enterprise/logging/service.py +0 -410
  277. unrealon_sdk/src/enterprise/multithreading_manager.py +0 -853
  278. unrealon_sdk/src/enterprise/performance_monitor.py +0 -539
  279. unrealon_sdk/src/enterprise/proxy_manager.py +0 -696
  280. unrealon_sdk/src/enterprise/rate_limiter.py +0 -652
  281. unrealon_sdk/src/enterprise/resource_pool.py +0 -763
  282. unrealon_sdk/src/enterprise/task_scheduler.py +0 -709
  283. unrealon_sdk/src/internal/__init__.py +0 -10
  284. unrealon_sdk/src/internal/command_router.py +0 -497
  285. unrealon_sdk/src/internal/connection_manager.py +0 -397
  286. unrealon_sdk/src/internal/http_client.py +0 -446
  287. unrealon_sdk/src/internal/websocket_client.py +0 -420
  288. unrealon_sdk/src/provider.py +0 -471
  289. unrealon_sdk/src/utils.py +0 -234
  290. /unrealon_browser/{src/cli → cli}/__init__.py +0 -0
  291. /unrealon_browser/{src/cli → cli}/interactive_mode.py +0 -0
  292. /unrealon_browser/{src/cli → cli}/main.py +0 -0
  293. /unrealon_browser/{src/core → core}/__init__.py +0 -0
  294. /unrealon_browser/{src/dto → dto}/__init__.py +0 -0
  295. /unrealon_browser/{src/dto → dto}/models/config.py +0 -0
  296. /unrealon_browser/{src/dto → dto}/models/core.py +0 -0
  297. /unrealon_browser/{src/dto → dto}/models/dataclasses.py +0 -0
  298. /unrealon_browser/{src/dto → dto}/models/detection.py +0 -0
  299. /unrealon_browser/{src/dto → dto}/models/enums.py +0 -0
  300. /unrealon_browser/{src/dto → dto}/models/statistics.py +0 -0
  301. /unrealon_browser/{src/managers → managers}/__init__.py +0 -0
  302. /unrealon_browser/{src/managers → managers}/stealth.py +0 -0
@@ -1,853 +0,0 @@
1
- """
2
- Multithreading Manager - Layer 4 Concurrency Service
3
-
4
- Enterprise-grade multithreading system with dynamic thread pool management,
5
- intelligent load balancing, and comprehensive monitoring. Provides high-performance
6
- concurrent execution with automatic scaling and deadlock prevention.
7
-
8
- Features:
9
- - Dynamic thread pool sizing with multiple strategies
10
- - Intelligent load balancing (least-loaded, performance-based, geographic)
11
- - Task priority queuing with dependency management
12
- - Automatic deadlock detection and resolution
13
- - Performance-based thread pool optimization
14
- - Resource pool integration and management
15
- - Real-time concurrency metrics and monitoring
16
- - Thread safety verification and race condition prevention
17
- - Graceful degradation under high load
18
- - Memory-efficient task scheduling
19
- """
20
-
21
- import asyncio
22
- import logging
23
- import threading
24
- import time
25
- import queue
26
- import weakref
27
- import psutil
28
- from typing import Dict, List, Optional, Any, Callable, Set, Union
29
- from datetime import datetime, timezone, timedelta
30
- from collections import defaultdict, deque
31
- from concurrent.futures import ThreadPoolExecutor, Future, as_completed
32
- from dataclasses import dataclass, field
33
-
34
- # Core SDK components
35
- from unrealon_sdk.src.core.config import AdapterConfig
36
- from unrealon_sdk.src.utils import generate_correlation_id
37
-
38
- # DTO models
39
- from unrealon_sdk.src.dto.logging import SDKEventType, SDKSeverity
40
- from unrealon_sdk.src.dto.concurrency import (
41
- ThreadPoolStrategy,
42
- LoadBalancingStrategy,
43
- TaskPriority,
44
- TaskStatus,
45
- ResourceType,
46
- ResourceStatus,
47
- ConcurrencyEventType,
48
- ThreadPoolConfig,
49
- ThreadInfo,
50
- Task,
51
- ResourcePool,
52
- ConcurrencyMetrics,
53
- LoadBalancingDecision,
54
- DeadlockDetection,
55
- )
56
-
57
- # Development logging
58
- from typing import TYPE_CHECKING
59
-
60
- if TYPE_CHECKING:
61
- from unrealon_sdk.src.enterprise.logging import DevelopmentLogger
62
-
63
- logger = logging.getLogger(__name__)
64
-
65
-
66
- @dataclass
67
- class ThreadContext:
68
- """Thread execution context."""
69
-
70
- thread_id: str
71
- thread_name: str
72
- pool_name: str
73
- created_at: datetime
74
- last_activity: datetime
75
- current_task: Optional[Task] = None
76
- task_history: deque[str] = field(default_factory=lambda: deque(maxlen=100))
77
- performance_metrics: Dict[str, float] = field(default_factory=dict)
78
- error_count: int = 0
79
- is_healthy: bool = True
80
-
81
-
82
- class TaskQueue:
83
- """Priority-based task queue with dependency management."""
84
-
85
- def __init__(self, max_size: int = 1000):
86
- self.max_size = max_size
87
- self._queues = {
88
- TaskPriority.CRITICAL: queue.PriorityQueue(),
89
- TaskPriority.HIGH: queue.PriorityQueue(),
90
- TaskPriority.NORMAL: queue.PriorityQueue(),
91
- TaskPriority.LOW: queue.PriorityQueue(),
92
- TaskPriority.BACKGROUND: queue.PriorityQueue(),
93
- }
94
- self._size = 0
95
- self._lock = threading.RLock()
96
- self._dependency_graph: Dict[str, Set[str]] = defaultdict(set)
97
- self._waiting_tasks: Dict[str, Task] = {}
98
-
99
- def put(self, task: Task, priority_boost: float = 0.0) -> bool:
100
- """Add task to queue with optional priority boost."""
101
- with self._lock:
102
- if self._size >= self.max_size:
103
- return False
104
-
105
- # Check dependencies
106
- if task.dependencies:
107
- unmet_deps = [dep for dep in task.dependencies if dep in self._waiting_tasks]
108
- if unmet_deps:
109
- self._waiting_tasks[task.task_id] = task
110
- for dep in task.dependencies:
111
- self._dependency_graph[dep].add(task.task_id)
112
- return True
113
-
114
- # Calculate priority score (lower = higher priority)
115
- priority_scores = {
116
- TaskPriority.CRITICAL: 0,
117
- TaskPriority.HIGH: 100,
118
- TaskPriority.NORMAL: 200,
119
- TaskPriority.LOW: 300,
120
- TaskPriority.BACKGROUND: 400,
121
- }
122
-
123
- score = priority_scores[task.priority] - priority_boost
124
- timestamp = time.time()
125
-
126
- self._queues[task.priority].put((score, timestamp, task))
127
- self._size += 1
128
- return True
129
-
130
- def get(self, timeout: Optional[float] = None) -> Optional[Task]:
131
- """Get next task from queue."""
132
- start_time = time.time()
133
-
134
- while timeout is None or (time.time() - start_time) < timeout:
135
- with self._lock:
136
- # Try each priority level
137
- for priority in TaskPriority:
138
- try:
139
- _, _, task = self._queues[priority].get_nowait()
140
- self._size -= 1
141
- return task
142
- except queue.Empty:
143
- continue
144
-
145
- # Wait a bit before trying again
146
- time.sleep(0.01)
147
-
148
- return None
149
-
150
- def task_completed(self, task_id: str) -> List[Task]:
151
- """Mark task as completed and return newly available tasks."""
152
- with self._lock:
153
- newly_available = []
154
-
155
- # Check if any waiting tasks can now run
156
- for dependent_id in self._dependency_graph.get(task_id, set()):
157
- if dependent_id in self._waiting_tasks:
158
- dependent_task = self._waiting_tasks[dependent_id]
159
- dependent_task.dependencies.remove(task_id)
160
-
161
- if not dependent_task.dependencies:
162
- # All dependencies met, add to queue
163
- del self._waiting_tasks[dependent_id]
164
- if self.put(dependent_task):
165
- newly_available.append(dependent_task)
166
-
167
- # Clean up dependency graph
168
- if task_id in self._dependency_graph:
169
- del self._dependency_graph[task_id]
170
-
171
- return newly_available
172
-
173
- def size(self) -> int:
174
- """Get current queue size."""
175
- return self._size
176
-
177
- def is_full(self) -> bool:
178
- """Check if queue is full."""
179
- return self._size >= self.max_size
180
-
181
-
182
- class MultithreadingManager:
183
- """
184
- Enterprise-grade multithreading manager.
185
-
186
- Provides dynamic thread pool management, intelligent load balancing,
187
- and comprehensive concurrency monitoring for UnrealOn SDK operations.
188
- """
189
-
190
- def __init__(
191
- self,
192
- config: AdapterConfig,
193
- pool_configs: Optional[Dict[str, ThreadPoolConfig]] = None,
194
- dev_logger: Optional["DevelopmentLogger"] = None,
195
- ):
196
- """Initialize multithreading manager."""
197
- self.config = config
198
- self.dev_logger = dev_logger
199
-
200
- # Thread pools
201
- self._pools: Dict[str, ThreadPoolExecutor] = {}
202
- self._pool_configs: Dict[str, ThreadPoolConfig] = pool_configs or {}
203
- self._thread_contexts: Dict[str, ThreadContext] = {}
204
-
205
- # Task management
206
- self._task_queues: Dict[str, TaskQueue] = {}
207
- self._active_tasks: Dict[str, Task] = {}
208
- self._task_futures: Dict[str, Future] = {}
209
-
210
- # Load balancing
211
- self._load_balancer_strategies: Dict[LoadBalancingStrategy, Callable] = {
212
- LoadBalancingStrategy.ROUND_ROBIN: self._round_robin_selection,
213
- LoadBalancingStrategy.LEAST_LOADED: self._least_loaded_selection,
214
- LoadBalancingStrategy.PERFORMANCE_BASED: self._performance_based_selection,
215
- LoadBalancingStrategy.RANDOM: self._random_selection,
216
- }
217
- self._last_assigned_thread: Dict[str, int] = defaultdict(int)
218
-
219
- # Monitoring and metrics
220
- self._metrics = ConcurrencyMetrics()
221
- self._thread_performance: Dict[str, deque[float]] = defaultdict(lambda: deque(maxlen=100))
222
- self._deadlock_detector_enabled = True
223
- self._resource_monitor_enabled = True
224
-
225
- # Background tasks
226
- self._monitor_task: Optional[asyncio.Task[None]] = None
227
- self._scaling_task: Optional[asyncio.Task[None]] = None
228
- self._deadlock_detection_task: Optional[asyncio.Task[None]] = None
229
- self._shutdown = False
230
-
231
- # Thread safety
232
- self._lock = threading.RLock()
233
-
234
- # Create default pool if none specified
235
- if not self._pool_configs:
236
- self._pool_configs["default"] = ThreadPoolConfig(
237
- pool_name="default",
238
- strategy=ThreadPoolStrategy.DYNAMIC,
239
- min_threads=2,
240
- max_threads=20,
241
- core_threads=5,
242
- )
243
-
244
- self._log_info("Multithreading manager initialized")
245
-
246
- async def start(self) -> None:
247
- """Start multithreading manager."""
248
- # Create initial thread pools
249
- for pool_name, pool_config in self._pool_configs.items():
250
- await self._create_thread_pool(pool_name, pool_config)
251
-
252
- # Start background monitoring tasks
253
- if self._monitor_task is None:
254
- self._monitor_task = asyncio.create_task(self._monitoring_loop())
255
-
256
- if self._scaling_task is None:
257
- self._scaling_task = asyncio.create_task(self._scaling_loop())
258
-
259
- if self._deadlock_detection_task is None and self._deadlock_detector_enabled:
260
- self._deadlock_detection_task = asyncio.create_task(self._deadlock_detection_loop())
261
-
262
- self._log_info("Multithreading manager started")
263
-
264
- async def stop(self) -> None:
265
- """Stop multithreading manager and cleanup."""
266
- self._shutdown = True
267
-
268
- # Cancel background tasks
269
- for task in [self._monitor_task, self._scaling_task, self._deadlock_detection_task]:
270
- if task:
271
- task.cancel()
272
- try:
273
- await task
274
- except asyncio.CancelledError:
275
- pass
276
-
277
- # Shutdown thread pools gracefully
278
- for pool_name, pool in self._pools.items():
279
- pool.shutdown(wait=True)
280
- self._log_info(f"Thread pool '{pool_name}' shut down")
281
-
282
- self._log_info("Multithreading manager stopped")
283
-
284
- async def submit_task(
285
- self,
286
- task_func: Callable,
287
- *args,
288
- pool_name: str = "default",
289
- priority: TaskPriority = TaskPriority.NORMAL,
290
- task_name: Optional[str] = None,
291
- dependencies: Optional[List[str]] = None,
292
- timeout_seconds: Optional[float] = None,
293
- context: Optional[Dict[str, Any]] = None,
294
- **kwargs,
295
- ) -> str:
296
- """Submit task for execution."""
297
-
298
- # Create task object
299
- task = Task(
300
- task_name=task_name or f"{task_func.__name__}",
301
- task_type=f"{task_func.__module__}.{task_func.__name__}",
302
- priority=priority,
303
- dependencies=dependencies or [],
304
- timeout_seconds=timeout_seconds,
305
- context=context or {},
306
- )
307
-
308
- # Store function and arguments in context
309
- task.context.update(
310
- {
311
- "func": task_func,
312
- "args": args,
313
- "kwargs": kwargs,
314
- }
315
- )
316
-
317
- with self._lock:
318
- # Add to appropriate queue
319
- if pool_name not in self._task_queues:
320
- self._task_queues[pool_name] = TaskQueue(
321
- max_size=self._pool_configs.get(
322
- pool_name, self._pool_configs["default"]
323
- ).max_queue_size
324
- )
325
-
326
- if self._task_queues[pool_name].put(task):
327
- task.status = TaskStatus.PENDING
328
- self._active_tasks[task.task_id] = task
329
-
330
- # Trigger task execution
331
- asyncio.create_task(self._process_task_queue(pool_name))
332
-
333
- self._log_task_event(ConcurrencyEventType.TASK_QUEUED, task)
334
- return task.task_id
335
- else:
336
- raise RuntimeError(f"Task queue for pool '{pool_name}' is full")
337
-
338
- async def _process_task_queue(self, pool_name: str) -> None:
339
- """Process tasks from queue."""
340
- if pool_name not in self._pools or pool_name not in self._task_queues:
341
- return
342
-
343
- pool = self._pools[pool_name]
344
- task_queue = self._task_queues[pool_name]
345
-
346
- # Get next task
347
- task = task_queue.get(timeout=0.1) # Non-blocking
348
- if not task:
349
- return
350
-
351
- # Select thread using load balancing
352
- pool_config = self._pool_configs[pool_name]
353
- thread_selection = self._select_thread(pool_name, task, pool_config.load_balancing_strategy)
354
-
355
- if thread_selection:
356
- task.assigned_thread_id = thread_selection.selected_thread_id
357
- task.pool_name = pool_name
358
-
359
- # Submit task to thread pool
360
- try:
361
- task.status = TaskStatus.RUNNING
362
- task.started_at = datetime.now(timezone.utc)
363
-
364
- future = pool.submit(self._execute_task_wrapper, task)
365
- self._task_futures[task.task_id] = future
366
-
367
- self._log_task_event(ConcurrencyEventType.TASK_STARTED, task)
368
-
369
- # Handle task completion asynchronously
370
- asyncio.create_task(self._handle_task_completion(task, future, task_queue))
371
-
372
- except Exception as e:
373
- task.status = TaskStatus.FAILED
374
- task.error_message = str(e)
375
- task.completed_at = datetime.now(timezone.utc)
376
- self._log_task_event(ConcurrencyEventType.TASK_FAILED, task)
377
-
378
- def _execute_task_wrapper(self, task: Task) -> Any:
379
- """Wrapper for task execution with monitoring."""
380
- thread_id = threading.get_ident()
381
- start_time = time.time()
382
-
383
- try:
384
- # Update thread context
385
- if str(thread_id) in self._thread_contexts:
386
- context = self._thread_contexts[str(thread_id)]
387
- context.current_task = task
388
- context.last_activity = datetime.now(timezone.utc)
389
-
390
- # Extract function and arguments
391
- func = task.context["func"]
392
- args = task.context.get("args", ())
393
- kwargs = task.context.get("kwargs", {})
394
-
395
- # Execute task with timeout if specified
396
- if task.timeout_seconds:
397
- # Note: This is a simplified timeout implementation
398
- # In production, you might want to use more sophisticated timeout handling
399
- result = func(*args, **kwargs)
400
- else:
401
- result = func(*args, **kwargs)
402
-
403
- # Calculate performance metrics
404
- duration = (time.time() - start_time) * 1000 # ms
405
- task.actual_duration_ms = duration
406
-
407
- # Update thread performance history
408
- if str(thread_id) in self._thread_contexts:
409
- self._thread_performance[str(thread_id)].append(duration)
410
-
411
- return result
412
-
413
- except Exception as e:
414
- task.error_message = str(e)
415
- raise
416
- finally:
417
- # Clean up thread context
418
- if str(thread_id) in self._thread_contexts:
419
- context = self._thread_contexts[str(thread_id)]
420
- context.current_task = None
421
- context.task_history.append(task.task_id)
422
- context.last_activity = datetime.now(timezone.utc)
423
-
424
- async def _handle_task_completion(
425
- self, task: Task, future: Future, task_queue: TaskQueue
426
- ) -> None:
427
- """Handle task completion."""
428
- try:
429
- # Wait for task completion
430
- result = await asyncio.wrap_future(future)
431
-
432
- task.status = TaskStatus.COMPLETED
433
- task.result = result
434
- task.completed_at = datetime.now(timezone.utc)
435
-
436
- self._log_task_event(ConcurrencyEventType.TASK_COMPLETED, task)
437
-
438
- except Exception as e:
439
- task.status = TaskStatus.FAILED
440
- task.error_message = str(e)
441
- task.completed_at = datetime.now(timezone.utc)
442
-
443
- # Handle retry logic
444
- if task.retry_count < task.max_retries:
445
- task.retry_count += 1
446
- task.status = TaskStatus.RETRYING
447
- # Re-queue task
448
- if task_queue.put(task):
449
- asyncio.create_task(self._process_task_queue(task.pool_name))
450
-
451
- self._log_task_event(ConcurrencyEventType.TASK_FAILED, task)
452
-
453
- finally:
454
- # Clean up and handle dependencies
455
- with self._lock:
456
- self._task_futures.pop(task.task_id, None)
457
-
458
- # Notify dependent tasks
459
- newly_available = task_queue.task_completed(task.task_id)
460
- for available_task in newly_available:
461
- asyncio.create_task(self._process_task_queue(available_task.pool_name))
462
-
463
- def _select_thread(
464
- self, pool_name: str, task: Task, strategy: LoadBalancingStrategy
465
- ) -> Optional[LoadBalancingDecision]:
466
- """Select thread for task execution using load balancing strategy."""
467
-
468
- if strategy not in self._load_balancer_strategies:
469
- strategy = LoadBalancingStrategy.LEAST_LOADED
470
-
471
- start_time = time.time()
472
-
473
- try:
474
- selection_func = self._load_balancer_strategies[strategy]
475
- thread_id = selection_func(pool_name, task)
476
-
477
- if thread_id:
478
- decision = LoadBalancingDecision(
479
- task_id=task.task_id,
480
- strategy_used=strategy,
481
- selected_thread_id=thread_id,
482
- selected_pool_name=pool_name,
483
- thread_utilization=self._get_thread_utilization(thread_id),
484
- load_score=self._calculate_load_score(thread_id),
485
- decision_time_ms=(time.time() - start_time) * 1000,
486
- )
487
- return decision
488
-
489
- except Exception as e:
490
- logger.error(f"Error in thread selection: {e}")
491
-
492
- return None
493
-
494
- def _round_robin_selection(self, pool_name: str, task: Task) -> Optional[str]:
495
- """Round-robin thread selection."""
496
- pool_threads = self._get_pool_threads(pool_name)
497
- if not pool_threads:
498
- return None
499
-
500
- last_index = self._last_assigned_thread[pool_name]
501
- next_index = (last_index + 1) % len(pool_threads)
502
- self._last_assigned_thread[pool_name] = next_index
503
-
504
- return pool_threads[next_index]
505
-
506
- def _least_loaded_selection(self, pool_name: str, task: Task) -> Optional[str]:
507
- """Select least loaded thread."""
508
- pool_threads = self._get_pool_threads(pool_name)
509
- if not pool_threads:
510
- return None
511
-
512
- # Find thread with lowest utilization
513
- best_thread = None
514
- lowest_utilization = float("inf")
515
-
516
- for thread_id in pool_threads:
517
- utilization = self._get_thread_utilization(thread_id)
518
- if utilization < lowest_utilization:
519
- lowest_utilization = utilization
520
- best_thread = thread_id
521
-
522
- return best_thread
523
-
524
- def _performance_based_selection(self, pool_name: str, task: Task) -> Optional[str]:
525
- """Select thread based on historical performance."""
526
- pool_threads = self._get_pool_threads(pool_name)
527
- if not pool_threads:
528
- return None
529
-
530
- # Find thread with best performance history
531
- best_thread = None
532
- best_performance = float("inf")
533
-
534
- for thread_id in pool_threads:
535
- if thread_id in self._thread_performance:
536
- avg_duration = sum(self._thread_performance[thread_id]) / len(
537
- self._thread_performance[thread_id]
538
- )
539
- if avg_duration < best_performance:
540
- best_performance = avg_duration
541
- best_thread = thread_id
542
-
543
- return best_thread or self._least_loaded_selection(pool_name, task)
544
-
545
- def _random_selection(self, pool_name: str, task: Task) -> Optional[str]:
546
- """Random thread selection."""
547
- import random
548
-
549
- pool_threads = self._get_pool_threads(pool_name)
550
- return random.choice(pool_threads) if pool_threads else None
551
-
552
- def _get_pool_threads(self, pool_name: str) -> List[str]:
553
- """Get list of thread IDs for pool."""
554
- # This is a simplified implementation
555
- # In practice, you'd track which threads belong to which pool
556
- return list(self._thread_contexts.keys())
557
-
558
- def _get_thread_utilization(self, thread_id: str) -> float:
559
- """Get thread utilization percentage."""
560
- if thread_id in self._thread_contexts:
561
- context = self._thread_contexts[thread_id]
562
- return 1.0 if context.current_task else 0.0
563
- return 0.0
564
-
565
- def _calculate_load_score(self, thread_id: str) -> float:
566
- """Calculate load score for thread."""
567
- utilization = self._get_thread_utilization(thread_id)
568
-
569
- # Factor in performance history
570
- if thread_id in self._thread_performance and self._thread_performance[thread_id]:
571
- avg_duration = sum(self._thread_performance[thread_id]) / len(
572
- self._thread_performance[thread_id]
573
- )
574
- performance_factor = min(avg_duration / 1000.0, 1.0) # Normalize to 0-1
575
- else:
576
- performance_factor = 0.5 # Neutral for new threads
577
-
578
- return utilization * 0.7 + performance_factor * 0.3
579
-
580
- async def _create_thread_pool(self, pool_name: str, config: ThreadPoolConfig) -> None:
581
- """Create and configure thread pool."""
582
- with self._lock:
583
- if pool_name in self._pools:
584
- return
585
-
586
- # Create thread pool executor
587
- initial_size = config.core_threads
588
- pool = ThreadPoolExecutor(
589
- max_workers=config.max_threads, thread_name_prefix=f"{pool_name}_worker"
590
- )
591
-
592
- self._pools[pool_name] = pool
593
- self._task_queues[pool_name] = TaskQueue(max_size=config.max_queue_size)
594
-
595
- # Create thread contexts for initial threads
596
- for i in range(initial_size):
597
- thread_id = f"{pool_name}_thread_{i}"
598
- context = ThreadContext(
599
- thread_id=thread_id,
600
- thread_name=f"{pool_name} Worker {i}",
601
- pool_name=pool_name,
602
- created_at=datetime.now(timezone.utc),
603
- last_activity=datetime.now(timezone.utc),
604
- )
605
- self._thread_contexts[thread_id] = context
606
-
607
- self._log_info(f"Created thread pool '{pool_name}' with {initial_size} threads")
608
-
609
- async def _monitoring_loop(self) -> None:
610
- """Background monitoring loop."""
611
- while not self._shutdown:
612
- try:
613
- await asyncio.sleep(10) # Monitor every 10 seconds
614
- await self._collect_metrics()
615
- await self._check_thread_health()
616
- except asyncio.CancelledError:
617
- break
618
- except Exception as e:
619
- logger.error(f"Error in monitoring loop: {e}")
620
-
621
- async def _scaling_loop(self) -> None:
622
- """Background scaling loop."""
623
- while not self._shutdown:
624
- try:
625
- await asyncio.sleep(30) # Scale every 30 seconds
626
- await self._auto_scale_pools()
627
- except asyncio.CancelledError:
628
- break
629
- except Exception as e:
630
- logger.error(f"Error in scaling loop: {e}")
631
-
632
- async def _deadlock_detection_loop(self) -> None:
633
- """Background deadlock detection loop."""
634
- while not self._shutdown:
635
- try:
636
- await asyncio.sleep(60) # Check every minute
637
- await self._detect_deadlocks()
638
- except asyncio.CancelledError:
639
- break
640
- except Exception as e:
641
- logger.error(f"Error in deadlock detection: {e}")
642
-
643
- async def _collect_metrics(self) -> None:
644
- """Collect concurrency metrics."""
645
- with self._lock:
646
- # Thread metrics
647
- total_threads = len(self._thread_contexts)
648
- active_threads = sum(1 for ctx in self._thread_contexts.values() if ctx.current_task)
649
- idle_threads = total_threads - active_threads
650
-
651
- # Task metrics
652
- total_tasks = len(self._active_tasks)
653
- running_tasks = sum(
654
- 1 for task in self._active_tasks.values() if task.status == TaskStatus.RUNNING
655
- )
656
- pending_tasks = sum(queue.size() for queue in self._task_queues.values())
657
-
658
- # Update metrics
659
- self._metrics.total_threads = total_threads
660
- self._metrics.active_threads = active_threads
661
- self._metrics.idle_threads = idle_threads
662
- self._metrics.thread_utilization_percent = (
663
- (active_threads / total_threads * 100) if total_threads > 0 else 0.0
664
- )
665
-
666
- self._metrics.total_tasks = total_tasks
667
- self._metrics.running_tasks = running_tasks
668
- self._metrics.pending_tasks = pending_tasks
669
-
670
- # System metrics
671
- try:
672
- self._metrics.cpu_usage_percent = psutil.cpu_percent()
673
- self._metrics.memory_usage_mb = psutil.virtual_memory().used / 1024 / 1024
674
- except Exception:
675
- pass
676
-
677
- async def _check_thread_health(self) -> None:
678
- """Check health of all threads."""
679
- current_time = datetime.now(timezone.utc)
680
- unhealthy_threshold = timedelta(minutes=5)
681
-
682
- for thread_id, context in self._thread_contexts.items():
683
- # Check if thread is responsive
684
- time_since_activity = current_time - context.last_activity
685
-
686
- if time_since_activity > unhealthy_threshold:
687
- context.is_healthy = False
688
- self._log_info(
689
- f"Thread {thread_id} marked as unhealthy (inactive for {time_since_activity})"
690
- )
691
- else:
692
- context.is_healthy = True
693
-
694
- async def _auto_scale_pools(self) -> None:
695
- """Automatically scale thread pools based on load."""
696
- for pool_name, config in self._pool_configs.items():
697
- if config.strategy != ThreadPoolStrategy.DYNAMIC:
698
- continue
699
-
700
- # Calculate current utilization
701
- pool_threads = [
702
- ctx for ctx in self._thread_contexts.values() if ctx.pool_name == pool_name
703
- ]
704
- if not pool_threads:
705
- continue
706
-
707
- active_count = sum(1 for ctx in pool_threads if ctx.current_task)
708
- utilization = active_count / len(pool_threads) if pool_threads else 0.0
709
-
710
- # Scale up if needed
711
- if utilization > config.scale_up_threshold and len(pool_threads) < config.max_threads:
712
- new_size = min(int(len(pool_threads) * config.scale_up_factor), config.max_threads)
713
- await self._scale_pool(pool_name, new_size)
714
- self._log_info(f"Scaled up pool '{pool_name}' to {new_size} threads")
715
-
716
- # Scale down if needed
717
- elif (
718
- utilization < config.scale_down_threshold and len(pool_threads) > config.min_threads
719
- ):
720
- new_size = max(
721
- int(len(pool_threads) * config.scale_down_factor), config.min_threads
722
- )
723
- await self._scale_pool(pool_name, new_size)
724
- self._log_info(f"Scaled down pool '{pool_name}' to {new_size} threads")
725
-
726
- async def _scale_pool(self, pool_name: str, new_size: int) -> None:
727
- """Scale thread pool to new size."""
728
- # This is a simplified implementation
729
- # In practice, you'd need more sophisticated pool resizing
730
- current_threads = [
731
- ctx for ctx in self._thread_contexts.values() if ctx.pool_name == pool_name
732
- ]
733
- current_size = len(current_threads)
734
-
735
- if new_size > current_size:
736
- # Add threads
737
- for i in range(current_size, new_size):
738
- thread_id = f"{pool_name}_thread_{i}"
739
- context = ThreadContext(
740
- thread_id=thread_id,
741
- thread_name=f"{pool_name} Worker {i}",
742
- pool_name=pool_name,
743
- created_at=datetime.now(timezone.utc),
744
- last_activity=datetime.now(timezone.utc),
745
- )
746
- self._thread_contexts[thread_id] = context
747
-
748
- elif new_size < current_size:
749
- # Remove threads (only idle ones)
750
- threads_to_remove = []
751
- for ctx in current_threads:
752
- if not ctx.current_task and len(threads_to_remove) < (current_size - new_size):
753
- threads_to_remove.append(ctx.thread_id)
754
-
755
- for thread_id in threads_to_remove:
756
- del self._thread_contexts[thread_id]
757
-
758
- async def _detect_deadlocks(self) -> None:
759
- """Detect potential deadlocks."""
760
- # Simplified deadlock detection
761
- # In practice, you'd implement more sophisticated deadlock detection algorithms
762
-
763
- long_running_threshold = timedelta(minutes=10)
764
- current_time = datetime.now(timezone.utc)
765
-
766
- potential_deadlocks = []
767
-
768
- for task in self._active_tasks.values():
769
- if (
770
- task.status == TaskStatus.RUNNING
771
- and task.started_at
772
- and current_time - task.started_at > long_running_threshold
773
- ):
774
- potential_deadlocks.append(task)
775
-
776
- if potential_deadlocks:
777
- self._log_info(f"Detected {len(potential_deadlocks)} potentially deadlocked tasks")
778
-
779
- def get_metrics(self) -> ConcurrencyMetrics:
780
- """Get current concurrency metrics."""
781
- return self._metrics.model_copy()
782
-
783
- def get_thread_info(
784
- self, thread_id: Optional[str] = None
785
- ) -> Union[ThreadInfo, List[ThreadInfo]]:
786
- """Get thread information."""
787
- if thread_id:
788
- if thread_id in self._thread_contexts:
789
- ctx = self._thread_contexts[thread_id]
790
- return ThreadInfo(
791
- thread_id=ctx.thread_id,
792
- thread_name=ctx.thread_name,
793
- pool_name=ctx.pool_name,
794
- is_alive=ctx.is_healthy,
795
- is_busy=ctx.current_task is not None,
796
- current_task_id=ctx.current_task.task_id if ctx.current_task else None,
797
- total_tasks_executed=len(ctx.task_history),
798
- last_activity=ctx.last_activity,
799
- created_at=ctx.created_at,
800
- error_count=ctx.error_count,
801
- )
802
- return None
803
- else:
804
- return [self.get_thread_info(tid) for tid in self._thread_contexts.keys()]
805
-
806
- def get_task_status(self, task_id: str) -> Optional[Task]:
807
- """Get task status."""
808
- return self._active_tasks.get(task_id)
809
-
810
- def cancel_task(self, task_id: str) -> bool:
811
- """Cancel task execution."""
812
- if task_id in self._task_futures:
813
- future = self._task_futures[task_id]
814
- return future.cancel()
815
- return False
816
-
817
- def _log_task_event(self, event_type: ConcurrencyEventType, task: Task) -> None:
818
- """Log task event."""
819
- message = f"Task {event_type.value}: {task.task_name} ({task.task_id})"
820
-
821
- if self.dev_logger:
822
- self.dev_logger.log_debug(
823
- SDKEventType.DEBUG_CHECKPOINT,
824
- message,
825
- details={
826
- "event_type": event_type.value,
827
- "task_id": task.task_id,
828
- "task_status": task.status.value,
829
- "pool_name": task.pool_name,
830
- },
831
- )
832
- else:
833
- logger.debug(message)
834
-
835
- def _log_info(self, message: str, **kwargs: Any) -> None:
836
- """Log info message."""
837
- if self.dev_logger:
838
- self.dev_logger.log_info(
839
- SDKEventType.PERFORMANCE_OPTIMIZATION_APPLIED, message, **kwargs
840
- )
841
- else:
842
- logger.info(message)
843
-
844
-
845
- __all__ = [
846
- # Main business logic class
847
- "MultithreadingManager",
848
- # Utility classes
849
- "TaskQueue",
850
- "ThreadContext",
851
- # Note: Concurrency models are available via DTO imports:
852
- # from unrealon_sdk.src.dto.concurrency import ...
853
- ]