unrealon 1.1.5__py3-none-any.whl → 2.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {unrealon-1.1.5.dist-info/licenses → unrealon-2.0.4.dist-info}/LICENSE +1 -1
- unrealon-2.0.4.dist-info/METADATA +491 -0
- unrealon-2.0.4.dist-info/RECORD +129 -0
- {unrealon-1.1.5.dist-info → unrealon-2.0.4.dist-info}/WHEEL +2 -1
- unrealon-2.0.4.dist-info/entry_points.txt +3 -0
- unrealon-2.0.4.dist-info/top_level.txt +3 -0
- unrealon_browser/__init__.py +5 -2
- unrealon_browser/cli/browser_cli.py +18 -9
- unrealon_browser/cli/interactive_mode.py +18 -7
- unrealon_browser/core/browser_manager.py +76 -13
- unrealon_browser/dto/__init__.py +21 -0
- unrealon_browser/dto/bot_detection.py +175 -0
- unrealon_browser/dto/models/config.py +14 -1
- unrealon_browser/managers/__init__.py +4 -1
- unrealon_browser/managers/logger_bridge.py +3 -6
- unrealon_browser/managers/page_wait_manager.py +198 -0
- unrealon_browser/stealth/__init__.py +27 -0
- unrealon_browser/stealth/bypass_techniques.pyc +0 -0
- unrealon_browser/stealth/manager.pyc +0 -0
- unrealon_browser/stealth/nodriver_stealth.pyc +0 -0
- unrealon_browser/stealth/playwright_stealth.pyc +0 -0
- unrealon_browser/stealth/scanner_tester.pyc +0 -0
- unrealon_browser/stealth/undetected_chrome.pyc +0 -0
- unrealon_core/__init__.py +160 -0
- unrealon_core/config/__init__.py +16 -0
- unrealon_core/config/environment.py +98 -0
- unrealon_core/config/urls.py +93 -0
- unrealon_core/enums/__init__.py +24 -0
- unrealon_core/enums/status.py +216 -0
- unrealon_core/enums/types.py +240 -0
- unrealon_core/error_handling/__init__.py +45 -0
- unrealon_core/error_handling/circuit_breaker.py +292 -0
- unrealon_core/error_handling/error_context.py +324 -0
- unrealon_core/error_handling/recovery.py +371 -0
- unrealon_core/error_handling/retry.py +268 -0
- unrealon_core/exceptions/__init__.py +46 -0
- unrealon_core/exceptions/base.py +292 -0
- unrealon_core/exceptions/communication.py +22 -0
- unrealon_core/exceptions/driver.py +11 -0
- unrealon_core/exceptions/proxy.py +11 -0
- unrealon_core/exceptions/task.py +12 -0
- unrealon_core/exceptions/validation.py +17 -0
- unrealon_core/models/__init__.py +98 -0
- unrealon_core/models/arq_context.py +252 -0
- unrealon_core/models/arq_responses.py +125 -0
- unrealon_core/models/base.py +291 -0
- unrealon_core/models/bridge_stats.py +58 -0
- unrealon_core/models/communication.py +39 -0
- unrealon_core/models/config.py +47 -0
- unrealon_core/models/connection_stats.py +47 -0
- unrealon_core/models/driver.py +30 -0
- unrealon_core/models/driver_details.py +98 -0
- unrealon_core/models/logging.py +28 -0
- unrealon_core/models/task.py +21 -0
- unrealon_core/models/typed_responses.py +210 -0
- unrealon_core/models/websocket/__init__.py +91 -0
- unrealon_core/models/websocket/base.py +49 -0
- unrealon_core/models/websocket/config.py +200 -0
- unrealon_core/models/websocket/driver.py +215 -0
- unrealon_core/models/websocket/errors.py +138 -0
- unrealon_core/models/websocket/heartbeat.py +100 -0
- unrealon_core/models/websocket/logging.py +261 -0
- unrealon_core/models/websocket/proxy.py +496 -0
- unrealon_core/models/websocket/tasks.py +275 -0
- unrealon_core/models/websocket/utils.py +153 -0
- unrealon_core/models/websocket_session.py +144 -0
- unrealon_core/monitoring/__init__.py +43 -0
- unrealon_core/monitoring/alerts.py +398 -0
- unrealon_core/monitoring/dashboard.py +307 -0
- unrealon_core/monitoring/health_check.py +354 -0
- unrealon_core/monitoring/metrics.py +352 -0
- unrealon_core/utils/__init__.py +11 -0
- unrealon_core/utils/time.py +61 -0
- unrealon_core/version.py +219 -0
- unrealon_driver/__init__.py +88 -50
- unrealon_driver/core_module/__init__.py +34 -0
- unrealon_driver/core_module/base.py +184 -0
- unrealon_driver/core_module/config.py +30 -0
- unrealon_driver/core_module/event_manager.py +127 -0
- unrealon_driver/core_module/protocols.py +98 -0
- unrealon_driver/core_module/registry.py +146 -0
- unrealon_driver/decorators/__init__.py +15 -0
- unrealon_driver/decorators/retry.py +117 -0
- unrealon_driver/decorators/schedule.py +137 -0
- unrealon_driver/decorators/task.py +61 -0
- unrealon_driver/decorators/timing.py +132 -0
- unrealon_driver/driver/__init__.py +20 -0
- unrealon_driver/driver/communication/__init__.py +10 -0
- unrealon_driver/driver/communication/session.py +203 -0
- unrealon_driver/driver/communication/websocket_client.py +197 -0
- unrealon_driver/driver/core/__init__.py +10 -0
- unrealon_driver/driver/core/config.py +85 -0
- unrealon_driver/driver/core/driver.py +221 -0
- unrealon_driver/driver/factory/__init__.py +9 -0
- unrealon_driver/driver/factory/manager_factory.py +130 -0
- unrealon_driver/driver/lifecycle/__init__.py +11 -0
- unrealon_driver/driver/lifecycle/daemon.py +76 -0
- unrealon_driver/driver/lifecycle/initialization.py +97 -0
- unrealon_driver/driver/lifecycle/shutdown.py +48 -0
- unrealon_driver/driver/monitoring/__init__.py +9 -0
- unrealon_driver/driver/monitoring/health.py +63 -0
- unrealon_driver/driver/utilities/__init__.py +10 -0
- unrealon_driver/driver/utilities/logging.py +51 -0
- unrealon_driver/driver/utilities/serialization.py +61 -0
- unrealon_driver/managers/__init__.py +32 -0
- unrealon_driver/managers/base.py +174 -0
- unrealon_driver/managers/browser.py +98 -0
- unrealon_driver/managers/cache.py +116 -0
- unrealon_driver/managers/http.py +107 -0
- unrealon_driver/managers/logger.py +286 -0
- unrealon_driver/managers/proxy.py +99 -0
- unrealon_driver/managers/registry.py +87 -0
- unrealon_driver/managers/threading.py +54 -0
- unrealon_driver/managers/update.py +107 -0
- unrealon_driver/utils/__init__.py +9 -0
- unrealon_driver/utils/time.py +10 -0
- unrealon/__init__.py +0 -40
- unrealon-1.1.5.dist-info/METADATA +0 -621
- unrealon-1.1.5.dist-info/RECORD +0 -54
- unrealon-1.1.5.dist-info/entry_points.txt +0 -9
- unrealon_browser/managers/stealth.py +0 -388
- unrealon_driver/exceptions.py +0 -33
- unrealon_driver/html_analyzer/__init__.py +0 -32
- unrealon_driver/html_analyzer/cleaner.py +0 -657
- unrealon_driver/html_analyzer/config.py +0 -64
- unrealon_driver/html_analyzer/manager.py +0 -247
- unrealon_driver/html_analyzer/models.py +0 -115
- unrealon_driver/html_analyzer/websocket_analyzer.py +0 -157
- unrealon_driver/models/__init__.py +0 -31
- unrealon_driver/models/websocket.py +0 -98
- unrealon_driver/parser/__init__.py +0 -36
- unrealon_driver/parser/cli_manager.py +0 -142
- unrealon_driver/parser/daemon_manager.py +0 -403
- unrealon_driver/parser/managers/__init__.py +0 -25
- unrealon_driver/parser/managers/config.py +0 -293
- unrealon_driver/parser/managers/error.py +0 -412
- unrealon_driver/parser/managers/result.py +0 -321
- unrealon_driver/parser/parser_manager.py +0 -458
- unrealon_driver/smart_logging/__init__.py +0 -24
- unrealon_driver/smart_logging/models.py +0 -44
- unrealon_driver/smart_logging/smart_logger.py +0 -406
- unrealon_driver/smart_logging/unified_logger.py +0 -525
- unrealon_driver/websocket/__init__.py +0 -31
- unrealon_driver/websocket/client.py +0 -249
- unrealon_driver/websocket/config.py +0 -188
- unrealon_driver/websocket/manager.py +0 -90
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Clean event manager for module communication.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
from typing import Dict, List, Callable, Any, Optional
|
|
8
|
+
from collections import defaultdict
|
|
9
|
+
|
|
10
|
+
from .protocols import ModuleEvent, EventType
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class EventManager:
|
|
16
|
+
"""
|
|
17
|
+
Clean event manager for module communication.
|
|
18
|
+
|
|
19
|
+
Provides pub/sub system for modules to communicate
|
|
20
|
+
without tight coupling.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self):
|
|
24
|
+
self.listeners: Dict[EventType, List[Callable]] = defaultdict(list)
|
|
25
|
+
self.event_queue: Optional[asyncio.Queue] = None
|
|
26
|
+
self._processor_task: Optional[asyncio.Task] = None
|
|
27
|
+
self._running = False
|
|
28
|
+
|
|
29
|
+
async def start(self):
|
|
30
|
+
"""Start event processing."""
|
|
31
|
+
if self._running:
|
|
32
|
+
return
|
|
33
|
+
|
|
34
|
+
self._running = True
|
|
35
|
+
# Only create task if there's a running event loop
|
|
36
|
+
try:
|
|
37
|
+
loop = asyncio.get_running_loop()
|
|
38
|
+
self.event_queue = asyncio.Queue()
|
|
39
|
+
self._processor_task = loop.create_task(self._process_events())
|
|
40
|
+
logger.info("Event manager started")
|
|
41
|
+
except RuntimeError:
|
|
42
|
+
# No running event loop - don't start background processing
|
|
43
|
+
logger.debug("No running event loop - EventManager running in sync mode")
|
|
44
|
+
|
|
45
|
+
async def stop(self):
|
|
46
|
+
"""Stop event processing."""
|
|
47
|
+
self._running = False
|
|
48
|
+
|
|
49
|
+
if self._processor_task and not self._processor_task.done():
|
|
50
|
+
self._processor_task.cancel()
|
|
51
|
+
try:
|
|
52
|
+
await self._processor_task
|
|
53
|
+
except asyncio.CancelledError:
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
# Clear queue
|
|
57
|
+
self.event_queue = None
|
|
58
|
+
self._processor_task = None
|
|
59
|
+
|
|
60
|
+
logger.info("Event manager stopped")
|
|
61
|
+
|
|
62
|
+
def subscribe(self, event_type: EventType, handler: Callable[[ModuleEvent], Any]):
|
|
63
|
+
"""Subscribe to event type."""
|
|
64
|
+
self.listeners[event_type].append(handler)
|
|
65
|
+
logger.debug(f"Subscribed handler to {event_type}")
|
|
66
|
+
|
|
67
|
+
def unsubscribe(self, event_type: EventType, handler: Callable):
|
|
68
|
+
"""Unsubscribe from event type."""
|
|
69
|
+
if handler in self.listeners[event_type]:
|
|
70
|
+
self.listeners[event_type].remove(handler)
|
|
71
|
+
logger.debug(f"Unsubscribed handler from {event_type}")
|
|
72
|
+
|
|
73
|
+
async def emit(self, event: ModuleEvent):
|
|
74
|
+
"""Emit event to queue."""
|
|
75
|
+
if self._running and self.event_queue:
|
|
76
|
+
await self.event_queue.put(event)
|
|
77
|
+
|
|
78
|
+
async def _process_events(self):
|
|
79
|
+
"""Background event processing."""
|
|
80
|
+
while self._running:
|
|
81
|
+
try:
|
|
82
|
+
# Get event from queue
|
|
83
|
+
event = await asyncio.wait_for(self.event_queue.get(), timeout=1.0)
|
|
84
|
+
|
|
85
|
+
# Process event
|
|
86
|
+
await self._handle_event(event)
|
|
87
|
+
|
|
88
|
+
except asyncio.TimeoutError:
|
|
89
|
+
# No events to process, continue
|
|
90
|
+
continue
|
|
91
|
+
except asyncio.CancelledError:
|
|
92
|
+
break
|
|
93
|
+
except Exception as e:
|
|
94
|
+
logger.error(f"Event processing error: {e}")
|
|
95
|
+
|
|
96
|
+
async def _handle_event(self, event: ModuleEvent):
|
|
97
|
+
"""Handle single event."""
|
|
98
|
+
try:
|
|
99
|
+
handlers = self.listeners.get(event.event_type, [])
|
|
100
|
+
|
|
101
|
+
if not handlers:
|
|
102
|
+
logger.debug(f"No handlers for event type: {event.event_type}")
|
|
103
|
+
return
|
|
104
|
+
|
|
105
|
+
# Execute all handlers
|
|
106
|
+
for handler in handlers:
|
|
107
|
+
try:
|
|
108
|
+
if asyncio.iscoroutinefunction(handler):
|
|
109
|
+
await handler(event)
|
|
110
|
+
else:
|
|
111
|
+
handler(event)
|
|
112
|
+
except Exception as e:
|
|
113
|
+
logger.error(f"Event handler error: {e}")
|
|
114
|
+
|
|
115
|
+
except Exception as e:
|
|
116
|
+
logger.error(f"Event handling error: {e}")
|
|
117
|
+
|
|
118
|
+
def get_stats(self) -> Dict[str, Any]:
|
|
119
|
+
"""Get event manager statistics."""
|
|
120
|
+
return {
|
|
121
|
+
"running": self._running,
|
|
122
|
+
"listeners": {
|
|
123
|
+
event_type.value: len(handlers)
|
|
124
|
+
for event_type, handlers in self.listeners.items()
|
|
125
|
+
},
|
|
126
|
+
"queue_size": self.event_queue.qsize()
|
|
127
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Clean protocols for module system.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from abc import abstractmethod
|
|
6
|
+
from typing import Protocol, Dict, Any, Optional
|
|
7
|
+
from datetime import datetime
|
|
8
|
+
from enum import Enum
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ModuleStatus(str, Enum):
|
|
14
|
+
"""Module lifecycle status."""
|
|
15
|
+
UNINITIALIZED = "uninitialized"
|
|
16
|
+
INITIALIZING = "initializing"
|
|
17
|
+
INITIALIZED = "initialized"
|
|
18
|
+
STARTING = "starting"
|
|
19
|
+
RUNNING = "running"
|
|
20
|
+
STOPPING = "stopping"
|
|
21
|
+
STOPPED = "stopped"
|
|
22
|
+
ERROR = "error"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class EventType(str, Enum):
|
|
26
|
+
"""Event types for module communication."""
|
|
27
|
+
MODULE_INITIALIZED = "module_initialized"
|
|
28
|
+
MODULE_STARTED = "module_started"
|
|
29
|
+
MODULE_STOPPED = "module_stopped"
|
|
30
|
+
MODULE_ERROR = "module_error"
|
|
31
|
+
TASK_STARTED = "task_started"
|
|
32
|
+
TASK_COMPLETED = "task_completed"
|
|
33
|
+
TASK_FAILED = "task_failed"
|
|
34
|
+
HEALTH_CHECK = "health_check"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class HealthStatus(str, Enum):
|
|
38
|
+
"""Health check status."""
|
|
39
|
+
HEALTHY = "healthy"
|
|
40
|
+
DEGRADED = "degraded"
|
|
41
|
+
UNHEALTHY = "unhealthy"
|
|
42
|
+
UNKNOWN = "unknown"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class ModuleEvent(BaseModel):
|
|
46
|
+
"""Event data structure."""
|
|
47
|
+
event_type: EventType
|
|
48
|
+
module_name: str
|
|
49
|
+
timestamp: datetime
|
|
50
|
+
data: Dict[str, Any] = {}
|
|
51
|
+
error: Optional[str] = None
|
|
52
|
+
|
|
53
|
+
model_config = {"extra": "forbid"}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class HealthCheckResult(BaseModel):
|
|
57
|
+
"""Health check result."""
|
|
58
|
+
status: HealthStatus
|
|
59
|
+
timestamp: datetime
|
|
60
|
+
details: Dict[str, Any] = {}
|
|
61
|
+
error: Optional[str] = None
|
|
62
|
+
response_time_ms: Optional[float] = None
|
|
63
|
+
|
|
64
|
+
model_config = {"extra": "forbid"}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class BaseModule(Protocol):
|
|
68
|
+
"""Base module protocol."""
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def name(self) -> str:
|
|
72
|
+
"""Module name."""
|
|
73
|
+
...
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def status(self) -> ModuleStatus:
|
|
77
|
+
"""Current module status."""
|
|
78
|
+
...
|
|
79
|
+
|
|
80
|
+
@abstractmethod
|
|
81
|
+
async def initialize(self) -> bool:
|
|
82
|
+
"""Initialize module."""
|
|
83
|
+
...
|
|
84
|
+
|
|
85
|
+
@abstractmethod
|
|
86
|
+
async def start(self) -> bool:
|
|
87
|
+
"""Start module."""
|
|
88
|
+
...
|
|
89
|
+
|
|
90
|
+
@abstractmethod
|
|
91
|
+
async def stop(self) -> None:
|
|
92
|
+
"""Stop module."""
|
|
93
|
+
...
|
|
94
|
+
|
|
95
|
+
@abstractmethod
|
|
96
|
+
async def health_check(self) -> HealthCheckResult:
|
|
97
|
+
"""Perform health check."""
|
|
98
|
+
...
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Clean module registry for lifecycle management.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
from typing import Dict, List, Optional, Any
|
|
8
|
+
|
|
9
|
+
from .protocols import BaseModule, ModuleStatus
|
|
10
|
+
from .event_manager import EventManager
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ModuleRegistry:
|
|
16
|
+
"""
|
|
17
|
+
Clean module registry.
|
|
18
|
+
|
|
19
|
+
Manages module lifecycle and provides centralized
|
|
20
|
+
access to all registered modules.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self, event_manager: Optional[EventManager] = None):
|
|
24
|
+
self.modules: Dict[str, BaseModule] = {}
|
|
25
|
+
self.event_manager = event_manager or EventManager()
|
|
26
|
+
self._initialized = False
|
|
27
|
+
|
|
28
|
+
def register(self, module: BaseModule):
|
|
29
|
+
"""Register a module."""
|
|
30
|
+
self.modules[module.name] = module
|
|
31
|
+
|
|
32
|
+
# Set event manager if module supports it
|
|
33
|
+
if hasattr(module, 'set_event_manager'):
|
|
34
|
+
module.set_event_manager(self.event_manager)
|
|
35
|
+
|
|
36
|
+
logger.info(f"Registered module: {module.name}")
|
|
37
|
+
|
|
38
|
+
def get(self, name: str) -> Optional[BaseModule]:
|
|
39
|
+
"""Get module by name."""
|
|
40
|
+
return self.modules.get(name)
|
|
41
|
+
|
|
42
|
+
def get_all(self) -> Dict[str, BaseModule]:
|
|
43
|
+
"""Get all registered modules."""
|
|
44
|
+
return self.modules.copy()
|
|
45
|
+
|
|
46
|
+
async def initialize_all(self) -> bool:
|
|
47
|
+
"""Initialize all registered modules."""
|
|
48
|
+
if self._initialized:
|
|
49
|
+
return True
|
|
50
|
+
|
|
51
|
+
# Start event manager
|
|
52
|
+
await self.event_manager.start()
|
|
53
|
+
|
|
54
|
+
success_count = 0
|
|
55
|
+
|
|
56
|
+
for name, module in self.modules.items():
|
|
57
|
+
try:
|
|
58
|
+
if await module.initialize():
|
|
59
|
+
success_count += 1
|
|
60
|
+
logger.info(f"Module {name} initialized successfully")
|
|
61
|
+
else:
|
|
62
|
+
logger.error(f"Module {name} initialization failed")
|
|
63
|
+
except Exception as e:
|
|
64
|
+
logger.error(f"Module {name} initialization error: {e}")
|
|
65
|
+
|
|
66
|
+
self._initialized = success_count == len(self.modules)
|
|
67
|
+
|
|
68
|
+
if self._initialized:
|
|
69
|
+
logger.info(f"All {len(self.modules)} modules initialized successfully")
|
|
70
|
+
else:
|
|
71
|
+
logger.warning(f"Only {success_count}/{len(self.modules)} modules initialized")
|
|
72
|
+
|
|
73
|
+
return self._initialized
|
|
74
|
+
|
|
75
|
+
async def start_all(self) -> bool:
|
|
76
|
+
"""Start all initialized modules."""
|
|
77
|
+
if not self._initialized:
|
|
78
|
+
logger.error("Cannot start modules - not all initialized")
|
|
79
|
+
return False
|
|
80
|
+
|
|
81
|
+
success_count = 0
|
|
82
|
+
|
|
83
|
+
for name, module in self.modules.items():
|
|
84
|
+
if module.status == ModuleStatus.INITIALIZED:
|
|
85
|
+
try:
|
|
86
|
+
if await module.start():
|
|
87
|
+
success_count += 1
|
|
88
|
+
logger.info(f"Module {name} started successfully")
|
|
89
|
+
else:
|
|
90
|
+
logger.error(f"Module {name} start failed")
|
|
91
|
+
except Exception as e:
|
|
92
|
+
logger.error(f"Module {name} start error: {e}")
|
|
93
|
+
|
|
94
|
+
logger.info(f"Started {success_count}/{len(self.modules)} modules")
|
|
95
|
+
return success_count > 0
|
|
96
|
+
|
|
97
|
+
async def stop_all(self):
|
|
98
|
+
"""Stop all running modules."""
|
|
99
|
+
for name, module in self.modules.items():
|
|
100
|
+
if module.status == ModuleStatus.RUNNING:
|
|
101
|
+
try:
|
|
102
|
+
await module.stop()
|
|
103
|
+
logger.info(f"Module {name} stopped successfully")
|
|
104
|
+
except Exception as e:
|
|
105
|
+
logger.error(f"Module {name} stop error: {e}")
|
|
106
|
+
|
|
107
|
+
# Stop event manager
|
|
108
|
+
await self.event_manager.stop()
|
|
109
|
+
|
|
110
|
+
self._initialized = False
|
|
111
|
+
logger.info("All modules stopped")
|
|
112
|
+
|
|
113
|
+
async def health_check_all(self) -> Dict[str, Any]:
|
|
114
|
+
"""Get health status of all modules."""
|
|
115
|
+
health_data = {}
|
|
116
|
+
|
|
117
|
+
for name, module in self.modules.items():
|
|
118
|
+
try:
|
|
119
|
+
result = await module.health_check()
|
|
120
|
+
health_data[name] = result.model_dump()
|
|
121
|
+
except Exception as e:
|
|
122
|
+
health_data[name] = {
|
|
123
|
+
"status": "error",
|
|
124
|
+
"error": str(e)
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return {
|
|
128
|
+
"modules": health_data,
|
|
129
|
+
"total": len(self.modules),
|
|
130
|
+
"initialized": self._initialized,
|
|
131
|
+
"event_manager": self.event_manager.get_stats()
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
def get_running_modules(self) -> List[str]:
|
|
135
|
+
"""Get list of running module names."""
|
|
136
|
+
return [
|
|
137
|
+
name for name, module in self.modules.items()
|
|
138
|
+
if module.status == ModuleStatus.RUNNING
|
|
139
|
+
]
|
|
140
|
+
|
|
141
|
+
def is_all_running(self) -> bool:
|
|
142
|
+
"""Check if all modules are running."""
|
|
143
|
+
return all(
|
|
144
|
+
module.status == ModuleStatus.RUNNING
|
|
145
|
+
for module in self.modules.values()
|
|
146
|
+
)
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Clean retry decorator with exponential backoff.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
from functools import wraps
|
|
8
|
+
from typing import Callable, Type, Tuple, Optional
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def retry(
|
|
14
|
+
max_attempts: int = 3,
|
|
15
|
+
delay: float = 1.0,
|
|
16
|
+
backoff: float = 2.0,
|
|
17
|
+
exceptions: Tuple[Type[Exception], ...] = (Exception,),
|
|
18
|
+
on_retry: Optional[Callable] = None
|
|
19
|
+
):
|
|
20
|
+
"""
|
|
21
|
+
Clean retry decorator.
|
|
22
|
+
|
|
23
|
+
Automatically retries function on specified exceptions
|
|
24
|
+
with exponential backoff.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
max_attempts: Maximum retry attempts
|
|
28
|
+
delay: Initial delay between retries
|
|
29
|
+
backoff: Backoff multiplier for delay
|
|
30
|
+
exceptions: Exception types to retry on
|
|
31
|
+
on_retry: Callback function called on each retry
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def decorator(func: Callable) -> Callable:
|
|
35
|
+
@wraps(func)
|
|
36
|
+
async def async_wrapper(*args, **kwargs):
|
|
37
|
+
"""Async retry wrapper."""
|
|
38
|
+
|
|
39
|
+
last_exception = None
|
|
40
|
+
current_delay = delay
|
|
41
|
+
|
|
42
|
+
for attempt in range(max_attempts):
|
|
43
|
+
try:
|
|
44
|
+
return await func(*args, **kwargs)
|
|
45
|
+
|
|
46
|
+
except exceptions as e:
|
|
47
|
+
last_exception = e
|
|
48
|
+
|
|
49
|
+
if attempt == max_attempts - 1:
|
|
50
|
+
# Last attempt failed
|
|
51
|
+
logger.error(f"Function {func.__name__} failed after {max_attempts} attempts: {e}")
|
|
52
|
+
raise
|
|
53
|
+
|
|
54
|
+
# Log retry attempt
|
|
55
|
+
logger.warning(f"Function {func.__name__} attempt {attempt + 1} failed: {e}. Retrying in {current_delay}s...")
|
|
56
|
+
|
|
57
|
+
# Call retry callback if provided
|
|
58
|
+
if on_retry:
|
|
59
|
+
try:
|
|
60
|
+
if asyncio.iscoroutinefunction(on_retry):
|
|
61
|
+
await on_retry(attempt + 1, e)
|
|
62
|
+
else:
|
|
63
|
+
on_retry(attempt + 1, e)
|
|
64
|
+
except Exception as callback_error:
|
|
65
|
+
logger.error(f"Retry callback error: {callback_error}")
|
|
66
|
+
|
|
67
|
+
# Wait before retry
|
|
68
|
+
await asyncio.sleep(current_delay)
|
|
69
|
+
current_delay *= backoff
|
|
70
|
+
|
|
71
|
+
# Should never reach here, but just in case
|
|
72
|
+
raise last_exception
|
|
73
|
+
|
|
74
|
+
@wraps(func)
|
|
75
|
+
def sync_wrapper(*args, **kwargs):
|
|
76
|
+
"""Sync retry wrapper."""
|
|
77
|
+
|
|
78
|
+
last_exception = None
|
|
79
|
+
current_delay = delay
|
|
80
|
+
|
|
81
|
+
for attempt in range(max_attempts):
|
|
82
|
+
try:
|
|
83
|
+
return func(*args, **kwargs)
|
|
84
|
+
|
|
85
|
+
except exceptions as e:
|
|
86
|
+
last_exception = e
|
|
87
|
+
|
|
88
|
+
if attempt == max_attempts - 1:
|
|
89
|
+
# Last attempt failed
|
|
90
|
+
logger.error(f"Function {func.__name__} failed after {max_attempts} attempts: {e}")
|
|
91
|
+
raise
|
|
92
|
+
|
|
93
|
+
# Log retry attempt
|
|
94
|
+
logger.warning(f"Function {func.__name__} attempt {attempt + 1} failed: {e}. Retrying in {current_delay}s...")
|
|
95
|
+
|
|
96
|
+
# Call retry callback if provided
|
|
97
|
+
if on_retry:
|
|
98
|
+
try:
|
|
99
|
+
on_retry(attempt + 1, e)
|
|
100
|
+
except Exception as callback_error:
|
|
101
|
+
logger.error(f"Retry callback error: {callback_error}")
|
|
102
|
+
|
|
103
|
+
# Wait before retry (sync)
|
|
104
|
+
import time
|
|
105
|
+
time.sleep(current_delay)
|
|
106
|
+
current_delay *= backoff
|
|
107
|
+
|
|
108
|
+
# Should never reach here, but just in case
|
|
109
|
+
raise last_exception
|
|
110
|
+
|
|
111
|
+
# Return appropriate wrapper based on function type
|
|
112
|
+
if asyncio.iscoroutinefunction(func):
|
|
113
|
+
return async_wrapper
|
|
114
|
+
else:
|
|
115
|
+
return sync_wrapper
|
|
116
|
+
|
|
117
|
+
return decorator
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Clean schedule decorator for cron-like scheduling.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
from functools import wraps
|
|
8
|
+
from typing import Callable, Optional
|
|
9
|
+
from datetime import datetime, timedelta
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def schedule(
|
|
15
|
+
interval: Optional[int] = None,
|
|
16
|
+
cron: Optional[str] = None,
|
|
17
|
+
run_once: bool = False,
|
|
18
|
+
start_immediately: bool = False
|
|
19
|
+
):
|
|
20
|
+
"""
|
|
21
|
+
Clean schedule decorator.
|
|
22
|
+
|
|
23
|
+
Schedules function execution at specified intervals or cron patterns.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
interval: Interval in seconds between executions
|
|
27
|
+
cron: Cron expression (simple format: "*/5 * * * *" for every 5 minutes)
|
|
28
|
+
run_once: Run only once then stop
|
|
29
|
+
start_immediately: Start immediately instead of waiting for first interval
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def decorator(func: Callable) -> Callable:
|
|
33
|
+
@wraps(func)
|
|
34
|
+
async def wrapper(*args, **kwargs):
|
|
35
|
+
"""Scheduled function wrapper."""
|
|
36
|
+
|
|
37
|
+
if not interval and not cron:
|
|
38
|
+
raise ValueError("Either interval or cron must be specified")
|
|
39
|
+
|
|
40
|
+
execution_count = 0
|
|
41
|
+
|
|
42
|
+
# Start immediately if requested
|
|
43
|
+
if start_immediately:
|
|
44
|
+
try:
|
|
45
|
+
logger.info(f"Executing scheduled function {func.__name__} (immediate start)")
|
|
46
|
+
|
|
47
|
+
if asyncio.iscoroutinefunction(func):
|
|
48
|
+
await func(*args, **kwargs)
|
|
49
|
+
else:
|
|
50
|
+
func(*args, **kwargs)
|
|
51
|
+
|
|
52
|
+
execution_count += 1
|
|
53
|
+
|
|
54
|
+
if run_once:
|
|
55
|
+
logger.info(f"Scheduled function {func.__name__} completed (run_once=True)")
|
|
56
|
+
return
|
|
57
|
+
|
|
58
|
+
except Exception as e:
|
|
59
|
+
logger.error(f"Scheduled function {func.__name__} failed: {e}")
|
|
60
|
+
|
|
61
|
+
# Main scheduling loop
|
|
62
|
+
while True:
|
|
63
|
+
try:
|
|
64
|
+
# Calculate next execution time
|
|
65
|
+
if interval:
|
|
66
|
+
await asyncio.sleep(interval)
|
|
67
|
+
elif cron:
|
|
68
|
+
# Simple cron parsing (just for intervals like "*/5 * * * *")
|
|
69
|
+
next_delay = _parse_simple_cron(cron)
|
|
70
|
+
await asyncio.sleep(next_delay)
|
|
71
|
+
|
|
72
|
+
# Execute function
|
|
73
|
+
logger.debug(f"Executing scheduled function {func.__name__}")
|
|
74
|
+
|
|
75
|
+
if asyncio.iscoroutinefunction(func):
|
|
76
|
+
await func(*args, **kwargs)
|
|
77
|
+
else:
|
|
78
|
+
func(*args, **kwargs)
|
|
79
|
+
|
|
80
|
+
execution_count += 1
|
|
81
|
+
|
|
82
|
+
if run_once:
|
|
83
|
+
logger.info(f"Scheduled function {func.__name__} completed after {execution_count} executions")
|
|
84
|
+
break
|
|
85
|
+
|
|
86
|
+
except asyncio.CancelledError:
|
|
87
|
+
logger.info(f"Scheduled function {func.__name__} cancelled after {execution_count} executions")
|
|
88
|
+
break
|
|
89
|
+
except Exception as e:
|
|
90
|
+
logger.error(f"Scheduled function {func.__name__} failed: {e}")
|
|
91
|
+
# Continue scheduling even if execution fails
|
|
92
|
+
|
|
93
|
+
# Store schedule metadata
|
|
94
|
+
wrapper._schedule_interval = interval
|
|
95
|
+
wrapper._schedule_cron = cron
|
|
96
|
+
wrapper._schedule_run_once = run_once
|
|
97
|
+
wrapper._is_scheduled = True
|
|
98
|
+
|
|
99
|
+
return wrapper
|
|
100
|
+
|
|
101
|
+
return decorator
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _parse_simple_cron(cron_expr: str) -> int:
|
|
105
|
+
"""
|
|
106
|
+
Parse simple cron expressions.
|
|
107
|
+
|
|
108
|
+
Currently supports only basic interval patterns like:
|
|
109
|
+
- "*/5 * * * *" (every 5 minutes)
|
|
110
|
+
- "0 * * * *" (every hour)
|
|
111
|
+
- "0 0 * * *" (every day)
|
|
112
|
+
|
|
113
|
+
Returns delay in seconds until next execution.
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
parts = cron_expr.strip().split()
|
|
117
|
+
if len(parts) != 5:
|
|
118
|
+
raise ValueError(f"Invalid cron expression: {cron_expr}")
|
|
119
|
+
|
|
120
|
+
minute, hour, day, month, weekday = parts
|
|
121
|
+
|
|
122
|
+
# Simple parsing for common patterns
|
|
123
|
+
if minute.startswith("*/"):
|
|
124
|
+
# Every N minutes
|
|
125
|
+
interval_minutes = int(minute[2:])
|
|
126
|
+
return interval_minutes * 60
|
|
127
|
+
elif minute == "0" and hour.startswith("*/"):
|
|
128
|
+
# Every N hours
|
|
129
|
+
interval_hours = int(hour[2:])
|
|
130
|
+
return interval_hours * 3600
|
|
131
|
+
elif minute == "0" and hour == "0":
|
|
132
|
+
# Daily
|
|
133
|
+
return 24 * 3600
|
|
134
|
+
else:
|
|
135
|
+
# Default to 5 minutes for unsupported patterns
|
|
136
|
+
logger.warning(f"Unsupported cron pattern {cron_expr}, defaulting to 5 minutes")
|
|
137
|
+
return 300
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Clean task decorator for registering task handlers.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import logging
|
|
7
|
+
from functools import wraps
|
|
8
|
+
from typing import Callable, Any, Optional
|
|
9
|
+
|
|
10
|
+
from unrealon_core.models.websocket import TaskAssignmentData, TaskResultData
|
|
11
|
+
from unrealon_core.enums import TaskStatus
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def task(task_type: str, description: Optional[str] = None):
|
|
17
|
+
"""
|
|
18
|
+
Clean task decorator.
|
|
19
|
+
|
|
20
|
+
Registers a function as a task handler and provides
|
|
21
|
+
automatic error handling and result formatting.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
task_type: Type of task this handler processes
|
|
25
|
+
description: Optional description of the task
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
def decorator(func: Callable) -> Callable:
|
|
29
|
+
@wraps(func)
|
|
30
|
+
async def wrapper(task_data: TaskAssignmentData, *args, **kwargs) -> Any:
|
|
31
|
+
"""Task wrapper with error handling."""
|
|
32
|
+
|
|
33
|
+
start_time = asyncio.get_event_loop().time()
|
|
34
|
+
|
|
35
|
+
try:
|
|
36
|
+
logger.info(f"Starting task {task_type}: {task_data.task_id}")
|
|
37
|
+
|
|
38
|
+
# Execute the task function
|
|
39
|
+
if asyncio.iscoroutinefunction(func):
|
|
40
|
+
result = await func(task_data, *args, **kwargs)
|
|
41
|
+
else:
|
|
42
|
+
result = func(task_data, *args, **kwargs)
|
|
43
|
+
|
|
44
|
+
duration = asyncio.get_event_loop().time() - start_time
|
|
45
|
+
logger.info(f"Task {task_type} completed in {duration:.2f}s: {task_data.task_id}")
|
|
46
|
+
|
|
47
|
+
return result
|
|
48
|
+
|
|
49
|
+
except Exception as e:
|
|
50
|
+
duration = asyncio.get_event_loop().time() - start_time
|
|
51
|
+
logger.error(f"Task {task_type} failed after {duration:.2f}s: {task_data.task_id} - {e}")
|
|
52
|
+
raise
|
|
53
|
+
|
|
54
|
+
# Store task metadata
|
|
55
|
+
wrapper._task_type = task_type
|
|
56
|
+
wrapper._task_description = description
|
|
57
|
+
wrapper._is_task_handler = True
|
|
58
|
+
|
|
59
|
+
return wrapper
|
|
60
|
+
|
|
61
|
+
return decorator
|