jettask 0.2.1__py3-none-any.whl → 0.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jettask/constants.py +213 -0
- jettask/core/app.py +525 -205
- jettask/core/cli.py +193 -185
- jettask/core/consumer_manager.py +126 -34
- jettask/core/context.py +3 -0
- jettask/core/enums.py +137 -0
- jettask/core/event_pool.py +501 -168
- jettask/core/message.py +147 -0
- jettask/core/offline_worker_recovery.py +181 -114
- jettask/core/task.py +10 -174
- jettask/core/task_batch.py +153 -0
- jettask/core/unified_manager_base.py +243 -0
- jettask/core/worker_scanner.py +54 -54
- jettask/executors/asyncio.py +184 -64
- jettask/webui/backend/config.py +51 -0
- jettask/webui/backend/data_access.py +2083 -92
- jettask/webui/backend/data_api.py +3294 -0
- jettask/webui/backend/dependencies.py +261 -0
- jettask/webui/backend/init_meta_db.py +158 -0
- jettask/webui/backend/main.py +1358 -69
- jettask/webui/backend/main_unified.py +78 -0
- jettask/webui/backend/main_v2.py +394 -0
- jettask/webui/backend/namespace_api.py +295 -0
- jettask/webui/backend/namespace_api_old.py +294 -0
- jettask/webui/backend/namespace_data_access.py +611 -0
- jettask/webui/backend/queue_backlog_api.py +727 -0
- jettask/webui/backend/queue_stats_v2.py +521 -0
- jettask/webui/backend/redis_monitor_api.py +476 -0
- jettask/webui/backend/unified_api_router.py +1601 -0
- jettask/webui/db_init.py +204 -32
- jettask/webui/frontend/package-lock.json +492 -1
- jettask/webui/frontend/package.json +4 -1
- jettask/webui/frontend/src/App.css +105 -7
- jettask/webui/frontend/src/App.jsx +49 -20
- jettask/webui/frontend/src/components/NamespaceSelector.jsx +166 -0
- jettask/webui/frontend/src/components/QueueBacklogChart.jsx +298 -0
- jettask/webui/frontend/src/components/QueueBacklogTrend.jsx +638 -0
- jettask/webui/frontend/src/components/QueueDetailsTable.css +65 -0
- jettask/webui/frontend/src/components/QueueDetailsTable.jsx +487 -0
- jettask/webui/frontend/src/components/QueueDetailsTableV2.jsx +465 -0
- jettask/webui/frontend/src/components/ScheduledTaskFilter.jsx +423 -0
- jettask/webui/frontend/src/components/TaskFilter.jsx +425 -0
- jettask/webui/frontend/src/components/TimeRangeSelector.css +21 -0
- jettask/webui/frontend/src/components/TimeRangeSelector.jsx +160 -0
- jettask/webui/frontend/src/components/layout/AppLayout.css +95 -0
- jettask/webui/frontend/src/components/layout/AppLayout.jsx +49 -0
- jettask/webui/frontend/src/components/layout/Header.css +34 -10
- jettask/webui/frontend/src/components/layout/Header.jsx +31 -23
- jettask/webui/frontend/src/components/layout/SideMenu.css +137 -0
- jettask/webui/frontend/src/components/layout/SideMenu.jsx +209 -0
- jettask/webui/frontend/src/components/layout/TabsNav.css +244 -0
- jettask/webui/frontend/src/components/layout/TabsNav.jsx +206 -0
- jettask/webui/frontend/src/components/layout/UserInfo.css +197 -0
- jettask/webui/frontend/src/components/layout/UserInfo.jsx +197 -0
- jettask/webui/frontend/src/contexts/NamespaceContext.jsx +72 -0
- jettask/webui/frontend/src/contexts/TabsContext.backup.jsx +245 -0
- jettask/webui/frontend/src/main.jsx +1 -0
- jettask/webui/frontend/src/pages/Alerts.jsx +684 -0
- jettask/webui/frontend/src/pages/Dashboard.jsx +1330 -0
- jettask/webui/frontend/src/pages/QueueDetail.jsx +1109 -10
- jettask/webui/frontend/src/pages/QueueMonitor.jsx +236 -115
- jettask/webui/frontend/src/pages/Queues.jsx +5 -1
- jettask/webui/frontend/src/pages/ScheduledTasks.jsx +809 -0
- jettask/webui/frontend/src/pages/Settings.jsx +800 -0
- jettask/webui/frontend/src/services/api.js +7 -5
- jettask/webui/frontend/src/utils/suppressWarnings.js +22 -0
- jettask/webui/frontend/src/utils/userPreferences.js +154 -0
- jettask/webui/multi_namespace_consumer.py +543 -0
- jettask/webui/pg_consumer.py +983 -246
- jettask/webui/static/dist/assets/index-7129cfe1.css +1 -0
- jettask/webui/static/dist/assets/index-8d1935cc.js +774 -0
- jettask/webui/static/dist/index.html +2 -2
- jettask/webui/task_center.py +216 -0
- jettask/webui/task_center_client.py +150 -0
- jettask/webui/unified_consumer_manager.py +193 -0
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/METADATA +1 -1
- jettask-0.2.4.dist-info/RECORD +134 -0
- jettask/webui/pg_consumer_slow.py +0 -1099
- jettask/webui/pg_consumer_test.py +0 -678
- jettask/webui/static/dist/assets/index-823408e8.css +0 -1
- jettask/webui/static/dist/assets/index-9968b0b8.js +0 -543
- jettask/webui/test_pg_consumer_recovery.py +0 -547
- jettask/webui/test_recovery_simple.py +0 -492
- jettask/webui/test_self_recovery.py +0 -467
- jettask-0.2.1.dist-info/RECORD +0 -91
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/WHEEL +0 -0
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/entry_points.txt +0 -0
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/licenses/LICENSE +0 -0
- {jettask-0.2.1.dist-info → jettask-0.2.4.dist-info}/top_level.txt +0 -0
@@ -5,8 +5,8 @@
|
|
5
5
|
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
|
6
6
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
7
7
|
<title>JetTask Monitor - 任务监控平台</title>
|
8
|
-
<script type="module" crossorigin src="/assets/index-
|
9
|
-
<link rel="stylesheet" href="/assets/index-
|
8
|
+
<script type="module" crossorigin src="/assets/index-8d1935cc.js"></script>
|
9
|
+
<link rel="stylesheet" href="/assets/index-7129cfe1.css">
|
10
10
|
</head>
|
11
11
|
<body>
|
12
12
|
<div id="root"></div>
|
@@ -0,0 +1,216 @@
|
|
1
|
+
"""
|
2
|
+
任务中心客户端 - 独立的、可复用的任务中心连接器
|
3
|
+
"""
|
4
|
+
import os
|
5
|
+
import aiohttp
|
6
|
+
from typing import Optional, Dict, Any
|
7
|
+
import logging
|
8
|
+
|
9
|
+
logger = logging.getLogger(__name__)
|
10
|
+
|
11
|
+
|
12
|
+
class TaskCenter:
|
13
|
+
"""独立的任务中心客户端"""
|
14
|
+
|
15
|
+
def __init__(self, namespace_url: str = None):
|
16
|
+
"""
|
17
|
+
初始化任务中心客户端
|
18
|
+
|
19
|
+
Args:
|
20
|
+
namespace_url: 命名空间的URL,如 http://localhost:8001/api/namespaces/{name}
|
21
|
+
"""
|
22
|
+
self.namespace_url = namespace_url
|
23
|
+
self._session: Optional[aiohttp.ClientSession] = None
|
24
|
+
self._config: Optional[Dict[str, Any]] = None
|
25
|
+
self._namespace_name: Optional[str] = None
|
26
|
+
self._initialized = False
|
27
|
+
|
28
|
+
# 从URL解析命名空间名称
|
29
|
+
if namespace_url:
|
30
|
+
self._parse_url(namespace_url)
|
31
|
+
|
32
|
+
def _parse_url(self, url: str):
|
33
|
+
"""解析URL获取命名空间名称"""
|
34
|
+
if url.startswith("http://") or url.startswith("https://"):
|
35
|
+
import re
|
36
|
+
# 匹配格式: /api/namespaces/{name}
|
37
|
+
match = re.search(r'/namespaces/([^/]+)$', url)
|
38
|
+
if match:
|
39
|
+
self._namespace_name = match.group(1)
|
40
|
+
elif url.startswith("taskcenter://"):
|
41
|
+
# 兼容旧格式 taskcenter://namespace/{name}
|
42
|
+
parts = url.replace("taskcenter://", "").split("/")
|
43
|
+
if len(parts) >= 2 and parts[0] == "namespace":
|
44
|
+
self._namespace_name = parts[1]
|
45
|
+
base_url = os.getenv("TASK_CENTER_BASE_URL", "http://localhost:8001")
|
46
|
+
self.namespace_url = f"{base_url}/api/namespaces/{self._namespace_name}"
|
47
|
+
|
48
|
+
@property
|
49
|
+
def is_enabled(self) -> bool:
|
50
|
+
"""是否启用任务中心"""
|
51
|
+
return self.namespace_url is not None
|
52
|
+
|
53
|
+
@property
|
54
|
+
def namespace_name(self) -> str:
|
55
|
+
"""获取命名空间名称"""
|
56
|
+
return self._namespace_name or "jettask"
|
57
|
+
|
58
|
+
@property
|
59
|
+
def redis_prefix(self) -> str:
|
60
|
+
"""获取Redis键前缀"""
|
61
|
+
return self.namespace_name
|
62
|
+
|
63
|
+
@property
|
64
|
+
def redis_config(self) -> Optional[Dict[str, Any]]:
|
65
|
+
"""获取Redis配置"""
|
66
|
+
return self._config.get('redis_config') if self._config else None
|
67
|
+
|
68
|
+
@property
|
69
|
+
def pg_config(self) -> Optional[Dict[str, Any]]:
|
70
|
+
"""获取PostgreSQL配置"""
|
71
|
+
return self._config.get('pg_config') if self._config else None
|
72
|
+
|
73
|
+
@property
|
74
|
+
def version(self) -> int:
|
75
|
+
"""获取配置版本"""
|
76
|
+
return self._config.get('version', 1) if self._config else 1
|
77
|
+
|
78
|
+
async def _get_session(self) -> aiohttp.ClientSession:
|
79
|
+
"""获取HTTP会话"""
|
80
|
+
if self._session is None or self._session.closed:
|
81
|
+
self._session = aiohttp.ClientSession()
|
82
|
+
return self._session
|
83
|
+
|
84
|
+
def connect(self, asyncio: bool = False):
|
85
|
+
"""
|
86
|
+
连接到任务中心并获取配置
|
87
|
+
|
88
|
+
Args:
|
89
|
+
asyncio: 是否使用异步模式,默认为False(同步模式)
|
90
|
+
|
91
|
+
Returns:
|
92
|
+
连接成功返回True,失败返回False(异步模式返回协程)
|
93
|
+
"""
|
94
|
+
if asyncio:
|
95
|
+
return self._connect_async()
|
96
|
+
else:
|
97
|
+
return self._connect_sync()
|
98
|
+
|
99
|
+
def _connect_sync(self) -> bool:
|
100
|
+
"""同步连接到任务中心"""
|
101
|
+
if not self.is_enabled:
|
102
|
+
return False
|
103
|
+
|
104
|
+
if self._initialized:
|
105
|
+
logger.debug(f"任务中心已初始化,使用缓存配置")
|
106
|
+
return True
|
107
|
+
|
108
|
+
try:
|
109
|
+
import requests
|
110
|
+
response = requests.get(self.namespace_url, timeout=10)
|
111
|
+
if response.status_code == 200:
|
112
|
+
data = response.json()
|
113
|
+
self._namespace_name = data.get('name')
|
114
|
+
self._config = {
|
115
|
+
'redis_config': data.get('redis_config'),
|
116
|
+
'pg_config': data.get('pg_config'),
|
117
|
+
'namespace_name': data.get('name'),
|
118
|
+
'version': data.get('version', 1)
|
119
|
+
}
|
120
|
+
self._initialized = True
|
121
|
+
logger.info(f"成功连接到任务中心命名空间: {self._namespace_name} (v{self.version})")
|
122
|
+
return True
|
123
|
+
else:
|
124
|
+
logger.error(f"无法连接到任务中心: HTTP {response.status_code}")
|
125
|
+
return False
|
126
|
+
except Exception as e:
|
127
|
+
logger.error(f"连接任务中心失败: {e}")
|
128
|
+
return False
|
129
|
+
|
130
|
+
async def _connect_async(self) -> bool:
|
131
|
+
"""异步连接到任务中心"""
|
132
|
+
if not self.is_enabled:
|
133
|
+
return False
|
134
|
+
|
135
|
+
if self._initialized:
|
136
|
+
logger.debug(f"任务中心已初始化,使用缓存配置")
|
137
|
+
return True
|
138
|
+
|
139
|
+
try:
|
140
|
+
session = await self._get_session()
|
141
|
+
async with session.get(self.namespace_url) as resp:
|
142
|
+
if resp.status == 200:
|
143
|
+
data = await resp.json()
|
144
|
+
self._namespace_name = data.get('name')
|
145
|
+
self._config = {
|
146
|
+
'redis_config': data.get('redis_config'),
|
147
|
+
'pg_config': data.get('pg_config'),
|
148
|
+
'namespace_name': data.get('name'),
|
149
|
+
'version': data.get('version', 1)
|
150
|
+
}
|
151
|
+
self._initialized = True
|
152
|
+
logger.info(f"成功连接到任务中心命名空间: {self._namespace_name} (v{self.version})")
|
153
|
+
return True
|
154
|
+
else:
|
155
|
+
logger.error(f"无法连接到任务中心: HTTP {resp.status}")
|
156
|
+
return False
|
157
|
+
except Exception as e:
|
158
|
+
logger.error(f"连接任务中心失败: {e}")
|
159
|
+
return False
|
160
|
+
|
161
|
+
def get_redis_url(self) -> Optional[str]:
|
162
|
+
"""
|
163
|
+
获取Redis连接URL
|
164
|
+
|
165
|
+
Returns:
|
166
|
+
Redis连接URL字符串
|
167
|
+
"""
|
168
|
+
if not self.redis_config:
|
169
|
+
return None
|
170
|
+
|
171
|
+
# 如果配置中直接有url字段,直接返回
|
172
|
+
if 'url' in self.redis_config:
|
173
|
+
return self.redis_config['url']
|
174
|
+
|
175
|
+
# 否则,从分离的字段构建URL
|
176
|
+
host = self.redis_config.get('host', 'localhost')
|
177
|
+
port = self.redis_config.get('port', 6379)
|
178
|
+
db = self.redis_config.get('db', 0)
|
179
|
+
password = self.redis_config.get('password', '')
|
180
|
+
|
181
|
+
if password:
|
182
|
+
return f"redis://:{password}@{host}:{port}/{db}"
|
183
|
+
else:
|
184
|
+
return f"redis://{host}:{port}/{db}"
|
185
|
+
|
186
|
+
def get_pg_url(self) -> Optional[str]:
|
187
|
+
"""
|
188
|
+
获取PostgreSQL连接URL
|
189
|
+
|
190
|
+
Returns:
|
191
|
+
PostgreSQL连接URL字符串
|
192
|
+
"""
|
193
|
+
if not self.pg_config:
|
194
|
+
return None
|
195
|
+
|
196
|
+
# 如果配置中直接有url字段,直接返回
|
197
|
+
if 'url' in self.pg_config:
|
198
|
+
return self.pg_config['url']
|
199
|
+
|
200
|
+
# 否则,从分离的字段构建URL
|
201
|
+
host = self.pg_config.get('host', 'localhost')
|
202
|
+
port = self.pg_config.get('port', 5432)
|
203
|
+
database = self.pg_config.get('database', 'jettask')
|
204
|
+
user = self.pg_config.get('user', 'jettask')
|
205
|
+
password = self.pg_config.get('password', '123456')
|
206
|
+
|
207
|
+
return f"postgresql://{user}:{password}@{host}:{port}/{database}"
|
208
|
+
|
209
|
+
async def close(self):
|
210
|
+
"""关闭客户端"""
|
211
|
+
if self._session:
|
212
|
+
await self._session.close()
|
213
|
+
self._session = None
|
214
|
+
|
215
|
+
def __repr__(self) -> str:
|
216
|
+
return f"<TaskCenter namespace='{self.namespace_name}' version={self.version} initialized={self._initialized}>"
|
@@ -0,0 +1,150 @@
|
|
1
|
+
"""
|
2
|
+
任务中心客户端 - JetTask App使用的客户端库
|
3
|
+
"""
|
4
|
+
import os
|
5
|
+
import aiohttp
|
6
|
+
import asyncio
|
7
|
+
from typing import Optional, Dict, Any
|
8
|
+
from .models.namespace import TaskCenterConfig
|
9
|
+
import logging
|
10
|
+
|
11
|
+
logger = logging.getLogger(__name__)
|
12
|
+
|
13
|
+
|
14
|
+
class TaskCenterClient:
|
15
|
+
"""任务中心客户端"""
|
16
|
+
|
17
|
+
def __init__(self, task_center_url: Optional[str] = None):
|
18
|
+
"""
|
19
|
+
初始化任务中心客户端
|
20
|
+
|
21
|
+
Args:
|
22
|
+
task_center_url: 任务中心配置URL,支持两种格式:
|
23
|
+
- HTTP格式: http://localhost:8001/api/namespaces/{namespace_id}
|
24
|
+
- 旧格式: taskcenter://namespace/{namespace_id} (向后兼容)
|
25
|
+
"""
|
26
|
+
self.task_center_url = task_center_url
|
27
|
+
self._session: Optional[aiohttp.ClientSession] = None
|
28
|
+
self._cached_config: Optional[Dict[str, Any]] = None
|
29
|
+
self._namespace_id: Optional[str] = None
|
30
|
+
self._namespace_name: Optional[str] = None # 添加命名空间名称
|
31
|
+
|
32
|
+
# 解析URL
|
33
|
+
self._config_url = task_center_url # 用于获取配置的URL
|
34
|
+
if task_center_url:
|
35
|
+
if task_center_url.startswith("http://") or task_center_url.startswith("https://"):
|
36
|
+
# HTTP格式,支持新格式: http://localhost:8001/api/namespaces/{name}
|
37
|
+
import re
|
38
|
+
# 匹配新格式:使用名称而非ID
|
39
|
+
match = re.search(r'/namespaces/([^/]+)$', task_center_url)
|
40
|
+
if match:
|
41
|
+
self._namespace_name = match.group(1)
|
42
|
+
# 新格式直接使用URL,不需要添加/config
|
43
|
+
elif task_center_url.startswith("taskcenter://"):
|
44
|
+
# 旧格式,转换为HTTP格式
|
45
|
+
parts = task_center_url.replace("taskcenter://", "").split("/")
|
46
|
+
if len(parts) >= 2 and parts[0] == "namespace":
|
47
|
+
# 旧格式使用的是ID,转换为按名称查找
|
48
|
+
self._namespace_name = parts[1] # 假设传入的也是名称
|
49
|
+
base_url = os.getenv("TASK_CENTER_BASE_URL", "http://localhost:8001")
|
50
|
+
self._config_url = f"{base_url}/api/namespaces/{self._namespace_name}"
|
51
|
+
|
52
|
+
@property
|
53
|
+
def is_enabled(self) -> bool:
|
54
|
+
"""是否启用任务中心"""
|
55
|
+
return self.task_center_url is not None
|
56
|
+
|
57
|
+
@property
|
58
|
+
def namespace_id(self) -> Optional[str]:
|
59
|
+
"""获取命名空间ID"""
|
60
|
+
return self._namespace_id
|
61
|
+
|
62
|
+
@property
|
63
|
+
def namespace_prefix(self) -> str:
|
64
|
+
"""获取Redis key前缀"""
|
65
|
+
# 优先使用namespace_name,不再添加tc:前缀
|
66
|
+
if self._namespace_name:
|
67
|
+
return self._namespace_name
|
68
|
+
# 如果配置已加载,使用其中的名称
|
69
|
+
elif self._cached_config and self._cached_config.get('namespace_name'):
|
70
|
+
return self._cached_config['namespace_name']
|
71
|
+
# 默认前缀
|
72
|
+
return "jettask"
|
73
|
+
|
74
|
+
async def _get_session(self) -> aiohttp.ClientSession:
|
75
|
+
"""获取HTTP会话"""
|
76
|
+
if self._session is None or self._session.closed:
|
77
|
+
self._session = aiohttp.ClientSession()
|
78
|
+
return self._session
|
79
|
+
|
80
|
+
async def get_config(self) -> Optional[Dict[str, Any]]:
|
81
|
+
"""
|
82
|
+
从任务中心获取配置
|
83
|
+
|
84
|
+
Returns:
|
85
|
+
包含redis_config和pg_config的字典
|
86
|
+
"""
|
87
|
+
if not self.is_enabled:
|
88
|
+
return None
|
89
|
+
|
90
|
+
# 如果已缓存,直接返回
|
91
|
+
if self._cached_config:
|
92
|
+
return self._cached_config
|
93
|
+
|
94
|
+
try:
|
95
|
+
session = await self._get_session()
|
96
|
+
# 使用配置URL(可能带有/config后缀)
|
97
|
+
async with session.get(self._config_url) as resp:
|
98
|
+
if resp.status == 200:
|
99
|
+
data = await resp.json()
|
100
|
+
self._namespace_name = data.get('name') # 保存命名空间名称
|
101
|
+
self._cached_config = {
|
102
|
+
'redis_config': data.get('redis_config'),
|
103
|
+
'pg_config': data.get('pg_config'),
|
104
|
+
'namespace_name': data.get('name'),
|
105
|
+
'namespace_id': self._namespace_id,
|
106
|
+
'version': data.get('version', 1) # 添加版本号
|
107
|
+
}
|
108
|
+
return self._cached_config
|
109
|
+
else:
|
110
|
+
logger.error(f"Failed to get config from task center: {resp.status}")
|
111
|
+
return None
|
112
|
+
except Exception as e:
|
113
|
+
logger.error(f"Error getting config from task center: {e}")
|
114
|
+
return None
|
115
|
+
|
116
|
+
async def get_task_result(self, task_id: str) -> Optional[Dict[str, Any]]:
|
117
|
+
"""
|
118
|
+
从任务中心获取任务结果(当Redis中不存在时)
|
119
|
+
|
120
|
+
Args:
|
121
|
+
task_id: 任务ID
|
122
|
+
|
123
|
+
Returns:
|
124
|
+
任务结果字典
|
125
|
+
"""
|
126
|
+
if not self.is_enabled or not self._namespace_id:
|
127
|
+
return None
|
128
|
+
|
129
|
+
try:
|
130
|
+
session = await self._get_session()
|
131
|
+
# 从配置URL推导出任务结果URL
|
132
|
+
base_url = self.task_center_url.replace(f"/namespace/{self._namespace_id}/config", "")
|
133
|
+
url = f"{base_url}/namespace/{self._namespace_id}/task/{task_id}/result"
|
134
|
+
async with session.get(url) as resp:
|
135
|
+
if resp.status == 200:
|
136
|
+
return await resp.json()
|
137
|
+
elif resp.status == 404:
|
138
|
+
return None
|
139
|
+
else:
|
140
|
+
logger.error(f"Failed to get task result from task center: {resp.status}")
|
141
|
+
return None
|
142
|
+
except Exception as e:
|
143
|
+
logger.error(f"Error getting task result from task center: {e}")
|
144
|
+
return None
|
145
|
+
|
146
|
+
|
147
|
+
async def close(self):
|
148
|
+
"""关闭客户端"""
|
149
|
+
if self._session:
|
150
|
+
await self._session.close()
|
@@ -0,0 +1,193 @@
|
|
1
|
+
"""
|
2
|
+
统一的数据消费者管理器
|
3
|
+
自动识别单命名空间和多命名空间模式
|
4
|
+
"""
|
5
|
+
import asyncio
|
6
|
+
import logging
|
7
|
+
import multiprocessing
|
8
|
+
from typing import Dict, Optional, Set
|
9
|
+
from jettask.core.unified_manager_base import UnifiedManagerBase
|
10
|
+
from .multi_namespace_consumer import NamespaceConsumerProcess
|
11
|
+
|
12
|
+
logger = logging.getLogger(__name__)
|
13
|
+
|
14
|
+
|
15
|
+
class UnifiedConsumerManager(UnifiedManagerBase):
|
16
|
+
"""
|
17
|
+
统一的消费者管理器
|
18
|
+
继承自 UnifiedManagerBase,实现消费者特定的逻辑
|
19
|
+
"""
|
20
|
+
|
21
|
+
def __init__(self,
|
22
|
+
task_center_url: str,
|
23
|
+
check_interval: int = 30,
|
24
|
+
debug: bool = False):
|
25
|
+
"""
|
26
|
+
初始化消费者管理器
|
27
|
+
|
28
|
+
Args:
|
29
|
+
task_center_url: 任务中心URL
|
30
|
+
check_interval: 命名空间检测间隔(秒)
|
31
|
+
debug: 是否启用调试模式
|
32
|
+
"""
|
33
|
+
super().__init__(task_center_url, check_interval, debug)
|
34
|
+
|
35
|
+
# 消费者进程管理
|
36
|
+
self.consumer_processes: Dict[str, NamespaceConsumerProcess] = {}
|
37
|
+
self.known_namespaces: Set[str] = set()
|
38
|
+
|
39
|
+
@property
|
40
|
+
def processes(self):
|
41
|
+
"""提供对外的进程访问接口"""
|
42
|
+
return self.consumer_processes
|
43
|
+
|
44
|
+
async def run_single_namespace(self, namespace_name: str):
|
45
|
+
"""
|
46
|
+
运行单命名空间模式
|
47
|
+
|
48
|
+
Args:
|
49
|
+
namespace_name: 命名空间名称
|
50
|
+
"""
|
51
|
+
logger.info(f"启动单命名空间消费者: {namespace_name}")
|
52
|
+
|
53
|
+
# 获取命名空间配置
|
54
|
+
namespaces = await self.fetch_namespaces_info({namespace_name})
|
55
|
+
|
56
|
+
if not namespaces:
|
57
|
+
logger.error(f"未找到命名空间配置: {namespace_name}")
|
58
|
+
return
|
59
|
+
|
60
|
+
ns_info = namespaces[0]
|
61
|
+
|
62
|
+
# 创建并启动消费进程
|
63
|
+
consumer = NamespaceConsumerProcess(ns_info)
|
64
|
+
consumer.start()
|
65
|
+
self.consumer_processes[namespace_name] = consumer
|
66
|
+
|
67
|
+
try:
|
68
|
+
# 保持运行,定期检查进程状态
|
69
|
+
while self.running:
|
70
|
+
await asyncio.sleep(10)
|
71
|
+
|
72
|
+
# 检查进程是否存活
|
73
|
+
if not consumer.is_alive():
|
74
|
+
logger.warning(f"命名空间 {namespace_name} 的消费进程已停止,尝试重启")
|
75
|
+
consumer.start()
|
76
|
+
except asyncio.CancelledError:
|
77
|
+
logger.info("收到取消信号")
|
78
|
+
|
79
|
+
async def run_multi_namespace(self, namespace_names: Optional[Set[str]]):
|
80
|
+
"""
|
81
|
+
运行多命名空间模式
|
82
|
+
|
83
|
+
Args:
|
84
|
+
namespace_names: 目标命名空间集合,None表示所有命名空间
|
85
|
+
"""
|
86
|
+
logger.info("启动多命名空间消费者管理")
|
87
|
+
|
88
|
+
# 获取初始命名空间配置
|
89
|
+
namespaces = await self.fetch_namespaces_info(namespace_names)
|
90
|
+
|
91
|
+
# 启动每个命名空间的消费者
|
92
|
+
for ns_info in namespaces:
|
93
|
+
try:
|
94
|
+
self._start_namespace_consumer(ns_info)
|
95
|
+
self.known_namespaces.add(ns_info['name'])
|
96
|
+
except Exception as e:
|
97
|
+
logger.error(f"启动命名空间 {ns_info['name']} 的消费者失败: {e}")
|
98
|
+
|
99
|
+
# 创建并发任务
|
100
|
+
try:
|
101
|
+
health_check_task = asyncio.create_task(self._health_check_loop())
|
102
|
+
namespace_check_task = asyncio.create_task(self._namespace_check_loop())
|
103
|
+
|
104
|
+
# 等待任一任务完成或出错
|
105
|
+
done, pending = await asyncio.wait(
|
106
|
+
[health_check_task, namespace_check_task],
|
107
|
+
return_when=asyncio.FIRST_EXCEPTION
|
108
|
+
)
|
109
|
+
|
110
|
+
# 取消所有未完成的任务
|
111
|
+
for task in pending:
|
112
|
+
task.cancel()
|
113
|
+
|
114
|
+
except asyncio.CancelledError:
|
115
|
+
logger.info("收到取消信号")
|
116
|
+
|
117
|
+
def _start_namespace_consumer(self, namespace_info: dict):
|
118
|
+
"""启动单个命名空间的消费者"""
|
119
|
+
name = namespace_info['name']
|
120
|
+
|
121
|
+
# 如果已存在,先停止
|
122
|
+
if name in self.consumer_processes:
|
123
|
+
self.consumer_processes[name].stop()
|
124
|
+
|
125
|
+
# 创建并启动新进程
|
126
|
+
consumer = NamespaceConsumerProcess(namespace_info)
|
127
|
+
consumer.start()
|
128
|
+
self.consumer_processes[name] = consumer
|
129
|
+
logger.info(f"启动命名空间 {name} 的消费进程")
|
130
|
+
|
131
|
+
async def _health_check_loop(self):
|
132
|
+
"""健康检查循环"""
|
133
|
+
while self.running:
|
134
|
+
try:
|
135
|
+
await asyncio.sleep(30) # 每30秒检查一次
|
136
|
+
|
137
|
+
# 检查所有消费进程的健康状态
|
138
|
+
for name, consumer in list(self.consumer_processes.items()):
|
139
|
+
if not consumer.is_alive():
|
140
|
+
logger.warning(f"命名空间 {name} 的消费进程已停止,尝试重启")
|
141
|
+
|
142
|
+
# 重新获取配置并重启
|
143
|
+
namespaces = await self.fetch_namespaces_info({name})
|
144
|
+
if namespaces:
|
145
|
+
self._start_namespace_consumer(namespaces[0])
|
146
|
+
else:
|
147
|
+
logger.error(f"无法获取命名空间 {name} 的配置")
|
148
|
+
|
149
|
+
except Exception as e:
|
150
|
+
logger.error(f"健康检查错误: {e}")
|
151
|
+
|
152
|
+
async def _namespace_check_loop(self):
|
153
|
+
"""命名空间检测循环(动态添加/移除)"""
|
154
|
+
while self.running:
|
155
|
+
try:
|
156
|
+
await asyncio.sleep(self.check_interval)
|
157
|
+
|
158
|
+
# 获取当前所有命名空间
|
159
|
+
current_namespaces = await self.fetch_namespaces_info()
|
160
|
+
current_names = {ns['name'] for ns in current_namespaces}
|
161
|
+
|
162
|
+
# 检测新增的命名空间
|
163
|
+
new_names = current_names - self.known_namespaces
|
164
|
+
for name in new_names:
|
165
|
+
logger.info(f"检测到新命名空间: {name}")
|
166
|
+
ns_info = next(ns for ns in current_namespaces if ns['name'] == name)
|
167
|
+
self._start_namespace_consumer(ns_info)
|
168
|
+
self.known_namespaces.add(name)
|
169
|
+
|
170
|
+
# 检测删除的命名空间
|
171
|
+
removed_names = self.known_namespaces - current_names
|
172
|
+
for name in removed_names:
|
173
|
+
logger.info(f"检测到命名空间已删除: {name}")
|
174
|
+
if name in self.consumer_processes:
|
175
|
+
self.consumer_processes[name].stop()
|
176
|
+
del self.consumer_processes[name]
|
177
|
+
self.known_namespaces.discard(name)
|
178
|
+
|
179
|
+
except Exception as e:
|
180
|
+
logger.error(f"命名空间检测错误: {e}")
|
181
|
+
|
182
|
+
async def cleanup(self):
|
183
|
+
"""清理资源"""
|
184
|
+
logger.info("停止所有消费进程")
|
185
|
+
|
186
|
+
for name, consumer in self.consumer_processes.items():
|
187
|
+
try:
|
188
|
+
consumer.stop()
|
189
|
+
logger.info(f"停止命名空间 {name} 的消费进程")
|
190
|
+
except Exception as e:
|
191
|
+
logger.error(f"停止命名空间 {name} 的消费进程失败: {e}")
|
192
|
+
|
193
|
+
self.consumer_processes.clear()
|