jettask 0.2.23__py3-none-any.whl → 0.2.24__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. jettask/__init__.py +2 -0
  2. jettask/cli.py +12 -8
  3. jettask/config/lua_scripts.py +37 -0
  4. jettask/config/nacos_config.py +1 -1
  5. jettask/core/app.py +313 -340
  6. jettask/core/container.py +4 -4
  7. jettask/{persistence → core}/namespace.py +93 -27
  8. jettask/core/task.py +16 -9
  9. jettask/core/unified_manager_base.py +136 -26
  10. jettask/db/__init__.py +67 -0
  11. jettask/db/base.py +137 -0
  12. jettask/{utils/db_connector.py → db/connector.py} +130 -26
  13. jettask/db/models/__init__.py +16 -0
  14. jettask/db/models/scheduled_task.py +196 -0
  15. jettask/db/models/task.py +77 -0
  16. jettask/db/models/task_run.py +85 -0
  17. jettask/executor/__init__.py +0 -15
  18. jettask/executor/core.py +76 -31
  19. jettask/executor/process_entry.py +29 -114
  20. jettask/executor/task_executor.py +4 -0
  21. jettask/messaging/event_pool.py +928 -685
  22. jettask/messaging/scanner.py +30 -0
  23. jettask/persistence/__init__.py +28 -103
  24. jettask/persistence/buffer.py +170 -0
  25. jettask/persistence/consumer.py +330 -249
  26. jettask/persistence/manager.py +304 -0
  27. jettask/persistence/persistence.py +391 -0
  28. jettask/scheduler/__init__.py +15 -3
  29. jettask/scheduler/{task_crud.py → database.py} +61 -57
  30. jettask/scheduler/loader.py +2 -2
  31. jettask/scheduler/{scheduler_coordinator.py → manager.py} +23 -6
  32. jettask/scheduler/models.py +14 -10
  33. jettask/scheduler/schedule.py +166 -0
  34. jettask/scheduler/scheduler.py +12 -11
  35. jettask/schemas/__init__.py +50 -1
  36. jettask/schemas/backlog.py +43 -6
  37. jettask/schemas/namespace.py +70 -19
  38. jettask/schemas/queue.py +19 -3
  39. jettask/schemas/responses.py +493 -0
  40. jettask/task/__init__.py +0 -2
  41. jettask/task/router.py +3 -0
  42. jettask/test_connection_monitor.py +1 -1
  43. jettask/utils/__init__.py +7 -5
  44. jettask/utils/db_init.py +8 -4
  45. jettask/utils/namespace_dep.py +167 -0
  46. jettask/utils/queue_matcher.py +186 -0
  47. jettask/utils/rate_limit/concurrency_limiter.py +7 -1
  48. jettask/utils/stream_backlog.py +1 -1
  49. jettask/webui/__init__.py +0 -1
  50. jettask/webui/api/__init__.py +4 -4
  51. jettask/webui/api/alerts.py +806 -71
  52. jettask/webui/api/example_refactored.py +400 -0
  53. jettask/webui/api/namespaces.py +390 -45
  54. jettask/webui/api/overview.py +300 -54
  55. jettask/webui/api/queues.py +971 -267
  56. jettask/webui/api/scheduled.py +1249 -56
  57. jettask/webui/api/settings.py +129 -7
  58. jettask/webui/api/workers.py +442 -0
  59. jettask/webui/app.py +46 -2329
  60. jettask/webui/middleware/__init__.py +6 -0
  61. jettask/webui/middleware/namespace_middleware.py +135 -0
  62. jettask/webui/services/__init__.py +146 -0
  63. jettask/webui/services/heartbeat_service.py +251 -0
  64. jettask/webui/services/overview_service.py +60 -51
  65. jettask/webui/services/queue_monitor_service.py +426 -0
  66. jettask/webui/services/redis_monitor_service.py +87 -0
  67. jettask/webui/services/settings_service.py +174 -111
  68. jettask/webui/services/task_monitor_service.py +222 -0
  69. jettask/webui/services/timeline_pg_service.py +452 -0
  70. jettask/webui/services/timeline_service.py +189 -0
  71. jettask/webui/services/worker_monitor_service.py +467 -0
  72. jettask/webui/utils/__init__.py +11 -0
  73. jettask/webui/utils/time_utils.py +122 -0
  74. jettask/worker/lifecycle.py +8 -2
  75. {jettask-0.2.23.dist-info → jettask-0.2.24.dist-info}/METADATA +1 -1
  76. jettask-0.2.24.dist-info/RECORD +142 -0
  77. jettask/executor/executor.py +0 -338
  78. jettask/persistence/backlog_monitor.py +0 -567
  79. jettask/persistence/base.py +0 -2334
  80. jettask/persistence/db_manager.py +0 -516
  81. jettask/persistence/maintenance.py +0 -81
  82. jettask/persistence/message_consumer.py +0 -259
  83. jettask/persistence/models.py +0 -49
  84. jettask/persistence/offline_recovery.py +0 -196
  85. jettask/persistence/queue_discovery.py +0 -215
  86. jettask/persistence/task_persistence.py +0 -218
  87. jettask/persistence/task_updater.py +0 -583
  88. jettask/scheduler/add_execution_count.sql +0 -11
  89. jettask/scheduler/add_priority_field.sql +0 -26
  90. jettask/scheduler/add_scheduler_id.sql +0 -25
  91. jettask/scheduler/add_scheduler_id_index.sql +0 -10
  92. jettask/scheduler/make_scheduler_id_required.sql +0 -28
  93. jettask/scheduler/migrate_interval_seconds.sql +0 -9
  94. jettask/scheduler/performance_optimization.sql +0 -45
  95. jettask/scheduler/run_scheduler.py +0 -186
  96. jettask/scheduler/schema.sql +0 -84
  97. jettask/task/task_executor.py +0 -318
  98. jettask/webui/api/analytics.py +0 -323
  99. jettask/webui/config.py +0 -90
  100. jettask/webui/models/__init__.py +0 -3
  101. jettask/webui/models/namespace.py +0 -63
  102. jettask/webui/namespace_manager/__init__.py +0 -10
  103. jettask/webui/namespace_manager/multi.py +0 -593
  104. jettask/webui/namespace_manager/unified.py +0 -193
  105. jettask/webui/run.py +0 -46
  106. jettask-0.2.23.dist-info/RECORD +0 -145
  107. {jettask-0.2.23.dist-info → jettask-0.2.24.dist-info}/WHEEL +0 -0
  108. {jettask-0.2.23.dist-info → jettask-0.2.24.dist-info}/entry_points.txt +0 -0
  109. {jettask-0.2.23.dist-info → jettask-0.2.24.dist-info}/licenses/LICENSE +0 -0
  110. {jettask-0.2.23.dist-info → jettask-0.2.24.dist-info}/top_level.txt +0 -0
@@ -1,323 +0,0 @@
1
- """
2
- 分析模块 - 数据分析、查询和报表
3
- 提供轻量级的路由入口,业务逻辑在 AnalyticsService 中实现
4
- """
5
- from fastapi import APIRouter, HTTPException, Request, Query
6
- from typing import List, Dict, Optional, Any
7
- from datetime import datetime
8
- import logging
9
-
10
- from jettask.schemas import TimeRangeQuery
11
- from jettask.webui.services.analytics_service import AnalyticsService
12
- from jettask.persistence.namespace import get_namespace_data_access
13
-
14
- router = APIRouter(prefix="/analytics", tags=["analytics"])
15
- logger = logging.getLogger(__name__)
16
-
17
- # 获取全局数据访问实例
18
- data_access = get_namespace_data_access()
19
-
20
-
21
- # ============ 命名空间管理 ============
22
-
23
- @router.get("/namespaces")
24
- async def get_all_namespaces():
25
- """
26
- 获取所有配置的命名空间列表
27
- """
28
- try:
29
- namespaces = await data_access.get_all_namespaces()
30
- return {
31
- "success": True,
32
- "data": namespaces
33
- }
34
- except Exception as e:
35
- logger.error(f"获取命名空间列表失败: {e}")
36
- raise HTTPException(status_code=500, detail=str(e))
37
-
38
-
39
- # ============ 队列统计 ============
40
-
41
- @router.get("/queue-stats/{namespace}")
42
- async def get_queue_stats(namespace: str):
43
- """
44
- 获取指定命名空间的队列统计信息
45
-
46
- Args:
47
- namespace: 命名空间名称
48
- """
49
- try:
50
- stats = await data_access.get_queue_stats(namespace)
51
- return {
52
- "success": True,
53
- "data": stats
54
- }
55
- except Exception as e:
56
- logger.error(f"获取队列统计失败: {e}")
57
- raise HTTPException(status_code=500, detail=str(e))
58
-
59
-
60
- @router.post("/queue-trends/{namespace}")
61
- async def get_queue_trends(namespace: str, request: Request):
62
- """
63
- 获取队列趋势数据
64
-
65
- Args:
66
- namespace: 命名空间名称
67
- """
68
- try:
69
- # 解析请求体
70
- body = await request.json() if await request.body() else {}
71
-
72
- # 从请求体中获取参数
73
- time_range = body.get('time_range', '1h')
74
- queue_name = body.get('queue_name')
75
- granularity = body.get('granularity', 'minute')
76
-
77
- # TODO: 实现队列趋势分析
78
- return {
79
- "success": True,
80
- "namespace": namespace,
81
- "time_range": time_range,
82
- "queue_name": queue_name,
83
- "data": [],
84
- "granularity": granularity
85
- }
86
- except Exception as e:
87
- logger.error(f"获取队列趋势失败: {e}")
88
- raise HTTPException(status_code=500, detail=str(e))
89
-
90
-
91
- # ============ 任务分析 ============
92
-
93
- @router.post("/task-analysis/{namespace}")
94
- async def analyze_tasks(namespace: str, request: Request):
95
- """
96
- 分析任务执行情况
97
-
98
- Args:
99
- namespace: 命名空间名称
100
- """
101
- try:
102
- body = await request.json()
103
-
104
- # TODO: 实现任务分析逻辑
105
- return {
106
- "success": True,
107
- "data": {
108
- "total_tasks": 0,
109
- "success_rate": 0.0,
110
- "avg_execution_time": 0.0,
111
- "error_distribution": {}
112
- }
113
- }
114
- except Exception as e:
115
- logger.error(f"任务分析失败: {e}")
116
- raise HTTPException(status_code=500, detail=str(e))
117
-
118
-
119
- @router.get("/task-distribution/{namespace}")
120
- async def get_task_distribution(
121
- namespace: str,
122
- time_range: str = Query("24h", description="时间范围")
123
- ):
124
- """
125
- 获取任务分布情况
126
-
127
- Args:
128
- namespace: 命名空间名称
129
- time_range: 时间范围
130
- """
131
- try:
132
- # TODO: 实现任务分布统计
133
- return {
134
- "success": True,
135
- "data": {
136
- "by_queue": {},
137
- "by_status": {},
138
- "by_worker": {},
139
- "by_hour": []
140
- }
141
- }
142
- except Exception as e:
143
- logger.error(f"获取任务分布失败: {e}")
144
- raise HTTPException(status_code=500, detail=str(e))
145
-
146
-
147
- # ============ 性能分析 ============
148
-
149
- @router.get("/performance-metrics/{namespace}")
150
- async def get_performance_metrics(
151
- namespace: str,
152
- time_range: str = Query("1h", description="时间范围")
153
- ):
154
- """
155
- 获取性能指标
156
-
157
- Args:
158
- namespace: 命名空间名称
159
- time_range: 时间范围
160
- """
161
- try:
162
- # TODO: 实现性能指标收集
163
- return {
164
- "success": True,
165
- "data": {
166
- "throughput": 0,
167
- "latency": {
168
- "p50": 0,
169
- "p95": 0,
170
- "p99": 0
171
- },
172
- "error_rate": 0,
173
- "queue_depth": 0
174
- }
175
- }
176
- except Exception as e:
177
- logger.error(f"获取性能指标失败: {e}")
178
- raise HTTPException(status_code=500, detail=str(e))
179
-
180
-
181
- @router.get("/bottlenecks/{namespace}")
182
- async def identify_bottlenecks(namespace: str):
183
- """
184
- 识别系统瓶颈
185
-
186
- Args:
187
- namespace: 命名空间名称
188
- """
189
- try:
190
- # TODO: 实现瓶颈分析
191
- return {
192
- "success": True,
193
- "data": {
194
- "slow_queues": [],
195
- "high_error_queues": [],
196
- "high_latency_tasks": [],
197
- "recommendations": []
198
- }
199
- }
200
- except Exception as e:
201
- logger.error(f"识别瓶颈失败: {e}")
202
- raise HTTPException(status_code=500, detail=str(e))
203
-
204
-
205
- # ============ 报表生成 ============
206
-
207
- @router.post("/reports/generate/{namespace}")
208
- async def generate_report(
209
- namespace: str,
210
- request: Request
211
- ):
212
- """
213
- 生成分析报表
214
-
215
- Args:
216
- namespace: 命名空间名称
217
- """
218
- try:
219
- body = await request.json()
220
- report_type = body.get("report_type", "daily")
221
- start_time = body.get("start_time")
222
- end_time = body.get("end_time")
223
-
224
- # TODO: 实现报表生成
225
- return {
226
- "success": True,
227
- "data": {
228
- "report_id": f"report_{namespace}_{datetime.now().strftime('%Y%m%d%H%M%S')}",
229
- "report_type": report_type,
230
- "status": "generated",
231
- "url": None
232
- }
233
- }
234
- except Exception as e:
235
- logger.error(f"生成报表失败: {e}")
236
- raise HTTPException(status_code=500, detail=str(e))
237
-
238
-
239
- @router.get("/reports/{report_id}")
240
- async def get_report(report_id: str):
241
- """
242
- 获取报表内容
243
-
244
- Args:
245
- report_id: 报表ID
246
- """
247
- try:
248
- # TODO: 实现报表获取
249
- return {
250
- "success": True,
251
- "data": {
252
- "report_id": report_id,
253
- "content": {},
254
- "generated_at": datetime.now().isoformat()
255
- }
256
- }
257
- except Exception as e:
258
- logger.error(f"获取报表失败: {e}")
259
- raise HTTPException(status_code=500, detail=str(e))
260
-
261
-
262
- # ============ 实时监控 ============
263
-
264
- @router.get("/realtime/{namespace}/stats")
265
- async def get_realtime_stats(namespace: str):
266
- """
267
- 获取实时统计数据
268
-
269
- Args:
270
- namespace: 命名空间名称
271
- """
272
- try:
273
- # TODO: 实现实时统计
274
- return {
275
- "success": True,
276
- "data": {
277
- "timestamp": datetime.now().isoformat(),
278
- "active_tasks": 0,
279
- "queued_tasks": 0,
280
- "completed_last_minute": 0,
281
- "failed_last_minute": 0,
282
- "avg_processing_time": 0.0
283
- }
284
- }
285
- except Exception as e:
286
- logger.error(f"获取实时统计失败: {e}")
287
- raise HTTPException(status_code=500, detail=str(e))
288
-
289
-
290
- # ============ 数据导出 ============
291
-
292
- @router.post("/export/{namespace}")
293
- async def export_data(
294
- namespace: str,
295
- request: Request
296
- ):
297
- """
298
- 导出分析数据
299
-
300
- Args:
301
- namespace: 命名空间名称
302
- """
303
- try:
304
- body = await request.json()
305
- export_format = body.get("format", "csv")
306
- data_type = body.get("data_type", "tasks")
307
-
308
- # TODO: 实现数据导出
309
- return {
310
- "success": True,
311
- "data": {
312
- "export_id": f"export_{namespace}_{datetime.now().strftime('%Y%m%d%H%M%S')}",
313
- "format": export_format,
314
- "status": "pending",
315
- "url": None
316
- }
317
- }
318
- except Exception as e:
319
- logger.error(f"导出数据失败: {e}")
320
- raise HTTPException(status_code=500, detail=str(e))
321
-
322
-
323
- __all__ = ['router']
jettask/webui/config.py DELETED
@@ -1,90 +0,0 @@
1
- import os
2
- from typing import Optional
3
- from dataclasses import dataclass
4
-
5
-
6
- @dataclass
7
- class PostgreSQLConfig:
8
- """PostgreSQL数据库配置"""
9
- host: str = "localhost"
10
- port: int = 5432
11
- database: str = "jettask"
12
- user: str = "jettask"
13
- password: str = "123456"
14
-
15
- @classmethod
16
- def from_env(cls) -> "PostgreSQLConfig":
17
- """从环境变量中读取配置"""
18
- return cls(
19
- host=os.getenv("JETTASK_PG_HOST", "localhost"),
20
- port=int(os.getenv("JETTASK_PG_PORT", "5432")),
21
- database=os.getenv("JETTASK_PG_DATABASE", "jettask"),
22
- user=os.getenv("JETTASK_PG_USER", "jettask"),
23
- password=os.getenv("JETTASK_PG_PASSWORD", "123456")
24
- )
25
-
26
- @classmethod
27
- def from_url(cls, url: str) -> "PostgreSQLConfig":
28
- """从数据库URL中解析配置
29
- 格式: postgresql://user:password@host:port/database
30
- """
31
- from urllib.parse import urlparse
32
-
33
- # 清理URL前缀(支持postgresql+asyncpg://)
34
- if url.startswith('postgresql+asyncpg://'):
35
- url = url.replace('postgresql+asyncpg://', 'postgresql://')
36
-
37
- parsed = urlparse(url)
38
-
39
- return cls(
40
- host=parsed.hostname or "localhost",
41
- port=parsed.port or 5432,
42
- database=parsed.path.lstrip("/") if parsed.path else "jettask",
43
- user=parsed.username or "jettask",
44
- password=parsed.password or "" # 如果没有密码返回空字符串
45
- )
46
-
47
- @property
48
- def dsn(self) -> str:
49
- """返回asyncpg连接字符串"""
50
- return f"postgresql://{self.user}:{self.password}@{self.host}:{self.port}/{self.database}"
51
-
52
- @property
53
- def psycopg_dsn(self) -> str:
54
- """返回psycopg连接字符串"""
55
- return f"host={self.host} port={self.port} dbname={self.database} user={self.user} password={self.password}"
56
-
57
-
58
- @dataclass
59
- class RedisConfig:
60
- """Redis配置"""
61
- host: str = "localhost"
62
- port: int = 6379
63
- db: int = 0
64
- password: Optional[str] = None
65
-
66
- @classmethod
67
- def from_env(cls) -> "RedisConfig":
68
- """从环境变量中读取配置"""
69
- return cls(
70
- host=os.getenv("JETTASK_REDIS_HOST", "localhost"),
71
- port=int(os.getenv("JETTASK_REDIS_PORT", "6379")),
72
- db=int(os.getenv("JETTASK_REDIS_DB", "0")),
73
- password=os.getenv("JETTASK_REDIS_PASSWORD")
74
- )
75
-
76
-
77
- @dataclass
78
- class WebUIConfig:
79
- """WebUI配置"""
80
- host: str = "0.0.0.0"
81
- port: int = 8000
82
- reload: bool = False
83
- postgres: PostgreSQLConfig = None
84
- redis: RedisConfig = None
85
-
86
- def __post_init__(self):
87
- if self.postgres is None:
88
- self.postgres = PostgreSQLConfig.from_env()
89
- if self.redis is None:
90
- self.redis = RedisConfig.from_env()
@@ -1,3 +0,0 @@
1
- """
2
- WebUI数据模型
3
- """
@@ -1,63 +0,0 @@
1
- """
2
- 任务中心数据模型
3
- """
4
- from dataclasses import dataclass
5
- from typing import Optional, Dict, Any
6
- from datetime import datetime
7
- import uuid
8
-
9
-
10
- @dataclass
11
- class Namespace:
12
- """命名空间模型"""
13
- id: int # 自增整数ID
14
- name: str # 唯一的命名空间名称
15
- connection_url: str # 专属连接URL
16
- redis_config: Dict[str, Any] # Redis配置
17
- pg_config: Dict[str, Any] # PostgreSQL配置
18
- created_at: datetime
19
- updated_at: datetime
20
- metadata: Optional[Dict[str, Any]] = None
21
-
22
- @classmethod
23
- def create(cls, name: str, redis_config: Dict, pg_config: Dict) -> 'Namespace':
24
- """创建新的命名空间"""
25
- # 使用名称作为URL,更直观
26
- connection_url = f"/api/namespaces/{name}"
27
- return cls(
28
- id=0, # 将由数据库自动分配
29
- name=name,
30
- connection_url=connection_url,
31
- redis_config=redis_config,
32
- pg_config=pg_config,
33
- created_at=datetime.utcnow(),
34
- updated_at=datetime.utcnow()
35
- )
36
-
37
-
38
- @dataclass
39
- class TaskCenterConfig:
40
- """任务中心配置"""
41
- task_center_url: Optional[str] = None # 任务中心连接URL
42
- namespace_id: Optional[str] = None # 从URL解析出的命名空间ID
43
- namespace_name: Optional[str] = None # 命名空间名称
44
-
45
- @property
46
- def is_enabled(self) -> bool:
47
- """是否启用任务中心"""
48
- return self.task_center_url is not None
49
-
50
- @classmethod
51
- def from_url(cls, url: str) -> 'TaskCenterConfig':
52
- """从连接URL创建配置"""
53
- if not url or not url.startswith("taskcenter://"):
54
- return cls()
55
-
56
- # 解析URL: taskcenter://namespace/{namespace_id}
57
- parts = url.replace("taskcenter://", "").split("/")
58
- if len(parts) >= 2 and parts[0] == "namespace":
59
- return cls(
60
- task_center_url=url,
61
- namespace_id=parts[1]
62
- )
63
- return cls(task_center_url=url)
@@ -1,10 +0,0 @@
1
- """
2
- 命名空间管理器模块
3
-
4
- 提供单命名空间和多命名空间的消费者管理
5
- """
6
-
7
- from .multi import NamespaceConsumerProcess
8
- from .unified import UnifiedConsumerManager
9
-
10
- __all__ = ['NamespaceConsumerProcess', 'UnifiedConsumerManager']