jettask 0.2.15__py3-none-any.whl → 0.2.17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jettask/__init__.py +14 -35
- jettask/{webui/__main__.py → __main__.py} +4 -4
- jettask/api/__init__.py +103 -0
- jettask/api/v1/__init__.py +29 -0
- jettask/api/v1/alerts.py +226 -0
- jettask/api/v1/analytics.py +323 -0
- jettask/api/v1/namespaces.py +134 -0
- jettask/api/v1/overview.py +136 -0
- jettask/api/v1/queues.py +530 -0
- jettask/api/v1/scheduled.py +420 -0
- jettask/api/v1/settings.py +44 -0
- jettask/{webui/api.py → api.py} +4 -46
- jettask/{webui/backend → backend}/main.py +21 -109
- jettask/{webui/backend → backend}/main_unified.py +1 -1
- jettask/{webui/backend → backend}/namespace_api_old.py +3 -30
- jettask/{webui/backend → backend}/namespace_data_access.py +2 -1
- jettask/{webui/backend → backend}/unified_api_router.py +14 -74
- jettask/{core/cli.py → cli.py} +106 -26
- jettask/config/nacos_config.py +386 -0
- jettask/core/app.py +8 -100
- jettask/core/db_manager.py +515 -0
- jettask/core/event_pool.py +5 -2
- jettask/core/unified_manager_base.py +59 -14
- jettask/{webui/db_init.py → db_init.py} +1 -1
- jettask/executors/asyncio.py +2 -2
- jettask/{webui/integrated_gradio_app.py → integrated_gradio_app.py} +1 -1
- jettask/{webui/multi_namespace_consumer.py → multi_namespace_consumer.py} +5 -2
- jettask/{webui/pg_consumer.py → pg_consumer.py} +137 -69
- jettask/{webui/run.py → run.py} +1 -1
- jettask/{webui/run_webui.py → run_webui.py} +4 -4
- jettask/scheduler/manager.py +6 -0
- jettask/scheduler/multi_namespace_scheduler.py +2 -2
- jettask/scheduler/unified_manager.py +5 -5
- jettask/scheduler/unified_scheduler_manager.py +20 -12
- jettask/schemas/__init__.py +166 -0
- jettask/schemas/alert.py +99 -0
- jettask/schemas/backlog.py +122 -0
- jettask/schemas/common.py +139 -0
- jettask/schemas/monitoring.py +181 -0
- jettask/schemas/namespace.py +168 -0
- jettask/schemas/queue.py +83 -0
- jettask/schemas/scheduled_task.py +128 -0
- jettask/schemas/task.py +70 -0
- jettask/services/__init__.py +24 -0
- jettask/services/alert_service.py +454 -0
- jettask/services/analytics_service.py +46 -0
- jettask/services/overview_service.py +978 -0
- jettask/services/queue_service.py +711 -0
- jettask/services/redis_monitor_service.py +151 -0
- jettask/services/scheduled_task_service.py +207 -0
- jettask/services/settings_service.py +758 -0
- jettask/services/task_service.py +157 -0
- jettask/{webui/task_center.py → task_center.py} +30 -8
- jettask/{webui/task_center_client.py → task_center_client.py} +1 -1
- jettask/{webui/config.py → webui_config.py} +6 -1
- jettask/webui_exceptions.py +67 -0
- jettask/webui_sql/verify_database.sql +72 -0
- {jettask-0.2.15.dist-info → jettask-0.2.17.dist-info}/METADATA +2 -1
- jettask-0.2.17.dist-info/RECORD +150 -0
- {jettask-0.2.15.dist-info → jettask-0.2.17.dist-info}/entry_points.txt +1 -1
- jettask/webui/backend/data_api.py +0 -3294
- jettask/webui/backend/namespace_api.py +0 -295
- jettask/webui/backend/queue_backlog_api.py +0 -727
- jettask/webui/backend/redis_monitor_api.py +0 -476
- jettask/webui/frontend/index.html +0 -13
- jettask/webui/frontend/package.json +0 -30
- jettask/webui/frontend/src/App.css +0 -109
- jettask/webui/frontend/src/App.jsx +0 -66
- jettask/webui/frontend/src/components/NamespaceSelector.jsx +0 -166
- jettask/webui/frontend/src/components/QueueBacklogChart.jsx +0 -298
- jettask/webui/frontend/src/components/QueueBacklogTrend.jsx +0 -638
- jettask/webui/frontend/src/components/QueueDetailsTable.css +0 -65
- jettask/webui/frontend/src/components/QueueDetailsTable.jsx +0 -487
- jettask/webui/frontend/src/components/QueueDetailsTableV2.jsx +0 -465
- jettask/webui/frontend/src/components/ScheduledTaskFilter.jsx +0 -423
- jettask/webui/frontend/src/components/TaskFilter.jsx +0 -425
- jettask/webui/frontend/src/components/TimeRangeSelector.css +0 -21
- jettask/webui/frontend/src/components/TimeRangeSelector.jsx +0 -160
- jettask/webui/frontend/src/components/charts/QueueChart.jsx +0 -111
- jettask/webui/frontend/src/components/charts/QueueTrendChart.jsx +0 -115
- jettask/webui/frontend/src/components/charts/WorkerChart.jsx +0 -40
- jettask/webui/frontend/src/components/common/StatsCard.jsx +0 -18
- jettask/webui/frontend/src/components/layout/AppLayout.css +0 -95
- jettask/webui/frontend/src/components/layout/AppLayout.jsx +0 -49
- jettask/webui/frontend/src/components/layout/Header.css +0 -106
- jettask/webui/frontend/src/components/layout/Header.jsx +0 -106
- jettask/webui/frontend/src/components/layout/SideMenu.css +0 -137
- jettask/webui/frontend/src/components/layout/SideMenu.jsx +0 -209
- jettask/webui/frontend/src/components/layout/TabsNav.css +0 -244
- jettask/webui/frontend/src/components/layout/TabsNav.jsx +0 -206
- jettask/webui/frontend/src/components/layout/UserInfo.css +0 -197
- jettask/webui/frontend/src/components/layout/UserInfo.jsx +0 -197
- jettask/webui/frontend/src/contexts/LoadingContext.jsx +0 -27
- jettask/webui/frontend/src/contexts/NamespaceContext.jsx +0 -72
- jettask/webui/frontend/src/contexts/TabsContext.backup.jsx +0 -245
- jettask/webui/frontend/src/index.css +0 -114
- jettask/webui/frontend/src/main.jsx +0 -22
- jettask/webui/frontend/src/pages/Alerts.jsx +0 -684
- jettask/webui/frontend/src/pages/Dashboard/index.css +0 -35
- jettask/webui/frontend/src/pages/Dashboard/index.jsx +0 -281
- jettask/webui/frontend/src/pages/Dashboard.jsx +0 -1330
- jettask/webui/frontend/src/pages/QueueDetail.jsx +0 -1117
- jettask/webui/frontend/src/pages/QueueMonitor.jsx +0 -527
- jettask/webui/frontend/src/pages/Queues.jsx +0 -12
- jettask/webui/frontend/src/pages/ScheduledTasks.jsx +0 -810
- jettask/webui/frontend/src/pages/Settings.jsx +0 -801
- jettask/webui/frontend/src/pages/Workers.jsx +0 -12
- jettask/webui/frontend/src/services/api.js +0 -159
- jettask/webui/frontend/src/services/queueTrend.js +0 -166
- jettask/webui/frontend/src/utils/suppressWarnings.js +0 -22
- jettask/webui/frontend/src/utils/userPreferences.js +0 -154
- jettask/webui/frontend/vite.config.js +0 -26
- jettask/webui/sql/init_database.sql +0 -640
- jettask-0.2.15.dist-info/RECORD +0 -172
- /jettask/{webui/backend → backend}/__init__.py +0 -0
- /jettask/{webui/backend → backend}/api/__init__.py +0 -0
- /jettask/{webui/backend → backend}/api/v1/__init__.py +0 -0
- /jettask/{webui/backend → backend}/api/v1/monitoring.py +0 -0
- /jettask/{webui/backend → backend}/api/v1/namespaces.py +0 -0
- /jettask/{webui/backend → backend}/api/v1/queues.py +0 -0
- /jettask/{webui/backend → backend}/api/v1/tasks.py +0 -0
- /jettask/{webui/backend → backend}/config.py +0 -0
- /jettask/{webui/backend → backend}/core/__init__.py +0 -0
- /jettask/{webui/backend → backend}/core/cache.py +0 -0
- /jettask/{webui/backend → backend}/core/database.py +0 -0
- /jettask/{webui/backend → backend}/core/exceptions.py +0 -0
- /jettask/{webui/backend → backend}/data_access.py +0 -0
- /jettask/{webui/backend → backend}/dependencies.py +0 -0
- /jettask/{webui/backend → backend}/init_meta_db.py +0 -0
- /jettask/{webui/backend → backend}/main_v2.py +0 -0
- /jettask/{webui/backend → backend}/models/__init__.py +0 -0
- /jettask/{webui/backend → backend}/models/requests.py +0 -0
- /jettask/{webui/backend → backend}/models/responses.py +0 -0
- /jettask/{webui/backend → backend}/queue_stats_v2.py +0 -0
- /jettask/{webui/backend → backend}/services/__init__.py +0 -0
- /jettask/{webui/backend → backend}/start.py +0 -0
- /jettask/{webui/cleanup_deprecated_tables.sql → cleanup_deprecated_tables.sql} +0 -0
- /jettask/{webui/gradio_app.py → gradio_app.py} +0 -0
- /jettask/{webui/__init__.py → main.py} +0 -0
- /jettask/{webui/models.py → models.py} +0 -0
- /jettask/{webui/run_monitor.py → run_monitor.py} +0 -0
- /jettask/{webui/schema.sql → schema.sql} +0 -0
- /jettask/{webui/unified_consumer_manager.py → unified_consumer_manager.py} +0 -0
- /jettask/{webui/models → webui_models}/__init__.py +0 -0
- /jettask/{webui/models → webui_models}/namespace.py +0 -0
- /jettask/{webui/sql → webui_sql}/batch_upsert_functions.sql +0 -0
- {jettask-0.2.15.dist-info → jettask-0.2.17.dist-info}/WHEEL +0 -0
- {jettask-0.2.15.dist-info → jettask-0.2.17.dist-info}/licenses/LICENSE +0 -0
- {jettask-0.2.15.dist-info → jettask-0.2.17.dist-info}/top_level.txt +0 -0
|
@@ -12,15 +12,19 @@ JetTask Monitor 独立后端API服务
|
|
|
12
12
|
- PostgreSQL:任务结果存储
|
|
13
13
|
- 由JetTask worker使用
|
|
14
14
|
"""
|
|
15
|
-
from fastapi import
|
|
16
|
-
from fastapi.middleware.cors import CORSMiddleware
|
|
17
|
-
from contextlib import asynccontextmanager
|
|
15
|
+
from fastapi import HTTPException, Request
|
|
18
16
|
from datetime import datetime, timedelta, timezone
|
|
19
17
|
from typing import List, Dict, Optional
|
|
20
|
-
from pydantic import BaseModel
|
|
21
18
|
from sqlalchemy import text
|
|
22
19
|
import logging
|
|
23
20
|
import traceback
|
|
21
|
+
from jettask.schemas import (
|
|
22
|
+
TimeRangeQuery,
|
|
23
|
+
QueueTimelineResponse,
|
|
24
|
+
TrimQueueRequest,
|
|
25
|
+
ScheduledTaskRequest,
|
|
26
|
+
AlertRuleRequest
|
|
27
|
+
)
|
|
24
28
|
|
|
25
29
|
import sys
|
|
26
30
|
import os
|
|
@@ -28,88 +32,26 @@ import os
|
|
|
28
32
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
29
33
|
|
|
30
34
|
from data_access import JetTaskDataAccess
|
|
31
|
-
from namespace_data_access import get_namespace_data_access
|
|
32
|
-
from
|
|
33
|
-
from namespace_api import router as namespace_router
|
|
34
|
-
from queue_stats_v2 import QueueStatsV2
|
|
35
|
-
from queue_backlog_api import router as backlog_router
|
|
36
|
-
from redis_monitor_api import router as redis_monitor_router
|
|
35
|
+
from jettask.backend.namespace_data_access import get_namespace_data_access
|
|
36
|
+
from jettask.backend.queue_stats_v2 import QueueStatsV2
|
|
37
37
|
|
|
38
38
|
# 设置日志
|
|
39
39
|
logging.basicConfig(level=logging.INFO)
|
|
40
40
|
logger = logging.getLogger(__name__)
|
|
41
41
|
|
|
42
|
-
#
|
|
42
|
+
# 从api包导入已配置的app
|
|
43
|
+
from jettask.api import app
|
|
44
|
+
|
|
45
|
+
# 创建全局数据访问实例供路由使用
|
|
43
46
|
data_access = JetTaskDataAccess()
|
|
44
|
-
# 新的命名空间数据访问
|
|
45
47
|
namespace_data_access = get_namespace_data_access()
|
|
46
48
|
|
|
49
|
+
# 将数据访问实例注入到app.state
|
|
50
|
+
app.state.data_access = data_access
|
|
51
|
+
app.state.namespace_data_access = namespace_data_access
|
|
47
52
|
|
|
48
|
-
@asynccontextmanager
|
|
49
|
-
async def lifespan(app: FastAPI):
|
|
50
|
-
"""应用生命周期管理"""
|
|
51
|
-
# 启动时初始化
|
|
52
|
-
try:
|
|
53
|
-
# 1. 初始化JetTask数据访问(用于查询任务数据)
|
|
54
|
-
await data_access.initialize()
|
|
55
|
-
|
|
56
|
-
# 2. 记录任务中心配置
|
|
57
|
-
from jettask.webui.backend.config import task_center_config
|
|
58
|
-
logger.info("=" * 60)
|
|
59
|
-
logger.info("任务中心配置:")
|
|
60
|
-
logger.info(f" 元数据库: {task_center_config.meta_db_host}:{task_center_config.meta_db_port}/{task_center_config.meta_db_name}")
|
|
61
|
-
logger.info(f" API服务: {task_center_config.api_host}:{task_center_config.api_port}")
|
|
62
|
-
logger.info(f" 基础URL: {task_center_config.base_url}")
|
|
63
|
-
logger.info("=" * 60)
|
|
64
|
-
|
|
65
|
-
logger.info("JetTask Monitor API 启动成功")
|
|
66
|
-
except Exception as e:
|
|
67
|
-
logger.error(f"启动失败: {e}")
|
|
68
|
-
traceback.print_exc()
|
|
69
|
-
raise
|
|
70
|
-
|
|
71
|
-
yield
|
|
72
|
-
|
|
73
|
-
# 关闭时清理资源
|
|
74
|
-
try:
|
|
75
|
-
await data_access.close()
|
|
76
|
-
logger.info("JetTask Monitor API 关闭完成")
|
|
77
|
-
except Exception as e:
|
|
78
|
-
logger.error(f"关闭时出错: {e}")
|
|
79
|
-
traceback.print_exc()
|
|
80
53
|
|
|
81
|
-
|
|
82
|
-
app = FastAPI(
|
|
83
|
-
title="JetTask Monitor API",
|
|
84
|
-
version="1.0.0",
|
|
85
|
-
lifespan=lifespan
|
|
86
|
-
)
|
|
87
|
-
|
|
88
|
-
# 配置CORS
|
|
89
|
-
app.add_middleware(
|
|
90
|
-
CORSMiddleware,
|
|
91
|
-
allow_origins=["*"], # 生产环境应该配置具体的前端地址
|
|
92
|
-
allow_credentials=True,
|
|
93
|
-
allow_methods=["*"],
|
|
94
|
-
allow_headers=["*"],
|
|
95
|
-
)
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
class TimeRangeQuery(BaseModel):
|
|
99
|
-
start_time: Optional[datetime] = None
|
|
100
|
-
end_time: Optional[datetime] = None
|
|
101
|
-
time_range: Optional[str] = "15m" # 默认15分钟
|
|
102
|
-
queues: Optional[List[str]] = None
|
|
103
|
-
filters: Optional[List[Dict]] = None # 筛选条件
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
class QueueTimelineResponse(BaseModel):
|
|
107
|
-
data: List[Dict]
|
|
108
|
-
granularity: str
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
class TrimQueueRequest(BaseModel):
|
|
112
|
-
max_length: int
|
|
54
|
+
# 模型已从 schemas 模块导入
|
|
113
55
|
|
|
114
56
|
|
|
115
57
|
@app.get("/")
|
|
@@ -440,18 +382,7 @@ async def health_check():
|
|
|
440
382
|
|
|
441
383
|
# ============= 定时任务管理API =============
|
|
442
384
|
|
|
443
|
-
|
|
444
|
-
"""定时任务请求模型"""
|
|
445
|
-
namespace: str # 命名空间
|
|
446
|
-
name: str
|
|
447
|
-
queue_name: str
|
|
448
|
-
task_data: dict
|
|
449
|
-
schedule_type: str # cron, interval, daily, weekly, monthly
|
|
450
|
-
schedule_config: dict # 根据schedule_type的不同配置
|
|
451
|
-
is_active: bool = True
|
|
452
|
-
description: Optional[str] = None
|
|
453
|
-
max_retry: int = 3
|
|
454
|
-
timeout: Optional[int] = None
|
|
385
|
+
# ScheduledTaskRequest 已从 schemas 模块导入
|
|
455
386
|
|
|
456
387
|
|
|
457
388
|
@app.get("/api/scheduled-tasks")
|
|
@@ -834,16 +765,7 @@ async def get_scheduled_task_execution_trend(
|
|
|
834
765
|
raise HTTPException(status_code=500, detail=str(e))
|
|
835
766
|
|
|
836
767
|
|
|
837
|
-
#
|
|
838
|
-
|
|
839
|
-
# 导入命名空间API路由
|
|
840
|
-
from jettask.webui.backend.namespace_api import router as namespace_router
|
|
841
|
-
|
|
842
|
-
# 注册命名空间路由
|
|
843
|
-
app.include_router(namespace_router)
|
|
844
|
-
app.include_router(data_router)
|
|
845
|
-
app.include_router(backlog_router)
|
|
846
|
-
app.include_router(redis_monitor_router)
|
|
768
|
+
# 路由已在 api/__init__.py 中注册
|
|
847
769
|
|
|
848
770
|
|
|
849
771
|
|
|
@@ -853,17 +775,7 @@ app.include_router(redis_monitor_router)
|
|
|
853
775
|
|
|
854
776
|
# ============= 告警管理API =============
|
|
855
777
|
|
|
856
|
-
|
|
857
|
-
"""告警规则请求模型"""
|
|
858
|
-
name: str
|
|
859
|
-
rule_type: str # queue_size, error_rate, response_time, custom
|
|
860
|
-
target_queues: List[str] # 监控的队列列表
|
|
861
|
-
condition: dict # 触发条件配置
|
|
862
|
-
action_type: str # webhook, email, sms
|
|
863
|
-
action_config: dict # 通知配置
|
|
864
|
-
is_active: bool = True
|
|
865
|
-
description: Optional[str] = None
|
|
866
|
-
check_interval: int = 60 # 检查间隔(秒)
|
|
778
|
+
# AlertRuleRequest 已从 schemas 模块导入
|
|
867
779
|
|
|
868
780
|
|
|
869
781
|
@app.get("/api/alert-rules")
|
|
@@ -14,7 +14,7 @@ try:
|
|
|
14
14
|
from unified_api_router import router as api_router
|
|
15
15
|
except ImportError:
|
|
16
16
|
# 如果相对导入失败,尝试绝对导入
|
|
17
|
-
from jettask.
|
|
17
|
+
from jettask.backend.unified_api_router import router as api_router
|
|
18
18
|
|
|
19
19
|
# 设置日志
|
|
20
20
|
logging.basicConfig(
|
|
@@ -3,12 +3,12 @@
|
|
|
3
3
|
使用数据库持久化命名空间数据
|
|
4
4
|
"""
|
|
5
5
|
from fastapi import APIRouter, HTTPException, Query
|
|
6
|
-
from pydantic import BaseModel
|
|
7
6
|
from typing import Dict, Any, Optional, List
|
|
8
7
|
from datetime import datetime
|
|
9
8
|
import json
|
|
10
9
|
from sqlalchemy import text
|
|
11
10
|
import logging
|
|
11
|
+
from jettask.schemas import NamespaceCreate, NamespaceUpdate, NamespaceResponse
|
|
12
12
|
|
|
13
13
|
logger = logging.getLogger(__name__)
|
|
14
14
|
|
|
@@ -17,7 +17,7 @@ router = APIRouter(prefix="/api/namespaces", tags=["namespaces"])
|
|
|
17
17
|
# 使用任务中心专用的元数据库连接
|
|
18
18
|
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
|
19
19
|
from sqlalchemy.orm import sessionmaker
|
|
20
|
-
from jettask.
|
|
20
|
+
from jettask.backend.config import task_center_config
|
|
21
21
|
|
|
22
22
|
# 创建异步引擎 - 连接到任务中心元数据库
|
|
23
23
|
# 注意:这是任务中心自己的数据库,不是JetTask应用的数据库
|
|
@@ -25,34 +25,7 @@ engine = create_async_engine(task_center_config.meta_database_url, echo=False)
|
|
|
25
25
|
AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
26
26
|
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
"""创建命名空间请求"""
|
|
30
|
-
name: str
|
|
31
|
-
description: Optional[str] = None
|
|
32
|
-
redis_config: Dict[str, Any]
|
|
33
|
-
pg_config: Dict[str, Any]
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
class NamespaceUpdate(BaseModel):
|
|
37
|
-
"""更新命名空间请求"""
|
|
38
|
-
description: Optional[str] = None
|
|
39
|
-
redis_config: Optional[Dict[str, Any]] = None
|
|
40
|
-
pg_config: Optional[Dict[str, Any]] = None
|
|
41
|
-
is_active: Optional[bool] = None
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
class NamespaceResponse(BaseModel):
|
|
45
|
-
"""命名空间响应"""
|
|
46
|
-
id: int # 改为整数类型
|
|
47
|
-
name: str
|
|
48
|
-
description: Optional[str]
|
|
49
|
-
redis_config: Dict[str, Any]
|
|
50
|
-
pg_config: Dict[str, Any]
|
|
51
|
-
is_active: bool
|
|
52
|
-
version: int # 添加版本号
|
|
53
|
-
created_at: datetime
|
|
54
|
-
updated_at: datetime
|
|
55
|
-
connection_url: str
|
|
28
|
+
# 模型已从 schemas 模块导入
|
|
56
29
|
|
|
57
30
|
|
|
58
31
|
@router.get("", response_model=List[NamespaceResponse])
|
|
@@ -46,6 +46,7 @@ class NamespaceConnection:
|
|
|
46
46
|
if dsn.startswith('postgresql://'):
|
|
47
47
|
dsn = dsn.replace('postgresql://', 'postgresql+psycopg://', 1)
|
|
48
48
|
|
|
49
|
+
print(f'{dsn=}')
|
|
49
50
|
self.async_engine = create_async_engine(
|
|
50
51
|
dsn,
|
|
51
52
|
pool_size=10,
|
|
@@ -185,7 +186,7 @@ class NamespaceDataAccessManager:
|
|
|
185
186
|
base_url = self.task_center_base_url
|
|
186
187
|
if 'localhost' in base_url:
|
|
187
188
|
base_url = base_url.replace('localhost', '127.0.0.1')
|
|
188
|
-
url = f"{base_url}/api/namespaces/{namespace_name}"
|
|
189
|
+
url = f"{base_url}/api/v1/namespaces/{namespace_name}"
|
|
189
190
|
|
|
190
191
|
try:
|
|
191
192
|
session = await self._get_session()
|
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
|
|
6
6
|
from fastapi import APIRouter, HTTPException, Query, Request, Depends
|
|
7
7
|
from typing import List, Dict, Optional, Any, Union
|
|
8
|
-
from pydantic import BaseModel, Field
|
|
9
8
|
from datetime import datetime, timedelta, timezone
|
|
10
9
|
import logging
|
|
11
10
|
import time
|
|
@@ -16,6 +15,17 @@ from sqlalchemy import text
|
|
|
16
15
|
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
|
17
16
|
from sqlalchemy.orm import sessionmaker
|
|
18
17
|
import traceback
|
|
18
|
+
from jettask.schemas import (
|
|
19
|
+
TimeRangeQuery,
|
|
20
|
+
QueueStatsResponse,
|
|
21
|
+
TaskDetailResponse,
|
|
22
|
+
DashboardOverviewRequest,
|
|
23
|
+
ScheduledTaskCreate,
|
|
24
|
+
ScheduledTaskUpdate,
|
|
25
|
+
AlertRuleCreate,
|
|
26
|
+
NamespaceCreate,
|
|
27
|
+
NamespaceUpdate
|
|
28
|
+
)
|
|
19
29
|
|
|
20
30
|
# 导入本地模块
|
|
21
31
|
try:
|
|
@@ -23,8 +33,8 @@ try:
|
|
|
23
33
|
from config import task_center_config
|
|
24
34
|
except ImportError:
|
|
25
35
|
# 如果相对导入失败,尝试绝对导入
|
|
26
|
-
from jettask.
|
|
27
|
-
from jettask.
|
|
36
|
+
from jettask.backend.namespace_data_access import get_namespace_data_access
|
|
37
|
+
from jettask.backend.config import task_center_config
|
|
28
38
|
|
|
29
39
|
# 设置日志
|
|
30
40
|
logger = logging.getLogger(__name__)
|
|
@@ -32,77 +42,7 @@ logger = logging.getLogger(__name__)
|
|
|
32
42
|
# 创建统一的路由器
|
|
33
43
|
router = APIRouter(prefix="/api", tags=["API"])
|
|
34
44
|
|
|
35
|
-
# ====================
|
|
36
|
-
|
|
37
|
-
class TimeRangeQuery(BaseModel):
|
|
38
|
-
"""时间范围查询参数"""
|
|
39
|
-
namespace: str = "default"
|
|
40
|
-
queue_name: Optional[str] = None
|
|
41
|
-
time_range: str = "1h"
|
|
42
|
-
granularity: Optional[str] = None
|
|
43
|
-
|
|
44
|
-
class QueueStatsResponse(BaseModel):
|
|
45
|
-
"""队列统计响应"""
|
|
46
|
-
queue_name: str
|
|
47
|
-
pending: int
|
|
48
|
-
running: int
|
|
49
|
-
completed: int
|
|
50
|
-
failed: int
|
|
51
|
-
|
|
52
|
-
class TaskDetailResponse(BaseModel):
|
|
53
|
-
"""任务详情响应"""
|
|
54
|
-
task_id: str
|
|
55
|
-
status: str
|
|
56
|
-
created_at: datetime
|
|
57
|
-
started_at: Optional[datetime]
|
|
58
|
-
completed_at: Optional[datetime]
|
|
59
|
-
error_message: Optional[str]
|
|
60
|
-
|
|
61
|
-
class DashboardOverviewRequest(BaseModel):
|
|
62
|
-
"""仪表板概览请求参数"""
|
|
63
|
-
time_range: str = "1h"
|
|
64
|
-
queues: Optional[List[str]] = None
|
|
65
|
-
|
|
66
|
-
class ScheduledTaskCreate(BaseModel):
|
|
67
|
-
"""创建定时任务的请求模型"""
|
|
68
|
-
task_name: str
|
|
69
|
-
queue_name: str
|
|
70
|
-
task_data: dict
|
|
71
|
-
cron_expression: Optional[str] = None
|
|
72
|
-
interval_seconds: Optional[int] = None
|
|
73
|
-
enabled: bool = True
|
|
74
|
-
|
|
75
|
-
class ScheduledTaskUpdate(BaseModel):
|
|
76
|
-
"""更新定时任务的请求模型"""
|
|
77
|
-
task_name: Optional[str] = None
|
|
78
|
-
queue_name: Optional[str] = None
|
|
79
|
-
task_data: Optional[dict] = None
|
|
80
|
-
cron_expression: Optional[str] = None
|
|
81
|
-
interval_seconds: Optional[int] = None
|
|
82
|
-
enabled: Optional[bool] = None
|
|
83
|
-
|
|
84
|
-
class AlertRuleCreate(BaseModel):
|
|
85
|
-
"""创建告警规则的请求模型"""
|
|
86
|
-
name: str
|
|
87
|
-
metric: str
|
|
88
|
-
threshold: float
|
|
89
|
-
condition: str # gt, lt, eq
|
|
90
|
-
duration: int # 持续时间(秒)
|
|
91
|
-
notification_channels: List[str]
|
|
92
|
-
enabled: bool = True
|
|
93
|
-
|
|
94
|
-
class NamespaceCreate(BaseModel):
|
|
95
|
-
"""创建命名空间的请求模型"""
|
|
96
|
-
name: str
|
|
97
|
-
redis_url: str
|
|
98
|
-
pg_url: Optional[str] = None
|
|
99
|
-
description: Optional[str] = None
|
|
100
|
-
|
|
101
|
-
class NamespaceUpdate(BaseModel):
|
|
102
|
-
"""更新命名空间的请求模型"""
|
|
103
|
-
redis_url: Optional[str] = None
|
|
104
|
-
pg_url: Optional[str] = None
|
|
105
|
-
description: Optional[str] = None
|
|
45
|
+
# ==================== 数据模型从 schemas 模块导入 ====================
|
|
106
46
|
|
|
107
47
|
# ==================== 辅助函数 ====================
|
|
108
48
|
|
jettask/{core/cli.py → cli.py}
RENAMED
|
@@ -9,16 +9,17 @@ import importlib
|
|
|
9
9
|
import importlib.util
|
|
10
10
|
import json
|
|
11
11
|
from pathlib import Path
|
|
12
|
+
from dotenv import load_dotenv
|
|
12
13
|
|
|
13
|
-
#
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
except ImportError:
|
|
17
|
-
# 如果相对导入失败,添加父目录到路径并尝试绝对导入
|
|
18
|
-
parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
14
|
+
# 处理直接运行时的路径问题
|
|
15
|
+
if __name__ == '__main__':
|
|
16
|
+
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
19
17
|
if parent_dir not in sys.path:
|
|
20
18
|
sys.path.insert(0, parent_dir)
|
|
21
|
-
|
|
19
|
+
|
|
20
|
+
from jettask.config.nacos_config import config
|
|
21
|
+
from jettask.core.app_importer import import_app, AppImporter
|
|
22
|
+
|
|
22
23
|
|
|
23
24
|
@click.group()
|
|
24
25
|
@click.version_option(version='0.1.0', prog_name='JetTask')
|
|
@@ -30,13 +31,28 @@ def cli():
|
|
|
30
31
|
@click.option('--host', default='0.0.0.0', help='服务器监听地址')
|
|
31
32
|
@click.option('--port', default=8001, type=int, help='服务器监听端口')
|
|
32
33
|
@click.option('--jettask-pg-url', envvar='JETTASK_PG_URL',
|
|
33
|
-
default='postgresql://jettask:123456@localhost:5432/jettask',
|
|
34
34
|
help='PostgreSQL连接URL')
|
|
35
|
+
@click.option('--use-nacos', is_flag=True, default=False,
|
|
36
|
+
help='从Nacos配置中心读取配置')
|
|
37
|
+
@click.option('--nacos-server', envvar='NACOS_SERVER',
|
|
38
|
+
help='Nacos服务器地址')
|
|
39
|
+
@click.option('--nacos-namespace', envvar='NACOS_NAMESPACE',
|
|
40
|
+
help='Nacos命名空间')
|
|
41
|
+
@click.option('--nacos-username', envvar='NACOS_USERNAME',
|
|
42
|
+
help='Nacos用户名')
|
|
43
|
+
@click.option('--nacos-password', envvar='NACOS_PASSWORD',
|
|
44
|
+
help='Nacos密码')
|
|
45
|
+
@click.option('--nacos-group', envvar='NACOS_GROUP',
|
|
46
|
+
help='Nacos配置组')
|
|
47
|
+
@click.option('--nacos-data-id', envvar='NACOS_DATA_ID',
|
|
48
|
+
help='Nacos配置ID')
|
|
35
49
|
@click.option('--reload', is_flag=True, default=False, help='启用自动重载')
|
|
36
50
|
@click.option('--log-level', default='info',
|
|
37
51
|
type=click.Choice(['debug', 'info', 'warning', 'error']),
|
|
38
52
|
help='日志级别')
|
|
39
|
-
def api(host, port, jettask_pg_url,
|
|
53
|
+
def api(host, port, jettask_pg_url, use_nacos, nacos_server, nacos_namespace,
|
|
54
|
+
nacos_username, nacos_password, nacos_group, nacos_data_id,
|
|
55
|
+
reload, log_level):
|
|
40
56
|
"""启动 API 服务和监控界面
|
|
41
57
|
|
|
42
58
|
示例:
|
|
@@ -47,7 +63,15 @@ def api(host, port, jettask_pg_url, reload, log_level):
|
|
|
47
63
|
# 指定端口和数据库
|
|
48
64
|
jettask api --host 0.0.0.0 --port 8080 --jettask-pg-url postgresql://user:pass@host/db
|
|
49
65
|
|
|
50
|
-
#
|
|
66
|
+
# 使用Nacos配置中心
|
|
67
|
+
jettask api --use-nacos --nacos-server 127.0.0.1:8848
|
|
68
|
+
|
|
69
|
+
# 通过环境变量配置Nacos
|
|
70
|
+
export NACOS_SERVER=127.0.0.1:8848
|
|
71
|
+
export NACOS_NAMESPACE=tamar_console_dev
|
|
72
|
+
jettask api --use-nacos
|
|
73
|
+
|
|
74
|
+
# 通过环境变量配置数据库
|
|
51
75
|
export JETTASK_PG_URL=postgresql://jettask:123456@localhost:5432/jettask
|
|
52
76
|
jettask api
|
|
53
77
|
|
|
@@ -57,12 +81,63 @@ def api(host, port, jettask_pg_url, reload, log_level):
|
|
|
57
81
|
import os
|
|
58
82
|
import uvicorn
|
|
59
83
|
|
|
84
|
+
print(f'{use_nacos=} {jettask_pg_url=}')
|
|
85
|
+
if not use_nacos and not jettask_pg_url:
|
|
86
|
+
raise ValueError("必须提供 --jettask-pg-url 或启用 --use-nacos")
|
|
87
|
+
# 如果使用Nacos,从Nacos获取配置
|
|
88
|
+
if use_nacos:
|
|
89
|
+
click.echo("正在从Nacos加载配置...")
|
|
90
|
+
load_dotenv()
|
|
91
|
+
# 设置Nacos环境变量
|
|
92
|
+
if nacos_server:
|
|
93
|
+
os.environ['NACOS_SERVER'] = nacos_server
|
|
94
|
+
if nacos_namespace:
|
|
95
|
+
os.environ['NACOS_NAMESPACE'] = nacos_namespace
|
|
96
|
+
if nacos_username:
|
|
97
|
+
os.environ['NACOS_USERNAME'] = nacos_username
|
|
98
|
+
if nacos_password:
|
|
99
|
+
os.environ['NACOS_PASSWORD'] = nacos_password
|
|
100
|
+
if nacos_group:
|
|
101
|
+
os.environ['NACOS_GROUP'] = nacos_group
|
|
102
|
+
if nacos_data_id:
|
|
103
|
+
os.environ['NACOS_DATA_ID'] = nacos_data_id
|
|
104
|
+
|
|
105
|
+
nacos_server = os.environ['NACOS_SERVER']
|
|
106
|
+
nacos_namespace = os.environ['NACOS_NAMESPACE']
|
|
107
|
+
nacos_username = os.environ['NACOS_USERNAME']
|
|
108
|
+
nacos_password = os.environ['NACOS_PASSWORD']
|
|
109
|
+
nacos_group = os.environ['NACOS_GROUP']
|
|
110
|
+
nacos_data_id = os.environ['NACOS_DATA_ID']
|
|
111
|
+
# 尝试从配置中获取JETTASK_PG_URL
|
|
112
|
+
nacos_pg_url = config.config.get('JETTASK_PG_URL')
|
|
113
|
+
print(f'{nacos_pg_url=}')
|
|
114
|
+
if not nacos_pg_url:
|
|
115
|
+
# 如果没有直接的URL,尝试从独立的配置项构建
|
|
116
|
+
pg_host = config.config.get('PG_DB_HOST', 'localhost')
|
|
117
|
+
pg_port = config.config.get('PG_DB_PORT', 5432)
|
|
118
|
+
pg_user = config.config.get('PG_DB_USERNAME', 'jettask')
|
|
119
|
+
pg_password = config.config.get('PG_DB_PASSWORD', '123456')
|
|
120
|
+
pg_database = config.config.get('PG_DB_DATABASE', 'jettask')
|
|
121
|
+
nacos_pg_url = f'postgresql://{pg_user}:{pg_password}@{pg_host}:{pg_port}/{pg_database}'
|
|
122
|
+
|
|
123
|
+
jettask_pg_url = nacos_pg_url
|
|
124
|
+
click.echo(f"✓ 从Nacos加载配置成功")
|
|
125
|
+
click.echo(f" 配置源: {nacos_server}/{nacos_namespace}/{nacos_group}/{nacos_data_id}")
|
|
126
|
+
|
|
127
|
+
# 将其他Nacos配置也设置到环境变量中
|
|
128
|
+
for key, value in config.config.items():
|
|
129
|
+
if isinstance(value, (str, int, float, bool)):
|
|
130
|
+
os.environ[key] = str(value)
|
|
131
|
+
|
|
132
|
+
|
|
60
133
|
# 设置环境变量(供应用内部使用)
|
|
61
134
|
os.environ['JETTASK_PG_URL'] = jettask_pg_url
|
|
62
|
-
|
|
135
|
+
|
|
136
|
+
# 设置是否使用Nacos的标志
|
|
137
|
+
os.environ['USE_NACOS'] = 'true' if use_nacos else 'false'
|
|
63
138
|
|
|
64
139
|
# 使用标准应用模块
|
|
65
|
-
app_module = "jettask.
|
|
140
|
+
app_module = "jettask.api:app"
|
|
66
141
|
click.echo(f"Starting JetTask API Server on {host}:{port}")
|
|
67
142
|
|
|
68
143
|
# 显示配置信息
|
|
@@ -73,6 +148,11 @@ def api(host, port, jettask_pg_url, reload, log_level):
|
|
|
73
148
|
click.echo(f"Port: {port}")
|
|
74
149
|
click.echo(f"Auto-reload: {reload}")
|
|
75
150
|
click.echo(f"Log level: {log_level}")
|
|
151
|
+
click.echo(f"Nacos: {'Enabled' if use_nacos else 'Disabled'}")
|
|
152
|
+
if use_nacos:
|
|
153
|
+
click.echo(f" Server: {nacos_server}")
|
|
154
|
+
click.echo(f" Namespace: {nacos_namespace}")
|
|
155
|
+
click.echo(f" Data ID: {nacos_data_id}")
|
|
76
156
|
click.echo(f"Database: {jettask_pg_url}")
|
|
77
157
|
click.echo("=" * 60)
|
|
78
158
|
click.echo(f"API Endpoint: http://{host}:{port}/api")
|
|
@@ -284,7 +364,7 @@ def worker(app_str, queues, executor, concurrency, prefetch, reload, config):
|
|
|
284
364
|
|
|
285
365
|
@cli.command('webui-consumer')
|
|
286
366
|
@click.option('--task-center', '-tc', envvar='JETTASK_CENTER_URL', required=True,
|
|
287
|
-
help='任务中心URL,如: http://localhost:8001 或 http://localhost:8001/api/namespaces/default')
|
|
367
|
+
help='任务中心URL,如: http://localhost:8001 或 http://localhost:8001/api/v1/namespaces/default')
|
|
288
368
|
@click.option('--check-interval', type=int, default=30,
|
|
289
369
|
help='命名空间检测间隔(秒),默认30秒')
|
|
290
370
|
@click.option('--debug', is_flag=True, help='启用调试模式')
|
|
@@ -292,7 +372,7 @@ def webui_consumer(task_center, check_interval, debug):
|
|
|
292
372
|
"""启动数据消费者(自动识别单/多命名空间)
|
|
293
373
|
|
|
294
374
|
根据URL格式自动判断运行模式:
|
|
295
|
-
- 单命名空间: http://localhost:8001/api/namespaces/{name}
|
|
375
|
+
- 单命名空间: http://localhost:8001/api/v1/namespaces/{name}
|
|
296
376
|
- 多命名空间: http://localhost:8001 或 http://localhost:8001/api
|
|
297
377
|
|
|
298
378
|
示例:
|
|
@@ -302,7 +382,7 @@ def webui_consumer(task_center, check_interval, debug):
|
|
|
302
382
|
jettask webui-consumer --task-center http://localhost:8001/api
|
|
303
383
|
|
|
304
384
|
# 为单个命名空间启动消费者
|
|
305
|
-
jettask webui-consumer --task-center http://localhost:8001/api/namespaces/default
|
|
385
|
+
jettask webui-consumer --task-center http://localhost:8001/api/v1/namespaces/default
|
|
306
386
|
|
|
307
387
|
# 自定义检测间隔
|
|
308
388
|
jettask webui-consumer --task-center http://localhost:8001 --check-interval 60
|
|
@@ -312,7 +392,7 @@ def webui_consumer(task_center, check_interval, debug):
|
|
|
312
392
|
jettask webui-consumer
|
|
313
393
|
"""
|
|
314
394
|
import asyncio
|
|
315
|
-
from jettask.
|
|
395
|
+
from jettask.unified_consumer_manager import UnifiedConsumerManager
|
|
316
396
|
|
|
317
397
|
# 运行消费者管理器
|
|
318
398
|
async def run_manager():
|
|
@@ -333,23 +413,23 @@ def webui_consumer(task_center, check_interval, debug):
|
|
|
333
413
|
def monitor():
|
|
334
414
|
"""启动系统监控器"""
|
|
335
415
|
click.echo("Starting JetTask Monitor")
|
|
336
|
-
from jettask.
|
|
416
|
+
from jettask.run_monitor import main as monitor_main
|
|
337
417
|
monitor_main()
|
|
338
418
|
|
|
339
419
|
@cli.command()
|
|
340
420
|
@click.option('--task-center', '-tc', envvar='JETTASK_CENTER_URL',
|
|
341
|
-
help='任务中心URL,如: http://localhost:8001/api/namespaces/default')
|
|
421
|
+
help='任务中心URL,如: http://localhost:8001/api/v1/namespaces/default')
|
|
342
422
|
def init(task_center):
|
|
343
423
|
"""初始化数据库和配置
|
|
344
424
|
|
|
345
425
|
示例:
|
|
346
426
|
\b
|
|
347
427
|
# 使用任务中心初始化
|
|
348
|
-
jettask init --task-center http://localhost:8001/api/namespaces/default
|
|
349
|
-
jettask init -tc http://localhost:8001/api/namespaces/production
|
|
428
|
+
jettask init --task-center http://localhost:8001/api/v1/namespaces/default
|
|
429
|
+
jettask init -tc http://localhost:8001/api/v1/namespaces/production
|
|
350
430
|
|
|
351
431
|
# 使用环境变量
|
|
352
|
-
export JETTASK_CENTER_URL=http://localhost:8001/api/namespaces/default
|
|
432
|
+
export JETTASK_CENTER_URL=http://localhost:8001/api/v1/namespaces/default
|
|
353
433
|
jettask init
|
|
354
434
|
|
|
355
435
|
# 不使用任务中心(仅使用本地环境变量)
|
|
@@ -366,7 +446,7 @@ def init(task_center):
|
|
|
366
446
|
|
|
367
447
|
# 尝试从任务中心获取数据库配置
|
|
368
448
|
try:
|
|
369
|
-
from jettask.
|
|
449
|
+
from jettask.task_center import TaskCenter
|
|
370
450
|
tc = TaskCenter(task_center)
|
|
371
451
|
if tc._connect_sync():
|
|
372
452
|
p_config = tc.pg_config
|
|
@@ -383,7 +463,7 @@ def init(task_center):
|
|
|
383
463
|
click.echo(" Falling back to local environment variables")
|
|
384
464
|
|
|
385
465
|
# 初始化数据库
|
|
386
|
-
from jettask.
|
|
466
|
+
from jettask.db_init import init_database
|
|
387
467
|
click.echo("\nInitializing database...")
|
|
388
468
|
init_database()
|
|
389
469
|
|
|
@@ -423,7 +503,7 @@ def status():
|
|
|
423
503
|
|
|
424
504
|
@cli.command()
|
|
425
505
|
@click.option('--task-center', '-tc', envvar='JETTASK_CENTER_URL', required=True,
|
|
426
|
-
help='任务中心URL,如: http://localhost:8001 或 http://localhost:8001/api/namespaces/default')
|
|
506
|
+
help='任务中心URL,如: http://localhost:8001 或 http://localhost:8001/api/v1/namespaces/default')
|
|
427
507
|
@click.option('--interval', '-i', type=float, default=0.1,
|
|
428
508
|
help='调度器扫描间隔(秒),默认0.1秒')
|
|
429
509
|
@click.option('--batch-size', '-b', type=int, default=100,
|
|
@@ -435,7 +515,7 @@ def scheduler(task_center, interval, batch_size, check_interval, debug):
|
|
|
435
515
|
"""启动定时任务调度器(自动识别单/多命名空间)
|
|
436
516
|
|
|
437
517
|
根据URL格式自动判断运行模式:
|
|
438
|
-
- 单命名空间: http://localhost:8001/api/namespaces/{name}
|
|
518
|
+
- 单命名空间: http://localhost:8001/api/v1/namespaces/{name}
|
|
439
519
|
- 多命名空间: http://localhost:8001 或 http://localhost:8001/api
|
|
440
520
|
|
|
441
521
|
示例:
|
|
@@ -445,7 +525,7 @@ def scheduler(task_center, interval, batch_size, check_interval, debug):
|
|
|
445
525
|
jettask scheduler --task-center http://localhost:8001/api
|
|
446
526
|
|
|
447
527
|
# 为单个命名空间启动调度器
|
|
448
|
-
jettask scheduler --task-center http://localhost:8001/api/namespaces/default
|
|
528
|
+
jettask scheduler --task-center http://localhost:8001/api/v1/namespaces/default
|
|
449
529
|
|
|
450
530
|
# 自定义配置
|
|
451
531
|
jettask scheduler --task-center http://localhost:8001 --check-interval 60 --interval 0.5
|