jettask 0.2.18__py3-none-any.whl → 0.2.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. jettask/__init__.py +60 -2
  2. jettask/cli.py +314 -228
  3. jettask/config/__init__.py +9 -1
  4. jettask/config/config.py +245 -0
  5. jettask/config/env_loader.py +381 -0
  6. jettask/config/lua_scripts.py +158 -0
  7. jettask/config/nacos_config.py +132 -5
  8. jettask/core/__init__.py +1 -1
  9. jettask/core/app.py +1573 -666
  10. jettask/core/app_importer.py +33 -16
  11. jettask/core/container.py +532 -0
  12. jettask/core/task.py +1 -4
  13. jettask/core/unified_manager_base.py +2 -2
  14. jettask/executor/__init__.py +38 -0
  15. jettask/executor/core.py +625 -0
  16. jettask/executor/executor.py +338 -0
  17. jettask/executor/orchestrator.py +290 -0
  18. jettask/executor/process_entry.py +638 -0
  19. jettask/executor/task_executor.py +317 -0
  20. jettask/messaging/__init__.py +68 -0
  21. jettask/messaging/event_pool.py +2188 -0
  22. jettask/messaging/reader.py +519 -0
  23. jettask/messaging/registry.py +266 -0
  24. jettask/messaging/scanner.py +369 -0
  25. jettask/messaging/sender.py +312 -0
  26. jettask/persistence/__init__.py +118 -0
  27. jettask/persistence/backlog_monitor.py +567 -0
  28. jettask/{backend/data_access.py → persistence/base.py} +58 -57
  29. jettask/persistence/consumer.py +315 -0
  30. jettask/{core → persistence}/db_manager.py +23 -22
  31. jettask/persistence/maintenance.py +81 -0
  32. jettask/persistence/message_consumer.py +259 -0
  33. jettask/{backend/namespace_data_access.py → persistence/namespace.py} +66 -98
  34. jettask/persistence/offline_recovery.py +196 -0
  35. jettask/persistence/queue_discovery.py +215 -0
  36. jettask/persistence/task_persistence.py +218 -0
  37. jettask/persistence/task_updater.py +583 -0
  38. jettask/scheduler/__init__.py +2 -2
  39. jettask/scheduler/loader.py +6 -5
  40. jettask/scheduler/run_scheduler.py +1 -1
  41. jettask/scheduler/scheduler.py +7 -7
  42. jettask/scheduler/{unified_scheduler_manager.py → scheduler_coordinator.py} +18 -13
  43. jettask/task/__init__.py +16 -0
  44. jettask/{router.py → task/router.py} +26 -8
  45. jettask/task/task_center/__init__.py +9 -0
  46. jettask/task/task_executor.py +318 -0
  47. jettask/task/task_registry.py +291 -0
  48. jettask/test_connection_monitor.py +73 -0
  49. jettask/utils/__init__.py +31 -1
  50. jettask/{monitor/run_backlog_collector.py → utils/backlog_collector.py} +1 -1
  51. jettask/utils/db_connector.py +1629 -0
  52. jettask/{db_init.py → utils/db_init.py} +1 -1
  53. jettask/utils/rate_limit/__init__.py +30 -0
  54. jettask/utils/rate_limit/concurrency_limiter.py +665 -0
  55. jettask/utils/rate_limit/config.py +145 -0
  56. jettask/utils/rate_limit/limiter.py +41 -0
  57. jettask/utils/rate_limit/manager.py +269 -0
  58. jettask/utils/rate_limit/qps_limiter.py +154 -0
  59. jettask/utils/rate_limit/task_limiter.py +384 -0
  60. jettask/utils/serializer.py +3 -0
  61. jettask/{monitor/stream_backlog_monitor.py → utils/stream_backlog.py} +14 -6
  62. jettask/utils/time_sync.py +173 -0
  63. jettask/webui/__init__.py +27 -0
  64. jettask/{api/v1 → webui/api}/alerts.py +1 -1
  65. jettask/{api/v1 → webui/api}/analytics.py +2 -2
  66. jettask/{api/v1 → webui/api}/namespaces.py +1 -1
  67. jettask/{api/v1 → webui/api}/overview.py +1 -1
  68. jettask/{api/v1 → webui/api}/queues.py +3 -3
  69. jettask/{api/v1 → webui/api}/scheduled.py +1 -1
  70. jettask/{api/v1 → webui/api}/settings.py +1 -1
  71. jettask/{api.py → webui/app.py} +253 -145
  72. jettask/webui/namespace_manager/__init__.py +10 -0
  73. jettask/{multi_namespace_consumer.py → webui/namespace_manager/multi.py} +69 -22
  74. jettask/{unified_consumer_manager.py → webui/namespace_manager/unified.py} +1 -1
  75. jettask/{run.py → webui/run.py} +2 -2
  76. jettask/{services → webui/services}/__init__.py +1 -3
  77. jettask/{services → webui/services}/overview_service.py +34 -16
  78. jettask/{services → webui/services}/queue_service.py +1 -1
  79. jettask/{backend → webui/services}/queue_stats_v2.py +1 -1
  80. jettask/{services → webui/services}/settings_service.py +1 -1
  81. jettask/worker/__init__.py +53 -0
  82. jettask/worker/lifecycle.py +1507 -0
  83. jettask/worker/manager.py +583 -0
  84. jettask/{core/offline_worker_recovery.py → worker/recovery.py} +268 -175
  85. {jettask-0.2.18.dist-info → jettask-0.2.20.dist-info}/METADATA +2 -71
  86. jettask-0.2.20.dist-info/RECORD +145 -0
  87. jettask/__main__.py +0 -140
  88. jettask/api/__init__.py +0 -103
  89. jettask/backend/__init__.py +0 -1
  90. jettask/backend/api/__init__.py +0 -3
  91. jettask/backend/api/v1/__init__.py +0 -17
  92. jettask/backend/api/v1/monitoring.py +0 -431
  93. jettask/backend/api/v1/namespaces.py +0 -504
  94. jettask/backend/api/v1/queues.py +0 -342
  95. jettask/backend/api/v1/tasks.py +0 -367
  96. jettask/backend/core/__init__.py +0 -3
  97. jettask/backend/core/cache.py +0 -221
  98. jettask/backend/core/database.py +0 -200
  99. jettask/backend/core/exceptions.py +0 -102
  100. jettask/backend/dependencies.py +0 -261
  101. jettask/backend/init_meta_db.py +0 -158
  102. jettask/backend/main.py +0 -1426
  103. jettask/backend/main_unified.py +0 -78
  104. jettask/backend/main_v2.py +0 -394
  105. jettask/backend/models/__init__.py +0 -3
  106. jettask/backend/models/requests.py +0 -236
  107. jettask/backend/models/responses.py +0 -230
  108. jettask/backend/namespace_api_old.py +0 -267
  109. jettask/backend/services/__init__.py +0 -3
  110. jettask/backend/start.py +0 -42
  111. jettask/backend/unified_api_router.py +0 -1541
  112. jettask/cleanup_deprecated_tables.sql +0 -16
  113. jettask/core/consumer_manager.py +0 -1695
  114. jettask/core/delay_scanner.py +0 -256
  115. jettask/core/event_pool.py +0 -1700
  116. jettask/core/heartbeat_process.py +0 -222
  117. jettask/core/task_batch.py +0 -153
  118. jettask/core/worker_scanner.py +0 -271
  119. jettask/executors/__init__.py +0 -5
  120. jettask/executors/asyncio.py +0 -876
  121. jettask/executors/base.py +0 -30
  122. jettask/executors/common.py +0 -148
  123. jettask/executors/multi_asyncio.py +0 -309
  124. jettask/gradio_app.py +0 -570
  125. jettask/integrated_gradio_app.py +0 -1088
  126. jettask/main.py +0 -0
  127. jettask/monitoring/__init__.py +0 -3
  128. jettask/pg_consumer.py +0 -1896
  129. jettask/run_monitor.py +0 -22
  130. jettask/run_webui.py +0 -148
  131. jettask/scheduler/multi_namespace_scheduler.py +0 -294
  132. jettask/scheduler/unified_manager.py +0 -450
  133. jettask/task_center_client.py +0 -150
  134. jettask/utils/serializer_optimized.py +0 -33
  135. jettask/webui_exceptions.py +0 -67
  136. jettask-0.2.18.dist-info/RECORD +0 -150
  137. /jettask/{constants.py → config/constants.py} +0 -0
  138. /jettask/{backend/config.py → config/task_center.py} +0 -0
  139. /jettask/{pg_consumer → messaging/pg_consumer}/pg_consumer_v2.py +0 -0
  140. /jettask/{pg_consumer → messaging/pg_consumer}/sql/add_execution_time_field.sql +0 -0
  141. /jettask/{pg_consumer → messaging/pg_consumer}/sql/create_new_tables.sql +0 -0
  142. /jettask/{pg_consumer → messaging/pg_consumer}/sql/create_tables_v3.sql +0 -0
  143. /jettask/{pg_consumer → messaging/pg_consumer}/sql/migrate_to_new_structure.sql +0 -0
  144. /jettask/{pg_consumer → messaging/pg_consumer}/sql/modify_time_fields.sql +0 -0
  145. /jettask/{pg_consumer → messaging/pg_consumer}/sql_utils.py +0 -0
  146. /jettask/{models.py → persistence/models.py} +0 -0
  147. /jettask/scheduler/{manager.py → task_crud.py} +0 -0
  148. /jettask/{schema.sql → schemas/schema.sql} +0 -0
  149. /jettask/{task_center.py → task/task_center/client.py} +0 -0
  150. /jettask/{monitoring → utils}/file_watcher.py +0 -0
  151. /jettask/{services/redis_monitor_service.py → utils/redis_monitor.py} +0 -0
  152. /jettask/{api/v1 → webui/api}/__init__.py +0 -0
  153. /jettask/{webui_config.py → webui/config.py} +0 -0
  154. /jettask/{webui_models → webui/models}/__init__.py +0 -0
  155. /jettask/{webui_models → webui/models}/namespace.py +0 -0
  156. /jettask/{services → webui/services}/alert_service.py +0 -0
  157. /jettask/{services → webui/services}/analytics_service.py +0 -0
  158. /jettask/{services → webui/services}/scheduled_task_service.py +0 -0
  159. /jettask/{services → webui/services}/task_service.py +0 -0
  160. /jettask/{webui_sql → webui/sql}/batch_upsert_functions.sql +0 -0
  161. /jettask/{webui_sql → webui/sql}/verify_database.sql +0 -0
  162. {jettask-0.2.18.dist-info → jettask-0.2.20.dist-info}/WHEEL +0 -0
  163. {jettask-0.2.18.dist-info → jettask-0.2.20.dist-info}/entry_points.txt +0 -0
  164. {jettask-0.2.18.dist-info → jettask-0.2.20.dist-info}/licenses/LICENSE +0 -0
  165. {jettask-0.2.18.dist-info → jettask-0.2.20.dist-info}/top_level.txt +0 -0
@@ -1,267 +0,0 @@
1
- """
2
- 命名空间管理API - 重构版本
3
- 使用数据库持久化命名空间数据
4
- """
5
- from fastapi import APIRouter, HTTPException, Query
6
- from typing import Dict, Any, Optional, List
7
- from datetime import datetime
8
- import json
9
- from sqlalchemy import text
10
- import logging
11
- from jettask.schemas import NamespaceCreate, NamespaceUpdate, NamespaceResponse
12
-
13
- logger = logging.getLogger(__name__)
14
-
15
- router = APIRouter(prefix="/api/namespaces", tags=["namespaces"])
16
-
17
- # 使用任务中心专用的元数据库连接
18
- from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
19
- from sqlalchemy.orm import sessionmaker
20
- from jettask.backend.config import task_center_config
21
-
22
- # 创建异步引擎 - 连接到任务中心元数据库
23
- # 注意:这是任务中心自己的数据库,不是JetTask应用的数据库
24
- engine = create_async_engine(task_center_config.meta_database_url, echo=False)
25
- AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
26
-
27
-
28
- # 模型已从 schemas 模块导入
29
-
30
-
31
- @router.get("", response_model=List[NamespaceResponse])
32
- async def list_namespaces(
33
- page: int = Query(1, ge=1),
34
- page_size: int = Query(20, ge=1, le=100),
35
- is_active: Optional[bool] = None
36
- ):
37
- """列出所有命名空间"""
38
- try:
39
- async with AsyncSessionLocal() as session:
40
- query = """
41
- SELECT id, name, description, redis_config, pg_config,
42
- is_active, version, created_at, updated_at
43
- FROM namespaces
44
- """
45
- params = {}
46
-
47
- if is_active is not None:
48
- query += " WHERE is_active = :is_active"
49
- params['is_active'] = is_active
50
-
51
- query += " ORDER BY created_at DESC"
52
- query += " LIMIT :limit OFFSET :offset"
53
- params['limit'] = page_size
54
- params['offset'] = (page - 1) * page_size
55
-
56
- result = await session.execute(text(query), params)
57
- rows = result.fetchall()
58
-
59
- namespaces = []
60
- for row in rows:
61
- namespaces.append(NamespaceResponse(
62
- id=row.id,
63
- name=row.name,
64
- description=row.description,
65
- redis_config=row.redis_config,
66
- pg_config=row.pg_config,
67
- is_active=row.is_active,
68
- version=row.version,
69
- created_at=row.created_at,
70
- updated_at=row.updated_at,
71
- connection_url=f"/api/namespaces/{row.name}" # 使用名称
72
- ))
73
-
74
- return namespaces
75
- except Exception as e:
76
- logger.error(f"Failed to list namespaces: {e}")
77
- raise HTTPException(status_code=500, detail=str(e))
78
-
79
-
80
- @router.post("", response_model=NamespaceResponse, status_code=201)
81
- async def create_namespace(namespace: NamespaceCreate):
82
- """创建新的命名空间"""
83
- try:
84
- async with AsyncSessionLocal() as session:
85
- # 检查名称是否已存在
86
- check_query = text("SELECT COUNT(*) FROM namespaces WHERE name = :name")
87
- result = await session.execute(check_query, {'name': namespace.name})
88
- if result.scalar() > 0:
89
- raise HTTPException(status_code=400, detail=f"命名空间 '{namespace.name}' 已存在")
90
-
91
- # 创建命名空间(使用自增ID)
92
- insert_query = text("""
93
- INSERT INTO namespaces (name, description, redis_config, pg_config, version)
94
- VALUES (:name, :description, :redis_config, :pg_config, 1)
95
- RETURNING id, name, description, redis_config, pg_config,
96
- is_active, version, created_at, updated_at
97
- """)
98
-
99
- result = await session.execute(insert_query, {
100
- 'name': namespace.name,
101
- 'description': namespace.description,
102
- 'redis_config': json.dumps(namespace.redis_config),
103
- 'pg_config': json.dumps(namespace.pg_config)
104
- })
105
-
106
- row = result.fetchone()
107
- await session.commit()
108
-
109
- return NamespaceResponse(
110
- id=str(row.id),
111
- name=row.name,
112
- description=row.description,
113
- redis_config=row.redis_config,
114
- pg_config=row.pg_config,
115
- is_active=row.is_active,
116
- version=row.version,
117
- created_at=row.created_at,
118
- updated_at=row.updated_at,
119
- connection_url=f"/api/namespaces/{str(row.id)}"
120
- )
121
- except HTTPException:
122
- raise
123
- except Exception as e:
124
- logger.error(f"Failed to create namespace: {e}")
125
- raise HTTPException(status_code=500, detail=str(e))
126
-
127
-
128
- @router.get("/{namespace_id}", response_model=NamespaceResponse)
129
- async def get_namespace(namespace_id: str):
130
- """获取指定命名空间的详细信息"""
131
- try:
132
- async with AsyncSessionLocal() as session:
133
- query = text("""
134
- SELECT id, name, description, redis_config, pg_config,
135
- is_active, version, created_at, updated_at
136
- FROM namespaces
137
- WHERE id = :id
138
- """)
139
-
140
- result = await session.execute(query, {'id': namespace_id})
141
- row = result.fetchone()
142
-
143
- if not row:
144
- raise HTTPException(status_code=404, detail="命名空间不存在")
145
-
146
- return NamespaceResponse(
147
- id=str(row.id),
148
- name=row.name,
149
- description=row.description,
150
- redis_config=row.redis_config,
151
- pg_config=row.pg_config,
152
- is_active=row.is_active,
153
- version=row.version,
154
- created_at=row.created_at,
155
- updated_at=row.updated_at,
156
- connection_url=f"/api/namespaces/{str(row.id)}"
157
- )
158
- except HTTPException:
159
- raise
160
- except Exception as e:
161
- logger.error(f"Failed to get namespace: {e}")
162
- raise HTTPException(status_code=500, detail=str(e))
163
-
164
-
165
- @router.put("/{namespace_id}", response_model=NamespaceResponse)
166
- async def update_namespace(namespace_id: str, namespace: NamespaceUpdate):
167
- """更新命名空间"""
168
- try:
169
- async with AsyncSessionLocal() as session:
170
- # 检查是否存在并获取名称
171
- check_query = text("SELECT name FROM namespaces WHERE id = :id")
172
- result = await session.execute(check_query, {'id': namespace_id})
173
- row = result.fetchone()
174
-
175
- if not row:
176
- raise HTTPException(status_code=404, detail="命名空间不存在")
177
-
178
- # 检查是否是默认命名空间
179
- if row.name == 'default':
180
- raise HTTPException(status_code=400, detail="默认命名空间不能编辑")
181
-
182
- # 构建更新语句
183
- updates = []
184
- params = {'id': namespace_id}
185
-
186
- if namespace.description is not None:
187
- updates.append("description = :description")
188
- params['description'] = namespace.description
189
-
190
- if namespace.redis_config is not None:
191
- updates.append("redis_config = :redis_config")
192
- params['redis_config'] = json.dumps(namespace.redis_config)
193
-
194
- if namespace.pg_config is not None:
195
- updates.append("pg_config = :pg_config")
196
- params['pg_config'] = json.dumps(namespace.pg_config)
197
-
198
- if namespace.is_active is not None:
199
- updates.append("is_active = :is_active")
200
- params['is_active'] = namespace.is_active
201
-
202
- if not updates:
203
- raise HTTPException(status_code=400, detail="没有要更新的字段")
204
-
205
- # 如果更新了redis_config或pg_config,递增版本号
206
- if 'redis_config' in params or 'pg_config' in params:
207
- updates.append("version = version + 1")
208
-
209
- update_query = text(f"""
210
- UPDATE namespaces
211
- SET {', '.join(updates)}
212
- WHERE id = :id
213
- RETURNING id, name, description, redis_config, pg_config,
214
- is_active, version, created_at, updated_at
215
- """)
216
-
217
- result = await session.execute(update_query, params)
218
- row = result.fetchone()
219
- await session.commit()
220
-
221
- return NamespaceResponse(
222
- id=str(row.id),
223
- name=row.name,
224
- description=row.description,
225
- redis_config=row.redis_config,
226
- pg_config=row.pg_config,
227
- is_active=row.is_active,
228
- version=row.version,
229
- created_at=row.created_at,
230
- updated_at=row.updated_at,
231
- connection_url=f"/api/namespaces/{str(row.id)}"
232
- )
233
- except HTTPException:
234
- raise
235
- except Exception as e:
236
- logger.error(f"Failed to update namespace: {e}")
237
- raise HTTPException(status_code=500, detail=str(e))
238
-
239
-
240
- @router.delete("/{namespace_id}")
241
- async def delete_namespace(namespace_id: str):
242
- """删除命名空间"""
243
- try:
244
- async with AsyncSessionLocal() as session:
245
- # 检查是否为默认命名空间
246
- check_query = text("SELECT name FROM namespaces WHERE id = :id")
247
- result = await session.execute(check_query, {'id': namespace_id})
248
- row = result.fetchone()
249
-
250
- if not row:
251
- raise HTTPException(status_code=404, detail="命名空间不存在")
252
-
253
- if row.name == 'default':
254
- raise HTTPException(status_code=400, detail="不能删除默认命名空间")
255
-
256
- # 删除命名空间
257
- delete_query = text("DELETE FROM namespaces WHERE id = :id")
258
- await session.execute(delete_query, {'id': namespace_id})
259
- await session.commit()
260
-
261
- return {"message": "命名空间已删除"}
262
- except HTTPException:
263
- raise
264
- except Exception as e:
265
- logger.error(f"Failed to delete namespace: {e}")
266
- raise HTTPException(status_code=500, detail=str(e))
267
-
@@ -1,3 +0,0 @@
1
- """
2
- Business logic services for JetTask WebUI Backend
3
- """
jettask/backend/start.py DELETED
@@ -1,42 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- 启动独立的JetTask Monitor后端API服务
4
- """
5
- import uvicorn
6
- import logging
7
- import sys
8
- import os
9
-
10
- # 添加项目根目录到Python路径
11
- current_dir = os.path.dirname(os.path.abspath(__file__))
12
- project_root = os.path.dirname(os.path.dirname(os.path.dirname(current_dir)))
13
- sys.path.insert(0, project_root)
14
-
15
- # 设置日志
16
- logging.basicConfig(
17
- level=logging.INFO,
18
- format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
19
- )
20
- logger = logging.getLogger(__name__)
21
-
22
- def main():
23
- """启动服务"""
24
- logger.info("启动JetTask Monitor独立后端API服务...")
25
-
26
- try:
27
- uvicorn.run(
28
- "main:app",
29
- host="0.0.0.0",
30
- port=8001,
31
- reload=True, # 开发模式,生产环境应设为False
32
- log_level="info",
33
- access_log=True
34
- )
35
- except KeyboardInterrupt:
36
- logger.info("服务被用户中断")
37
- except Exception as e:
38
- logger.error(f"服务启动失败: {e}")
39
- sys.exit(1)
40
-
41
- if __name__ == "__main__":
42
- main()