jettask 0.2.19__py3-none-any.whl → 0.2.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. jettask/__init__.py +10 -3
  2. jettask/cli.py +314 -228
  3. jettask/config/__init__.py +9 -1
  4. jettask/config/config.py +245 -0
  5. jettask/config/env_loader.py +381 -0
  6. jettask/config/lua_scripts.py +158 -0
  7. jettask/config/nacos_config.py +132 -5
  8. jettask/core/__init__.py +1 -1
  9. jettask/core/app.py +1573 -666
  10. jettask/core/app_importer.py +33 -16
  11. jettask/core/container.py +532 -0
  12. jettask/core/task.py +1 -4
  13. jettask/core/unified_manager_base.py +2 -2
  14. jettask/executor/__init__.py +38 -0
  15. jettask/executor/core.py +625 -0
  16. jettask/executor/executor.py +338 -0
  17. jettask/executor/orchestrator.py +290 -0
  18. jettask/executor/process_entry.py +638 -0
  19. jettask/executor/task_executor.py +317 -0
  20. jettask/messaging/__init__.py +68 -0
  21. jettask/messaging/event_pool.py +2188 -0
  22. jettask/messaging/reader.py +519 -0
  23. jettask/messaging/registry.py +266 -0
  24. jettask/messaging/scanner.py +369 -0
  25. jettask/messaging/sender.py +312 -0
  26. jettask/persistence/__init__.py +118 -0
  27. jettask/persistence/backlog_monitor.py +567 -0
  28. jettask/{backend/data_access.py → persistence/base.py} +58 -57
  29. jettask/persistence/consumer.py +315 -0
  30. jettask/{core → persistence}/db_manager.py +23 -22
  31. jettask/persistence/maintenance.py +81 -0
  32. jettask/persistence/message_consumer.py +259 -0
  33. jettask/{backend/namespace_data_access.py → persistence/namespace.py} +66 -98
  34. jettask/persistence/offline_recovery.py +196 -0
  35. jettask/persistence/queue_discovery.py +215 -0
  36. jettask/persistence/task_persistence.py +218 -0
  37. jettask/persistence/task_updater.py +583 -0
  38. jettask/scheduler/__init__.py +2 -2
  39. jettask/scheduler/loader.py +6 -5
  40. jettask/scheduler/run_scheduler.py +1 -1
  41. jettask/scheduler/scheduler.py +7 -7
  42. jettask/scheduler/{unified_scheduler_manager.py → scheduler_coordinator.py} +18 -13
  43. jettask/task/__init__.py +16 -0
  44. jettask/{router.py → task/router.py} +26 -8
  45. jettask/task/task_center/__init__.py +9 -0
  46. jettask/task/task_executor.py +318 -0
  47. jettask/task/task_registry.py +291 -0
  48. jettask/test_connection_monitor.py +73 -0
  49. jettask/utils/__init__.py +31 -1
  50. jettask/{monitor/run_backlog_collector.py → utils/backlog_collector.py} +1 -1
  51. jettask/utils/db_connector.py +1629 -0
  52. jettask/{db_init.py → utils/db_init.py} +1 -1
  53. jettask/utils/rate_limit/__init__.py +30 -0
  54. jettask/utils/rate_limit/concurrency_limiter.py +665 -0
  55. jettask/utils/rate_limit/config.py +145 -0
  56. jettask/utils/rate_limit/limiter.py +41 -0
  57. jettask/utils/rate_limit/manager.py +269 -0
  58. jettask/utils/rate_limit/qps_limiter.py +154 -0
  59. jettask/utils/rate_limit/task_limiter.py +384 -0
  60. jettask/utils/serializer.py +3 -0
  61. jettask/{monitor/stream_backlog_monitor.py → utils/stream_backlog.py} +14 -6
  62. jettask/utils/time_sync.py +173 -0
  63. jettask/webui/__init__.py +27 -0
  64. jettask/{api/v1 → webui/api}/alerts.py +1 -1
  65. jettask/{api/v1 → webui/api}/analytics.py +2 -2
  66. jettask/{api/v1 → webui/api}/namespaces.py +1 -1
  67. jettask/{api/v1 → webui/api}/overview.py +1 -1
  68. jettask/{api/v1 → webui/api}/queues.py +3 -3
  69. jettask/{api/v1 → webui/api}/scheduled.py +1 -1
  70. jettask/{api/v1 → webui/api}/settings.py +1 -1
  71. jettask/{api.py → webui/app.py} +253 -145
  72. jettask/webui/namespace_manager/__init__.py +10 -0
  73. jettask/{multi_namespace_consumer.py → webui/namespace_manager/multi.py} +69 -22
  74. jettask/{unified_consumer_manager.py → webui/namespace_manager/unified.py} +1 -1
  75. jettask/{run.py → webui/run.py} +2 -2
  76. jettask/{services → webui/services}/__init__.py +1 -3
  77. jettask/{services → webui/services}/overview_service.py +34 -16
  78. jettask/{services → webui/services}/queue_service.py +1 -1
  79. jettask/{backend → webui/services}/queue_stats_v2.py +1 -1
  80. jettask/{services → webui/services}/settings_service.py +1 -1
  81. jettask/worker/__init__.py +53 -0
  82. jettask/worker/lifecycle.py +1507 -0
  83. jettask/worker/manager.py +583 -0
  84. jettask/{core/offline_worker_recovery.py → worker/recovery.py} +268 -175
  85. {jettask-0.2.19.dist-info → jettask-0.2.20.dist-info}/METADATA +2 -71
  86. jettask-0.2.20.dist-info/RECORD +145 -0
  87. jettask/__main__.py +0 -140
  88. jettask/api/__init__.py +0 -103
  89. jettask/backend/__init__.py +0 -1
  90. jettask/backend/api/__init__.py +0 -3
  91. jettask/backend/api/v1/__init__.py +0 -17
  92. jettask/backend/api/v1/monitoring.py +0 -431
  93. jettask/backend/api/v1/namespaces.py +0 -504
  94. jettask/backend/api/v1/queues.py +0 -342
  95. jettask/backend/api/v1/tasks.py +0 -367
  96. jettask/backend/core/__init__.py +0 -3
  97. jettask/backend/core/cache.py +0 -221
  98. jettask/backend/core/database.py +0 -200
  99. jettask/backend/core/exceptions.py +0 -102
  100. jettask/backend/dependencies.py +0 -261
  101. jettask/backend/init_meta_db.py +0 -158
  102. jettask/backend/main.py +0 -1426
  103. jettask/backend/main_unified.py +0 -78
  104. jettask/backend/main_v2.py +0 -394
  105. jettask/backend/models/__init__.py +0 -3
  106. jettask/backend/models/requests.py +0 -236
  107. jettask/backend/models/responses.py +0 -230
  108. jettask/backend/namespace_api_old.py +0 -267
  109. jettask/backend/services/__init__.py +0 -3
  110. jettask/backend/start.py +0 -42
  111. jettask/backend/unified_api_router.py +0 -1541
  112. jettask/cleanup_deprecated_tables.sql +0 -16
  113. jettask/core/consumer_manager.py +0 -1695
  114. jettask/core/delay_scanner.py +0 -256
  115. jettask/core/event_pool.py +0 -1700
  116. jettask/core/heartbeat_process.py +0 -222
  117. jettask/core/task_batch.py +0 -153
  118. jettask/core/worker_scanner.py +0 -271
  119. jettask/executors/__init__.py +0 -5
  120. jettask/executors/asyncio.py +0 -876
  121. jettask/executors/base.py +0 -30
  122. jettask/executors/common.py +0 -148
  123. jettask/executors/multi_asyncio.py +0 -309
  124. jettask/gradio_app.py +0 -570
  125. jettask/integrated_gradio_app.py +0 -1088
  126. jettask/main.py +0 -0
  127. jettask/monitoring/__init__.py +0 -3
  128. jettask/pg_consumer.py +0 -1896
  129. jettask/run_monitor.py +0 -22
  130. jettask/run_webui.py +0 -148
  131. jettask/scheduler/multi_namespace_scheduler.py +0 -294
  132. jettask/scheduler/unified_manager.py +0 -450
  133. jettask/task_center_client.py +0 -150
  134. jettask/utils/serializer_optimized.py +0 -33
  135. jettask/webui_exceptions.py +0 -67
  136. jettask-0.2.19.dist-info/RECORD +0 -150
  137. /jettask/{constants.py → config/constants.py} +0 -0
  138. /jettask/{backend/config.py → config/task_center.py} +0 -0
  139. /jettask/{pg_consumer → messaging/pg_consumer}/pg_consumer_v2.py +0 -0
  140. /jettask/{pg_consumer → messaging/pg_consumer}/sql/add_execution_time_field.sql +0 -0
  141. /jettask/{pg_consumer → messaging/pg_consumer}/sql/create_new_tables.sql +0 -0
  142. /jettask/{pg_consumer → messaging/pg_consumer}/sql/create_tables_v3.sql +0 -0
  143. /jettask/{pg_consumer → messaging/pg_consumer}/sql/migrate_to_new_structure.sql +0 -0
  144. /jettask/{pg_consumer → messaging/pg_consumer}/sql/modify_time_fields.sql +0 -0
  145. /jettask/{pg_consumer → messaging/pg_consumer}/sql_utils.py +0 -0
  146. /jettask/{models.py → persistence/models.py} +0 -0
  147. /jettask/scheduler/{manager.py → task_crud.py} +0 -0
  148. /jettask/{schema.sql → schemas/schema.sql} +0 -0
  149. /jettask/{task_center.py → task/task_center/client.py} +0 -0
  150. /jettask/{monitoring → utils}/file_watcher.py +0 -0
  151. /jettask/{services/redis_monitor_service.py → utils/redis_monitor.py} +0 -0
  152. /jettask/{api/v1 → webui/api}/__init__.py +0 -0
  153. /jettask/{webui_config.py → webui/config.py} +0 -0
  154. /jettask/{webui_models → webui/models}/__init__.py +0 -0
  155. /jettask/{webui_models → webui/models}/namespace.py +0 -0
  156. /jettask/{services → webui/services}/alert_service.py +0 -0
  157. /jettask/{services → webui/services}/analytics_service.py +0 -0
  158. /jettask/{services → webui/services}/scheduled_task_service.py +0 -0
  159. /jettask/{services → webui/services}/task_service.py +0 -0
  160. /jettask/{webui_sql → webui/sql}/batch_upsert_functions.sql +0 -0
  161. /jettask/{webui_sql → webui/sql}/verify_database.sql +0 -0
  162. {jettask-0.2.19.dist-info → jettask-0.2.20.dist-info}/WHEEL +0 -0
  163. {jettask-0.2.19.dist-info → jettask-0.2.20.dist-info}/entry_points.txt +0 -0
  164. {jettask-0.2.19.dist-info → jettask-0.2.20.dist-info}/licenses/LICENSE +0 -0
  165. {jettask-0.2.19.dist-info → jettask-0.2.20.dist-info}/top_level.txt +0 -0
jettask/cli.py CHANGED
@@ -7,9 +7,8 @@ import sys
7
7
  import os
8
8
  import importlib
9
9
  import importlib.util
10
- import json
10
+ import multiprocessing
11
11
  from pathlib import Path
12
- from dotenv import load_dotenv
13
12
 
14
13
  # 处理直接运行时的路径问题
15
14
  if __name__ == '__main__':
@@ -18,142 +17,248 @@ if __name__ == '__main__':
18
17
  sys.path.insert(0, parent_dir)
19
18
 
20
19
  from jettask.config.nacos_config import config
20
+ from jettask.config.env_loader import EnvLoader
21
21
  from jettask.core.app_importer import import_app, AppImporter
22
22
 
23
23
 
24
24
  @click.group()
25
25
  @click.version_option(version='0.1.0', prog_name='JetTask')
26
- def cli():
27
- """JetTask - 高性能分布式任务队列系统"""
28
- pass
29
-
30
- @cli.command()
31
- @click.option('--host', default='0.0.0.0', help='服务器监听地址')
32
- @click.option('--port', default=8001, type=int, help='服务器监听端口')
33
- @click.option('--jettask-pg-url', envvar='JETTASK_PG_URL',
34
- help='PostgreSQL连接URL')
35
- @click.option('--use-nacos', is_flag=True, default=False,
26
+ @click.option('--env-file', '-e', envvar='JETTASK_ENV_FILE',
27
+ help='环境变量文件路径 (.env 格式),可通过 JETTASK_ENV_FILE 环境变量指定')
28
+ @click.option('--use-nacos', is_flag=True,
36
29
  help='从Nacos配置中心读取配置')
37
- @click.option('--nacos-server', envvar='NACOS_SERVER',
38
- help='Nacos服务器地址')
39
- @click.option('--nacos-namespace', envvar='NACOS_NAMESPACE',
30
+ @click.option('--nacos-server',
31
+ help='Nacos服务器地址 (如: 127.0.0.1:8848)')
32
+ @click.option('--nacos-namespace',
40
33
  help='Nacos命名空间')
41
- @click.option('--nacos-username', envvar='NACOS_USERNAME',
34
+ @click.option('--nacos-username',
42
35
  help='Nacos用户名')
43
- @click.option('--nacos-password', envvar='NACOS_PASSWORD',
36
+ @click.option('--nacos-password',
44
37
  help='Nacos密码')
45
- @click.option('--nacos-group', envvar='NACOS_GROUP',
38
+ @click.option('--nacos-group',
46
39
  help='Nacos配置组')
47
- @click.option('--nacos-data-id', envvar='NACOS_DATA_ID',
48
- help='Nacos配置ID')
49
- @click.option('--reload', is_flag=True, default=False, help='启用自动重载')
50
- @click.option('--log-level', default='info',
40
+ @click.option('--nacos-data-id',
41
+ help='Nacos配置ID(api/webui-consumer/scheduler 需要)')
42
+ @click.pass_context
43
+ def cli(ctx, env_file, use_nacos, nacos_server, nacos_namespace,
44
+ nacos_username, nacos_password, nacos_group, nacos_data_id):
45
+ """JetTask - 高性能分布式任务队列系统
46
+
47
+ 配置加载优先级(从高到低):
48
+ 1. 命令行参数
49
+ 2. 环境变量
50
+ 3. .env文件
51
+ 4. 默认值
52
+
53
+ Nacos 必需参数:
54
+ --nacos-server Nacos服务器地址
55
+ --nacos-namespace Nacos命名空间
56
+
57
+ Nacos 可选参数:
58
+ --nacos-group 配置组(默认: DEFAULT_GROUP)
59
+ --nacos-data-id 配置ID(api/webui-consumer/scheduler 需要)
60
+ --nacos-username 用户名
61
+ --nacos-password 密码
62
+
63
+ 说明:
64
+ • worker 命令不使用全局 Nacos 配置,而是通过 app 读取配置
65
+ • api/webui-consumer/scheduler 需要 nacos-data-id 来读取应用配置
66
+
67
+ 示例:
68
+ \b
69
+ # 使用.env文件
70
+ jettask -e .env worker --queues default
71
+
72
+ # 使用Nacos(从.env读取配置)
73
+ jettask --use-nacos worker --queues default
74
+
75
+ # 使用Nacos(命令行指定最小配置)
76
+ jettask --use-nacos \\
77
+ --nacos-server 127.0.0.1:8848 \\
78
+ --nacos-namespace prod \\
79
+ worker --queues default
80
+
81
+ # 混合使用:先加载.env,再从Nacos覆盖
82
+ jettask -e .env --use-nacos worker --queues default
83
+ """
84
+ # 在所有命令执行前加载环境变量
85
+ ctx.ensure_object(dict)
86
+
87
+ # 初始化 context(即使不使用 Nacos 也要设置)
88
+ ctx.obj['use_nacos'] = use_nacos
89
+ ctx.obj['nacos_data_id'] = None
90
+
91
+ loader = EnvLoader()
92
+
93
+ # 如果手动指定了env文件,只加载指定的文件;否则自动搜索加载
94
+ if env_file:
95
+ # 手动指定了文件,直接加载,不进行自动搜索
96
+ try:
97
+ loader.load_env_file(env_file, override=True)
98
+ click.echo(f"✓ 已加载环境变量文件: {env_file}")
99
+ except FileNotFoundError:
100
+ click.echo(f"Error: 环境变量文件不存在: {env_file}", err=True)
101
+ raise click.Abort()
102
+ except Exception as e:
103
+ click.echo(f"Error: 加载环境变量文件失败: {e}", err=True)
104
+ raise click.Abort()
105
+ else:
106
+ # 未指定文件,自动搜索并加载 .env, .env.{ENV}, .env.local
107
+ loader.auto_load()
108
+
109
+ # 3. 如果启用了Nacos,从Nacos读取配置并更新环境变量
110
+ if use_nacos:
111
+ try:
112
+ # 在加载 .env 之后,再从环境变量读取 Nacos 配置
113
+ # 优先级: 命令行参数 > .env 文件中的环境变量
114
+ nacos_server = nacos_server or os.environ.get('NACOS_SERVER')
115
+ nacos_namespace = nacos_namespace or os.environ.get('NACOS_NAMESPACE')
116
+ nacos_username = nacos_username or os.environ.get('NACOS_USERNAME')
117
+ nacos_password = nacos_password or os.environ.get('NACOS_PASSWORD')
118
+ nacos_group = nacos_group or os.environ.get('NACOS_GROUP', 'DEFAULT_GROUP')
119
+ nacos_data_id = nacos_data_id or os.environ.get('NACOS_DATA_ID')
120
+
121
+ # 更新 Nacos 配置到 context
122
+ ctx.obj['nacos_data_id'] = nacos_data_id
123
+
124
+ # 检查必需的配置是否完整(nacos_data_id 改为可选)
125
+ if not all([nacos_server, nacos_namespace, nacos_group]):
126
+ missing = []
127
+ if not nacos_server: missing.append("--nacos-server 或 NACOS_SERVER")
128
+ if not nacos_namespace: missing.append("--nacos-namespace 或 NACOS_NAMESPACE")
129
+ if not nacos_group: missing.append("--nacos-group 或 NACOS_GROUP")
130
+
131
+ click.echo("Error: Nacos配置不完整,缺少以下配置:", err=True)
132
+ for item in missing:
133
+ click.echo(f" - {item}", err=True)
134
+ raise click.Abort()
135
+
136
+ # 将命令行参数或环境变量的Nacos配置写入os.environ
137
+ # 这样nacos_config模块才能读取到
138
+ os.environ['NACOS_SERVER'] = nacos_server
139
+ os.environ['NACOS_NAMESPACE'] = nacos_namespace
140
+ os.environ['NACOS_GROUP'] = nacos_group
141
+ if nacos_data_id:
142
+ os.environ['NACOS_DATA_ID'] = nacos_data_id
143
+ if nacos_username:
144
+ os.environ['NACOS_USERNAME'] = nacos_username
145
+ if nacos_password:
146
+ os.environ['NACOS_PASSWORD'] = nacos_password
147
+
148
+ click.echo(f"正在从Nacos加载配置 ({nacos_server}/{nacos_namespace})...")
149
+
150
+ # 加载Nacos配置
151
+ nacos_pg_url = config.config.get('JETTASK_PG_URL')
152
+ if not nacos_pg_url:
153
+ # 如果没有直接的URL,尝试从独立的配置项构建
154
+ pg_host = config.config.get('PG_DB_HOST', 'localhost')
155
+ pg_port = config.config.get('PG_DB_PORT', 5432)
156
+ pg_user = config.config.get('PG_DB_USERNAME', 'jettask')
157
+ pg_password = config.config.get('PG_DB_PASSWORD', '123456')
158
+ pg_database = config.config.get('PG_DB_DATABASE', 'jettask')
159
+ nacos_pg_url = f'postgresql://{pg_user}:{pg_password}@{pg_host}:{pg_port}/{pg_database}'
160
+
161
+ # 将Nacos配置写入环境变量(会覆盖.env中的配置)
162
+ if nacos_pg_url:
163
+ os.environ['JETTASK_PG_URL'] = nacos_pg_url
164
+
165
+ # 将其他Nacos配置也设置到环境变量中
166
+ for key, value in config.config.items():
167
+ if isinstance(value, (str, int, float, bool)):
168
+ os.environ[key] = str(value)
169
+
170
+ click.echo(f"✓ 从Nacos加载配置成功")
171
+ except click.Abort:
172
+ raise
173
+ except Exception as e:
174
+ click.echo(f"Error: 从Nacos加载配置失败: {e}", err=True)
175
+ import traceback
176
+ traceback.print_exc()
177
+ raise click.Abort()
178
+
179
+ # 保存loader到context,供子命令使用
180
+ ctx.obj['env_loader'] = loader
181
+ ctx.obj['use_nacos'] = use_nacos
182
+
183
+ @cli.command()
184
+ @click.option('--host', default='0.0.0.0', envvar='JETTASK_API_HOST', help='服务器监听地址')
185
+ @click.option('--port', default=8001, type=int, envvar='JETTASK_API_PORT', help='服务器监听端口')
186
+ @click.option('--redis-url', envvar='JETTASK_REDIS_URL', help='Redis连接URL')
187
+ @click.option('--redis-prefix', envvar='JETTASK_REDIS_PREFIX', help='Redis键前缀')
188
+ @click.option('--jettask-pg-url', envvar='JETTASK_PG_URL', help='PostgreSQL连接URL')
189
+ @click.option('--reload', is_flag=True, envvar='JETTASK_API_RELOAD', help='启用自动重载')
190
+ @click.option('--log-level', default='info', envvar='JETTASK_LOG_LEVEL',
51
191
  type=click.Choice(['debug', 'info', 'warning', 'error']),
52
192
  help='日志级别')
53
- def api(host, port, jettask_pg_url, use_nacos, nacos_server, nacos_namespace,
54
- nacos_username, nacos_password, nacos_group, nacos_data_id,
55
- reload, log_level):
193
+ @click.pass_context
194
+ def api(ctx, host, port, redis_url, redis_prefix, jettask_pg_url, reload, log_level):
56
195
  """启动 API 服务和监控界面
57
-
196
+
58
197
  示例:
59
198
  \b
60
199
  # 使用默认配置启动
61
200
  jettask api
62
-
63
- # 指定端口和数据库
64
- jettask api --host 0.0.0.0 --port 8080 --jettask-pg-url postgresql://user:pass@host/db
65
-
66
- # 使用Nacos配置中心
67
- jettask api --use-nacos --nacos-server 127.0.0.1:8848
68
-
69
- # 通过环境变量配置Nacos
70
- export NACOS_SERVER=127.0.0.1:8848
71
- export NACOS_NAMESPACE=tamar_console_dev
72
- jettask api --use-nacos
73
-
74
- # 通过环境变量配置数据库
75
- export JETTASK_PG_URL=postgresql://jettask:123456@localhost:5432/jettask
76
- jettask api
77
-
201
+
202
+ # 指定配置
203
+ jettask api --host 0.0.0.0 --port 8080 \
204
+ --redis-url redis://localhost:6379/0 \
205
+ --redis-prefix my_app \
206
+ --jettask-pg-url postgresql://user:pass@host/db
207
+
208
+ # 使用Nacos配置中心(全局参数)
209
+ jettask --use-nacos api
210
+
211
+ # 通过环境变量配置(.env文件)
212
+ jettask -e .env api
213
+
78
214
  # 启用开发模式(自动重载)
79
215
  jettask api --reload --log-level debug
80
216
  """
81
217
  import os
82
218
  import uvicorn
83
-
84
- print(f'{use_nacos=} {jettask_pg_url=}')
85
- if not use_nacos and not jettask_pg_url:
86
- raise ValueError("必须提供 --jettask-pg-url 或启用 --use-nacos")
87
- # 如果使用Nacos,从Nacos获取配置
88
- if use_nacos:
89
- click.echo("正在从Nacos加载配置...")
90
- load_dotenv()
91
- # 设置Nacos环境变量
92
- if nacos_server:
93
- os.environ['NACOS_SERVER'] = nacos_server
94
- if nacos_namespace:
95
- os.environ['NACOS_NAMESPACE'] = nacos_namespace
96
- if nacos_username:
97
- os.environ['NACOS_USERNAME'] = nacos_username
98
- if nacos_password:
99
- os.environ['NACOS_PASSWORD'] = nacos_password
100
- if nacos_group:
101
- os.environ['NACOS_GROUP'] = nacos_group
102
- if nacos_data_id:
103
- os.environ['NACOS_DATA_ID'] = nacos_data_id
104
-
105
- nacos_server = os.environ['NACOS_SERVER']
106
- nacos_namespace = os.environ['NACOS_NAMESPACE']
107
- nacos_username = os.environ['NACOS_USERNAME']
108
- nacos_password = os.environ['NACOS_PASSWORD']
109
- nacos_group = os.environ['NACOS_GROUP']
110
- nacos_data_id = os.environ['NACOS_DATA_ID']
111
- # 尝试从配置中获取JETTASK_PG_URL
112
- nacos_pg_url = config.config.get('JETTASK_PG_URL')
113
- print(f'{nacos_pg_url=}')
114
- if not nacos_pg_url:
115
- # 如果没有直接的URL,尝试从独立的配置项构建
116
- pg_host = config.config.get('PG_DB_HOST', 'localhost')
117
- pg_port = config.config.get('PG_DB_PORT', 5432)
118
- pg_user = config.config.get('PG_DB_USERNAME', 'jettask')
119
- pg_password = config.config.get('PG_DB_PASSWORD', '123456')
120
- pg_database = config.config.get('PG_DB_DATABASE', 'jettask')
121
- nacos_pg_url = f'postgresql://{pg_user}:{pg_password}@{pg_host}:{pg_port}/{pg_database}'
122
-
123
- jettask_pg_url = nacos_pg_url
124
- click.echo(f"✓ 从Nacos加载配置成功")
125
- click.echo(f" 配置源: {nacos_server}/{nacos_namespace}/{nacos_group}/{nacos_data_id}")
126
-
127
- # 将其他Nacos配置也设置到环境变量中
128
- for key, value in config.config.items():
129
- if isinstance(value, (str, int, float, bool)):
130
- os.environ[key] = str(value)
131
219
 
132
-
220
+ # Click的envvar已经自动处理了环境变量读取
221
+ # 全局的--use-nacos也已经处理了Nacos配置加载
222
+
223
+ # 获取use_nacos状态(从全局context)
224
+ use_nacos = ctx.obj.get('use_nacos', False)
225
+
226
+ # 检查必需参数
227
+ if not jettask_pg_url:
228
+ raise ValueError(
229
+ "必须提供 --jettask-pg-url 或在环境变量中设置 JETTASK_PG_URL\n"
230
+ "或使用 jettask --use-nacos api 从Nacos加载配置"
231
+ )
232
+
233
+ if not redis_url:
234
+ raise ValueError(
235
+ "必须提供 --redis-url 或在环境变量中设置 JETTASK_REDIS_URL\n"
236
+ "或使用 jettask --use-nacos api 从Nacos加载配置"
237
+ )
238
+
133
239
  # 设置环境变量(供应用内部使用)
240
+ os.environ['JETTASK_REDIS_URL'] = redis_url
241
+ if redis_prefix:
242
+ os.environ['JETTASK_REDIS_PREFIX'] = redis_prefix
134
243
  os.environ['JETTASK_PG_URL'] = jettask_pg_url
135
-
136
- # 设置是否使用Nacos的标志
137
244
  os.environ['USE_NACOS'] = 'true' if use_nacos else 'false'
138
245
 
139
246
  # 使用标准应用模块
140
- app_module = "jettask.api:app"
247
+ app_module = "jettask.webui.app:app"
141
248
  click.echo(f"Starting JetTask API Server on {host}:{port}")
142
249
 
143
250
  # 显示配置信息
144
251
  click.echo("=" * 60)
145
252
  click.echo("API Server Configuration")
146
253
  click.echo("=" * 60)
147
- click.echo(f"Host: {host}")
148
- click.echo(f"Port: {port}")
149
- click.echo(f"Auto-reload: {reload}")
150
- click.echo(f"Log level: {log_level}")
151
- click.echo(f"Nacos: {'Enabled' if use_nacos else 'Disabled'}")
152
- if use_nacos:
153
- click.echo(f" Server: {nacos_server}")
154
- click.echo(f" Namespace: {nacos_namespace}")
155
- click.echo(f" Data ID: {nacos_data_id}")
156
- click.echo(f"Database: {jettask_pg_url}")
254
+ click.echo(f"Host: {host}")
255
+ click.echo(f"Port: {port}")
256
+ click.echo(f"Redis URL: {redis_url}")
257
+ click.echo(f"Redis Prefix: {redis_prefix or 'jettask (default)'}")
258
+ click.echo(f"Database: {jettask_pg_url}")
259
+ click.echo(f"Auto-reload: {reload}")
260
+ click.echo(f"Log level: {log_level}")
261
+ click.echo(f"Nacos: {'Enabled' if use_nacos else 'Disabled'}")
157
262
  click.echo("=" * 60)
158
263
  click.echo(f"API Endpoint: http://{host}:{port}/api")
159
264
  click.echo(f"WebUI: http://{host}:{port}/")
@@ -215,47 +320,61 @@ def find_jettask_app(module):
215
320
  return None
216
321
 
217
322
  @cli.command()
218
- @click.argument('app_str', required=False, default=None)
219
- @click.option('--queues', '-q', help='队列名称(逗号分隔,如: queue1,queue2)')
220
- @click.option('--executor', '-e',
221
- type=click.Choice(['asyncio', 'multi_asyncio']),
222
- default='asyncio',
223
- help='执行器类型')
224
- @click.option('--concurrency', '-c', type=int, default=4, help='并发数')
225
- @click.option('--prefetch', '-p', type=int, default=100, help='预取倍数')
226
- @click.option('--reload', '-r', is_flag=True, help='自动重载')
227
- @click.option('--config', help='配置文件 (JSON格式)')
228
- def worker(app_str, queues, executor, concurrency, prefetch, reload, config):
323
+ @click.option('--app', '-a', 'app_str', envvar='JETTASK_APP',
324
+ help='应用位置 (如: module:app, path/to/file.py:app, 或目录名)')
325
+ @click.option('--queues', '-q', envvar='JETTASK_QUEUES', help='队列名称(逗号分隔,如: queue1,queue2)')
326
+ @click.option('--concurrency', '-c', type=int, envvar='JETTASK_CONCURRENCY', help='并发数(默认为CPU核心数的一半)')
327
+ @click.option('--prefetch', '-p', type=int, default=100, envvar='JETTASK_PREFETCH', help='预取倍数')
328
+ @click.option('--reload', '-r', is_flag=True, envvar='JETTASK_RELOAD', help='自动重载')
329
+ @click.pass_context
330
+ def worker(ctx, app_str, queues, concurrency, prefetch, reload):
229
331
  """启动任务处理 Worker
230
-
332
+
231
333
  示例:
232
334
  \b
233
- # 显式指定 app
234
- jettask worker main:app --queues async_queue
235
- jettask worker tasks.py:app --queues queue1,queue2
236
- jettask worker myapp.tasks --queues high,normal,low
237
-
335
+ # 使用 -a 或 --app 参数指定 app
336
+ jettask worker -a main:app --queues async_queue
337
+ jettask worker --app tasks.py:app --queues queue1,queue2
338
+ jettask worker -a myapp --queues high,normal,low
339
+
238
340
  # 自动发现 app(从当前目录的 app.py 或 main.py)
239
341
  jettask worker --queues async_queue
240
-
342
+
241
343
  # 使用环境变量
242
344
  export JETTASK_APP=myapp:app
243
345
  jettask worker --queues async_queue
346
+
347
+ # 加载环境变量文件(全局 -e 参数)
348
+ jettask -e .env worker --queues async_queue
349
+ jettask --env-file production.env worker -a main:app --queues high,low
350
+
351
+ # 或使用环境变量
352
+ export JETTASK_ENV_FILE=.env
353
+ jettask worker --queues async_queue
244
354
  """
245
-
246
- # 如果提供了配置文件,从中加载配置
247
- if config:
248
- click.echo(f"Loading configuration from {config}")
249
- with open(config, 'r') as f:
250
- config_data = json.load(f)
251
-
252
- # 从配置文件读取参数(命令行参数优先)
253
- queues = queues or ','.join(config_data.get('queues', [])) if config_data.get('queues') else None
254
- executor = executor or config_data.get('executor', 'asyncio')
255
- concurrency = concurrency if concurrency != 4 else config_data.get('concurrency', 4)
256
- prefetch = prefetch if prefetch != 100 else config_data.get('prefetch', 100)
257
- reload = reload or config_data.get('reload', False)
258
-
355
+
356
+ # Click的envvar已经自动处理了环境变量读取
357
+
358
+ # 检查必需的 queues 参数
359
+ if not queues:
360
+ click.echo("Error: 必须指定 --queues 参数或设置 JETTASK_QUEUES 环境变量", err=True)
361
+ click.echo("\n示例:", err=True)
362
+ click.echo(" jettask worker --queues default", err=True)
363
+ click.echo(" jettask worker -q high,normal,low", err=True)
364
+ click.echo(" export JETTASK_QUEUES=default && jettask worker", err=True)
365
+ raise click.Abort()
366
+
367
+ # 设置默认 concurrency 值(如果没有通过CLI或环境变量指定)
368
+ cpu_count = multiprocessing.cpu_count()
369
+ if concurrency is None:
370
+ concurrency = max(1, cpu_count // 4)
371
+ click.echo(f"Using default concurrency: {concurrency} (1/4 of {cpu_count} CPU cores)")
372
+ if concurrency>cpu_count:
373
+ click.echo(f"Error: Specified concurrency {concurrency} exceeds CPU cores {cpu_count}", err=True)
374
+ raise click.Abort()
375
+ # 固定使用 multi_asyncio 执行器
376
+ executor = 'multi_asyncio'
377
+
259
378
  # 加载应用
260
379
  try:
261
380
  if app_str:
@@ -266,10 +385,13 @@ def worker(app_str, queues, executor, concurrency, prefetch, reload, config):
266
385
  click.echo("Searching in: app.py, main.py, server.py, worker.py")
267
386
  app = import_app() # 自动发现
268
387
 
388
+
389
+ # app.redis_prefix = app.redis_prefix or 'jettask'
269
390
  # 显示应用信息
270
391
  app_info = AppImporter.get_app_info(app)
271
392
  click.echo(f"\nFound Jettask app:")
272
393
  click.echo(f" Tasks: {app_info['tasks']} registered")
394
+
273
395
  if app_info.get('task_names') and app_info['tasks'] > 0:
274
396
  task_preview = app_info['task_names'][:3]
275
397
  click.echo(f" Names: {', '.join(task_preview)}" +
@@ -294,6 +416,7 @@ def worker(app_str, queues, executor, concurrency, prefetch, reload, config):
294
416
  sys.exit(1)
295
417
  except Exception as e:
296
418
  import traceback
419
+ traceback.print_exc()
297
420
  click.echo(f"Error loading app: {e}", err=True)
298
421
 
299
422
  # 对于所有异常都显示堆栈信息
@@ -307,24 +430,8 @@ def worker(app_str, queues, executor, concurrency, prefetch, reload, config):
307
430
  click.echo("Please check the traceback above for details.", err=True)
308
431
  sys.exit(1)
309
432
 
310
- # 处理队列参数
311
- if queues:
312
- # 解析队列列表(支持逗号分隔)
313
- queue_list = [q.strip() for q in queues.split(',') if q.strip()]
314
- else:
315
- # 如果没有指定队列,尝试从 app 获取
316
- if hasattr(app, 'ep') and hasattr(app.ep, 'queues'):
317
- queue_list = list(app.ep.queues)
318
- if queue_list:
319
- click.echo(f"Using queues from app: {', '.join(queue_list)}")
320
- else:
321
- queue_list = []
322
-
323
- if not queue_list:
324
- click.echo("Error: No queues specified", err=True)
325
- click.echo(" Use --queues to specify queues, e.g.: --queues queue1,queue2", err=True)
326
- click.echo(" Or define queues in your app configuration", err=True)
327
- sys.exit(1)
433
+ # 解析队列列表(支持逗号分隔)
434
+ queue_list = [q.strip() for q in queues.split(',') if q.strip()]
328
435
 
329
436
  # 从 app 实例中获取实际配置
330
437
  redis_url = app.redis_url if hasattr(app, 'redis_url') else 'Not configured'
@@ -364,36 +471,36 @@ def worker(app_str, queues, executor, concurrency, prefetch, reload, config):
364
471
 
365
472
  @cli.command('webui-consumer')
366
473
  @click.option('--task-center', '-tc', envvar='JETTASK_CENTER_URL', required=True,
367
- help='任务中心URL,如: http://localhost:8001 或 http://localhost:8001/api/v1/namespaces/default')
474
+ help='任务中心URL,如: http://localhost:8001 或 http://localhost:8001/api/namespaces/default')
368
475
  @click.option('--check-interval', type=int, default=30,
369
476
  help='命名空间检测间隔(秒),默认30秒')
370
477
  @click.option('--debug', is_flag=True, help='启用调试模式')
371
478
  def webui_consumer(task_center, check_interval, debug):
372
479
  """启动数据消费者(自动识别单/多命名空间)
373
-
480
+
374
481
  根据URL格式自动判断运行模式:
375
- - 单命名空间: http://localhost:8001/api/v1/namespaces/{name}
482
+ - 单命名空间: http://localhost:8001/api/namespaces/{name}
376
483
  - 多命名空间: http://localhost:8001 或 http://localhost:8001/api
377
-
484
+
378
485
  示例:
379
486
  \b
380
487
  # 为所有命名空间启动消费者(自动检测)
381
488
  jettask webui-consumer --task-center http://localhost:8001
382
489
  jettask webui-consumer --task-center http://localhost:8001/api
383
-
490
+
384
491
  # 为单个命名空间启动消费者
385
- jettask webui-consumer --task-center http://localhost:8001/api/v1/namespaces/default
386
-
492
+ jettask webui-consumer --task-center http://localhost:8001/api/namespaces/default
493
+
387
494
  # 自定义检测间隔
388
495
  jettask webui-consumer --task-center http://localhost:8001 --check-interval 60
389
-
496
+
390
497
  # 使用环境变量
391
498
  export JETTASK_CENTER_URL=http://localhost:8001
392
499
  jettask webui-consumer
393
500
  """
394
501
  import asyncio
395
- from jettask.unified_consumer_manager import UnifiedConsumerManager
396
-
502
+ from jettask.webui.namespace_manager.unified import UnifiedConsumerManager
503
+
397
504
  # 运行消费者管理器
398
505
  async def run_manager():
399
506
  """运行统一的消费者管理器"""
@@ -403,70 +510,51 @@ def webui_consumer(task_center, check_interval, debug):
403
510
  debug=debug
404
511
  )
405
512
  await manager.run()
406
-
513
+
407
514
  try:
408
515
  asyncio.run(run_manager())
409
516
  except KeyboardInterrupt:
410
517
  click.echo("\nShutdown complete")
411
518
 
412
- @cli.command()
413
- def monitor():
414
- """启动系统监控器"""
415
- click.echo("Starting JetTask Monitor")
416
- from jettask.run_monitor import main as monitor_main
417
- monitor_main()
519
+ # @cli.command()
520
+ # def monitor():
521
+ # """启动系统监控器"""
522
+ # click.echo("Starting JetTask Monitor")
523
+ # # 已删除 run_monitor 模块
524
+ # click.echo("Monitor command is deprecated and has been removed.")
418
525
 
419
526
  @cli.command()
420
- @click.option('--task-center', '-tc', envvar='JETTASK_CENTER_URL',
421
- help='任务中心URL,如: http://localhost:8001/api/v1/namespaces/default')
422
- def init(task_center):
527
+ @click.option('--pg-url', envvar='JETTASK_PG_URL',
528
+ help='PostgreSQL连接URL (如: postgresql://user:pass@localhost/db)')
529
+ def init(pg_url):
423
530
  """初始化数据库和配置
424
-
531
+
425
532
  示例:
426
533
  \b
427
- # 使用任务中心初始化
428
- jettask init --task-center http://localhost:8001/api/v1/namespaces/default
429
- jettask init -tc http://localhost:8001/api/v1/namespaces/production
430
-
534
+ # 使用命令行参数
535
+ jettask init --pg-url postgresql://user:pass@localhost/jettask
536
+
431
537
  # 使用环境变量
432
- export JETTASK_CENTER_URL=http://localhost:8001/api/v1/namespaces/default
433
- jettask init
434
-
435
- # 不使用任务中心(仅使用本地环境变量)
538
+ export JETTASK_PG_URL=postgresql://user:pass@localhost/jettask
436
539
  jettask init
540
+
541
+ # 使用 .env 文件
542
+ jettask -e .env init
437
543
  """
438
544
  click.echo("Initializing JetTask...")
439
-
545
+
440
546
  import os
441
-
442
- # 如果提供了任务中心URL,尝试从任务中心获取配置
443
- if task_center:
444
- os.environ['JETTASK_CENTER_URL'] = task_center
445
- click.echo(f"Using Task Center: {task_center}")
446
-
447
- # 尝试从任务中心获取数据库配置
448
- try:
449
- from jettask.task_center import TaskCenter
450
- tc = TaskCenter(task_center)
451
- if tc._connect_sync():
452
- p_config = tc.pg_config
453
- # 从任务中心获取的配置中提取数据库连接参数
454
- os.environ['JETTASK_PG_HOST'] = p_config['host']
455
- os.environ['JETTASK_PG_PORT'] = str(p_config['port'])
456
- os.environ['JETTASK_PG_DATABASE'] = p_config['database']
457
- os.environ['JETTASK_PG_USER'] = p_config['user']
458
- os.environ['JETTASK_PG_PASSWORD'] = p_config['password']
459
- else:
460
- click.echo("⚠ Failed to connect to Task Center, using local configuration", err=True)
461
- except Exception as e:
462
- click.echo(f"⚠ Could not get configuration from Task Center: {e}", err=True)
463
- click.echo(" Falling back to local environment variables")
464
-
547
+
548
+ # 如果提供了 PG URL,设置到环境变量
549
+ if pg_url:
550
+ os.environ['JETTASK_PG_URL'] = pg_url
551
+ click.echo(f"Using PostgreSQL: {pg_url.split('@')[1] if '@' in pg_url else pg_url}")
552
+
465
553
  # 初始化数据库
466
- from jettask.db_init import init_database
554
+ from jettask.utils.db_init import init_database
467
555
  click.echo("\nInitializing database...")
468
556
  init_database()
469
-
557
+
470
558
  click.echo("\n✓ JetTask initialized successfully!")
471
559
 
472
560
  @cli.command()
@@ -477,12 +565,12 @@ def status():
477
565
 
478
566
  # 检查 Redis 连接
479
567
  try:
480
- import redis
481
- r = redis.Redis(host='localhost', port=6379, db=0)
568
+ from jettask.utils.db_connector import get_sync_redis_client
569
+ r = get_sync_redis_client('redis://localhost:6379/0', decode_responses=True)
482
570
  r.ping()
483
571
  click.echo("✓ Redis: Connected")
484
- except:
485
- click.echo("✗ Redis: Not connected")
572
+ except Exception as e:
573
+ click.echo(f"✗ Redis: Not connected ({e})")
486
574
 
487
575
  # 检查 PostgreSQL 连接
488
576
  try:
@@ -503,8 +591,8 @@ def status():
503
591
 
504
592
  @cli.command()
505
593
  @click.option('--task-center', '-tc', envvar='JETTASK_CENTER_URL', required=True,
506
- help='任务中心URL,如: http://localhost:8001 或 http://localhost:8001/api/v1/namespaces/default')
507
- @click.option('--interval', '-i', type=float, default=0.1,
594
+ help='任务中心URL,如: http://localhost:8001 或 http://localhost:8001/api/namespaces/default')
595
+ @click.option('--interval', '-i', type=float, default=0.1,
508
596
  help='调度器扫描间隔(秒),默认0.1秒')
509
597
  @click.option('--batch-size', '-b', type=int, default=100,
510
598
  help='每批处理的最大任务数,默认100')
@@ -513,30 +601,30 @@ def status():
513
601
  @click.option('--debug', is_flag=True, help='启用调试模式')
514
602
  def scheduler(task_center, interval, batch_size, check_interval, debug):
515
603
  """启动定时任务调度器(自动识别单/多命名空间)
516
-
604
+
517
605
  根据URL格式自动判断运行模式:
518
- - 单命名空间: http://localhost:8001/api/v1/namespaces/{name}
606
+ - 单命名空间: http://localhost:8001/api/namespaces/{name}
519
607
  - 多命名空间: http://localhost:8001 或 http://localhost:8001/api
520
-
608
+
521
609
  示例:
522
610
  \b
523
611
  # 为所有命名空间启动调度器(自动检测)
524
612
  jettask scheduler --task-center http://localhost:8001
525
613
  jettask scheduler --task-center http://localhost:8001/api
526
-
614
+
527
615
  # 为单个命名空间启动调度器
528
- jettask scheduler --task-center http://localhost:8001/api/v1/namespaces/default
529
-
616
+ jettask scheduler --task-center http://localhost:8001/api/namespaces/default
617
+
530
618
  # 自定义配置
531
619
  jettask scheduler --task-center http://localhost:8001 --check-interval 60 --interval 0.5
532
-
620
+
533
621
  # 使用环境变量
534
622
  export JETTASK_CENTER_URL=http://localhost:8001
535
623
  jettask scheduler
536
624
  """
537
625
  import asyncio
538
- from jettask.scheduler.unified_scheduler_manager import UnifiedSchedulerManager
539
-
626
+ from jettask.scheduler.scheduler_coordinator import UnifiedSchedulerManager
627
+
540
628
  # 运行调度器管理器
541
629
  async def run_manager():
542
630
  """运行统一的调度器管理器"""
@@ -548,7 +636,7 @@ def scheduler(task_center, interval, batch_size, check_interval, debug):
548
636
  debug=debug
549
637
  )
550
638
  await manager.run()
551
-
639
+
552
640
  try:
553
641
  asyncio.run(run_manager())
554
642
  except KeyboardInterrupt:
@@ -710,8 +798,6 @@ def frontend(port, host, api_url, auto_install, force_install, build_only):
710
798
  import http.server
711
799
  import socketserver
712
800
  import urllib.request
713
- import urllib.parse
714
- import json
715
801
 
716
802
  class ProxyHandler(http.server.SimpleHTTPRequestHandler):
717
803
  def __init__(self, *args, **kwargs):