jettask 0.2.19__py3-none-any.whl → 0.2.23__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. jettask/__init__.py +12 -3
  2. jettask/cli.py +314 -228
  3. jettask/config/__init__.py +9 -1
  4. jettask/config/config.py +245 -0
  5. jettask/config/env_loader.py +381 -0
  6. jettask/config/lua_scripts.py +158 -0
  7. jettask/config/nacos_config.py +132 -5
  8. jettask/core/__init__.py +1 -1
  9. jettask/core/app.py +1573 -666
  10. jettask/core/app_importer.py +33 -16
  11. jettask/core/container.py +532 -0
  12. jettask/core/task.py +1 -4
  13. jettask/core/unified_manager_base.py +2 -2
  14. jettask/executor/__init__.py +38 -0
  15. jettask/executor/core.py +625 -0
  16. jettask/executor/executor.py +338 -0
  17. jettask/executor/orchestrator.py +290 -0
  18. jettask/executor/process_entry.py +638 -0
  19. jettask/executor/task_executor.py +317 -0
  20. jettask/messaging/__init__.py +68 -0
  21. jettask/messaging/event_pool.py +2188 -0
  22. jettask/messaging/reader.py +519 -0
  23. jettask/messaging/registry.py +266 -0
  24. jettask/messaging/scanner.py +369 -0
  25. jettask/messaging/sender.py +312 -0
  26. jettask/persistence/__init__.py +118 -0
  27. jettask/persistence/backlog_monitor.py +567 -0
  28. jettask/{backend/data_access.py → persistence/base.py} +58 -57
  29. jettask/persistence/consumer.py +315 -0
  30. jettask/{core → persistence}/db_manager.py +23 -22
  31. jettask/persistence/maintenance.py +81 -0
  32. jettask/persistence/message_consumer.py +259 -0
  33. jettask/{backend/namespace_data_access.py → persistence/namespace.py} +66 -98
  34. jettask/persistence/offline_recovery.py +196 -0
  35. jettask/persistence/queue_discovery.py +215 -0
  36. jettask/persistence/task_persistence.py +218 -0
  37. jettask/persistence/task_updater.py +583 -0
  38. jettask/scheduler/__init__.py +2 -2
  39. jettask/scheduler/loader.py +6 -5
  40. jettask/scheduler/run_scheduler.py +1 -1
  41. jettask/scheduler/scheduler.py +7 -7
  42. jettask/scheduler/{unified_scheduler_manager.py → scheduler_coordinator.py} +18 -13
  43. jettask/task/__init__.py +16 -0
  44. jettask/{router.py → task/router.py} +26 -8
  45. jettask/task/task_center/__init__.py +9 -0
  46. jettask/task/task_executor.py +318 -0
  47. jettask/task/task_registry.py +291 -0
  48. jettask/test_connection_monitor.py +73 -0
  49. jettask/utils/__init__.py +31 -1
  50. jettask/{monitor/run_backlog_collector.py → utils/backlog_collector.py} +1 -1
  51. jettask/utils/db_connector.py +1629 -0
  52. jettask/{db_init.py → utils/db_init.py} +1 -1
  53. jettask/utils/rate_limit/__init__.py +30 -0
  54. jettask/utils/rate_limit/concurrency_limiter.py +665 -0
  55. jettask/utils/rate_limit/config.py +145 -0
  56. jettask/utils/rate_limit/limiter.py +41 -0
  57. jettask/utils/rate_limit/manager.py +269 -0
  58. jettask/utils/rate_limit/qps_limiter.py +154 -0
  59. jettask/utils/rate_limit/task_limiter.py +384 -0
  60. jettask/utils/serializer.py +3 -0
  61. jettask/{monitor/stream_backlog_monitor.py → utils/stream_backlog.py} +14 -6
  62. jettask/utils/time_sync.py +173 -0
  63. jettask/webui/__init__.py +27 -0
  64. jettask/{api/v1 → webui/api}/alerts.py +1 -1
  65. jettask/{api/v1 → webui/api}/analytics.py +2 -2
  66. jettask/{api/v1 → webui/api}/namespaces.py +1 -1
  67. jettask/{api/v1 → webui/api}/overview.py +1 -1
  68. jettask/{api/v1 → webui/api}/queues.py +3 -3
  69. jettask/{api/v1 → webui/api}/scheduled.py +1 -1
  70. jettask/{api/v1 → webui/api}/settings.py +1 -1
  71. jettask/{api.py → webui/app.py} +253 -145
  72. jettask/webui/namespace_manager/__init__.py +10 -0
  73. jettask/{multi_namespace_consumer.py → webui/namespace_manager/multi.py} +69 -22
  74. jettask/{unified_consumer_manager.py → webui/namespace_manager/unified.py} +1 -1
  75. jettask/{run.py → webui/run.py} +2 -2
  76. jettask/{services → webui/services}/__init__.py +1 -3
  77. jettask/{services → webui/services}/overview_service.py +34 -16
  78. jettask/{services → webui/services}/queue_service.py +1 -1
  79. jettask/{backend → webui/services}/queue_stats_v2.py +1 -1
  80. jettask/{services → webui/services}/settings_service.py +1 -1
  81. jettask/worker/__init__.py +53 -0
  82. jettask/worker/lifecycle.py +1507 -0
  83. jettask/worker/manager.py +583 -0
  84. jettask/{core/offline_worker_recovery.py → worker/recovery.py} +268 -175
  85. {jettask-0.2.19.dist-info → jettask-0.2.23.dist-info}/METADATA +2 -71
  86. jettask-0.2.23.dist-info/RECORD +145 -0
  87. jettask/__main__.py +0 -140
  88. jettask/api/__init__.py +0 -103
  89. jettask/backend/__init__.py +0 -1
  90. jettask/backend/api/__init__.py +0 -3
  91. jettask/backend/api/v1/__init__.py +0 -17
  92. jettask/backend/api/v1/monitoring.py +0 -431
  93. jettask/backend/api/v1/namespaces.py +0 -504
  94. jettask/backend/api/v1/queues.py +0 -342
  95. jettask/backend/api/v1/tasks.py +0 -367
  96. jettask/backend/core/__init__.py +0 -3
  97. jettask/backend/core/cache.py +0 -221
  98. jettask/backend/core/database.py +0 -200
  99. jettask/backend/core/exceptions.py +0 -102
  100. jettask/backend/dependencies.py +0 -261
  101. jettask/backend/init_meta_db.py +0 -158
  102. jettask/backend/main.py +0 -1426
  103. jettask/backend/main_unified.py +0 -78
  104. jettask/backend/main_v2.py +0 -394
  105. jettask/backend/models/__init__.py +0 -3
  106. jettask/backend/models/requests.py +0 -236
  107. jettask/backend/models/responses.py +0 -230
  108. jettask/backend/namespace_api_old.py +0 -267
  109. jettask/backend/services/__init__.py +0 -3
  110. jettask/backend/start.py +0 -42
  111. jettask/backend/unified_api_router.py +0 -1541
  112. jettask/cleanup_deprecated_tables.sql +0 -16
  113. jettask/core/consumer_manager.py +0 -1695
  114. jettask/core/delay_scanner.py +0 -256
  115. jettask/core/event_pool.py +0 -1700
  116. jettask/core/heartbeat_process.py +0 -222
  117. jettask/core/task_batch.py +0 -153
  118. jettask/core/worker_scanner.py +0 -271
  119. jettask/executors/__init__.py +0 -5
  120. jettask/executors/asyncio.py +0 -876
  121. jettask/executors/base.py +0 -30
  122. jettask/executors/common.py +0 -148
  123. jettask/executors/multi_asyncio.py +0 -309
  124. jettask/gradio_app.py +0 -570
  125. jettask/integrated_gradio_app.py +0 -1088
  126. jettask/main.py +0 -0
  127. jettask/monitoring/__init__.py +0 -3
  128. jettask/pg_consumer.py +0 -1896
  129. jettask/run_monitor.py +0 -22
  130. jettask/run_webui.py +0 -148
  131. jettask/scheduler/multi_namespace_scheduler.py +0 -294
  132. jettask/scheduler/unified_manager.py +0 -450
  133. jettask/task_center_client.py +0 -150
  134. jettask/utils/serializer_optimized.py +0 -33
  135. jettask/webui_exceptions.py +0 -67
  136. jettask-0.2.19.dist-info/RECORD +0 -150
  137. /jettask/{constants.py → config/constants.py} +0 -0
  138. /jettask/{backend/config.py → config/task_center.py} +0 -0
  139. /jettask/{pg_consumer → messaging/pg_consumer}/pg_consumer_v2.py +0 -0
  140. /jettask/{pg_consumer → messaging/pg_consumer}/sql/add_execution_time_field.sql +0 -0
  141. /jettask/{pg_consumer → messaging/pg_consumer}/sql/create_new_tables.sql +0 -0
  142. /jettask/{pg_consumer → messaging/pg_consumer}/sql/create_tables_v3.sql +0 -0
  143. /jettask/{pg_consumer → messaging/pg_consumer}/sql/migrate_to_new_structure.sql +0 -0
  144. /jettask/{pg_consumer → messaging/pg_consumer}/sql/modify_time_fields.sql +0 -0
  145. /jettask/{pg_consumer → messaging/pg_consumer}/sql_utils.py +0 -0
  146. /jettask/{models.py → persistence/models.py} +0 -0
  147. /jettask/scheduler/{manager.py → task_crud.py} +0 -0
  148. /jettask/{schema.sql → schemas/schema.sql} +0 -0
  149. /jettask/{task_center.py → task/task_center/client.py} +0 -0
  150. /jettask/{monitoring → utils}/file_watcher.py +0 -0
  151. /jettask/{services/redis_monitor_service.py → utils/redis_monitor.py} +0 -0
  152. /jettask/{api/v1 → webui/api}/__init__.py +0 -0
  153. /jettask/{webui_config.py → webui/config.py} +0 -0
  154. /jettask/{webui_models → webui/models}/__init__.py +0 -0
  155. /jettask/{webui_models → webui/models}/namespace.py +0 -0
  156. /jettask/{services → webui/services}/alert_service.py +0 -0
  157. /jettask/{services → webui/services}/analytics_service.py +0 -0
  158. /jettask/{services → webui/services}/scheduled_task_service.py +0 -0
  159. /jettask/{services → webui/services}/task_service.py +0 -0
  160. /jettask/{webui_sql → webui/sql}/batch_upsert_functions.sql +0 -0
  161. /jettask/{webui_sql → webui/sql}/verify_database.sql +0 -0
  162. {jettask-0.2.19.dist-info → jettask-0.2.23.dist-info}/WHEEL +0 -0
  163. {jettask-0.2.19.dist-info → jettask-0.2.23.dist-info}/entry_points.txt +0 -0
  164. {jettask-0.2.19.dist-info → jettask-0.2.23.dist-info}/licenses/LICENSE +0 -0
  165. {jettask-0.2.19.dist-info → jettask-0.2.23.dist-info}/top_level.txt +0 -0
@@ -1,158 +0,0 @@
1
- """
2
- 初始化任务中心元数据库
3
- 这个脚本用于创建任务中心自己的数据库和表结构
4
- """
5
- import psycopg2
6
- import sys
7
- from config import task_center_config
8
- import logging
9
-
10
- logging.basicConfig(level=logging.INFO)
11
- logger = logging.getLogger(__name__)
12
-
13
-
14
- def create_meta_database():
15
- """创建任务中心元数据库"""
16
- # 连接到postgres数据库来创建新数据库
17
- conn = psycopg2.connect(
18
- host=task_center_config.meta_db_host,
19
- port=task_center_config.meta_db_port,
20
- user=task_center_config.meta_db_user,
21
- password=task_center_config.meta_db_password,
22
- database='postgres'
23
- )
24
- conn.autocommit = True
25
- cur = conn.cursor()
26
-
27
- try:
28
- # 检查数据库是否存在
29
- cur.execute(
30
- "SELECT 1 FROM pg_database WHERE datname = %s",
31
- (task_center_config.meta_db_name,)
32
- )
33
-
34
- if not cur.fetchone():
35
- # 创建数据库
36
- cur.execute(f"CREATE DATABASE {task_center_config.meta_db_name}")
37
- logger.info(f"Created database: {task_center_config.meta_db_name}")
38
- else:
39
- logger.info(f"Database already exists: {task_center_config.meta_db_name}")
40
-
41
- finally:
42
- cur.close()
43
- conn.close()
44
-
45
-
46
- def init_meta_tables():
47
- """初始化元数据表"""
48
- conn = psycopg2.connect(task_center_config.sync_meta_database_url)
49
- cur = conn.cursor()
50
-
51
- try:
52
- # 创建命名空间表
53
- cur.execute("""
54
- CREATE TABLE IF NOT EXISTS namespaces (
55
- id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
56
- name VARCHAR(255) NOT NULL UNIQUE,
57
- description TEXT,
58
- -- JetTask应用使用的Redis配置
59
- redis_config JSONB NOT NULL COMMENT 'JetTask应用的Redis配置',
60
- -- JetTask应用使用的PostgreSQL配置
61
- pg_config JSONB NOT NULL COMMENT 'JetTask应用的PostgreSQL配置',
62
- is_active BOOLEAN DEFAULT true,
63
- version INTEGER DEFAULT 1,
64
- created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
65
- updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
66
- );
67
- """)
68
-
69
- # 创建索引
70
- cur.execute("CREATE INDEX IF NOT EXISTS idx_namespaces_name ON namespaces(name);")
71
- cur.execute("CREATE INDEX IF NOT EXISTS idx_namespaces_is_active ON namespaces(is_active);")
72
- cur.execute("CREATE INDEX IF NOT EXISTS idx_namespaces_version ON namespaces(version);")
73
-
74
- # 创建更新时间触发器
75
- cur.execute("""
76
- CREATE OR REPLACE FUNCTION update_namespaces_updated_at()
77
- RETURNS TRIGGER AS $$
78
- BEGIN
79
- NEW.updated_at = CURRENT_TIMESTAMP;
80
- RETURN NEW;
81
- END;
82
- $$ LANGUAGE plpgsql;
83
- """)
84
-
85
- cur.execute("""
86
- DO $$
87
- BEGIN
88
- IF NOT EXISTS (
89
- SELECT 1 FROM pg_trigger
90
- WHERE tgname = 'update_namespaces_updated_at'
91
- ) THEN
92
- CREATE TRIGGER update_namespaces_updated_at
93
- BEFORE UPDATE ON namespaces
94
- FOR EACH ROW
95
- EXECUTE FUNCTION update_namespaces_updated_at();
96
- END IF;
97
- END;
98
- $$;
99
- """)
100
-
101
- # 插入默认命名空间
102
- cur.execute("""
103
- INSERT INTO namespaces (id, name, description, redis_config, pg_config)
104
- VALUES (
105
- 'a8f10720-068b-4264-bcbb-e00ceba370e9',
106
- 'default',
107
- '默认命名空间 - JetTask应用使用此配置',
108
- '{"host": "localhost", "port": 6379, "password": null, "db": 0}',
109
- '{"host": "localhost", "port": 5432, "user": "jettask", "password": "123456", "database": "jettask"}'
110
- )
111
- ON CONFLICT (name) DO NOTHING;
112
- """)
113
-
114
- conn.commit()
115
- logger.info("Initialized meta tables successfully")
116
-
117
- except Exception as e:
118
- conn.rollback()
119
- logger.error(f"Failed to initialize meta tables: {e}")
120
- raise
121
- finally:
122
- cur.close()
123
- conn.close()
124
-
125
-
126
- def main():
127
- """主函数"""
128
- logger.info("=" * 60)
129
- logger.info("任务中心元数据库初始化")
130
- logger.info("=" * 60)
131
- logger.info(f"元数据库地址: {task_center_config.meta_db_host}:{task_center_config.meta_db_port}")
132
- logger.info(f"元数据库名称: {task_center_config.meta_db_name}")
133
- logger.info(f"元数据库用户: {task_center_config.meta_db_user}")
134
- logger.info("-" * 60)
135
-
136
- try:
137
- # 1. 创建数据库
138
- create_meta_database()
139
-
140
- # 2. 初始化表结构
141
- init_meta_tables()
142
-
143
- logger.info("-" * 60)
144
- logger.info("✅ 元数据库初始化完成!")
145
- logger.info("")
146
- logger.info("说明:")
147
- logger.info("1. 任务中心元数据库用于存储命名空间配置")
148
- logger.info("2. 每个命名空间定义了JetTask应用使用的Redis和PostgreSQL配置")
149
- logger.info("3. JetTask应用通过命名空间ID获取其专用的数据库配置")
150
- logger.info("=" * 60)
151
-
152
- except Exception as e:
153
- logger.error(f"初始化失败: {e}")
154
- sys.exit(1)
155
-
156
-
157
- if __name__ == "__main__":
158
- main()