jettask 0.2.19__py3-none-any.whl → 0.2.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jettask/__init__.py +10 -3
- jettask/cli.py +314 -228
- jettask/config/__init__.py +9 -1
- jettask/config/config.py +245 -0
- jettask/config/env_loader.py +381 -0
- jettask/config/lua_scripts.py +158 -0
- jettask/config/nacos_config.py +132 -5
- jettask/core/__init__.py +1 -1
- jettask/core/app.py +1573 -666
- jettask/core/app_importer.py +33 -16
- jettask/core/container.py +532 -0
- jettask/core/task.py +1 -4
- jettask/core/unified_manager_base.py +2 -2
- jettask/executor/__init__.py +38 -0
- jettask/executor/core.py +625 -0
- jettask/executor/executor.py +338 -0
- jettask/executor/orchestrator.py +290 -0
- jettask/executor/process_entry.py +638 -0
- jettask/executor/task_executor.py +317 -0
- jettask/messaging/__init__.py +68 -0
- jettask/messaging/event_pool.py +2188 -0
- jettask/messaging/reader.py +519 -0
- jettask/messaging/registry.py +266 -0
- jettask/messaging/scanner.py +369 -0
- jettask/messaging/sender.py +312 -0
- jettask/persistence/__init__.py +118 -0
- jettask/persistence/backlog_monitor.py +567 -0
- jettask/{backend/data_access.py → persistence/base.py} +58 -57
- jettask/persistence/consumer.py +315 -0
- jettask/{core → persistence}/db_manager.py +23 -22
- jettask/persistence/maintenance.py +81 -0
- jettask/persistence/message_consumer.py +259 -0
- jettask/{backend/namespace_data_access.py → persistence/namespace.py} +66 -98
- jettask/persistence/offline_recovery.py +196 -0
- jettask/persistence/queue_discovery.py +215 -0
- jettask/persistence/task_persistence.py +218 -0
- jettask/persistence/task_updater.py +583 -0
- jettask/scheduler/__init__.py +2 -2
- jettask/scheduler/loader.py +6 -5
- jettask/scheduler/run_scheduler.py +1 -1
- jettask/scheduler/scheduler.py +7 -7
- jettask/scheduler/{unified_scheduler_manager.py → scheduler_coordinator.py} +18 -13
- jettask/task/__init__.py +16 -0
- jettask/{router.py → task/router.py} +26 -8
- jettask/task/task_center/__init__.py +9 -0
- jettask/task/task_executor.py +318 -0
- jettask/task/task_registry.py +291 -0
- jettask/test_connection_monitor.py +73 -0
- jettask/utils/__init__.py +31 -1
- jettask/{monitor/run_backlog_collector.py → utils/backlog_collector.py} +1 -1
- jettask/utils/db_connector.py +1629 -0
- jettask/{db_init.py → utils/db_init.py} +1 -1
- jettask/utils/rate_limit/__init__.py +30 -0
- jettask/utils/rate_limit/concurrency_limiter.py +665 -0
- jettask/utils/rate_limit/config.py +145 -0
- jettask/utils/rate_limit/limiter.py +41 -0
- jettask/utils/rate_limit/manager.py +269 -0
- jettask/utils/rate_limit/qps_limiter.py +154 -0
- jettask/utils/rate_limit/task_limiter.py +384 -0
- jettask/utils/serializer.py +3 -0
- jettask/{monitor/stream_backlog_monitor.py → utils/stream_backlog.py} +14 -6
- jettask/utils/time_sync.py +173 -0
- jettask/webui/__init__.py +27 -0
- jettask/{api/v1 → webui/api}/alerts.py +1 -1
- jettask/{api/v1 → webui/api}/analytics.py +2 -2
- jettask/{api/v1 → webui/api}/namespaces.py +1 -1
- jettask/{api/v1 → webui/api}/overview.py +1 -1
- jettask/{api/v1 → webui/api}/queues.py +3 -3
- jettask/{api/v1 → webui/api}/scheduled.py +1 -1
- jettask/{api/v1 → webui/api}/settings.py +1 -1
- jettask/{api.py → webui/app.py} +253 -145
- jettask/webui/namespace_manager/__init__.py +10 -0
- jettask/{multi_namespace_consumer.py → webui/namespace_manager/multi.py} +69 -22
- jettask/{unified_consumer_manager.py → webui/namespace_manager/unified.py} +1 -1
- jettask/{run.py → webui/run.py} +2 -2
- jettask/{services → webui/services}/__init__.py +1 -3
- jettask/{services → webui/services}/overview_service.py +34 -16
- jettask/{services → webui/services}/queue_service.py +1 -1
- jettask/{backend → webui/services}/queue_stats_v2.py +1 -1
- jettask/{services → webui/services}/settings_service.py +1 -1
- jettask/worker/__init__.py +53 -0
- jettask/worker/lifecycle.py +1507 -0
- jettask/worker/manager.py +583 -0
- jettask/{core/offline_worker_recovery.py → worker/recovery.py} +268 -175
- {jettask-0.2.19.dist-info → jettask-0.2.20.dist-info}/METADATA +2 -71
- jettask-0.2.20.dist-info/RECORD +145 -0
- jettask/__main__.py +0 -140
- jettask/api/__init__.py +0 -103
- jettask/backend/__init__.py +0 -1
- jettask/backend/api/__init__.py +0 -3
- jettask/backend/api/v1/__init__.py +0 -17
- jettask/backend/api/v1/monitoring.py +0 -431
- jettask/backend/api/v1/namespaces.py +0 -504
- jettask/backend/api/v1/queues.py +0 -342
- jettask/backend/api/v1/tasks.py +0 -367
- jettask/backend/core/__init__.py +0 -3
- jettask/backend/core/cache.py +0 -221
- jettask/backend/core/database.py +0 -200
- jettask/backend/core/exceptions.py +0 -102
- jettask/backend/dependencies.py +0 -261
- jettask/backend/init_meta_db.py +0 -158
- jettask/backend/main.py +0 -1426
- jettask/backend/main_unified.py +0 -78
- jettask/backend/main_v2.py +0 -394
- jettask/backend/models/__init__.py +0 -3
- jettask/backend/models/requests.py +0 -236
- jettask/backend/models/responses.py +0 -230
- jettask/backend/namespace_api_old.py +0 -267
- jettask/backend/services/__init__.py +0 -3
- jettask/backend/start.py +0 -42
- jettask/backend/unified_api_router.py +0 -1541
- jettask/cleanup_deprecated_tables.sql +0 -16
- jettask/core/consumer_manager.py +0 -1695
- jettask/core/delay_scanner.py +0 -256
- jettask/core/event_pool.py +0 -1700
- jettask/core/heartbeat_process.py +0 -222
- jettask/core/task_batch.py +0 -153
- jettask/core/worker_scanner.py +0 -271
- jettask/executors/__init__.py +0 -5
- jettask/executors/asyncio.py +0 -876
- jettask/executors/base.py +0 -30
- jettask/executors/common.py +0 -148
- jettask/executors/multi_asyncio.py +0 -309
- jettask/gradio_app.py +0 -570
- jettask/integrated_gradio_app.py +0 -1088
- jettask/main.py +0 -0
- jettask/monitoring/__init__.py +0 -3
- jettask/pg_consumer.py +0 -1896
- jettask/run_monitor.py +0 -22
- jettask/run_webui.py +0 -148
- jettask/scheduler/multi_namespace_scheduler.py +0 -294
- jettask/scheduler/unified_manager.py +0 -450
- jettask/task_center_client.py +0 -150
- jettask/utils/serializer_optimized.py +0 -33
- jettask/webui_exceptions.py +0 -67
- jettask-0.2.19.dist-info/RECORD +0 -150
- /jettask/{constants.py → config/constants.py} +0 -0
- /jettask/{backend/config.py → config/task_center.py} +0 -0
- /jettask/{pg_consumer → messaging/pg_consumer}/pg_consumer_v2.py +0 -0
- /jettask/{pg_consumer → messaging/pg_consumer}/sql/add_execution_time_field.sql +0 -0
- /jettask/{pg_consumer → messaging/pg_consumer}/sql/create_new_tables.sql +0 -0
- /jettask/{pg_consumer → messaging/pg_consumer}/sql/create_tables_v3.sql +0 -0
- /jettask/{pg_consumer → messaging/pg_consumer}/sql/migrate_to_new_structure.sql +0 -0
- /jettask/{pg_consumer → messaging/pg_consumer}/sql/modify_time_fields.sql +0 -0
- /jettask/{pg_consumer → messaging/pg_consumer}/sql_utils.py +0 -0
- /jettask/{models.py → persistence/models.py} +0 -0
- /jettask/scheduler/{manager.py → task_crud.py} +0 -0
- /jettask/{schema.sql → schemas/schema.sql} +0 -0
- /jettask/{task_center.py → task/task_center/client.py} +0 -0
- /jettask/{monitoring → utils}/file_watcher.py +0 -0
- /jettask/{services/redis_monitor_service.py → utils/redis_monitor.py} +0 -0
- /jettask/{api/v1 → webui/api}/__init__.py +0 -0
- /jettask/{webui_config.py → webui/config.py} +0 -0
- /jettask/{webui_models → webui/models}/__init__.py +0 -0
- /jettask/{webui_models → webui/models}/namespace.py +0 -0
- /jettask/{services → webui/services}/alert_service.py +0 -0
- /jettask/{services → webui/services}/analytics_service.py +0 -0
- /jettask/{services → webui/services}/scheduled_task_service.py +0 -0
- /jettask/{services → webui/services}/task_service.py +0 -0
- /jettask/{webui_sql → webui/sql}/batch_upsert_functions.sql +0 -0
- /jettask/{webui_sql → webui/sql}/verify_database.sql +0 -0
- {jettask-0.2.19.dist-info → jettask-0.2.20.dist-info}/WHEEL +0 -0
- {jettask-0.2.19.dist-info → jettask-0.2.20.dist-info}/entry_points.txt +0 -0
- {jettask-0.2.19.dist-info → jettask-0.2.20.dist-info}/licenses/LICENSE +0 -0
- {jettask-0.2.19.dist-info → jettask-0.2.20.dist-info}/top_level.txt +0 -0
jettask/backend/init_meta_db.py
DELETED
@@ -1,158 +0,0 @@
|
|
1
|
-
"""
|
2
|
-
初始化任务中心元数据库
|
3
|
-
这个脚本用于创建任务中心自己的数据库和表结构
|
4
|
-
"""
|
5
|
-
import psycopg2
|
6
|
-
import sys
|
7
|
-
from config import task_center_config
|
8
|
-
import logging
|
9
|
-
|
10
|
-
logging.basicConfig(level=logging.INFO)
|
11
|
-
logger = logging.getLogger(__name__)
|
12
|
-
|
13
|
-
|
14
|
-
def create_meta_database():
|
15
|
-
"""创建任务中心元数据库"""
|
16
|
-
# 连接到postgres数据库来创建新数据库
|
17
|
-
conn = psycopg2.connect(
|
18
|
-
host=task_center_config.meta_db_host,
|
19
|
-
port=task_center_config.meta_db_port,
|
20
|
-
user=task_center_config.meta_db_user,
|
21
|
-
password=task_center_config.meta_db_password,
|
22
|
-
database='postgres'
|
23
|
-
)
|
24
|
-
conn.autocommit = True
|
25
|
-
cur = conn.cursor()
|
26
|
-
|
27
|
-
try:
|
28
|
-
# 检查数据库是否存在
|
29
|
-
cur.execute(
|
30
|
-
"SELECT 1 FROM pg_database WHERE datname = %s",
|
31
|
-
(task_center_config.meta_db_name,)
|
32
|
-
)
|
33
|
-
|
34
|
-
if not cur.fetchone():
|
35
|
-
# 创建数据库
|
36
|
-
cur.execute(f"CREATE DATABASE {task_center_config.meta_db_name}")
|
37
|
-
logger.info(f"Created database: {task_center_config.meta_db_name}")
|
38
|
-
else:
|
39
|
-
logger.info(f"Database already exists: {task_center_config.meta_db_name}")
|
40
|
-
|
41
|
-
finally:
|
42
|
-
cur.close()
|
43
|
-
conn.close()
|
44
|
-
|
45
|
-
|
46
|
-
def init_meta_tables():
|
47
|
-
"""初始化元数据表"""
|
48
|
-
conn = psycopg2.connect(task_center_config.sync_meta_database_url)
|
49
|
-
cur = conn.cursor()
|
50
|
-
|
51
|
-
try:
|
52
|
-
# 创建命名空间表
|
53
|
-
cur.execute("""
|
54
|
-
CREATE TABLE IF NOT EXISTS namespaces (
|
55
|
-
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
56
|
-
name VARCHAR(255) NOT NULL UNIQUE,
|
57
|
-
description TEXT,
|
58
|
-
-- JetTask应用使用的Redis配置
|
59
|
-
redis_config JSONB NOT NULL COMMENT 'JetTask应用的Redis配置',
|
60
|
-
-- JetTask应用使用的PostgreSQL配置
|
61
|
-
pg_config JSONB NOT NULL COMMENT 'JetTask应用的PostgreSQL配置',
|
62
|
-
is_active BOOLEAN DEFAULT true,
|
63
|
-
version INTEGER DEFAULT 1,
|
64
|
-
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
65
|
-
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
|
66
|
-
);
|
67
|
-
""")
|
68
|
-
|
69
|
-
# 创建索引
|
70
|
-
cur.execute("CREATE INDEX IF NOT EXISTS idx_namespaces_name ON namespaces(name);")
|
71
|
-
cur.execute("CREATE INDEX IF NOT EXISTS idx_namespaces_is_active ON namespaces(is_active);")
|
72
|
-
cur.execute("CREATE INDEX IF NOT EXISTS idx_namespaces_version ON namespaces(version);")
|
73
|
-
|
74
|
-
# 创建更新时间触发器
|
75
|
-
cur.execute("""
|
76
|
-
CREATE OR REPLACE FUNCTION update_namespaces_updated_at()
|
77
|
-
RETURNS TRIGGER AS $$
|
78
|
-
BEGIN
|
79
|
-
NEW.updated_at = CURRENT_TIMESTAMP;
|
80
|
-
RETURN NEW;
|
81
|
-
END;
|
82
|
-
$$ LANGUAGE plpgsql;
|
83
|
-
""")
|
84
|
-
|
85
|
-
cur.execute("""
|
86
|
-
DO $$
|
87
|
-
BEGIN
|
88
|
-
IF NOT EXISTS (
|
89
|
-
SELECT 1 FROM pg_trigger
|
90
|
-
WHERE tgname = 'update_namespaces_updated_at'
|
91
|
-
) THEN
|
92
|
-
CREATE TRIGGER update_namespaces_updated_at
|
93
|
-
BEFORE UPDATE ON namespaces
|
94
|
-
FOR EACH ROW
|
95
|
-
EXECUTE FUNCTION update_namespaces_updated_at();
|
96
|
-
END IF;
|
97
|
-
END;
|
98
|
-
$$;
|
99
|
-
""")
|
100
|
-
|
101
|
-
# 插入默认命名空间
|
102
|
-
cur.execute("""
|
103
|
-
INSERT INTO namespaces (id, name, description, redis_config, pg_config)
|
104
|
-
VALUES (
|
105
|
-
'a8f10720-068b-4264-bcbb-e00ceba370e9',
|
106
|
-
'default',
|
107
|
-
'默认命名空间 - JetTask应用使用此配置',
|
108
|
-
'{"host": "localhost", "port": 6379, "password": null, "db": 0}',
|
109
|
-
'{"host": "localhost", "port": 5432, "user": "jettask", "password": "123456", "database": "jettask"}'
|
110
|
-
)
|
111
|
-
ON CONFLICT (name) DO NOTHING;
|
112
|
-
""")
|
113
|
-
|
114
|
-
conn.commit()
|
115
|
-
logger.info("Initialized meta tables successfully")
|
116
|
-
|
117
|
-
except Exception as e:
|
118
|
-
conn.rollback()
|
119
|
-
logger.error(f"Failed to initialize meta tables: {e}")
|
120
|
-
raise
|
121
|
-
finally:
|
122
|
-
cur.close()
|
123
|
-
conn.close()
|
124
|
-
|
125
|
-
|
126
|
-
def main():
|
127
|
-
"""主函数"""
|
128
|
-
logger.info("=" * 60)
|
129
|
-
logger.info("任务中心元数据库初始化")
|
130
|
-
logger.info("=" * 60)
|
131
|
-
logger.info(f"元数据库地址: {task_center_config.meta_db_host}:{task_center_config.meta_db_port}")
|
132
|
-
logger.info(f"元数据库名称: {task_center_config.meta_db_name}")
|
133
|
-
logger.info(f"元数据库用户: {task_center_config.meta_db_user}")
|
134
|
-
logger.info("-" * 60)
|
135
|
-
|
136
|
-
try:
|
137
|
-
# 1. 创建数据库
|
138
|
-
create_meta_database()
|
139
|
-
|
140
|
-
# 2. 初始化表结构
|
141
|
-
init_meta_tables()
|
142
|
-
|
143
|
-
logger.info("-" * 60)
|
144
|
-
logger.info("✅ 元数据库初始化完成!")
|
145
|
-
logger.info("")
|
146
|
-
logger.info("说明:")
|
147
|
-
logger.info("1. 任务中心元数据库用于存储命名空间配置")
|
148
|
-
logger.info("2. 每个命名空间定义了JetTask应用使用的Redis和PostgreSQL配置")
|
149
|
-
logger.info("3. JetTask应用通过命名空间ID获取其专用的数据库配置")
|
150
|
-
logger.info("=" * 60)
|
151
|
-
|
152
|
-
except Exception as e:
|
153
|
-
logger.error(f"初始化失败: {e}")
|
154
|
-
sys.exit(1)
|
155
|
-
|
156
|
-
|
157
|
-
if __name__ == "__main__":
|
158
|
-
main()
|