jettask 0.2.23__py3-none-any.whl → 0.2.24__py3-none-any.whl

jettask/webui/utils/time_utils.py

@@ -0,0 +1,122 @@
+"""
+时间处理工具函数
+"""
+import json
+from datetime import datetime, timezone
+from typing import Dict, Any, List
+
+
+def parse_iso_datetime(time_str: str) -> datetime:
+    """
+    解析 ISO 格式的时间字符串，确保返回 UTC 时间
+
+    Args:
+        time_str: ISO 格式的时间字符串
+
+    Returns:
+        UTC 时间的 datetime 对象
+    """
+    if time_str.endswith('Z'):
+        # Z 表示 UTC 时间
+        dt = datetime.fromisoformat(time_str.replace('Z', '+00:00'))
+    else:
+        dt = datetime.fromisoformat(time_str)
+
+    # 如果没有时区信息，假定为 UTC
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    # 如果有时区信息，转换为 UTC
+    elif dt.tzinfo != timezone.utc:
+        dt = dt.astimezone(timezone.utc)
+
+    return dt
+
+
+def format_task_timestamps(task: Dict[str, Any], fields: List[str] = None) -> Dict[str, Any]:
+    """
+    将任务对象中的时间字段转换为 ISO 格式字符串
+
+    Args:
+        task: 任务字典
+        fields: 需要转换的字段列表，默认为 ['created_at', 'started_at', 'completed_at']
+
+    Returns:
+        转换后的任务字典
+    """
+    if fields is None:
+        fields = ['created_at', 'started_at', 'completed_at']
+
+    for field in fields:
+        if task.get(field):
+            # PostgreSQL 的 TIMESTAMP WITH TIME ZONE 会返回 aware datetime
+            if task[field].tzinfo is None:
+                # 如果没有时区信息，假定为 UTC
+                task[field] = task[field].replace(tzinfo=timezone.utc)
+            task[field] = task[field].isoformat()
+
+    return task
+
+
+def parse_task_json_fields(task: Dict[str, Any], fields: List[str] = None) -> Dict[str, Any]:
+    """
+    解析任务对象中的 JSON 字段
+
+    Args:
+        task: 任务字典
+        fields: 需要解析的字段列表，默认为 ['task_data', 'result', 'metadata']
+
+    Returns:
+        解析后的任务字典
+    """
+    if fields is None:
+        fields = ['task_data', 'result', 'metadata']
+
+    for field in fields:
+        if task.get(field) and isinstance(task[field], str):
+            try:
+                task[field] = json.loads(task[field])
+            except:
+                pass
+
+    return task
+
+
+def task_obj_to_dict(task_obj) -> Dict[str, Any]:
+    """
+    将 Task ORM 对象转换为字典
+
+    Args:
+        task_obj: Task ORM 对象
+
+    Returns:
+        任务字典
+    """
+    task = {
+        'id': task_obj.id,
+        'queue_name': task_obj.queue_name,
+        'task_name': task_obj.task_name,
+        'task_data': task_obj.task_data,
+        'priority': task_obj.priority,
+        'retry_count': task_obj.retry_count,
+        'max_retry': task_obj.max_retry,
+        'status': task_obj.status,
+        'result': task_obj.result,
+        'error_message': task_obj.error_message,
+        'created_at': task_obj.created_at,
+        'started_at': task_obj.started_at,
+        'completed_at': task_obj.completed_at,
+        'worker_id': task_obj.worker_id,
+        'execution_time': task_obj.execution_time,
+        'duration': task_obj.duration,
+        'metadata': task_obj.task_metadata,
+        'next_sync_time': task_obj.next_sync_time,
+        'sync_check_count': task_obj.sync_check_count
+    }
+
+    # 转换时间戳为 ISO 格式
+    task = format_task_timestamps(task)
+
+    # 解析 JSON 字段
+    task = parse_task_json_fields(task)
+
+    return task

jettask/worker/lifecycle.py

@@ -366,6 +366,12 @@ class WorkerStateManager:
         pubsub = self.redis.pubsub()
         await pubsub.subscribe(self.worker_state_channel)
 
+        # 标记PubSub连接，防止被空闲连接清理
+        if hasattr(pubsub, 'connection') and pubsub.connection:
+            pubsub.connection._is_pubsub_connection = True
+            socket_timeout = pubsub.connection.socket_timeout if hasattr(pubsub.connection, 'socket_timeout') else 'N/A'
+            logger.info(f"Marked PubSub connection {id(pubsub.connection)} to prevent cleanup, socket_timeout={socket_timeout}")
+
         logger.debug(f"Created and subscribed to Redis Pub/Sub channel: {self.worker_state_channel}")
         return pubsub
 
@@ -739,7 +745,7 @@ class HeartbeatThreadManager:
                  heartbeat_timeout=15.0):
         """初始化心跳线程管理器"""
         if redis_url is not None:
-            from jettask.utils.db_connector import get_sync_redis_client
+            from jettask.db.connector import get_sync_redis_client
             self.redis_client = get_sync_redis_client(redis_url, decode_responses=True)
             self.redis_url = redis_url
             self.consumer_id = consumer_id
@@ -992,7 +998,7 @@ class HeartbeatThreadManager:
                         pass
                     try:
                         if self.redis_url:
-                            from jettask.utils.db_connector import get_sync_redis_client
+                            from jettask.db.connector import get_sync_redis_client
                             self.redis_client = get_sync_redis_client(
                                 redis_url=self.redis_url,
                                 decode_responses=True,

{jettask-0.2.23.dist-info → jettask-0.2.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jettask
-Version: 0.2.23
+Version: 0.2.24
 Summary: A high-performance distributed task queue system with web monitoring
 Author-email: JetTask Team <support@jettask.io>
 License-Expression: MIT

jettask-0.2.24.dist-info/RECORD

@@ -0,0 +1,142 @@
+jettask/__init__.py,sha256=4TsONE4CfUIRgLvZMDvCOoEZWCA2qx11ZONaaqmhsls,2012
+jettask/cli.py,sha256=0pxM2Kvoa-hM2gE3q_nYXjfWJy_8pXXZSxIjKe3z8uw,38708
+jettask/exceptions.py,sha256=aQTUpiF2ftiZwGORUHXatAKbDEzptUf6iP7ZOXBQ3tQ,1971
+jettask/test_connection_monitor.py,sha256=aJmWDcC8XoJVEKfnQbixof9VysxFUMfHvIoI2TV1eS0,2189
+jettask/config/__init__.py,sha256=gCfoHVfTWIRPbtY1-iE4ydf2lfOmEwTU3EoSJ7NbLIU,273
+jettask/config/config.py,sha256=YdResg9uUD1rbmoumoIE-FkhMZ_skoRLgTlUjkLEEzQ,7111
+jettask/config/constants.py,sha256=rw06U4WPSFk6f80blC0ebkqnPUS_eteKQesTSBXKn6A,7028
+jettask/config/env_loader.py,sha256=P2uO63mAHfhu9Pqa_Hh0JKWmeFDptudSHgBI-fyr0sA,10558
+jettask/config/lua_scripts.py,sha256=40nFzJNm6cZHfKZ7ovYRMVofS5bxWFGU-Dq0RS4PCdo,6070
+jettask/config/nacos_config.py,sha256=PPmb-2gYTfgrdianjQdB9B4xYLAfRSihHEalccZXpo8,19689
+jettask/config/performance.py,sha256=bOdLEskfB_6cRfS10IRgmtKEsJw_CaIZsPHbXxaHwbU,5686
+jettask/config/task_center.py,sha256=dJ9UkWRR1kulHFhfSVXmT_bM-N7yEMJdp0JylUkrLZ4,2062
+jettask/core/__init__.py,sha256=3qnUjllxsvy9gqJEKnJbQeJcSEmxdWUwwzumdJG5vKI,198
+jettask/core/app.py,sha256=ycR5G9JBSMi941MyCnC0RG_3RyI4wpl1XQgq20F16NQ,103722
+jettask/core/app_importer.py,sha256=04wRDCp5NFNiDkDhMOM7CdaHS_0b0cYylnYpXie8fwM,11018
+jettask/core/container.py,sha256=Q4eGqkaw6C8jW3PyrmG5RrcCz3NA1ky2HelPx4Nn-xU,18108
+jettask/core/context.py,sha256=XI4Q1s75NIWImcrHIkCLgGo_kpuJIk8MlBoNIJuEfF0,993
+jettask/core/enums.py,sha256=ufXI5OcGZoOSbLEyAWtnkZVO-LcqNbIOygyjLLa6ce8,4632
+jettask/core/message.py,sha256=WDOY8kvoV6at4MsIP01c3GJmIwHvRnUHimli6NKcw8w,4340
+jettask/core/namespace.py,sha256=_EVMGWB4BrHf5Z6Zmdof3Uim9uX7-4IQ40HjLmE_WRM,28616
+jettask/core/retry.py,sha256=lX56Z0fvKjEhVbYPHYdoN-W4XUPVplHlfIOupBylEeI,10711
+jettask/core/task.py,sha256=Ada-wJLZLexrZtKLVqRDFSQOsvAnQpZnxPTDdoJnQq0,5712
+jettask/core/unified_manager_base.py,sha256=JBBZUEolQHlHbeuNU3k8TwOBh7kxgs5z4yb8FbhbdKg,18987
+jettask/db/__init__.py,sha256=FOYiPJs8B2ek-SgaszCTcBuzWaMKKTAwPTfqBHBDIJo,1388
+jettask/db/base.py,sha256=vhrPErPKIkSsAEcF7l3FCZ4e4iNUjocwH0mkQ7MECQI,3769
+jettask/db/connector.py,sha256=f3yfOhjadXDQEU9hBHIciV80tze3ugkaFn1uQgQz7To,63906
+jettask/db/models/__init__.py,sha256=M9WlVfOOvovX5b_4aO0_NsEiQrxcV8zih2nITSbWiis,277
+jettask/db/models/scheduled_task.py,sha256=Xxz3ayhyAdoFhCPEJUmDpNgp6pZdhhzCByp6PI9DiRc,7263
+jettask/db/models/task.py,sha256=oliM4ZHM6bIANvlYGLjhivgmcGAvG_686--aRxCHljE,2441
+jettask/db/models/task_run.py,sha256=SYIJtrAnnQMO6VvrK0_eD4z24gVHgRS6uoUwQwQDhLs,2826
+jettask/executor/__init__.py,sha256=mdGkv9IsTxSJEfMg8b8mzw_RIGuPT05o2OxHxpKUWWQ,498
+jettask/executor/core.py,sha256=Nf7tZ0ZxuSfKDqYzwQTIlFtdsNjWTbxj6-UHVVdTAVs,27119
+jettask/executor/orchestrator.py,sha256=DS5dYnYnp7sISLb5hLBcPNhP-Tqvd5NxhoBE-4iWxRQ,9986
+jettask/executor/process_entry.py,sha256=wr_0qXD5O_wZfkXRHRaW7ZJXZ97iWtPcEjt9c4njdnI,19510
+jettask/executor/task_executor.py,sha256=RqHE6K0bcvZ72-Wkc5oGl41KIuFvZ--6Y8bqi35vtXg,11270
+jettask/messaging/__init__.py,sha256=hWv1p5kmud8nTbx-KceKkmaYbvYWhzJWNR8OzTvoRP0,1961
+jettask/messaging/event_pool.py,sha256=m2xacXiaflsTCRCaWWbH07_RHDezNgDYONY2EK_lm2I,118256
+jettask/messaging/reader.py,sha256=hlOYzU1LJrm7OhKb5iBrUjqHjsN3sN5FGKYHLiHGHes,17556
+jettask/messaging/registry.py,sha256=wG6SvyoCFMDggGd_SaDlLKaBcUPWzlblcGLMRhJwx90,10290
+jettask/messaging/scanner.py,sha256=jRr2-RXfMNTLfU6v9ZX4QEPPBZdTqjwq9JSvMz8WkRQ,15496
+jettask/messaging/sender.py,sha256=cz98GiSMQhKxUyEkPmOHUIG78GFMW_f-P4j-KUCVMrE,10051
+jettask/messaging/pg_consumer/pg_consumer_v2.py,sha256=ajd6ZM_HY9YWK_I5BNbGuWxNqSS892d3ZzvDlTSQgwc,16211
+jettask/messaging/pg_consumer/sql_utils.py,sha256=dZM_8kIzUmRq0me4yLwb3UeoDvIVKJdIc9VNP1nbS0E,6461
+jettask/messaging/pg_consumer/sql/add_execution_time_field.sql,sha256=z2Jx-0uG-IW5RsTGxcLFu2_BcMReluEptMJWB6Qrj08,1184
+jettask/messaging/pg_consumer/sql/create_new_tables.sql,sha256=layb08_MFGVSE-NQEcjlQBklZ307JtJaccrnqkcj7Vg,6566
+jettask/messaging/pg_consumer/sql/create_tables_v3.sql,sha256=ZwBjxKWU6imWACoWMCIACLsYUPiPep9WmJsdIjI2a58,8173
+jettask/messaging/pg_consumer/sql/migrate_to_new_structure.sql,sha256=elhObWvbtqAhb4osJC6D5JhxcOxhGJgIBqx4ItB_xO8,5716
+jettask/messaging/pg_consumer/sql/modify_time_fields.sql,sha256=mRfoXbP9iXbAJBj-xNume8-4AV_UPQcBALNUqEf3aKw,2572
+jettask/persistence/__init__.py,sha256=9vU0k8ANmQxMBsd0Vtj-gd_ZNQoS8jJ2Bl8Cj2AsWTE,1228
+jettask/persistence/buffer.py,sha256=Pd0EBT23c2mSdpSG5j4PGguj1mdIpBjyx6kkAVHooz8,5347
+jettask/persistence/consumer.py,sha256=Krt_xY2he88Ge08uYq_idEnFb-yD_523Z1lygOYX8Ek,15589
+jettask/persistence/manager.py,sha256=gPjdVu135lDniBuceHyaRyX2HhFC9uNvbyUhiNeycYY,11660
+jettask/persistence/persistence.py,sha256=S_GEIRAdLVM60uYF93WUBWe5H5LJ01ZvnuALNY8Mxkw,16494
+jettask/scheduler/__init__.py,sha256=vlNbwGS0r3ednD73ZD9nM4kFdbPk8WMQHIsM3qfMP5A,856
+jettask/scheduler/database.py,sha256=HA3mAD9IN-zpjD7Ps70V0-gTG5g3WHq225LW0fRvAEk,26736
+jettask/scheduler/loader.py,sha256=KtDdKnn2cF1Qegv5fU-BAfeCRlJR89VcB75JzjawBM8,8808
+jettask/scheduler/manager.py,sha256=MQmSN7okmxrBq-i10fj_H1yarcxWgDDLfVauEbMbSrU,13003
+jettask/scheduler/models.py,sha256=sSp_ItOb6d4ZW0wGN9LxrR475I0dJSuF2u9gHjrZEWs,7497
+jettask/scheduler/schedule.py,sha256=OE0z4NCa85LOQYT6eRm37rlhsgSPYh-rXG6Qfgq4aEE,5718
+jettask/scheduler/scheduler.py,sha256=Zj0nqEvDp-QfS5xl03oKzkJWU3G4PHxIVfcV36BYSZw,27842
+jettask/schemas/__init__.py,sha256=COaNqPbLi71ColUk8PsZVV8VSzdAiS7Ad1IkBh5IL9s,4329
+jettask/schemas/alert.py,sha256=Od29QjDrnLN1Tw_F2rPFSGsqRvbBcO2hf1d8GO1_WEQ,4958
+jettask/schemas/backlog.py,sha256=SJVzNG7Vwm1tOWW4aAZMGLwsOxfv1FlmCK6HfOoWTws,6600
+jettask/schemas/common.py,sha256=e1lrLnOpxKT25FH7QnDQy38Qj4SGqd_cwHvRObTuRZw,6038
+jettask/schemas/monitoring.py,sha256=zPfNv6YXw3uufj61PseQTaQkpwa9azgsnCGUaF4HNUk,8084
+jettask/schemas/namespace.py,sha256=9kYRBXGBqGV6i8EBmB4PTAmTG0ex2xOffZaUgMIDxvw,10870
+jettask/schemas/queue.py,sha256=qy8Ty8BseRPyXmfDOpFgojTvYhsaBAJnpzFK2ZSYF_s,4516
+jettask/schemas/responses.py,sha256=StGoD6Y2pNtU6V9t1xpbHoSWjigk_YBddai09SYDwqM,16522
+jettask/schemas/scheduled_task.py,sha256=MPAXvDeqmyQTWhL31yS94mRuzJWOV4njukp6BQrX6K4,7130
+jettask/schemas/schema.sql,sha256=IY-vLzj3TYoR4p51ChLuSxzo4KU1PQNFNDutc7p0nHg,4490
+jettask/schemas/task.py,sha256=niW2F_84HvstAjX3vo2pZswoRwMCF3DIoONuHwWNwtk,3172
+jettask/task/__init__.py,sha256=IR3c7HJ3NK25YzwrlKBhpyUcbv5kGnDQ9ogzcmR1zdU,246
+jettask/task/router.py,sha256=nyHR4SAAFsWbeQh3Hza5U5a_w7l0WyZgsbUki9TzzhM,5900
+jettask/task/task_registry.py,sha256=HYq6S9XWvavhy7UJeOGQSTR5LF0V_vyACcZ12yIMS68,7987
+jettask/task/task_center/__init__.py,sha256=Ed_D427tD59wTeqfbjckB-qKrCPnRLuzWovpQRW9uFs,132
+jettask/task/task_center/client.py,sha256=BcJOaKAeUKDl75GQzoQdBkZdBiC4gPxLrNa5VbKC86I,8537
+jettask/utils/__init__.py,sha256=qyE5dRmgC5qYWtjGUHpIid9LOG65oHcv-Nx1uptfIRI,960
+jettask/utils/backlog_collector.py,sha256=U3_TGHtcD4CbKOyFkbOShht8wkEeCwJ38QiZTvDgEn0,2523
+jettask/utils/db_init.py,sha256=9E7hk8V-AYgffJ3WNbt34_7TKR1enOF3kBgFTcv1o2E,15486
+jettask/utils/error_handler.py,sha256=9iYHCInUkEkP9o07dUgPxQr4aWqR5FUhnNWLgB0TEqM,2794
+jettask/utils/exception_hook.py,sha256=Fp0m71_jjmmZvyoELARSsPIvSVPCmMQMVC1FxMFOoHQ,1765
+jettask/utils/file_watcher.py,sha256=r3Mgekb_5sOssDrnFBCbbyvpWkwD2ZPA_j22ztzRCT0,1207
+jettask/utils/helpers.py,sha256=uTdyRFZsdwSQzuEcNoGgADuVnHaDEYq6JQP6zVxAj04,3368
+jettask/utils/logger.py,sha256=PcZgUtx7I9pe90Y8WgPQtKhMefd1oQ14pgUlKCFpCXo,843
+jettask/utils/namespace_dep.py,sha256=JVcBqRPYwhGtmWFg-OKoPkxkeYCU4IxsGjx6PRF_jqM,5168
+jettask/utils/queue_matcher.py,sha256=AP7Zh1WDMh2vG8XB7JCMAaepAb37rGi0O2HuMgBJxng,5430
+jettask/utils/redis_monitor.py,sha256=2l6fF2qAtctS9HUaEV3acJMJ5zhkvT4iygJGhGBCoKw,3660
+jettask/utils/serializer.py,sha256=Omnnbpp9FuRDugqan2QK75kVV91Rv07kN-hzqCC_3G4,856
+jettask/utils/stream_backlog.py,sha256=4F-AiRngdby9CW8evdwWGLYrMEeTbNrCFMrbfB7RHcg,15971
+jettask/utils/task_logger.py,sha256=kauCgiDDzhuge3wk-BXR_KwytOfmrrVBKe1S4XUWBXU,13628
+jettask/utils/time_sync.py,sha256=DcZWboLKa9DwyQgDGGZGNLrQrMRVaJXJLk0xCDUW0B4,5209
+jettask/utils/traceback_filter.py,sha256=2svLo5_OoosAbS08UpceeWtjSKkdx-Fp6U-ysJQkroY,3025
+jettask/utils/rate_limit/__init__.py,sha256=Ahl96s0BqSofOIsQUIHfc2QpO46GezjtLXC7QzOFpKs,810
+jettask/utils/rate_limit/concurrency_limiter.py,sha256=NPHlPwhudoceobgIb2BsuKrLxUaOsZIF0G9lcZoKVDA,27718
+jettask/utils/rate_limit/config.py,sha256=ChOQ2AWUBMbq3lTImYZ59UBwDPAetJ3UcV0WuhHPUgY,3862
+jettask/utils/rate_limit/limiter.py,sha256=itH_APz7m5qBntJtlfF120OvkD1N7hKuiwPtu6OF28U,1285
+jettask/utils/rate_limit/manager.py,sha256=R9dqczrJYpVhcecMASEYdnPDKmSVaNwmdnzrzLvadZg,9535
+jettask/utils/rate_limit/qps_limiter.py,sha256=npo6_Sw51AT9I4OvRfsYZddViF6cZQl5DpW8XDWRRBo,4787
+jettask/utils/rate_limit/task_limiter.py,sha256=7QaK1IR8rF-h1e-4-loxFL5tBiQPceFXLCdqcK8Rpl4,14072
+jettask/webui/__init__.py,sha256=OcPwYBgPX1PSWF_Hdnxaq_2N2jX-VCdyrs-MYrwkmds,570
+jettask/webui/app.py,sha256=vcQ7-gYnN8mf8z9BCMqzNuq11S9crd2eAQ12AU8CiNo,4657
+jettask/webui/api/__init__.py,sha256=GYl42r7ZtGtqOtiFUeyrKsNDYKTo8jS03Pci-sQxqm8,1200
+jettask/webui/api/alerts.py,sha256=Vy9PwihQyaFtoFXeuyxNZcVQxJpIR1HIAkVYzpj37hY,29540
+jettask/webui/api/example_refactored.py,sha256=aVaGDuIrrdlfjmrXnfusWWCRz4SdDGIj4CxCBDcUY5Y,13603
+jettask/webui/api/namespaces.py,sha256=hY3tSplUjteQdITjKdgrd0X06T8k8-EAPW29lQSW-zQ,14637
+jettask/webui/api/overview.py,sha256=tyf9xlW7r-ztnZDmgacih_QUNhR7DFhwpTrpasT8ZdQ,12014
+jettask/webui/api/queues.py,sha256=SU1mjf3VL6qL3biWAQVkX9NhHwcgoSZ-GJuPQH_LQeQ,42341
+jettask/webui/api/scheduled.py,sha256=XKqgnRwrRrMZ1zGtdO0x6aKMBOwBEy1-2BXdDzvSE6o,54370
+jettask/webui/api/settings.py,sha256=vPg1p8TkNK0E9tml6nZY8yxdUo4zJ0pL2s0eqBjuPaA,4435
+jettask/webui/api/workers.py,sha256=pK7G5bvFic5Nhgot1HqhJBtcygUkMS34RnENldHyUHg,14101
+jettask/webui/middleware/__init__.py,sha256=v_5RM0XlGEYL4VGiPG8Jy12bIkOtHzhkkXXzh_uGJh0,119
+jettask/webui/middleware/namespace_middleware.py,sha256=kZ4k1rnlOibuYzhSejcBc_G5X5nmh6vK3rndJZXHnd4,4926
+jettask/webui/services/__init__.py,sha256=PyUMoa6tEEYL7VFJkblIB7r_4ojr4vNls3qUfvx3ooU,6109
+jettask/webui/services/alert_service.py,sha256=AIk4hjx-yoFnl5vPff7TMwjgjheKu91da9mZ5gYDroo,14169
+jettask/webui/services/analytics_service.py,sha256=zLTtGUJ7wYIU7O5NP_VS_-wk_IXNfxKikAvQ3aaeCmI,1101
+jettask/webui/services/heartbeat_service.py,sha256=Srp3IoVJaoz2EF-dobH9Eq0bSVJGz8s9LEZlT1ByxyY,9386
+jettask/webui/services/overview_service.py,sha256=kuNz-AxRFa_z6bAytA05J797MNBHYlp2qvQbMjBOgEA,37102
+jettask/webui/services/queue_monitor_service.py,sha256=XkJuwFrVO3GqKcfC9Ll3d-XV2Nxt1bISAfJJF_RBuZw,14927
+jettask/webui/services/queue_service.py,sha256=3ONDfGos2xHtHZlWcAlaXzEo1Km-HTIllCpN3GPKKG0,27419
+jettask/webui/services/queue_stats_v2.py,sha256=DW46_w1-3k9PozZScLUen0-wbHMTgxo346DLVmxsw5Q,24409
+jettask/webui/services/redis_monitor_service.py,sha256=Ot1POh4GS3qbFMx03kjDDox53tnINxTepcUQmt8KfTU,2524
+jettask/webui/services/scheduled_task_service.py,sha256=ZFFtcUdk_IDGm17maHA1Hb0p00_56QqQ9ETFAF7T1bU,6226
+jettask/webui/services/settings_service.py,sha256=nyeTPF0DaLtNfk6h2uHfzIzf7L53GV9zhfL4bstxE_Q,35592
+jettask/webui/services/task_monitor_service.py,sha256=lLWGi9eO1gmemV0Wxa2h5S1uP0J5gXAGYYUb6ZMiJUU,7804
+jettask/webui/services/task_service.py,sha256=vfSEyhFxDcEzOuA4sHEg1sYvLriUUvJ08SQ2ICG7MSU,4751
+jettask/webui/services/timeline_pg_service.py,sha256=6IScSTnK_ZCzg7_EGs0CLrac1RTqLzhp7KsrnPjsOFM,17467
+jettask/webui/services/timeline_service.py,sha256=79QqB7nAQGj3vHl1Ubf1ao7ALRFahiaPr2px4wHCnOI,6434
+jettask/webui/services/worker_monitor_service.py,sha256=KTjJsMLHrHKIRYWi9RHKNjIXIHzMNAC-nqdXh0XdSnM,17323
+jettask/webui/sql/batch_upsert_functions.sql,sha256=5eWOhOD8gWHhtcop_BrCpZTxPFeyBHtt_leNQZO89Cs,6615
+jettask/webui/sql/verify_database.sql,sha256=HtTup3xHWbOo1BTU_u4i41E9LrPEXB8qYbChL9WeWOc,2313
+jettask/webui/utils/__init__.py,sha256=4Ne5wAOwejOzKZVk6DRQ6apOI9mTS_tu9TOvKAFL5gk,208
+jettask/webui/utils/time_utils.py,sha256=UwBkt0ZKH5wpDlRZx6wvTlacSlqAbMQ9cIhlDZnArgc,3426
+jettask/worker/__init__.py,sha256=HUwUjZEWV6oaGJSUg26PPwVsIacAB8Tk8qzQQYvlMLM,1218
+jettask/worker/lifecycle.py,sha256=ioSIw2TidBnGpHrryIzQE4cgSDjiUAlClPBUtrAlf1s,58989
+jettask/worker/manager.py,sha256=fIayGpH9nLqvxTv-hDwrVj41sQqid96ZzNjENCLR1Nk,19913
+jettask/worker/recovery.py,sha256=I7Au9DPpsEhlILxYcnnC0H-Qw51rr97sUsy2pn43Czw,29120
+jettask-0.2.24.dist-info/licenses/LICENSE,sha256=sKR8OPWvnqxzcHAmnaVSANpRpeM0Z52PNLCB0ZlFN6c,1062
+jettask-0.2.24.dist-info/METADATA,sha256=Yd9LfUdxKRpsHlYmSx2vvqG0gd0GxhHm8pqbqVWbAWY,1949
+jettask-0.2.24.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+jettask-0.2.24.dist-info/entry_points.txt,sha256=9-8eTLJPgzpBgeGC8WHdMYtldZxHAXFspMeqArVUdew,148
+jettask-0.2.24.dist-info/top_level.txt,sha256=uymyRUF87-OsSurk5NhpeTW0jy3Wltnn91Zoa6jmAaw,8
+jettask-0.2.24.dist-info/RECORD,,

jettask/executor/executor.py

@@ -1,338 +0,0 @@
-"""
-统一执行器
-
-整合了单进程和多进程执行模式的统一接口
-"""
-
-import asyncio
-import multiprocessing
-import logging
-import os
-import time
-from collections import deque
-from typing import List, Optional
-
-from .core import ExecutionMode, ExecutorCore
-from .orchestrator import ProcessConfig, ProcessOrchestrator
-from ..worker.lifecycle import WorkerStateManager
-from ..utils.rate_limit.manager import RateLimiterManager
-
-logger = logging.getLogger('app')
-
-# Try to use uvloop for better performance
-try:
-    import uvloop
-    uvloop.install()
-    logger.debug("Using uvloop for better performance")
-except ImportError:
-    pass
-
-
-class UnifiedExecutor:
-    """
-    统一执行器
-
-    整合AsyncioExecutor和MultiAsyncioExecutor的功能
-    支持单进程和多进程两种执行模式
-
-    职责:
-    1. 提供统一的执行器接口
-    2. 根据模式选择ExecutorCore或ProcessOrchestrator
-    3. 管理事件队列和任务分发
-    """
-
-    def __init__(self, event_queue, app, concurrency=100,
-                 mode: ExecutionMode = ExecutionMode.SINGLE_PROCESS,
-                 task_name: str = None):
-        """
-        初始化统一执行器
-
-        Args:
-            event_queue: 事件队列
-            app: Application实例
-            concurrency: 并发数
-            mode: 执行模式
-            task_name: 任务名称(单进程模式必需)
-        """
-        self.event_queue = event_queue
-        self.app = app
-        self.concurrency = concurrency
-        self.mode = mode
-        self.task_name = task_name
-
-        # 根据模式初始化核心组件
-        if mode == ExecutionMode.SINGLE_PROCESS:
-            if not task_name:
-                raise ValueError("task_name is required for SINGLE_PROCESS mode")
-
-            self.executor_core = ExecutorCore(
-                app=app,
-                task_name=task_name,
-                concurrency=concurrency
-            )
-            self.orchestrator = None
-            logger.debug(f"UnifiedExecutor initialized in SINGLE_PROCESS mode for task {task_name}")
-
-        elif mode == ExecutionMode.MULTI_PROCESS:
-            self.executor_core = None
-            self.orchestrator = ProcessOrchestrator(
-                app=app,
-                num_processes=concurrency
-            )
-            logger.debug(f"UnifiedExecutor initialized in MULTI_PROCESS mode with {concurrency} processes")
-
-        else:
-            raise ValueError(f"Unsupported execution mode: {mode}")
-
-        # 活动任务集合(单进程模式使用)
-        self._active_tasks = set()
-
-    def logic(self, *args, **kwargs):
-        """
-        BaseExecutor接口方法
-        在单进程模式下不使用,多进程模式委托给ProcessOrchestrator
-        """
-        pass
-
-    async def loop(self):
-        """主循环 - 单进程模式"""
-        if self.mode != ExecutionMode.SINGLE_PROCESS:
-            raise RuntimeError("loop() is only for SINGLE_PROCESS mode")
-
-        # 初始化限流器
-        self.app.consumer_manager._heartbeat_strategy._ensure_consumer_id()
-        worker_id = self.app.consumer_manager._heartbeat_strategy.consumer_id
-        registry_manager = self.app.consumer_manager
-
-        if not self.app.worker_state_manager:
-            self.app.worker_state_manager = WorkerStateManager(
-                redis_client=self.app.ep.async_redis_client,
-                redis_prefix=self.executor_core.prefix,
-                event_pool=self.app.ep  # 传入 EventPool 实例，启用事件驱动的消息恢复
-            )
-            await self.app.worker_state_manager.start_listener()
-            logger.debug(f"WorkerStateManager started for worker {worker_id}")
-
-        # # 初始化时间同步
-        from jettask.utils.time_sync import init_time_sync
-        time_sync = await init_time_sync(self.app.ep.async_redis_client)
-        logger.debug(f"TimeSync initialized, offset={time_sync.get_offset():.6f}s")
-
-        self.executor_core.rate_limiter_manager = RateLimiterManager(
-            redis_client=self.app.ep.async_redis_client,
-            worker_id=worker_id,
-            redis_prefix=self.executor_core.prefix,
-            registry_manager=registry_manager,
-            worker_state_manager=self.app.worker_state_manager
-        )
-        logger.debug(f"RateLimiterManager initialized for worker {worker_id}")
-
-        await self.executor_core.rate_limiter_manager.load_config_from_redis()
-
-        tasks_batch = []
-        max_buffer_size = 5000
-
-        try:
-            while True:
-                # 检查退出信号
-                if hasattr(self.app, '_should_exit') and self.app._should_exit:
-                    logger.debug("UnifiedExecutor detected shutdown signal")
-                    break
-
-                # 检查父进程
-                if hasattr(os, 'getppid') and os.getppid() == 1:
-                    logger.debug("Parent process died, exiting...")
-                    break
-
-                current_time = time.time()
-
-                # 获取事件
-                event = None
-                try:
-                    event = await asyncio.wait_for(self.event_queue.get(), timeout=0.1)
-                except asyncio.TimeoutError:
-                    event = None
-
-                if event:
-                    event.pop("execute_time", None)
-                    tasks_batch.append(event)
-                    logger.debug(f"[EVENT] Got event: {event.get('event_id', 'unknown')}, task_name={event.get('event_data', {}).get('_task_name')}")
-
-                # 批量创建任务
-                if tasks_batch:
-                    for event in tasks_batch:
-                        event_data = event.get('event_data', {})
-                        event_task_name = event_data.get("_task_name") or event_data.get("name")
-
-                        if not event_task_name:
-                            logger.error(f"No task_name in event {event.get('event_id')}")
-                            continue
-
-                        # 验证任务名称匹配
-                        if event_task_name != self.task_name:
-                            logger.error(f"Task name mismatch: {event_task_name} != {self.task_name}")
-                            continue
-
-                        # 限流控制
-                        logger.debug(f"[TASK] Attempting to acquire rate limit for {self.task_name}, event_id={event.get('event_id')}")
-                        rate_limit_token = await self.executor_core.rate_limiter_manager.acquire(
-                            task_name=self.task_name,
-                            timeout=None
-                        )
-                        print(f'{rate_limit_token=}')
-                        if not rate_limit_token:
-                            logger.error(f"Failed to acquire token for {self.task_name}")
-                            continue
-                        logger.debug(f"[TASK] Successfully acquired rate limit for {self.task_name}, token={rate_limit_token}, starting execution")
-
-                        self.executor_core.batch_counter += 1
-
-                        # 创建任务包装器，在任务完成时自动释放限流许可
-                        async def execute_with_release(event_data, token):
-                            try:
-                                await self.executor_core.execute_task(**event_data)
-                            finally:
-                                # 无论任务成功还是失败，都释放并发许可
-                                await self.executor_core.rate_limiter_manager.release(self.task_name, task_id=token)
-
-                        task = asyncio.create_task(execute_with_release(event, rate_limit_token))
-                        self._active_tasks.add(task)
-                        task.add_done_callback(self._active_tasks.discard)
-
-                    tasks_batch.clear()
-
-                # 智能缓冲区管理
-                buffer_full = (
-                    len(self.executor_core.pending_acks) >= max_buffer_size or
-                    len(self.executor_core.status_updates) >= max_buffer_size or
-                    len(self.executor_core.data_updates) >= max_buffer_size or
-                    len(self.executor_core.task_info_updates) >= max_buffer_size
-                )
-
-                should_flush_periodic = False
-                has_pending_data = (
-                    self.executor_core.pending_acks or
-                    self.executor_core.status_updates or
-                    self.executor_core.data_updates or
-                    self.executor_core.task_info_updates
-                )
-
-                if has_pending_data:
-                    for data_type, config in self.executor_core.pipeline_config.items():
-                        time_since_flush = current_time - self.executor_core.last_pipeline_flush[data_type]
-
-                        if data_type == 'ack' and self.executor_core.pending_acks:
-                            if time_since_flush >= config['max_delay']:
-                                should_flush_periodic = True
-                                break
-                        elif data_type == 'task_info' and self.executor_core.task_info_updates:
-                            if time_since_flush >= config['max_delay']:
-                                should_flush_periodic = True
-                                break
-                        elif data_type == 'status' and self.executor_core.status_updates:
-                            if time_since_flush >= config['max_delay']:
-                                should_flush_periodic = True
-                                break
-                        elif data_type == 'data' and self.executor_core.data_updates:
-                            if time_since_flush >= config['max_delay']:
-                                should_flush_periodic = True
-                                break
-
-                if buffer_full or should_flush_periodic:
-                    asyncio.create_task(self.executor_core._flush_all_buffers())
-
-                # 智能休眠
-                has_events = False
-                if isinstance(self.event_queue, deque):
-                    has_events = bool(self.event_queue)
-                elif isinstance(self.event_queue, asyncio.Queue):
-                    has_events = not self.event_queue.empty()
-
-                if has_events:
-                    await asyncio.sleep(0)
-                else:
-                    if has_pending_data:
-                        await self.executor_core._flush_all_buffers()
-                    await asyncio.sleep(0.001)
-
-        except KeyboardInterrupt:
-            logger.debug("UnifiedExecutor received KeyboardInterrupt")
-        except Exception as e:
-            logger.error(f"UnifiedExecutor loop error: {e}")
-        finally:
-            await self._cleanup_single_process()
-
-    async def _cleanup_single_process(self):
-        """清理单进程模式资源"""
-        logger.debug("UnifiedExecutor cleaning up...")
-
-        # 设置停止标志
-        if hasattr(self.app.ep, '_stop_reading'):
-            self.app.ep._stop_reading = True
-
-        # 取消活动任务
-        if self._active_tasks:
-            logger.debug(f"Cancelling {len(self._active_tasks)} active tasks...")
-            for task in self._active_tasks:
-                if not task.done():
-                    task.cancel()
-
-            if self._active_tasks:
-                try:
-                    await asyncio.wait_for(
-                        asyncio.gather(*self._active_tasks, return_exceptions=True),
-                        timeout=0.2
-                    )
-                except asyncio.TimeoutError:
-                    logger.debug("Some tasks did not complete in time")
-
-        # 清理ExecutorCore
-        await self.executor_core.cleanup()
-
-        # 清理event_pool
-        if hasattr(self.app.ep, 'cleanup'):
-            try:
-                self.app.ep.cleanup()
-            except Exception as e:
-                logger.error(f"Error cleaning up EventPool: {e}")
-
-        # 标记worker离线
-        if self.app.consumer_manager:
-            try:
-                self.app.consumer_manager.cleanup()
-                logger.debug("Worker marked as offline")
-            except Exception as e:
-                logger.error(f"Error marking worker offline: {e}")
-
-        logger.debug("UnifiedExecutor stopped")
-
-    def start_multi_process(self, queues: List[str], prefetch_multiplier: int = 100, worker_id: str = None, worker_key: str = None):
-        """启动多进程模式
-
-        Args:
-            queues: 队列列表
-            prefetch_multiplier: 预取倍数
-            worker_id: Worker ID（主进程生成，子进程复用）
-            worker_key: Worker Key（主进程生成，子进程复用）
-        """
-        if self.mode != ExecutionMode.MULTI_PROCESS:
-            raise RuntimeError("start_multi_process() is only for MULTI_PROCESS mode")
-
-        self.orchestrator.start(queues, prefetch_multiplier, worker_id, worker_key)
-
-    def shutdown(self):
-        """
-        关闭执行器
-
-        根据执行模式调用相应的关闭方法
-        """
-        if self.mode == ExecutionMode.MULTI_PROCESS:
-            if self.orchestrator:
-                self.orchestrator.shutdown()
-        elif self.mode == ExecutionMode.SINGLE_PROCESS:
-            # 单进程模式的清理在 _cleanup_single_process 中处理
-            # 这里只是一个占位符，实际清理由事件循环完成
-            logger.debug("UnifiedExecutor shutdown called in SINGLE_PROCESS mode")
-
-
-__all__ = ['UnifiedExecutor']