funboost 47.9__tar.gz → 48.1__tar.gz

{funboost-47.9/funboost.egg-info → funboost-48.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 47.9
+Version: 48.1
 Summary: pip install funboost，python全功能分布式函数调度框架,funboost的功能是全面性重量级，用户能想得到的功能99%全都有;funboost的使用方式是轻量级，只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs
@@ -26,97 +26,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: Software Development :: Libraries
 Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: nb_log>=13.2
-Requires-Dist: nb_libs>=1.8
-Requires-Dist: nb_time>=2.1
-Requires-Dist: pymongo>=4.6.3
-Requires-Dist: AMQPStorm==2.10.6
-Requires-Dist: rabbitpy==2.0.1
-Requires-Dist: decorator==5.1.1
-Requires-Dist: tomorrow3==1.1.0
-Requires-Dist: persist-queue>=0.4.2
-Requires-Dist: apscheduler>=3.10.1
-Requires-Dist: pikav0
-Requires-Dist: pikav1
-Requires-Dist: redis2
-Requires-Dist: redis3
-Requires-Dist: redis5
-Requires-Dist: redis
-Requires-Dist: setuptools_rust
-Requires-Dist: fabric2==2.6.0
-Requires-Dist: nb_filelock
-Requires-Dist: pysnooper
-Requires-Dist: deprecated
-Requires-Dist: cryptography
-Requires-Dist: auto_run_on_remote
-Requires-Dist: frozenlist
-Requires-Dist: fire
-Requires-Dist: pydantic
-Requires-Dist: orjson
-Requires-Dist: async-timeout
-Requires-Dist: typing-extensions
 Provides-Extra: all
-Requires-Dist: confluent_kafka==1.7.0; extra == "all"
-Requires-Dist: pulsar-client==3.1.0; python_version >= "3.7" and extra == "all"
-Requires-Dist: celery; extra == "all"
-Requires-Dist: flower; extra == "all"
-Requires-Dist: nameko==2.14.1; extra == "all"
-Requires-Dist: sqlalchemy==1.4.13; extra == "all"
-Requires-Dist: sqlalchemy_utils==0.36.1; extra == "all"
-Requires-Dist: dramatiq==1.14.2; extra == "all"
-Requires-Dist: huey==2.4.5; extra == "all"
-Requires-Dist: rq==1.15.0; extra == "all"
-Requires-Dist: kombu; extra == "all"
-Requires-Dist: eventlet==0.33.3; extra == "all"
-Requires-Dist: gevent==22.10.2; extra == "all"
-Requires-Dist: elasticsearch; extra == "all"
-Requires-Dist: gnsq==1.0.1; extra == "all"
-Requires-Dist: psutil; extra == "all"
-Requires-Dist: peewee==3.17.3; extra == "all"
-Requires-Dist: nats-python; extra == "all"
-Requires-Dist: aiohttp==3.8.3; extra == "all"
-Requires-Dist: paho-mqtt; extra == "all"
-Requires-Dist: rocketmq; extra == "all"
-Requires-Dist: zmq; extra == "all"
-Requires-Dist: pyzmq; extra == "all"
-Requires-Dist: kafka-python==2.0.2; extra == "all"
-Requires-Dist: flask; extra == "all"
-Requires-Dist: flask_bootstrap; extra == "all"
-Requires-Dist: flask_wtf; extra == "all"
-Requires-Dist: wtforms; extra == "all"
-Requires-Dist: flask_login; extra == "all"
-Provides-Extra: extra-brokers
-Requires-Dist: confluent_kafka==1.7.0; extra == "extra-brokers"
-Requires-Dist: pulsar-client==3.1.0; python_version >= "3.7" and extra == "extra-brokers"
-Requires-Dist: celery; extra == "extra-brokers"
-Requires-Dist: flower; extra == "extra-brokers"
-Requires-Dist: nameko==2.14.1; extra == "extra-brokers"
-Requires-Dist: sqlalchemy==1.4.13; extra == "extra-brokers"
-Requires-Dist: sqlalchemy_utils==0.36.1; extra == "extra-brokers"
-Requires-Dist: dramatiq==1.14.2; extra == "extra-brokers"
-Requires-Dist: huey==2.4.5; extra == "extra-brokers"
-Requires-Dist: rq==1.15.0; extra == "extra-brokers"
-Requires-Dist: kombu; extra == "extra-brokers"
-Requires-Dist: eventlet==0.33.3; extra == "extra-brokers"
-Requires-Dist: gevent==22.10.2; extra == "extra-brokers"
-Requires-Dist: elasticsearch; extra == "extra-brokers"
-Requires-Dist: gnsq==1.0.1; extra == "extra-brokers"
-Requires-Dist: psutil; extra == "extra-brokers"
-Requires-Dist: peewee==3.17.3; extra == "extra-brokers"
-Requires-Dist: nats-python; extra == "extra-brokers"
-Requires-Dist: aiohttp==3.8.3; extra == "extra-brokers"
-Requires-Dist: paho-mqtt; extra == "extra-brokers"
-Requires-Dist: rocketmq; extra == "extra-brokers"
-Requires-Dist: zmq; extra == "extra-brokers"
-Requires-Dist: pyzmq; extra == "extra-brokers"
-Requires-Dist: kafka-python==2.0.2; extra == "extra-brokers"
+Provides-Extra: extra_brokers
 Provides-Extra: flask
-Requires-Dist: flask; extra == "flask"
-Requires-Dist: flask_bootstrap; extra == "flask"
-Requires-Dist: flask_wtf; extra == "flask"
-Requires-Dist: wtforms; extra == "flask"
-Requires-Dist: flask_login; extra == "flask"
+License-File: LICENSE
 
 
 
@@ -146,7 +59,7 @@ funboost的功能是全面性重量级，用户能想得到的功能99%全都有
 
 python函数加速器，框架包罗万象，一统编程思维，兼容50% python编程业务场景，适用范围广。
 python万能分布式函数调度框架，支持5种并发模式，30+种消息队列中间件(或任务队列框架)，
-30种任务控制功能。给任意python函数赋能。 
+30种任务控制功能。给任意python函数赋能。
 用途概念就是常规经典的 生产者 + 消息队列中间件 + 消费者 编程思想。
 
 框架只需要学习@boost这一个装饰器的入参就可以，所有用法几乎和1.3例子一摸一样，非常简化简单。
@@ -528,7 +441,9 @@ if __name__ == "__main__":
 
 <a href="https://imgse.com/i/pkFkCUe"><img src="https://s21.ax1x.com/2024/04/29/pkFkCUe.png" alt="pkFkCUe.png" border="0" /></a>
 
-<a href="https://imgse.com/i/pkE6IYR"><img src="https://s21.ax1x.com/2024/05/07/pkE6IYR.png" alt="pkE6IYR.png" border="0" /></a>
+[![pEGDim8.png](https://s21.ax1x.com/2025/03/03/pEGDim8.png)](https://imgse.com/i/pEGDim8)
+[![pEGD96P.png](https://s21.ax1x.com/2025/03/03/pEGD96P.png)](https://imgse.com/i/pEGD96P)
+[![pEGDCOf.png](https://s21.ax1x.com/2025/03/03/pEGDCOf.png)](https://imgse.com/i/pEGDCOf)
 
 ## 1.4  python分布式函数执行为什么重要？
 

{funboost-47.9 → funboost-48.1}/README.md

@@ -26,7 +26,7 @@ funboost的功能是全面性重量级，用户能想得到的功能99%全都有
 
 python函数加速器，框架包罗万象，一统编程思维，兼容50% python编程业务场景，适用范围广。
 python万能分布式函数调度框架，支持5种并发模式，30+种消息队列中间件(或任务队列框架)，
-30种任务控制功能。给任意python函数赋能。 
+30种任务控制功能。给任意python函数赋能。
 用途概念就是常规经典的 生产者 + 消息队列中间件 + 消费者 编程思想。
 
 框架只需要学习@boost这一个装饰器的入参就可以，所有用法几乎和1.3例子一摸一样，非常简化简单。
@@ -408,7 +408,9 @@ if __name__ == "__main__":
 
 <a href="https://imgse.com/i/pkFkCUe"><img src="https://s21.ax1x.com/2024/04/29/pkFkCUe.png" alt="pkFkCUe.png" border="0" /></a>
 
-<a href="https://imgse.com/i/pkE6IYR"><img src="https://s21.ax1x.com/2024/05/07/pkE6IYR.png" alt="pkE6IYR.png" border="0" /></a>
+[![pEGDim8.png](https://s21.ax1x.com/2025/03/03/pEGDim8.png)](https://imgse.com/i/pEGDim8)
+[![pEGD96P.png](https://s21.ax1x.com/2025/03/03/pEGD96P.png)](https://imgse.com/i/pEGD96P)
+[![pEGDCOf.png](https://s21.ax1x.com/2025/03/03/pEGDCOf.png)](https://imgse.com/i/pEGDCOf)
 
 ## 1.4  python分布式函数执行为什么重要？
 
@@ -473,4 +475,4 @@ funboost通过支持celery作为broker_kind,使celer框架变成了funboost的
 
 ![](https://visitor-badge.glitch.me/badge?page_id=distributed_framework)
 
-<div> </div>
+<div> </div>

{funboost-47.9 → funboost-48.1}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "47.9"
+__version__ = "48.1"
 
 from funboost.set_frame_config import show_frame_config
 
@@ -42,7 +42,7 @@ from funboost.factories.broker_kind__publsiher_consumer_type_map import register
 from funboost.factories.publisher_factotry import get_publisher
 from funboost.factories.consumer_factory import get_consumer
 
-from funboost.timing_job import fsdf_background_scheduler, timing_publish_deco, funboost_aps_scheduler
+from funboost.timing_job import fsdf_background_scheduler, timing_publish_deco, funboost_aps_scheduler,ApsJobAdder
 from funboost.constant import BrokerEnum, ConcurrentModeEnum
 
 from funboost.core.booster import boost, Booster, BoostersManager
@@ -58,7 +58,7 @@ from funboost.utils.ctrl_c_end import ctrl_c_recv
 from funboost.utils.redis_manager import RedisMixin
 from funboost.concurrent_pool.custom_threadpool_executor import show_current_threads_num
 
-from funboost.core.current_task import funboost_current_task
+from funboost.core.current_task import funboost_current_task,fct,get_current_taskid
 
 
 # atexit.register(ctrl_c_recv)  # 还是需要用户自己在代码末尾加才可以.

{funboost-47.9 → funboost-48.1}/funboost/assist/celery_helper.py

@@ -85,7 +85,7 @@ class CeleryHelper:
 
         if is_start_consume_all_queues is False:
             to_be_start_work_celery_queue_name_set_new = copy.copy(cls.to_be_start_work_celery_queue_name_set)
-            to_be_start_work_celery_queue_name_set_new.update(set(start_consume_queue_name_list))
+            to_be_start_work_celery_queue_name_set_new.update(set(start_consume_queue_name_list or []))
         else:
             from funboost import BoostersManager
             # print(BoostersManager.get_all_queues())

{funboost-47.9 → funboost-48.1}/funboost/constant.py

@@ -1,4 +1,7 @@
 # coding= utf-8
+from calendar import c
+
+
 class BrokerEnum:
     EMPTY = 'empty'  # 空的实现，需要搭配 boost入参的 consumer_override_cls 和 publisher_override_cls使用，或者被继承。
 
@@ -106,3 +109,7 @@ class ConstStrForClassMethod:
     CLS_MODULE = 'cls_module'
     CLS_FILE = 'cls_file'
 
+class RedisKeys:
+    REDIS_KEY_PAUSE_FLAG  = 'funboost_pause_flag' 
+    REDIS_KEY_STOP_FLAG = 'funboost_stop_flag'
+    QUEUE__MSG_COUNT_MAP = 'funboost_queue__msg_count_map'

{funboost-47.9 → funboost-48.1}/funboost/consumers/base_consumer.py

@@ -74,7 +74,7 @@ from funboost.consumers.redis_filter import RedisFilter, RedisImpermanencyFilter
 from funboost.factories.publisher_factotry import get_publisher
 
 from funboost.utils import decorators, time_util, redis_manager
-from funboost.constant import ConcurrentModeEnum, BrokerEnum, ConstStrForClassMethod
+from funboost.constant import ConcurrentModeEnum, BrokerEnum, ConstStrForClassMethod,RedisKeys
 from funboost.core import kill_remote_task
 from funboost.core.exceptions import ExceptionForRequeue, ExceptionForPushToDlxqueue
 
@@ -164,19 +164,25 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         filter_class = RedisFilter if consumer_params.task_filtering_expire_seconds == 0 else RedisImpermanencyFilter
         self._redis_filter = filter_class(self._redis_filter_key_name, consumer_params.task_filtering_expire_seconds)
 
-        self._unit_time_for_count = 10  # 每隔多少秒计数，显示单位时间内执行多少次，暂时固定为10秒。
-        self._execute_task_times_every_unit_time = 0  # 每单位时间执行了多少次任务。
         self._lock_for_count_execute_task_times_every_unit_time = Lock()
-        self._current_time_for_execute_task_times_every_unit_time = time.time()
-        self._consuming_function_cost_time_total_every_unit_time = 0
-        self._last_execute_task_time = time.time()  # 最近一次执行任务的时间。
-
-        self._last_show_remaining_execution_time = 0
-        self._show_remaining_execution_time_interval = 300
-
-        self._msg_num_in_broker = 0
-        self._last_timestamp_when_has_task_in_queue = 0
-        self._last_timestamp_print_msg_num = 0
+        # self._unit_time_for_count = 10  # 每隔多少秒计数，显示单位时间内执行多少次，暂时固定为10秒。
+        # self._execute_task_times_every_unit_time = 0  # 每单位时间执行了多少次任务。
+        # self._execute_task_times_every_unit_time_fail =0  # 每单位时间执行了多少次任务失败。
+        # self._lock_for_count_execute_task_times_every_unit_time = Lock()
+        # self._current_time_for_execute_task_times_every_unit_time = time.time()
+        # self._consuming_function_cost_time_total_every_unit_time = 0
+        # self._last_execute_task_time = time.time()  # 最近一次执行任务的时间。
+        # self._last_10s_execute_count = 0
+        # self._last_10s_execute_count_fail = 0
+        #
+        # self._last_show_remaining_execution_time = 0
+        # self._show_remaining_execution_time_interval = 300
+        #
+        # self._msg_num_in_broker = 0
+        # self._last_timestamp_when_has_task_in_queue = 0
+        # self._last_timestamp_print_msg_num = 0
+
+        self.metric_calculation = MetricCalculation(self)
 
         self._result_persistence_helper: ResultPersistenceHelper
         self._check_broker_exclusive_config()
@@ -189,8 +195,8 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         self._stop_flag = None
         self._pause_flag = None  # 暂停消费标志，从reids读取
         self._last_show_pause_log_time = 0
-        self._redis_key_stop_flag = f'funboost_stop_flag:{self.queue_name}'
-        self._redis_key_pause_flag = f'funboost_pause_flag:{self.queue_name}'
+        # self._redis_key_stop_flag = f'funboost_stop_flag'
+        # self._redis_key_pause_flag = RedisKeys.REDIS_KEY_PAUSE_FLAG
 
         # 控频要用到的成员变量
         self._last_submit_task_timestamp = 0
@@ -358,7 +364,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             self._distributed_consumer_statistics.run()
             self.logger.warning(f'启动了分布式环境 使用 redis 的键 hearbeat:{self._queue_name} 统计活跃消费者 ，当前消费者唯一标识为 {self.consumer_identification}')
 
-        self.keep_circulating(60, block=False, daemon=False)(self.check_heartbeat_and_message_count)()  # 间隔时间最好比self._unit_time_for_count小整数倍，不然日志不准。
+        self.keep_circulating(60, block=False, daemon=False)(self.check_heartbeat_and_message_count)()
         if self.consumer_params.is_support_remote_kill_task:
             kill_remote_task.RemoteTaskKiller(self.queue_name, None).start_cycle_kill_task()
             self.consumer_params.is_show_message_get_from_broker = True  # 方便用户看到从消息队列取出来的消息的task_id,然后使用task_id杀死运行中的消息。
@@ -403,13 +409,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
     logger_apscheduler = get_logger('push_for_apscheduler_use_database_store', log_filename='push_for_apscheduler_use_database_store.log')
 
     @classmethod
-    def _push_apscheduler_task_to_broker(cls, queue_name, msg, runonce_uuid):
+    def _push_apscheduler_task_to_broker(cls, queue_name, msg):
         funboost_lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
-        # key = 'apscheduler.redisjobstore_runonce'
-        # if RedisMixin().redis_db_frame.sadd(key, runonce_uuid):  # 这样可以阻止多次启动同队列名消费者 redis jobstore多次运行函数.
-        #     cls.logger_apscheduler.debug(f'延时任务用普通消息重新发布到普通队列 {msg}')
-        #     funboost_lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
-
+       
     @abc.abstractmethod
     def _shedual_task(self):
         """
@@ -512,7 +514,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             # print(msg_no_delay)
             # 数据库作为apscheduler的jobstores时候， 不能用 self.pbulisher_of_same_queue.publish，self不能序列化
             self._delay_task_scheduler.add_job(self._push_apscheduler_task_to_broker, 'date', run_date=run_date,
-                                               kwargs={'queue_name': self.queue_name, 'msg': msg_no_delay, 'runonce_uuid': str(uuid.uuid4())},
+                                               kwargs={'queue_name': self.queue_name, 'msg': msg_no_delay, },
                                                misfire_grace_time=misfire_grace_time,
                                               )
             self._confirm_consume(kw)
@@ -681,26 +683,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                         p.execute()
 
             with self._lock_for_count_execute_task_times_every_unit_time:
-                self._execute_task_times_every_unit_time += 1
-                self._consuming_function_cost_time_total_every_unit_time += time.time() - t_start_run_fun
-                self._last_execute_task_time = time.time()
-                if time.time() - self._current_time_for_execute_task_times_every_unit_time > self._unit_time_for_count:
-                    avarage_function_spend_time = round(self._consuming_function_cost_time_total_every_unit_time / self._execute_task_times_every_unit_time, 4)
-                    msg = f'{self._unit_time_for_count} 秒内执行了 {self._execute_task_times_every_unit_time} 次函数 [ {self.consuming_function.__name__} ] ,' \
-                          f'函数平均运行耗时 {avarage_function_spend_time} 秒。 '
-                    self.logger.info(msg)
-                    if time.time() - self._last_show_remaining_execution_time > self._show_remaining_execution_time_interval:
-                        self._msg_num_in_broker = self.publisher_of_same_queue.get_message_count()
-                        if self._msg_num_in_broker != -1:  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
-                            # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
-                            need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /
-                                                                                self._distributed_consumer_statistics.active_consumer_num)
-                            msg += f''' 预计还需要 {need_time} 时间 才能执行完成 队列 {self.queue_name} 中的 {self._msg_num_in_broker} 个剩余任务'''
-                            self.logger.info(msg)
-                            self._last_show_remaining_execution_time = time.time()
-                    self._current_time_for_execute_task_times_every_unit_time = time.time()
-                    self._consuming_function_cost_time_total_every_unit_time = 0
-                    self._execute_task_times_every_unit_time = 0
+                self.metric_calculation.cal(t_start_run_fun,current_function_result_status)
             self.user_custom_record_process_info_func(current_function_result_status)  # 两种方式都可以自定义,记录结果,建议继承方式,不使用boost中指定 user_custom_record_process_info_func
             if self.consumer_params.user_custom_record_process_info_func:
                 self.consumer_params.user_custom_record_process_info_func(current_function_result_status)
@@ -854,24 +837,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                     await simple_run_in_executor(push_result)
 
             # 异步执行不存在线程并发，不需要加锁。
-            self._execute_task_times_every_unit_time += 1
-            self._consuming_function_cost_time_total_every_unit_time += time.time() - t_start_run_fun
-            self._last_execute_task_time = time.time()
-            if time.time() - self._current_time_for_execute_task_times_every_unit_time > self._unit_time_for_count:
-                avarage_function_spend_time = round(self._consuming_function_cost_time_total_every_unit_time / self._execute_task_times_every_unit_time, 4)
-                msg = f'{self._unit_time_for_count} 秒内执行了 {self._execute_task_times_every_unit_time} 次函数 [ {self.consuming_function.__name__} ] ,' \
-                      f'函数平均运行耗时 {avarage_function_spend_time} 秒。 '
-                self.logger.info(msg)
-                if self._msg_num_in_broker != -1 and time.time() - self._last_show_remaining_execution_time > self._show_remaining_execution_time_interval:  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
-                    # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
-                    need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /
-                                                                        self._distributed_consumer_statistics.active_consumer_num)
-                    msg += f''' 预计还需要 {need_time} 时间 才能执行完成 队列 {self.queue_name} 中的 {self._msg_num_in_broker} 个剩余任务'''
-                    self.logger.info(msg)
-                    self._last_show_remaining_execution_time = time.time()
-                self._current_time_for_execute_task_times_every_unit_time = time.time()
-                self._consuming_function_cost_time_total_every_unit_time = 0
-                self._execute_task_times_every_unit_time = 0
+            self.metric_calculation.cal(t_start_run_fun, current_function_result_status)
 
             self.user_custom_record_process_info_func(current_function_result_status)  # 两种方式都可以自定义,记录结果.建议使用文档4.21.b的方式继承来重写
             await self.aio_user_custom_record_process_info_func(current_function_result_status)
@@ -958,14 +924,15 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         raise NotImplementedError
 
     def check_heartbeat_and_message_count(self):
-        self._msg_num_in_broker = self.publisher_of_same_queue.get_message_count()
-        if time.time() - self._last_timestamp_print_msg_num > 600:
-            if self._msg_num_in_broker != -1:
-                self.logger.info(f'队列 [{self._queue_name}] 中还有 [{self._msg_num_in_broker}] 个任务')
-            self._last_timestamp_print_msg_num = time.time()
-        if self._msg_num_in_broker != 0:
-            self._last_timestamp_when_has_task_in_queue = time.time()
-        return self._msg_num_in_broker
+        self.metric_calculation.msg_num_in_broker = self.publisher_of_same_queue.get_message_count()
+        self.metric_calculation.last_get_msg_num_ts = time.time()
+        if time.time() - self.metric_calculation.last_timestamp_print_msg_num > 600:
+            if self.metric_calculation.msg_num_in_broker != -1:
+                self.logger.info(f'队列 [{self._queue_name}] 中还有 [{self.metric_calculation.msg_num_in_broker}] 个任务')
+            self.metric_calculation.last_timestamp_print_msg_num = time.time()
+        if self.metric_calculation.msg_num_in_broker != 0:
+            self.metric_calculation.last_timestamp_when_has_task_in_queue = time.time()
+        return self.metric_calculation.msg_num_in_broker
 
     @abc.abstractmethod
     def _requeue(self, kw):
@@ -1003,11 +970,11 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
 
     def pause_consume(self):
         """从远程机器可以设置队列为暂停消费状态，funboost框架会自动停止消费，此功能需要配置好redis"""
-        RedisMixin().redis_db_frame.set(self._redis_key_pause_flag, 1)
+        RedisMixin().redis_db_frame.hset(RedisKeys.REDIS_KEY_PAUSE_FLAG, self.queue_name,'1')
 
     def continue_consume(self):
         """从远程机器可以设置队列为暂停消费状态，funboost框架会自动继续消费，此功能需要配置好redis"""
-        RedisMixin().redis_db_frame.set(self._redis_key_pause_flag, 0)
+        RedisMixin().redis_db_frame.hset(RedisKeys.REDIS_KEY_PAUSE_FLAG, self.queue_name,'0')
 
     @decorators.FunctionResultCacher.cached_function_result_for_a_time(120)
     def _judge_is_daylight(self):
@@ -1031,9 +998,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         no_task_time = 0
         while 1:
             # noinspection PyBroadException
-            message_count = self._msg_num_in_broker
+            message_count = self.metric_calculation.msg_num_in_broker
             # print(message_count,self._last_execute_task_time,time.time() - self._last_execute_task_time,no_task_time)
-            if message_count == 0 and self._last_execute_task_time != 0 and (time.time() - self._last_execute_task_time) > minutes * 60:
+            if message_count == 0 and self.metric_calculation.last_execute_task_time != 0 and (time.time() - self.metric_calculation.last_execute_task_time) > minutes * 60:
                 no_task_time += 30
             else:
                 no_task_time = 0
@@ -1154,6 +1121,77 @@ def wait_for_possible_has_finish_all_tasks_by_conusmer_list(consumer_list: typin
             pool.submit(consumer.wait_for_possible_has_finish_all_tasks(minutes))
 
 
+class MetricCalculation:
+    UNIT_TIME_FOR_COUNT = 10 # 这个不要随意改,需要其他地方配合,每隔多少秒计数，显示单位时间内执行多少次，暂时固定为10秒。
+
+    def __init__(self,conusmer:AbstractConsumer) -> None:
+        self.consumer = conusmer
+
+        self.unit_time_for_count = self.UNIT_TIME_FOR_COUNT  # 
+        self.execute_task_times_every_unit_time_temp = 0  # 每单位时间执行了多少次任务。
+        self.execute_task_times_every_unit_time_temp_fail =0  # 每单位时间执行了多少次任务失败。
+        self.current_time_for_execute_task_times_every_unit_time = time.time()
+        self.consuming_function_cost_time_total_every_unit_time = 0
+        self.last_execute_task_time = time.time()  # 最近一次执行任务的时间。
+        self.last_x_s_execute_count = 0
+        self.last_x_s_execute_count_fail = 0
+        self.last_show_remaining_execution_time = 0
+        self.show_remaining_execution_time_interval = 300
+        self.msg_num_in_broker = 0
+        self.last_get_msg_num_ts = 0
+        self.last_timestamp_when_has_task_in_queue = 0
+        self.last_timestamp_print_msg_num = 0
+        self.avarage_function_spend_time = None
+        self.total_consume_count_from_start =0
+        self.total_consume_count_from_start_fail =0
+
+    def cal(self,t_start_run_fun:float,current_function_result_status:FunctionResultStatus):
+        self.execute_task_times_every_unit_time_temp += 1
+        self.total_consume_count_from_start  +=1
+        if current_function_result_status.success is False:
+            self.execute_task_times_every_unit_time_temp_fail += 1
+            self.total_consume_count_from_start_fail +=1
+        self.consuming_function_cost_time_total_every_unit_time += time.time() - t_start_run_fun
+        self.last_execute_task_time = time.time()
+        if time.time() - self.current_time_for_execute_task_times_every_unit_time > self.unit_time_for_count:
+            self.last_x_s_execute_count = self.execute_task_times_every_unit_time_temp
+            self.last_x_s_execute_count_fail = self.execute_task_times_every_unit_time_temp_fail
+            self.avarage_function_spend_time = round(self.consuming_function_cost_time_total_every_unit_time / self.last_x_s_execute_count, 4)
+            msg = f'{self.unit_time_for_count} 秒内执行了 {self.last_x_s_execute_count} 次函数 [ {self.consumer.consuming_function.__name__} ] ,' \
+                  f'失败了{self.last_x_s_execute_count_fail} 次,函数平均运行耗时 {self.avarage_function_spend_time} 秒。 '
+            self.consumer.logger.info(msg)
+            if time.time() - self.last_show_remaining_execution_time > self.show_remaining_execution_time_interval:
+                self.msg_num_in_broker = self.consumer.publisher_of_same_queue.get_message_count()
+                self.last_get_msg_num_ts = time.time()
+                if self.msg_num_in_broker != -1:  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
+                    # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
+                    need_time = time_util.seconds_to_hour_minute_second(self.msg_num_in_broker / (self.execute_task_times_every_unit_time_temp / self.unit_time_for_count) /
+                                                                        self.consumer._distributed_consumer_statistics.active_consumer_num)
+                    msg += f''' 预计还需要 {need_time} 时间 才能执行完成 队列 {self.consumer.queue_name} 中的 {self.msg_num_in_broker} 个剩余任务'''
+                    self.consumer.logger.info(msg)
+                    self.last_show_remaining_execution_time = time.time()
+            self.current_time_for_execute_task_times_every_unit_time = time.time()
+            self.consuming_function_cost_time_total_every_unit_time = 0
+            self.execute_task_times_every_unit_time_temp = 0
+            self.execute_task_times_every_unit_time_temp_fail = 0
+
+    def get_report_hearbeat_info(self) ->dict:
+        return {
+            'unit_time_for_count':self.unit_time_for_count,
+            'last_x_s_execute_count':self.last_x_s_execute_count,
+            'last_x_s_execute_count_fail':self.last_x_s_execute_count_fail,
+            'last_execute_task_time':self.last_execute_task_time,
+            # 'last_show_remaining_execution_time':self.last_show_remaining_execution_time,
+            # 'msg_num_in_broker':self.msg_num_in_broker,
+            'current_time_for_execute_task_times_every_unit_time':self.current_time_for_execute_task_times_every_unit_time,
+            'last_timestamp_when_has_task_in_queue':self.last_timestamp_when_has_task_in_queue,
+            'avarage_function_spend_time':self.avarage_function_spend_time,
+            'total_consume_count_from_start':self.total_consume_count_from_start,
+            'total_consume_count_from_start_fail':self.total_consume_count_from_start_fail
+
+        }
+
+
 class DistributedConsumerStatistics(RedisMixin, FunboostFileLoggerMixin):
     """
     为了兼容模拟mq的中间件（例如redis，他没有实现amqp协议，redis的list结构和真mq差远了），获取一个队列有几个连接活跃消费者数量。
@@ -1190,9 +1228,18 @@ class DistributedConsumerStatistics(RedisMixin, FunboostFileLoggerMixin):
         self._server__consumer_identification_map_key_name = f'funboost_hearbeat_server__dict:{nb_log_config_default.computer_ip}'
 
     def run(self):
+        self._send_consumer_params()
         self.send_heartbeat()
         self._consumer.keep_circulating(self.SEND_HEARTBEAT_INTERVAL, block=False, daemon=False)(self.send_heartbeat)()
 
+    def _send_consumer_params(self):
+        """
+        保存队列的消费者参数，以便在web界面查看。
+        :return:
+        """
+        self.redis_db_frame.hmset('funboost_queue__consumer_parmas',{self._consumer.queue_name: self._consumer.consumer_params.json_str_value()})
+
+
     def _send_heartbeat_with_dict_value(self, redis_key, ):
         # 发送当前消费者进程心跳的，值是字典，按一个机器或者一个队列运行了哪些进程。
 
@@ -1206,10 +1253,19 @@ class DistributedConsumerStatistics(RedisMixin, FunboostFileLoggerMixin):
                     p.srem(redis_key, result)
             self._consumer_identification_map['hearbeat_datetime_str'] = time_util.DatetimeConverter().datetime_str
             self._consumer_identification_map['hearbeat_timestamp'] = self.timestamp()
+            self._consumer_identification_map.update(self._consumer.metric_calculation.get_report_hearbeat_info())
             value = Serialization.to_json_str(self._consumer_identification_map, )
             p.sadd(redis_key, value)
             p.execute()
 
+
+    def _send_msg_num(self):
+        dic = {'msg_num_in_broker':self._consumer.metric_calculation.msg_num_in_broker,
+               'last_get_msg_num_ts':self._consumer.metric_calculation.last_get_msg_num_ts,
+               'report_ts':time.time(),
+               }
+        self.redis_db_frame.hset(RedisKeys.QUEUE__MSG_COUNT_MAP, self._consumer.queue_name, json.dumps(dic))
+
     def send_heartbeat(self):
         # 根据队列名心跳的，值是字符串，方便值作为其他redis的键名
 
@@ -1226,6 +1282,7 @@ class DistributedConsumerStatistics(RedisMixin, FunboostFileLoggerMixin):
         self._send_heartbeat_with_dict_value(self._server__consumer_identification_map_key_name)
         self._show_active_consumer_num()
         self._get_stop_and_pause_flag_from_redis()
+        self._send_msg_num()
 
     def _show_active_consumer_num(self):
         self.active_consumer_num = self.redis_db_frame.scard(self._redis_key_name) or 1
@@ -1241,13 +1298,13 @@ class DistributedConsumerStatistics(RedisMixin, FunboostFileLoggerMixin):
 
     # noinspection PyProtectedMember
     def _get_stop_and_pause_flag_from_redis(self):
-        stop_flag = self.redis_db_frame.get(self._consumer._redis_key_stop_flag)
+        stop_flag = self.redis_db_frame.hget(RedisKeys.REDIS_KEY_STOP_FLAG,self._consumer.queue_name)
         if stop_flag is not None and int(stop_flag) == 1:
             self._consumer._stop_flag = 1
         else:
             self._consumer._stop_flag = 0
 
-        pause_flag = self.redis_db_frame.get(self._consumer._redis_key_pause_flag)
+        pause_flag = self.redis_db_frame.hget(RedisKeys.REDIS_KEY_PAUSE_FLAG,self._consumer.queue_name)
         if pause_flag is not None and int(pause_flag) == 1:
             self._consumer._pause_flag = 1
         else:

{funboost-47.9 → funboost-48.1}/funboost/consumers/redis_consumer_ack_able.py

@@ -126,7 +126,7 @@ class RedisConsumerAckAble(ConsumerConfirmMixinWithTheHelpOfRedisByHearbeat, Abs
         pull_msg_batch_size = self.consumer_params.broker_exclusive_config['pull_msg_batch_size']
         lua = f'''
                      local task_list = redis.call("lrange", KEYS[1],0,{pull_msg_batch_size-1})
-                     redis.call("ltrim", KEYS[1],100,-1)
+                     redis.call("ltrim", KEYS[1],{pull_msg_batch_size},-1)
                      if (#task_list > 0) then
                         for task_index,task_value in ipairs(task_list)
                         do

{funboost-47.9 → funboost-48.1}/funboost/core/active_cousumer_info_getter.py

@@ -1,9 +1,14 @@
 import json
+import time
 import typing
 
+from pydantic import main
+
 from funboost.utils.redis_manager import RedisMixin
 
 from funboost.core.loggers import FunboostFileLoggerMixin,nb_log_config_default
+from funboost.core.serialization import Serialization
+from funboost.constant import RedisKeys
 
 class ActiveCousumerProcessInfoGetter(RedisMixin, FunboostFileLoggerMixin):
     """
@@ -23,6 +28,9 @@ class ActiveCousumerProcessInfoGetter(RedisMixin, FunboostFileLoggerMixin):
             result_dict = json.loads(result)
             if self.timestamp() - result_dict['hearbeat_timestamp'] < 15:
                 active_consumers_processor_info_list.append(result_dict)
+                if self.timestamp() - result_dict['current_time_for_execute_task_times_every_unit_time'] > 30:
+                    result_dict['last_x_s_execute_count'] = 0
+                    result_dict['last_x_s_execute_count_fail'] = 0
         return active_consumers_processor_info_list
 
     def get_all_hearbeat_info_by_queue_name(self, queue_name) -> typing.List[typing.Dict]:
@@ -67,6 +75,9 @@ class ActiveCousumerProcessInfoGetter(RedisMixin, FunboostFileLoggerMixin):
                 info_dict = json.loads(info_str)
                 if self.timestamp() - info_dict['hearbeat_timestamp'] < 15:
                     infos_map[dict_key].append(info_dict)
+                    if self.timestamp() - info_dict['current_time_for_execute_task_times_every_unit_time'] > 30:
+                        info_dict['last_x_s_execute_count'] = 0
+                        info_dict['last_x_s_execute_count_fail'] = 0
         return infos_map
 
     def get_all_hearbeat_info_partition_by_queue_name(self) -> typing.Dict[typing.AnyStr, typing.List[typing.Dict]]:
@@ -80,3 +91,52 @@ class ActiveCousumerProcessInfoGetter(RedisMixin, FunboostFileLoggerMixin):
         infos_map = self._get_all_hearbeat_info_partition_by_redis_key_prefix('funboost_hearbeat_server__dict:')
         self.logger.info(f'获取所有机器ip对应的活跃消费者进程信息，按机器ip划分，结果是 {json.dumps(infos_map, indent=4)}')
         return infos_map
+
+
+
+class QueueConusmerParamsGetter(RedisMixin, FunboostFileLoggerMixin):
+
+    def get_queue_params(self):
+        queue__consumer_params_map  = self.redis_db_frame.hgetall('funboost_queue__consumer_parmas')
+        return {k:Serialization.to_dict(v)  for k,v in queue__consumer_params_map.items()}
+
+    def get_pause_flag(self):
+        queue__pause_map = self.redis_db_frame.hgetall(RedisKeys.REDIS_KEY_PAUSE_FLAG)
+        return {k:int(v)  for k,v in queue__pause_map.items()}
+
+    def get_msg_num(self):
+        queue__msg_count_info_map = self.redis_db_frame.hgetall(RedisKeys.QUEUE__MSG_COUNT_MAP)
+        queue__msg_count_dict = {}
+        for queue_name,info_json in queue__msg_count_info_map.items():
+            info_dict = json.loads(info_json)
+            if info_dict['report_ts'] > time.time() - 15 and info_dict['last_get_msg_num_ts'] > time.time() - 1200:
+                queue__msg_count_dict[queue_name] = info_dict['msg_num_in_broker']
+        return queue__msg_count_dict
+    
+    def get_queue_params_and_active_consumers(self):
+        queue__active_consumers_map = ActiveCousumerProcessInfoGetter().get_all_hearbeat_info_partition_by_queue_name()
+        queue__consumer_params_map  = self.get_queue_params()
+        queue__pause_map = self.get_pause_flag()
+        queue__msg_count_dict = self.get_msg_num()
+        queue_params_and_active_consumers = {}
+
+        for queue, consumer_params in  queue__consumer_params_map.items():
+            active_consumers = queue__active_consumers_map.get(queue, [])
+            all_consumers_last_x_s_execute_count = 0
+            all_consumers_last_x_s_execute_count_fail =0
+            for c in active_consumers:
+                all_consumers_last_x_s_execute_count += c['last_x_s_execute_count']
+                all_consumers_last_x_s_execute_count_fail += c['last_x_s_execute_count_fail']
+            queue_params_and_active_consumers[queue] = {
+            'queue_params':consumer_params,
+            'active_consumers':active_consumers,
+            'pause_flag':queue__pause_map.get(queue,-1),
+            'msg_num_in_broker':queue__msg_count_dict.get(queue,None),
+                'all_consumers_last_x_s_execute_count':all_consumers_last_x_s_execute_count,
+                'all_consumers_last_x_s_execute_count_fail':all_consumers_last_x_s_execute_count_fail,
+            }
+        return queue_params_and_active_consumers
+
+
+if __name__ == '__main__':
+    print(Serialization.to_json_str(QueueConusmerParamsGetter().get_queue_params_and_active_consumers()))