funboost 48.8__py3-none-any.whl → 49.0__py3-none-any.whl

funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "48.8"
+__version__ = "49.0"
 
 from funboost.set_frame_config import show_frame_config
 

funboost/concurrent_pool/custom_threadpool_executor.py

@@ -98,7 +98,7 @@ class ThreadPoolExecutorShrinkAble(Executor, FunboostFileLoggerMixin, LoggerLeve
     # MIN_WORKERS = 5   # 最小值可以设置为0，代表线程池无论多久没有任务最少要保持多少个线程待命。
     # KEEP_ALIVE_TIME = 60  # 这个参数表名，当前线程从queue.get(block=True, timeout=KEEP_ALIVE_TIME)多久没任务，就线程结束。
 
-    MIN_WORKERS = 5
+    MIN_WORKERS = 1
     KEEP_ALIVE_TIME = 60
 
     def __init__(self, max_workers: int = None, thread_name_prefix='',work_queue_maxsize=10):

funboost/constant.py

@@ -108,9 +108,23 @@ class ConstStrForClassMethod:
     OBJ_INIT_PARAMS = 'obj_init_params'
     CLS_MODULE = 'cls_module'
     CLS_FILE = 'cls_file'
-
 class RedisKeys:
     REDIS_KEY_PAUSE_FLAG  = 'funboost_pause_flag' 
     REDIS_KEY_STOP_FLAG = 'funboost_stop_flag'
     QUEUE__MSG_COUNT_MAP = 'funboost_queue__msg_count_map'
-    FUNBOOST_QUEUE__CONSUMER_PARAMS= 'funboost_queue__consumer_parmas'
+    FUNBOOST_QUEUE__CONSUMER_PARAMS= 'funboost_queue__consumer_parmas'
+    FUNBOOST_QUEUE__RUN_COUNT_MAP = 'funboost_queue__run_count_map'
+    FUNBOOST_QUEUE__RUN_FAIL_COUNT_MAP = 'funboost_queue__run_fail_count_map'
+    FUNBOOST_ALL_QUEUE_NAMES = 'funboost_all_queue_names'
+    FUNBOOST_ALL_IPS = 'funboost_all_ips'
+
+    FUNBOOST_HEARTBEAT_QUEUE__DICT_PREFIX = 'funboost_hearbeat_queue__dict:'
+    FUNBOOST_HEARTBEAT_SERVER__DICT_PREFIX = 'funboost_hearbeat_server__dict:'
+
+    @staticmethod
+    def gen_funboost_hearbeat_queue__dict_key_by_queue_name(queue_name):
+        return f'{RedisKeys.FUNBOOST_HEARTBEAT_QUEUE__DICT_PREFIX}{queue_name}'
+
+    @staticmethod
+    def gen_funboost_hearbeat_server__dict_key_by_ip(ip):
+        return f'{RedisKeys.FUNBOOST_HEARTBEAT_SERVER__DICT_PREFIX}{ip}'

funboost/consumers/base_consumer.py

@@ -194,7 +194,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         self.consumer_params.broker_exclusive_config = broker_exclusive_config_merge
 
         self._stop_flag = None
-        self._pause_flag = None  # 暂停消费标志，从reids读取
+        self._pause_flag = threading.Event()  # 暂停消费标志，从reids读取
         self._last_show_pause_log_time = 0
         # self._redis_key_stop_flag = f'funboost_stop_flag'
         # self._redis_key_pause_flag = RedisKeys.REDIS_KEY_PAUSE_FLAG
@@ -243,12 +243,27 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         # print(self.publisher_params)
         if is_main_process:
             self.logger.info(f'{self.queue_name} consumer 的消费者配置:\n {self.consumer_params.json_str_value()}')
+
         atexit.register(self.join_shedual_task_thread)
 
+        self._save_consumer_params()
+
         if self.consumer_params.is_auto_start_consuming_message:
             _ = self.publisher_of_same_queue
             self.start_consuming_message()
 
+    def _save_consumer_params(self):
+        """
+        保存队列的消费者参数，以便在web界面查看。
+        :return:
+        """
+        if self.consumer_params.is_send_consumer_hearbeat_to_redis:
+            RedisMixin().redis_db_frame.sadd(RedisKeys.FUNBOOST_ALL_QUEUE_NAMES,self.queue_name)
+            RedisMixin().redis_db_frame.hmset(RedisKeys.FUNBOOST_QUEUE__CONSUMER_PARAMS,
+                                    {self.queue_name: self.consumer_params.json_str_value()})
+            RedisMixin().redis_db_frame.sadd(RedisKeys.FUNBOOST_ALL_IPS,nb_log_config_default.computer_ip)
+        
+    
     def _build_logger(self):
         logger_prefix = self.consumer_params.logger_prefix
         if logger_prefix != '':
@@ -456,15 +471,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         return msg
 
     def _submit_task(self, kw):
-        while 1:  # 这一块的代码为支持暂停消费。
-            # print(self._pause_flag)
-            if self._pause_flag == 1:
-                time.sleep(5)
-                if time.time() - self._last_show_pause_log_time > 60:
-                    self.logger.warning(f'已设置 {self.queue_name} 队列中的任务为暂停消费')
-                    self._last_show_pause_log_time = time.time()
-            else:
-                break
+
         kw['body'] = self.convert_msg_before_run(kw['body'])
         self._print_message_get_from_broker(kw['body'])
         if self._judge_is_daylight():
@@ -473,9 +480,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             return
         function_only_params = delete_keys_and_return_new_dict(kw['body'], )
         if self._get_priority_conf(kw, 'do_task_filtering') and self._redis_filter.check_value_exists(
-                function_only_params):  # 对函数的参数进行检查，过滤已经执行过并且成功的任务。
+                function_only_params,self._get_priority_conf(kw, 'filter_str')):  # 对函数的参数进行检查，过滤已经执行过并且成功的任务。
             self.logger.warning(f'redis的 [{self._redis_filter_key_name}] 键 中 过滤任务 {kw["body"]}')
-            self._confirm_consume(kw)
+            self._confirm_consume(kw) # 不运行就必须确认消费，否则会发不能确认消费，导致消息队列中间件认为消息没有被消费。
             return
         publish_time = get_publish_time(kw['body'])
         msg_expire_senconds_priority = self._get_priority_conf(kw, 'msg_expire_senconds')
@@ -529,6 +536,16 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         else:
             self._frequency_control(self.consumer_params.qps, self._msg_schedule_time_intercal)
 
+        while 1:  # 这一块的代码为支持暂停消费。
+            # print(self._pause_flag)
+            if self._pause_flag.is_set():
+                if time.time() - self._last_show_pause_log_time > 60:
+                    self.logger.warning(f'已设置 {self.queue_name} 队列中的任务为暂停消费')
+                    self._last_show_pause_log_time = time.time()
+                time.sleep(5)
+            else:
+                break
+
     def __delete_eta_countdown(self, msg_body: dict):
         self.__dict_pop(msg_body.get('extra', {}), 'eta')
         self.__dict_pop(msg_body.get('extra', {}), 'countdown')
@@ -661,7 +678,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             current_function_result_status.run_status = RunStatus.finish
             self._result_persistence_helper.save_function_result_to_mongo(current_function_result_status)
             if self._get_priority_conf(kw, 'do_task_filtering'):
-                self._redis_filter.add_a_value(function_only_params)  # 函数执行成功后，添加函数的参数排序后的键值对字符串到set中。
+                self._redis_filter.add_a_value(function_only_params,self._get_priority_conf(kw, 'filter_str'))  # 函数执行成功后，添加函数的参数排序后的键值对字符串到set中。
             if current_function_result_status.success is False and current_retry_times == max_retry_times:
                 log_msg = f'函数 {self.consuming_function.__name__} 达到最大重试次数 {self._get_priority_conf(kw, "max_retry_times")} 后,仍然失败， 入参是  {function_only_params} '
                 if self.consumer_params.is_push_to_dlx_queue_when_retry_max_times:
@@ -707,7 +724,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         fct_context = FctContext(function_params=function_only_params,
                                  full_msg=kw['body'],
                                  function_result_status=function_result_status,
-                                 logger=self.logger, )
+                                 logger=self.logger, queue_name=self.queue_name,)
 
         try:
             function_run = self.consuming_function
@@ -715,6 +732,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 fct_context.asyncio_use_thread_concurrent_mode = True
                 function_run = sync_or_async_fun_deco(function_run)
             else:
+                pass
                 fct_context.asynco_use_thread_concurrent_mode = False
             fct.set_fct_context(fct_context)
             function_timeout = self._get_priority_conf(kw, 'function_timeout')
@@ -814,7 +832,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             await simple_run_in_executor(self._result_persistence_helper.save_function_result_to_mongo, current_function_result_status)
             if self._get_priority_conf(kw, 'do_task_filtering'):
                 # self._redis_filter.add_a_value(function_only_params)  # 函数执行成功后，添加函数的参数排序后的键值对字符串到set中。
-                await simple_run_in_executor(self._redis_filter.add_a_value, function_only_params)
+                await simple_run_in_executor(self._redis_filter.add_a_value, function_only_params,self._get_priority_conf(kw, 'filter_str'))
             if current_function_result_status.success is False and current_retry_times == max_retry_times:
                 log_msg = f'函数 {self.consuming_function.__name__} 达到最大重试次数 {self._get_priority_conf(kw, "max_retry_times")} 后,仍然失败， 入参是  {function_only_params} '
                 if self.consumer_params.is_push_to_dlx_queue_when_retry_max_times:
@@ -866,7 +884,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         fct_context = FctContext(function_params=function_only_params,
                                  full_msg=kw['body'],
                                  function_result_status=function_result_status,
-                                 logger=self.logger, )
+                                 logger=self.logger,queue_name=self.queue_name,)
         fct.set_fct_context(fct_context)
         try:
             corotinue_obj = self.consuming_function(**self._convert_real_function_only_params_by_conusuming_function_kind(function_only_params))
@@ -1177,6 +1195,10 @@ class MetricCalculation:
                     msg += f''' 预计还需要 {need_time} 时间 才能执行完成 队列 {self.consumer.queue_name} 中的 {self.msg_num_in_broker} 个剩余任务'''
                     self.consumer.logger.info(msg)
                 self.last_show_remaining_execution_time = time.time()
+            if self.consumer.consumer_params.is_send_consumer_hearbeat_to_redis is True:
+                RedisMixin().redis_db_frame.hincrby(RedisKeys.FUNBOOST_QUEUE__RUN_COUNT_MAP,self.consumer.queue_name,self.execute_task_times_every_unit_time_temp)
+                RedisMixin().redis_db_frame.hincrby(RedisKeys.FUNBOOST_QUEUE__RUN_FAIL_COUNT_MAP,self.consumer.queue_name,self.execute_task_times_every_unit_time_temp_fail)
+
             self.current_time_for_execute_task_times_every_unit_time = time.time()
             self.consuming_function_cost_time_total_every_unit_time_tmp = 0
             self.execute_task_times_every_unit_time_temp = 0
@@ -1233,22 +1255,13 @@ class DistributedConsumerStatistics(RedisMixin, FunboostFileLoggerMixin):
         self.active_consumer_num = 1
         self._last_show_consumer_num_timestamp = 0
 
-        self._queue__consumer_identification_map_key_name = f'funboost_hearbeat_queue__dict:{self._queue_name}'
-        self._server__consumer_identification_map_key_name = f'funboost_hearbeat_server__dict:{nb_log_config_default.computer_ip}'
+        self._queue__consumer_identification_map_key_name = RedisKeys.gen_funboost_hearbeat_queue__dict_key_by_queue_name(self._queue_name)
+        self._server__consumer_identification_map_key_name = RedisKeys.gen_funboost_hearbeat_server__dict_key_by_ip(nb_log_config_default.computer_ip)
 
     def run(self):
-        self._send_consumer_params()
         self.send_heartbeat()
         self._consumer.keep_circulating(self.SEND_HEARTBEAT_INTERVAL, block=False, daemon=False)(self.send_heartbeat)()
 
-    def _send_consumer_params(self):
-        """
-        保存队列的消费者参数，以便在web界面查看。
-        :return:
-        """
-        self.redis_db_frame.hmset('funboost_queue__consumer_parmas',{self._consumer.queue_name: self._consumer.consumer_params.json_str_value()})
-
-
     def _send_heartbeat_with_dict_value(self, redis_key, ):
         # 发送当前消费者进程心跳的，值是字典，按一个机器或者一个队列运行了哪些进程。
 
@@ -1315,8 +1328,8 @@ class DistributedConsumerStatistics(RedisMixin, FunboostFileLoggerMixin):
 
         pause_flag = self.redis_db_frame.hget(RedisKeys.REDIS_KEY_PAUSE_FLAG,self._consumer.queue_name)
         if pause_flag is not None and int(pause_flag) == 1:
-            self._consumer._pause_flag = 1
+            self._consumer._pause_flag.set()
         else:
-            self._consumer._pause_flag = 0
+            self._consumer._pause_flag.clear()
   
       

funboost/consumers/rabbitmq_amqpstorm_consumer.py

@@ -30,6 +30,7 @@ class RabbitmqConsumerAmqpStorm(AbstractConsumer):
         rp.channel_wrapper_by_ampqstormbaic.qos(self.consumer_params.concurrent_num)
         rp.channel_wrapper_by_ampqstormbaic.consume(callback=callback, queue=self.queue_name, no_ack=self.consumer_params.broker_exclusive_config['no_ack'],
                                                     )
+        self._rp=rp
         rp.channel.start_consuming(auto_decode=True)
 
     def _confirm_consume(self, kw):
@@ -44,3 +45,7 @@ class RabbitmqConsumerAmqpStorm(AbstractConsumer):
         # amqpstorm.Message.delivery_tag
         # print(kw['amqpstorm_message'].delivery_tag)
         kw['amqpstorm_message'].nack(requeue=True)
+        # kw['amqpstorm_message'].reject(requeue=True)
+        # kw['amqpstorm_message'].ack()
+        # self.publisher_of_same_queue.publish(kw['body'])
+

funboost/consumers/redis_filter.py

@@ -32,23 +32,36 @@ class RedisFilter(RedisMixin, FunboostFileLoggerMixin):
         self._redis_key_name = redis_key_name
         self._redis_filter_task_expire_seconds = redis_filter_task_expire_seconds
 
+        # @staticmethod
+        # def _get_ordered_str(value):
+        #     """对json的键值对在redis中进行过滤，需要先把键值对排序，否则过滤会不准确如 {"a":1,"b":2} 和 {"b":2,"a":1}"""
+        #     value = Serialization.to_dict(value)
+        #     ordered_dict = OrderedDict()
+        #     for k in sorted(value):
+        #         ordered_dict[k] = value[k]
+        #     return json.dumps(ordered_dict)
+    
     @staticmethod
-    def _get_ordered_str(value):
+    def generate_filter_str(value: typing.Union[str, dict],  filter_str: typing.Optional[str] = None):
         """对json的键值对在redis中进行过滤，需要先把键值对排序，否则过滤会不准确如 {"a":1,"b":2} 和 {"b":2,"a":1}"""
+        if filter_str: # 如果用户单独指定了过滤字符串，就使用使用户指定的过滤字符串，否则使用排序后的键值对字符串
+            return filter_str
         value = Serialization.to_dict(value)
         ordered_dict = OrderedDict()
         for k in sorted(value):
             ordered_dict[k] = value[k]
+        # print(ordered_dict,filter_str)
         return json.dumps(ordered_dict)
 
-    def add_a_value(self, value: typing.Union[str, dict]):
-        self.redis_db_filter_and_rpc_result.sadd(self._redis_key_name, self._get_ordered_str(value))
 
-    def manual_delete_a_value(self, value: typing.Union[str, dict]):
-        self.redis_db_filter_and_rpc_result.srem(self._redis_key_name, self._get_ordered_str(value))
+    def add_a_value(self, value: typing.Union[str, dict], filter_str: typing.Optional[str] = None):
+        self.redis_db_filter_and_rpc_result.sadd(self._redis_key_name, self.generate_filter_str(value, filter_str))
 
-    def check_value_exists(self, value):
-        return self.redis_db_filter_and_rpc_result.sismember(self._redis_key_name, self._get_ordered_str(value))
+    def manual_delete_a_value(self, value: typing.Union[str, dict], filter_str: typing.Optional[str] = None):
+        self.redis_db_filter_and_rpc_result.srem(self._redis_key_name, self.generate_filter_str(value, filter_str))
+
+    def check_value_exists(self, value, filter_str: typing.Optional[str] = None):
+        return self.redis_db_filter_and_rpc_result.sismember(self._redis_key_name, self.generate_filter_str(value, filter_str))
 
     def delete_expire_filter_task_cycle(self):
         pass
@@ -61,15 +74,17 @@ class RedisImpermanencyFilter(RedisFilter):
     如果是30分钟内发布过这个任务，则不执行1 + 2，现在把这个逻辑集成到框架，一般用于接口缓存。
     """
 
-    def add_a_value(self, value: typing.Union[str, dict]):
-        self.redis_db_filter_and_rpc_result.zadd(self._redis_key_name, {self._get_ordered_str(value):time.time()})
+    def add_a_value(self, value: typing.Union[str, dict], filter_str: typing.Optional[str] = None):
+        self.redis_db_filter_and_rpc_result.zadd(self._redis_key_name, {self.generate_filter_str(value, filter_str):time.time()})
 
-    def manual_delete_a_value(self, value: typing.Union[str, dict]):
-        self.redis_db_filter_and_rpc_result.zrem(self._redis_key_name, self._get_ordered_str(value))
+    def manual_delete_a_value(self, value: typing.Union[str, dict], filter_str: typing.Optional[str] = None):
+        self.redis_db_filter_and_rpc_result.zrem(self._redis_key_name, self.generate_filter_str(value, filter_str))
 
-    def check_value_exists(self, value):
-        # print(self.redis_db_filter_and_rpc_result.zrank(self._redis_key_name, self._get_ordered_str(value)))
-        return False if self.redis_db_filter_and_rpc_result.zrank(self._redis_key_name, self._get_ordered_str(value)) is None else True
+    def check_value_exists(self, value, filter_str: typing.Optional[str] = None):
+        # print(self.redis_db_filter_and_rpc_result.zrank(self._redis_key_name, self.generate_filter_str(value, filter_str)))
+        is_exists = False if self.redis_db_filter_and_rpc_result.zscore(self._redis_key_name, self.generate_filter_str(value, filter_str)) is None else True
+        # print(is_exists,value,filter_str,self.generate_filter_str(value, filter_str))
+        return is_exists   
 
     @decorators.keep_circulating(60, block=False)
     def delete_expire_filter_task_cycle000(self):
@@ -111,16 +126,16 @@ class RedisImpermanencyFilterUsingRedisKey(RedisFilter):
         """
         return f'{self._redis_key_name}:{value.replace(":", "：")}'  # 任务是json，带有：会形成很多树，换成中文冒号。
 
-    def add_a_value(self, value: typing.Union[str, dict]):
-        redis_key = self.__add_dir_prefix(self._get_ordered_str(value))
+    def add_a_value(self, value: typing.Union[str, dict], filter_str: typing.Optional[str] = None):
+        redis_key = self.__add_dir_prefix(self.generate_filter_str(value, filter_str))
         self.redis_db_filter_and_rpc_result.set(redis_key, 1)
         self.redis_db_filter_and_rpc_result.expire(redis_key, self._redis_filter_task_expire_seconds)
 
-    def manual_delete_a_value(self, value: typing.Union[str, dict]):
-        self.redis_db_filter_and_rpc_result.delete(self.__add_dir_prefix(self._get_ordered_str(value)))
+    def manual_delete_a_value(self, value: typing.Union[str, dict], filter_str: typing.Optional[str] = None):
+        self.redis_db_filter_and_rpc_result.delete(self.__add_dir_prefix(self.generate_filter_str(value, filter_str)))
 
-    def check_value_exists(self, value):
-        return True if self.redis_db_filter_and_rpc_result.exists(self.__add_dir_prefix(self._get_ordered_str(value))) else True
+    def check_value_exists(self, value, filter_str: typing.Optional[str] = None):
+        return True if self.redis_db_filter_and_rpc_result.exists(self.__add_dir_prefix(self.generate_filter_str(value, filter_str))) else True
 
     def delete_expire_filter_task_cycle(self):
         """
@@ -131,16 +146,17 @@ class RedisImpermanencyFilterUsingRedisKey(RedisFilter):
 
 
 if __name__ == '__main__':
-    # filter = RedisFilter('filter_set:abcdefgh', 120)
-    params_filter = RedisImpermanencyFilter('filter_zset:abcdef', 120)
+    # params_filter = RedisFilter('filter_set:abcdefgh2', 120)
+    params_filter = RedisImpermanencyFilter('filter_zset:abcdef2', 120)
     # params_filter = RedisImpermanencyFilterUsingRedisKey('filter_dir', 300)
     for i in range(10):
-        params_filter.add_a_value({'x': i, 'y': i * 2})
-
-    params_filter.manual_delete_a_value({'a': 1, 'b': 2})
-    print(params_filter.check_value_exists({'a': 1, 'b': 2}))
-    params_filter.delete_expire_filter_task_cycle()
-    params_filter.add_a_value({'a': 1, 'b': 5})
-    print(params_filter.check_value_exists({'a': 1, 'b': 2}))
-    time.sleep(130)
-    print(params_filter.check_value_exists({'a': 1, 'b': 2}))
+        # params_filter.add_a_value({'x': i, 'y': i * 2},str(i))
+        params_filter.add_a_value({'x': i, 'y': i * 2},None)
+
+    # params_filter.manual_delete_a_value({'a': 1, 'b': 2})
+    print(params_filter.check_value_exists({'x': 1, 'y': 2}))
+    # params_filter.delete_expire_filter_task_cycle()
+    # params_filter.add_a_value({'a': 1, 'b': 5})
+    # print(params_filter.check_value_exists({'a': 1, 'b': 2}))
+    # time.sleep(130)
+    # print(params_filter.check_value_exists({'a': 1, 'b': 2}))

funboost/core/active_cousumer_info_getter.py

@@ -52,7 +52,7 @@ class ActiveCousumerProcessInfoGetter(RedisMixin, FunboostFileLoggerMixin):
                 "start_timestamp": 1640604084.0780013
             }, ...............]
         """
-        redis_key = f'funboost_hearbeat_queue__dict:{queue_name}'
+        redis_key = RedisKeys.gen_funboost_hearbeat_queue__dict_key_by_queue_name(queue_name)
         return self._get_all_hearbeat_info_by_redis_key_name(redis_key)
 
     def get_all_hearbeat_info_by_ip(self, ip=None) -> typing.List[typing.Dict]:
@@ -61,15 +61,38 @@ class ActiveCousumerProcessInfoGetter(RedisMixin, FunboostFileLoggerMixin):
         返回结果的格式和上面的 get_all_hearbeat_dict_by_queue_name 方法相同。
         """
         ip = ip or nb_log_config_default.computer_ip
-        redis_key = f'funboost_hearbeat_server__dict:{ip}'
+        redis_key = RedisKeys.gen_funboost_hearbeat_server__dict_key_by_ip(ip)
         return self._get_all_hearbeat_info_by_redis_key_name(redis_key)
 
-    def _get_all_hearbeat_info_partition_by_redis_key_prefix(self, redis_key_prefix):
-        keys = self.redis_db_frame.scan(0, f'{redis_key_prefix}*', count=10000)[1]
+    # def _get_all_hearbeat_info_partition_by_redis_key_prefix(self, redis_key_prefix):
+    #     keys = self.redis_db_frame.scan(0, f'{redis_key_prefix}*', count=10000)[1]
+    #     infos_map = {}
+    #     for key in keys:
+    #         infos = self.redis_db_frame.smembers(key)
+    #         dict_key = key.replace(redis_key_prefix, '')
+    #         infos_map[dict_key] = []
+    #         for info_str in infos:
+    #             info_dict = json.loads(info_str)
+    #             if self.timestamp() - info_dict['hearbeat_timestamp'] < 15:
+    #                 infos_map[dict_key].append(info_dict)
+    #                 if self.timestamp() - info_dict['current_time_for_execute_task_times_every_unit_time'] > 30:
+    #                     info_dict['last_x_s_execute_count'] = 0
+    #                     info_dict['last_x_s_execute_count_fail'] = 0
+    #     return infos_map
+
+    def get_all_queue_names(self):
+        return self.redis_db_frame.smembers(RedisKeys.FUNBOOST_ALL_QUEUE_NAMES)
+    
+    def get_all_ips(self):
+        return self.redis_db_frame.smembers(RedisKeys.FUNBOOST_ALL_IPS)
+    
+    def _get_all_hearbeat_info_partition_by_redis_keys(self, keys):
+        
+        # keys = [f'{redis_key_prefix}{queue_name}' for queue_name in queue_names]
         infos_map = {}
         for key in keys:
             infos = self.redis_db_frame.smembers(key)
-            dict_key = key.replace(redis_key_prefix, '')
+            dict_key = key.replace(RedisKeys.FUNBOOST_HEARTBEAT_QUEUE__DICT_PREFIX, '').replace(RedisKeys.FUNBOOST_HEARTBEAT_SERVER__DICT_PREFIX, '')
             infos_map[dict_key] = []
             for info_str in infos:
                 info_dict = json.loads(info_str)
@@ -82,13 +105,15 @@ class ActiveCousumerProcessInfoGetter(RedisMixin, FunboostFileLoggerMixin):
 
     def get_all_hearbeat_info_partition_by_queue_name(self) -> typing.Dict[typing.AnyStr, typing.List[typing.Dict]]:
         """获取所有队列对应的活跃消费者进程信息，按队列名划分,不需要传入队列名，自动扫描redis键。请不要在 funboost_config.py 的redis 指定的db中放太多其他业务的缓存键值对"""
-        infos_map = self._get_all_hearbeat_info_partition_by_redis_key_prefix('funboost_hearbeat_queue__dict:')
+        queue_names = self.get_all_queue_names()
+        infos_map = self._get_all_hearbeat_info_partition_by_redis_keys([RedisKeys.gen_funboost_hearbeat_queue__dict_key_by_queue_name(queue_name) for queue_name in queue_names])
         self.logger.info(f'获取所有队列对应的活跃消费者进程信息，按队列名划分，结果是 {json.dumps(infos_map, indent=4)}')
         return infos_map
 
     def get_all_hearbeat_info_partition_by_ip(self) -> typing.Dict[typing.AnyStr, typing.List[typing.Dict]]:
         """获取所有机器ip对应的活跃消费者进程信息，按机器ip划分,不需要传入机器ip，自动扫描redis键。请不要在 funboost_config.py 的redis 指定的db中放太多其他业务的缓存键值对 """
-        infos_map = self._get_all_hearbeat_info_partition_by_redis_key_prefix('funboost_hearbeat_server__dict:')
+        ips = self.get_all_ips()
+        infos_map = self._get_all_hearbeat_info_partition_by_redis_keys([RedisKeys.gen_funboost_hearbeat_server__dict_key_by_ip(ip) for ip in ips])
         self.logger.info(f'获取所有机器ip对应的活跃消费者进程信息，按机器ip划分，结果是 {json.dumps(infos_map, indent=4)}')
         return infos_map
 
@@ -123,8 +148,19 @@ class QueueConusmerParamsGetter(RedisMixin, FunboostFileLoggerMixin):
                 s+=c[filed]
         return s
     
+    def get_queues_history_run_count(self,):
+        return self.redis_db_frame.hgetall(RedisKeys.FUNBOOST_QUEUE__RUN_COUNT_MAP)
+    
+    def get_queues_history_run_fail_count(self,):
+        return self.redis_db_frame.hgetall(RedisKeys.FUNBOOST_QUEUE__RUN_FAIL_COUNT_MAP)
+    
     def get_queue_params_and_active_consumers(self):
         queue__active_consumers_map = ActiveCousumerProcessInfoGetter().get_all_hearbeat_info_partition_by_queue_name()
+
+        queue_name_list = list(queue__active_consumers_map.keys())
+        queue__history_run_count_map = self.get_queues_history_run_count()
+        queue__history_run_fail_count_map = self.get_queues_history_run_fail_count()
+
         queue__consumer_params_map  = self.get_queue_params()
         queue__pause_map = self.get_pause_flag()
         queue__msg_count_dict = self.get_msg_num(ignore_report_ts=True)
@@ -148,6 +184,10 @@ class QueueConusmerParamsGetter(RedisMixin, FunboostFileLoggerMixin):
                 'active_consumers':active_consumers,
                 'pause_flag':queue__pause_map.get(queue,-1),
                 'msg_num_in_broker':queue__msg_count_dict.get(queue,None),
+                
+                'history_run_count':queue__history_run_count_map.get(queue,None),
+                'history_run_fail_count':queue__history_run_fail_count_map.get(queue,None),
+
                 'all_consumers_last_x_s_execute_count':all_consumers_last_x_s_execute_count,
                 'all_consumers_last_x_s_execute_count_fail':all_consumers_last_x_s_execute_count_fail,
                 'all_consumers_last_x_s_avarage_function_spend_time':all_consumers_last_x_s_avarage_function_spend_time,

funboost/core/booster.py

@@ -100,6 +100,7 @@ class Booster:
             consuming_function = args[0]
             self.boost_params.consuming_function = consuming_function
             self.boost_params.consuming_function_raw = consuming_function
+            self.boost_params.consuming_function_name = consuming_function.__name__
             # print(consuming_function)
             # print(ClsHelper.get_method_kind(consuming_function))
             # print(inspect.getsourcelines(consuming_function))

funboost/core/current_task.py

@@ -1,4 +1,5 @@
 import abc
+import typing
 import contextvars
 from dataclasses import dataclass
 import logging
@@ -7,6 +8,8 @@ import asyncio
 
 from funboost.core.function_result_status_saver import FunctionResultStatus
 
+
+
 """ 用法例子 
     '''
     fct = funboost_current_task()
@@ -60,7 +63,9 @@ class FctContext:
     full_msg: dict
     function_result_status: FunctionResultStatus
     logger: logging.Logger
+    queue_name: str
     asyncio_use_thread_concurrent_mode: bool = False
+    
 
 # class FctContext:
 #     """
@@ -107,6 +112,11 @@ class _BaseCurrentTask(metaclass=abc.ABCMeta):
     @property
     def logger(self) -> logging.Logger:
         return self.get_fct_context().logger
+    
+    @property
+    def queue_name(self) -> str:
+        return self.get_fct_context().queue_name
+    
 
     def __str__(self):
         return f'<{self.__class__.__name__} [{self.function_result_status.get_status_dict()}]>'
@@ -184,6 +194,13 @@ class _FctProxy:
     @property
     def logger(self) -> logging.Logger:
         return self.fct_context.logger
+    
+    @property
+    def queue_name(self) -> str:
+        return self.fct_context.queue_name
+    
+
+
 
     def __str__(self):
         return f'<{self.__class__.__name__} [{self.function_result_status.get_status_dict()}]>'