funboost 48.7__py3-none-any.whl → 48.9__py3-none-any.whl

funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "48.7"
+__version__ = "48.9"
 
 from funboost.set_frame_config import show_frame_config
 

funboost/__main__.py

@@ -18,6 +18,7 @@ def _check_pass_params():
 
 def main():
     _check_pass_params()
+    
 
     fire.Fire(BoosterFire, )
 
@@ -31,4 +32,5 @@ python -m funboost  --project_root_path=/codes/funboost   --booster_dirs_str=tes
 python -m funboost  --project_root_path=/codes/funboost   --booster_dirs_str=test_frame/test_funboost_cli/test_find_boosters --max_depth=2  push test_find_queue1 --x=1 --y=2
 
 
+python -m funboost  --project_root_path=/codes/funboost  start_web
 '''

funboost/concurrent_pool/custom_threadpool_executor.py

@@ -98,7 +98,7 @@ class ThreadPoolExecutorShrinkAble(Executor, FunboostFileLoggerMixin, LoggerLeve
     # MIN_WORKERS = 5   # 最小值可以设置为0，代表线程池无论多久没有任务最少要保持多少个线程待命。
     # KEEP_ALIVE_TIME = 60  # 这个参数表名，当前线程从queue.get(block=True, timeout=KEEP_ALIVE_TIME)多久没任务，就线程结束。
 
-    MIN_WORKERS = 5
+    MIN_WORKERS = 1
     KEEP_ALIVE_TIME = 60
 
     def __init__(self, max_workers: int = None, thread_name_prefix='',work_queue_maxsize=10):

funboost/constant.py

@@ -108,8 +108,23 @@ class ConstStrForClassMethod:
     OBJ_INIT_PARAMS = 'obj_init_params'
     CLS_MODULE = 'cls_module'
     CLS_FILE = 'cls_file'
-
 class RedisKeys:
     REDIS_KEY_PAUSE_FLAG  = 'funboost_pause_flag' 
     REDIS_KEY_STOP_FLAG = 'funboost_stop_flag'
-    QUEUE__MSG_COUNT_MAP = 'funboost_queue__msg_count_map'
+    QUEUE__MSG_COUNT_MAP = 'funboost_queue__msg_count_map'
+    FUNBOOST_QUEUE__CONSUMER_PARAMS= 'funboost_queue__consumer_parmas'
+    FUNBOOST_QUEUE__RUN_COUNT_MAP = 'funboost_queue__run_count_map'
+    FUNBOOST_QUEUE__RUN_FAIL_COUNT_MAP = 'funboost_queue__run_fail_count_map'
+    FUNBOOST_ALL_QUEUE_NAMES = 'funboost_all_queue_names'
+    FUNBOOST_ALL_IPS = 'funboost_all_ips'
+
+    FUNBOOST_HEARTBEAT_QUEUE__DICT_PREFIX = 'funboost_hearbeat_queue__dict:'
+    FUNBOOST_HEARTBEAT_SERVER__DICT_PREFIX = 'funboost_hearbeat_server__dict:'
+
+    @staticmethod
+    def gen_funboost_hearbeat_queue__dict_key_by_queue_name(queue_name):
+        return f'{RedisKeys.FUNBOOST_HEARTBEAT_QUEUE__DICT_PREFIX}{queue_name}'
+
+    @staticmethod
+    def gen_funboost_hearbeat_server__dict_key_by_ip(ip):
+        return f'{RedisKeys.FUNBOOST_HEARTBEAT_SERVER__DICT_PREFIX}{ip}'

funboost/consumers/base_consumer.py

@@ -165,6 +165,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         self._redis_filter = filter_class(self._redis_filter_key_name, consumer_params.task_filtering_expire_seconds)
 
         self._lock_for_count_execute_task_times_every_unit_time = Lock()
+        self._async_lock_for_count_execute_task_times_every_unit_time = asyncio.Lock()
         # self._unit_time_for_count = 10  # 每隔多少秒计数，显示单位时间内执行多少次，暂时固定为10秒。
         # self._execute_task_times_every_unit_time = 0  # 每单位时间执行了多少次任务。
         # self._execute_task_times_every_unit_time_fail =0  # 每单位时间执行了多少次任务失败。
@@ -193,7 +194,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         self.consumer_params.broker_exclusive_config = broker_exclusive_config_merge
 
         self._stop_flag = None
-        self._pause_flag = None  # 暂停消费标志，从reids读取
+        self._pause_flag = threading.Event()  # 暂停消费标志，从reids读取
         self._last_show_pause_log_time = 0
         # self._redis_key_stop_flag = f'funboost_stop_flag'
         # self._redis_key_pause_flag = RedisKeys.REDIS_KEY_PAUSE_FLAG
@@ -242,12 +243,27 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         # print(self.publisher_params)
         if is_main_process:
             self.logger.info(f'{self.queue_name} consumer 的消费者配置:\n {self.consumer_params.json_str_value()}')
+
         atexit.register(self.join_shedual_task_thread)
 
+        self._save_consumer_params()
+
         if self.consumer_params.is_auto_start_consuming_message:
             _ = self.publisher_of_same_queue
             self.start_consuming_message()
 
+    def _save_consumer_params(self):
+        """
+        保存队列的消费者参数，以便在web界面查看。
+        :return:
+        """
+        if self.consumer_params.is_send_consumer_hearbeat_to_redis:
+            RedisMixin().redis_db_frame.sadd(RedisKeys.FUNBOOST_ALL_QUEUE_NAMES,self.queue_name)
+            RedisMixin().redis_db_frame.hmset(RedisKeys.FUNBOOST_QUEUE__CONSUMER_PARAMS,
+                                    {self.queue_name: self.consumer_params.json_str_value()})
+            RedisMixin().redis_db_frame.sadd(RedisKeys.FUNBOOST_ALL_IPS,nb_log_config_default.computer_ip)
+        
+    
     def _build_logger(self):
         logger_prefix = self.consumer_params.logger_prefix
         if logger_prefix != '':
@@ -455,15 +471,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         return msg
 
     def _submit_task(self, kw):
-        while 1:  # 这一块的代码为支持暂停消费。
-            # print(self._pause_flag)
-            if self._pause_flag == 1:
-                time.sleep(5)
-                if time.time() - self._last_show_pause_log_time > 60:
-                    self.logger.warning(f'已设置 {self.queue_name} 队列中的任务为暂停消费')
-                    self._last_show_pause_log_time = time.time()
-            else:
-                break
+
         kw['body'] = self.convert_msg_before_run(kw['body'])
         self._print_message_get_from_broker(kw['body'])
         if self._judge_is_daylight():
@@ -528,6 +536,16 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         else:
             self._frequency_control(self.consumer_params.qps, self._msg_schedule_time_intercal)
 
+        while 1:  # 这一块的代码为支持暂停消费。
+            # print(self._pause_flag)
+            if self._pause_flag.is_set():
+                if time.time() - self._last_show_pause_log_time > 60:
+                    self.logger.warning(f'已设置 {self.queue_name} 队列中的任务为暂停消费')
+                    self._last_show_pause_log_time = time.time()
+                time.sleep(5)
+            else:
+                break
+
     def __delete_eta_countdown(self, msg_body: dict):
         self.__dict_pop(msg_body.get('extra', {}), 'eta')
         self.__dict_pop(msg_body.get('extra', {}), 'countdown')
@@ -706,7 +724,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         fct_context = FctContext(function_params=function_only_params,
                                  full_msg=kw['body'],
                                  function_result_status=function_result_status,
-                                 logger=self.logger, )
+                                 logger=self.logger, queue_name=self.queue_name,)
 
         try:
             function_run = self.consuming_function
@@ -714,6 +732,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 fct_context.asyncio_use_thread_concurrent_mode = True
                 function_run = sync_or_async_fun_deco(function_run)
             else:
+                pass
                 fct_context.asynco_use_thread_concurrent_mode = False
             fct.set_fct_context(fct_context)
             function_timeout = self._get_priority_conf(kw, 'function_timeout')
@@ -836,8 +855,8 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 if (current_function_result_status.success is False and current_retry_times == max_retry_times) or current_function_result_status.success is True:
                     await simple_run_in_executor(push_result)
 
-            # 异步执行不存在线程并发，不需要加锁。
-            self.metric_calculation.cal(t_start_run_fun, current_function_result_status)
+            async with self._async_lock_for_count_execute_task_times_every_unit_time:
+                self.metric_calculation.cal(t_start_run_fun, current_function_result_status)
 
             self.user_custom_record_process_info_func(current_function_result_status)  # 两种方式都可以自定义,记录结果.建议使用文档4.21.b的方式继承来重写
             await self.aio_user_custom_record_process_info_func(current_function_result_status)
@@ -865,7 +884,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         fct_context = FctContext(function_params=function_only_params,
                                  full_msg=kw['body'],
                                  function_result_status=function_result_status,
-                                 logger=self.logger, )
+                                 logger=self.logger,queue_name=self.queue_name,)
         fct.set_fct_context(fct_context)
         try:
             corotinue_obj = self.consuming_function(**self._convert_real_function_only_params_by_conusuming_function_kind(function_only_params))
@@ -1175,7 +1194,11 @@ class MetricCalculation:
                                                                         self.consumer._distributed_consumer_statistics.active_consumer_num)
                     msg += f''' 预计还需要 {need_time} 时间 才能执行完成 队列 {self.consumer.queue_name} 中的 {self.msg_num_in_broker} 个剩余任务'''
                     self.consumer.logger.info(msg)
-                    self.last_show_remaining_execution_time = time.time()
+                self.last_show_remaining_execution_time = time.time()
+            if self.consumer.consumer_params.is_send_consumer_hearbeat_to_redis is True:
+                RedisMixin().redis_db_frame.hincrby(RedisKeys.FUNBOOST_QUEUE__RUN_COUNT_MAP,self.consumer.queue_name,self.execute_task_times_every_unit_time_temp)
+                RedisMixin().redis_db_frame.hincrby(RedisKeys.FUNBOOST_QUEUE__RUN_FAIL_COUNT_MAP,self.consumer.queue_name,self.execute_task_times_every_unit_time_temp_fail)
+
             self.current_time_for_execute_task_times_every_unit_time = time.time()
             self.consuming_function_cost_time_total_every_unit_time_tmp = 0
             self.execute_task_times_every_unit_time_temp = 0
@@ -1232,22 +1255,13 @@ class DistributedConsumerStatistics(RedisMixin, FunboostFileLoggerMixin):
         self.active_consumer_num = 1
         self._last_show_consumer_num_timestamp = 0
 
-        self._queue__consumer_identification_map_key_name = f'funboost_hearbeat_queue__dict:{self._queue_name}'
-        self._server__consumer_identification_map_key_name = f'funboost_hearbeat_server__dict:{nb_log_config_default.computer_ip}'
+        self._queue__consumer_identification_map_key_name = RedisKeys.gen_funboost_hearbeat_queue__dict_key_by_queue_name(self._queue_name)
+        self._server__consumer_identification_map_key_name = RedisKeys.gen_funboost_hearbeat_server__dict_key_by_ip(nb_log_config_default.computer_ip)
 
     def run(self):
-        self._send_consumer_params()
         self.send_heartbeat()
         self._consumer.keep_circulating(self.SEND_HEARTBEAT_INTERVAL, block=False, daemon=False)(self.send_heartbeat)()
 
-    def _send_consumer_params(self):
-        """
-        保存队列的消费者参数，以便在web界面查看。
-        :return:
-        """
-        self.redis_db_frame.hmset('funboost_queue__consumer_parmas',{self._consumer.queue_name: self._consumer.consumer_params.json_str_value()})
-
-
     def _send_heartbeat_with_dict_value(self, redis_key, ):
         # 发送当前消费者进程心跳的，值是字典，按一个机器或者一个队列运行了哪些进程。
 
@@ -1314,8 +1328,8 @@ class DistributedConsumerStatistics(RedisMixin, FunboostFileLoggerMixin):
 
         pause_flag = self.redis_db_frame.hget(RedisKeys.REDIS_KEY_PAUSE_FLAG,self._consumer.queue_name)
         if pause_flag is not None and int(pause_flag) == 1:
-            self._consumer._pause_flag = 1
+            self._consumer._pause_flag.set()
         else:
-            self._consumer._pause_flag = 0
+            self._consumer._pause_flag.clear()
   
       

funboost/consumers/rabbitmq_amqpstorm_consumer.py

@@ -30,6 +30,7 @@ class RabbitmqConsumerAmqpStorm(AbstractConsumer):
         rp.channel_wrapper_by_ampqstormbaic.qos(self.consumer_params.concurrent_num)
         rp.channel_wrapper_by_ampqstormbaic.consume(callback=callback, queue=self.queue_name, no_ack=self.consumer_params.broker_exclusive_config['no_ack'],
                                                     )
+        self._rp=rp
         rp.channel.start_consuming(auto_decode=True)
 
     def _confirm_consume(self, kw):
@@ -42,5 +43,9 @@ class RabbitmqConsumerAmqpStorm(AbstractConsumer):
 
     def _requeue(self, kw):
         # amqpstorm.Message.delivery_tag
-        print(kw['amqpstorm_message'].delivery_tag)
+        # print(kw['amqpstorm_message'].delivery_tag)
         kw['amqpstorm_message'].nack(requeue=True)
+        # kw['amqpstorm_message'].reject(requeue=True)
+        # kw['amqpstorm_message'].ack()
+        # self.publisher_of_same_queue.publish(kw['body'])
+

funboost/core/active_cousumer_info_getter.py

@@ -52,7 +52,7 @@ class ActiveCousumerProcessInfoGetter(RedisMixin, FunboostFileLoggerMixin):
                 "start_timestamp": 1640604084.0780013
             }, ...............]
         """
-        redis_key = f'funboost_hearbeat_queue__dict:{queue_name}'
+        redis_key = RedisKeys.gen_funboost_hearbeat_queue__dict_key_by_queue_name(queue_name)
         return self._get_all_hearbeat_info_by_redis_key_name(redis_key)
 
     def get_all_hearbeat_info_by_ip(self, ip=None) -> typing.List[typing.Dict]:
@@ -61,15 +61,38 @@ class ActiveCousumerProcessInfoGetter(RedisMixin, FunboostFileLoggerMixin):
         返回结果的格式和上面的 get_all_hearbeat_dict_by_queue_name 方法相同。
         """
         ip = ip or nb_log_config_default.computer_ip
-        redis_key = f'funboost_hearbeat_server__dict:{ip}'
+        redis_key = RedisKeys.gen_funboost_hearbeat_server__dict_key_by_ip(ip)
         return self._get_all_hearbeat_info_by_redis_key_name(redis_key)
 
-    def _get_all_hearbeat_info_partition_by_redis_key_prefix(self, redis_key_prefix):
-        keys = self.redis_db_frame.scan(0, f'{redis_key_prefix}*', count=10000)[1]
+    # def _get_all_hearbeat_info_partition_by_redis_key_prefix(self, redis_key_prefix):
+    #     keys = self.redis_db_frame.scan(0, f'{redis_key_prefix}*', count=10000)[1]
+    #     infos_map = {}
+    #     for key in keys:
+    #         infos = self.redis_db_frame.smembers(key)
+    #         dict_key = key.replace(redis_key_prefix, '')
+    #         infos_map[dict_key] = []
+    #         for info_str in infos:
+    #             info_dict = json.loads(info_str)
+    #             if self.timestamp() - info_dict['hearbeat_timestamp'] < 15:
+    #                 infos_map[dict_key].append(info_dict)
+    #                 if self.timestamp() - info_dict['current_time_for_execute_task_times_every_unit_time'] > 30:
+    #                     info_dict['last_x_s_execute_count'] = 0
+    #                     info_dict['last_x_s_execute_count_fail'] = 0
+    #     return infos_map
+
+    def get_all_queue_names(self):
+        return self.redis_db_frame.smembers(RedisKeys.FUNBOOST_ALL_QUEUE_NAMES)
+    
+    def get_all_ips(self):
+        return self.redis_db_frame.smembers(RedisKeys.FUNBOOST_ALL_IPS)
+    
+    def _get_all_hearbeat_info_partition_by_redis_keys(self, keys):
+        
+        # keys = [f'{redis_key_prefix}{queue_name}' for queue_name in queue_names]
         infos_map = {}
         for key in keys:
             infos = self.redis_db_frame.smembers(key)
-            dict_key = key.replace(redis_key_prefix, '')
+            dict_key = key.replace(RedisKeys.FUNBOOST_HEARTBEAT_QUEUE__DICT_PREFIX, '').replace(RedisKeys.FUNBOOST_HEARTBEAT_SERVER__DICT_PREFIX, '')
             infos_map[dict_key] = []
             for info_str in infos:
                 info_dict = json.loads(info_str)
@@ -82,13 +105,15 @@ class ActiveCousumerProcessInfoGetter(RedisMixin, FunboostFileLoggerMixin):
 
     def get_all_hearbeat_info_partition_by_queue_name(self) -> typing.Dict[typing.AnyStr, typing.List[typing.Dict]]:
         """获取所有队列对应的活跃消费者进程信息，按队列名划分,不需要传入队列名，自动扫描redis键。请不要在 funboost_config.py 的redis 指定的db中放太多其他业务的缓存键值对"""
-        infos_map = self._get_all_hearbeat_info_partition_by_redis_key_prefix('funboost_hearbeat_queue__dict:')
+        queue_names = self.get_all_queue_names()
+        infos_map = self._get_all_hearbeat_info_partition_by_redis_keys([RedisKeys.gen_funboost_hearbeat_queue__dict_key_by_queue_name(queue_name) for queue_name in queue_names])
         self.logger.info(f'获取所有队列对应的活跃消费者进程信息，按队列名划分，结果是 {json.dumps(infos_map, indent=4)}')
         return infos_map
 
     def get_all_hearbeat_info_partition_by_ip(self) -> typing.Dict[typing.AnyStr, typing.List[typing.Dict]]:
         """获取所有机器ip对应的活跃消费者进程信息，按机器ip划分,不需要传入机器ip，自动扫描redis键。请不要在 funboost_config.py 的redis 指定的db中放太多其他业务的缓存键值对 """
-        infos_map = self._get_all_hearbeat_info_partition_by_redis_key_prefix('funboost_hearbeat_server__dict:')
+        ips = self.get_all_ips()
+        infos_map = self._get_all_hearbeat_info_partition_by_redis_keys([RedisKeys.gen_funboost_hearbeat_server__dict_key_by_ip(ip) for ip in ips])
         self.logger.info(f'获取所有机器ip对应的活跃消费者进程信息，按机器ip划分，结果是 {json.dumps(infos_map, indent=4)}')
         return infos_map
 
@@ -104,12 +129,12 @@ class QueueConusmerParamsGetter(RedisMixin, FunboostFileLoggerMixin):
         queue__pause_map = self.redis_db_frame.hgetall(RedisKeys.REDIS_KEY_PAUSE_FLAG)
         return {k:int(v)  for k,v in queue__pause_map.items()}
 
-    def get_msg_num(self):
+    def get_msg_num(self,ignore_report_ts=False):
         queue__msg_count_info_map = self.redis_db_frame.hgetall(RedisKeys.QUEUE__MSG_COUNT_MAP)
         queue__msg_count_dict = {}
         for queue_name,info_json in queue__msg_count_info_map.items():
             info_dict = json.loads(info_json)
-            if info_dict['report_ts'] > time.time() - 15 and info_dict['last_get_msg_num_ts'] > time.time() - 1200:
+            if ignore_report_ts or (info_dict['report_ts'] > time.time() - 15 and info_dict['last_get_msg_num_ts'] > time.time() - 1200):
                 queue__msg_count_dict[queue_name] = info_dict['msg_num_in_broker']
         return queue__msg_count_dict
 
@@ -123,11 +148,22 @@ class QueueConusmerParamsGetter(RedisMixin, FunboostFileLoggerMixin):
                 s+=c[filed]
         return s
     
+    def get_queues_history_run_count(self,):
+        return self.redis_db_frame.hgetall(RedisKeys.FUNBOOST_QUEUE__RUN_COUNT_MAP)
+    
+    def get_queues_history_run_fail_count(self,):
+        return self.redis_db_frame.hgetall(RedisKeys.FUNBOOST_QUEUE__RUN_FAIL_COUNT_MAP)
+    
     def get_queue_params_and_active_consumers(self):
         queue__active_consumers_map = ActiveCousumerProcessInfoGetter().get_all_hearbeat_info_partition_by_queue_name()
+
+        queue_name_list = list(queue__active_consumers_map.keys())
+        queue__history_run_count_map = self.get_queues_history_run_count()
+        queue__history_run_fail_count_map = self.get_queues_history_run_fail_count()
+
         queue__consumer_params_map  = self.get_queue_params()
         queue__pause_map = self.get_pause_flag()
-        queue__msg_count_dict = self.get_msg_num()
+        queue__msg_count_dict = self.get_msg_num(ignore_report_ts=True)
         queue_params_and_active_consumers = {}
 
         for queue, consumer_params in  queue__consumer_params_map.items():
@@ -148,6 +184,10 @@ class QueueConusmerParamsGetter(RedisMixin, FunboostFileLoggerMixin):
                 'active_consumers':active_consumers,
                 'pause_flag':queue__pause_map.get(queue,-1),
                 'msg_num_in_broker':queue__msg_count_dict.get(queue,None),
+                
+                'history_run_count':queue__history_run_count_map.get(queue,None),
+                'history_run_fail_count':queue__history_run_fail_count_map.get(queue,None),
+
                 'all_consumers_last_x_s_execute_count':all_consumers_last_x_s_execute_count,
                 'all_consumers_last_x_s_execute_count_fail':all_consumers_last_x_s_execute_count_fail,
                 'all_consumers_last_x_s_avarage_function_spend_time':all_consumers_last_x_s_avarage_function_spend_time,

funboost/core/booster.py

@@ -100,6 +100,7 @@ class Booster:
             consuming_function = args[0]
             self.boost_params.consuming_function = consuming_function
             self.boost_params.consuming_function_raw = consuming_function
+            self.boost_params.consuming_function_name = consuming_function.__name__
             # print(consuming_function)
             # print(ClsHelper.get_method_kind(consuming_function))
             # print(inspect.getsourcelines(consuming_function))
@@ -307,7 +308,7 @@ class BoostersManager:
         跨不同的项目，发布消息。例如proj1中定义有fun1消费函数，但proj2无法直接到日proj1的函数，无法直接 fun1.push 来发布消息
         可以使用这个方法，获取一个publisher。
 
-        publisher = BoostersManager.get_cross_project_publisher(PublisherParams(queue_name='proj1_queue', broker_kind=BrokerEnum.SQLITE_QUEUE))
+        publisher = BoostersManager.get_cross_project_publisher(PublisherParams(queue_name='proj1_queue', broker_kind=publisher_params.broker_kind))
         publisher.publish({'x': aaa})
         """
         pid = os.getpid()

funboost/core/cli/funboost_fire.py

@@ -127,3 +127,13 @@ class BoosterFire(object):
         """
         for queue_name in queue_names:
             BoostersManager.get_booster(queue_name).continue_consume()
+    
+    def start_funboost_web_manager(self):
+        """
+        启动funboost web管理器;
+        例子: start_funboost_web_manager
+        """
+        from funboost.function_result_web.app import start_funboost_web_manager
+        start_funboost_web_manager()
+
+    start_web = start_funboost_web_manager

funboost/core/current_task.py

@@ -1,4 +1,5 @@
 import abc
+import typing
 import contextvars
 from dataclasses import dataclass
 import logging
@@ -7,6 +8,8 @@ import asyncio
 
 from funboost.core.function_result_status_saver import FunctionResultStatus
 
+
+
 """ 用法例子 
     '''
     fct = funboost_current_task()
@@ -60,7 +63,9 @@ class FctContext:
     full_msg: dict
     function_result_status: FunctionResultStatus
     logger: logging.Logger
+    queue_name: str
     asyncio_use_thread_concurrent_mode: bool = False
+    
 
 # class FctContext:
 #     """
@@ -107,6 +112,11 @@ class _BaseCurrentTask(metaclass=abc.ABCMeta):
     @property
     def logger(self) -> logging.Logger:
         return self.get_fct_context().logger
+    
+    @property
+    def queue_name(self) -> str:
+        return self.get_fct_context().queue_name
+    
 
     def __str__(self):
         return f'<{self.__class__.__name__} [{self.function_result_status.get_status_dict()}]>'
@@ -184,6 +194,13 @@ class _FctProxy:
     @property
     def logger(self) -> logging.Logger:
         return self.fct_context.logger
+    
+    @property
+    def queue_name(self) -> str:
+        return self.fct_context.queue_name
+    
+
+
 
     def __str__(self):
         return f'<{self.__class__.__name__} [{self.function_result_status.get_status_dict()}]>'

funboost/core/func_params_model.py

@@ -127,17 +127,18 @@ class BoosterParams(BaseJsonAbleModel):
     """
 
     queue_name: str  # 队列名字,必传项,每个函数要使用不同的队列名字.
+    broker_kind: str = BrokerEnum.SQLITE_QUEUE  # 中间件选型见3.1章节 https://funboost.readthedocs.io/zh-cn/latest/articles/c3.html
 
     """如果设置了qps，并且cocurrent_num是默认的50，会自动开了500并发，由于是采用的智能线程池任务少时候不会真开那么多线程而且会自动缩小线程数量。具体看ThreadPoolExecutorShrinkAble的说明
     由于有很好用的qps控制运行频率和智能扩大缩小的线程池，此框架建议不需要理会和设置并发数量只需要关心qps就行了，框架的并发是自适应并发数量，这一点很强很好用。"""
     concurrent_mode: str = ConcurrentModeEnum.THREADING  # 并发模式,支持THREADING,GEVENT,EVENTLET,ASYNC,SINGLE_THREAD并发,multi_process_consume 支持协程/线程 叠加多进程并发,性能炸裂.
     concurrent_num: int = 50  # 并发数量，并发种类由concurrent_mode决定
     specify_concurrent_pool: typing.Optional[FunboostBaseConcurrentPool] = None  # 使用指定的线程池/携程池，可以多个消费者共使用一个线程池,节约线程.不为None时候。threads_num失效
-    specify_async_loop: asyncio.AbstractEventLoop = None  # 指定的async的loop循环，设置并发模式为async才能起作用。 有些包例如aiohttp,请求和httpclient的实例化不能处在两个不同的loop中,可以传过来.
+    specify_async_loop: typing.Optional[asyncio.AbstractEventLoop] = None  # 指定的async的loop循环，设置并发模式为async才能起作用。 有些包例如aiohttp,发送请求和httpclient的实例化不能处在两个不同的loop中,可以传过来.
 
     """qps:
     强悍的控制功能,指定1秒内的函数执行次数，例如可以是小数0.01代表每100秒执行一次，也可以是50代表1秒执行50次.为None则不控频。 设置qps时候,不需要指定并发数量,funboost的能够自适应智能动态调节并发池大小."""
-    qps: typing.Union[float, int] = None
+    qps: typing.Union[float, int, None] = None
     """is_using_distributed_frequency_control:
     是否使用分布式空频（依赖redis统计消费者数量，然后频率平分），默认只对当前实例化的消费者空频有效。假如实例化了2个qps为10的使用同一队列名的消费者，并且都启动，则每秒运行次数会达到20。
     如果使用分布式空频则所有消费者加起来的总运行次数是10。"""
@@ -155,7 +156,7 @@ class BoosterParams(BaseJsonAbleModel):
     is_push_to_dlx_queue_when_retry_max_times: bool = False  # 函数达到最大重试次数仍然没成功，是否发送到死信队列,死信队列的名字是 队列名字 + _dlx。
 
 
-    consumin_function_decorator: typing.Callable = None  # 函数的装饰器。因为此框架做参数自动转指点，需要获取精准的入参名称，不支持在消费函数上叠加 @ *args  **kwargs的装饰器，如果想用装饰器可以这里指定。
+    consumin_function_decorator: typing.Optional[typing.Callable] = None  # 函数的装饰器。因为此框架做参数自动转指点，需要获取精准的入参名称，不支持在消费函数上叠加 @ *args  **kwargs的装饰器，如果想用装饰器可以这里指定。
     function_timeout: typing.Union[int, float,None] = None  # 超时秒数，函数运行超过这个时间，则自动杀死函数。为0是不限制。 谨慎使用,非必要别去设置超时时间,设置后性能会降低(因为需要把用户函数包装到另一个线单独的程中去运行),而且突然强制超时杀死运行中函数,可能会造成死锁.(例如用户函数在获得线程锁后突然杀死函数,别的线程再也无法获得锁了)
 
     log_level: int = logging.DEBUG  # 消费者和发布者的日志级别,建议设置DEBUG级别,不然无法知道正在运行什么消息
@@ -166,7 +167,7 @@ class BoosterParams(BaseJsonAbleModel):
     is_show_message_get_from_broker: bool = False  # 运行时候,是否记录从消息队列获取出来的消息内容
     is_print_detail_exception: bool = True  # 消费函数出错时候,是否打印详细的报错堆栈,为False则只打印简略的报错信息不包含堆栈.
 
-    msg_expire_senconds: typing.Union[float, int] = None  # 消息过期时间,可以设置消息是多久之前发布的就丢弃这条消息,不运行. 为None则永不丢弃
+    msg_expire_senconds: typing.Union[float, int,None] = None  # 消息过期时间,可以设置消息是多久之前发布的就丢弃这条消息,不运行. 为None则永不丢弃
 
     do_task_filtering: bool = False  # 是否对函数入参进行过滤去重.
     task_filtering_expire_seconds: int = 0  # 任务过滤的失效期，为0则永久性过滤任务。例如设置过滤过期时间是1800秒 ， 30分钟前发布过1 + 2 的任务，现在仍然执行，如果是30分钟以内执行过这个任务，则不执行1 + 2
@@ -174,7 +175,7 @@ class BoosterParams(BaseJsonAbleModel):
     function_result_status_persistance_conf: FunctionResultStatusPersistanceConfig = FunctionResultStatusPersistanceConfig(
         is_save_result=False, is_save_status=False, expire_seconds=7 * 24 * 3600, is_use_bulk_insert=False)  # 是否保存函数的入参，运行结果和运行状态到mongodb。这一步用于后续的参数追溯，任务统计和web展示，需要安装mongo。
 
-    user_custom_record_process_info_func: typing.Callable = None  # 提供一个用户自定义的保存消息处理记录到某个地方例如mysql数据库的函数，函数仅仅接受一个入参，入参类型是 FunctionResultStatus，用户可以打印参数
+    user_custom_record_process_info_func: typing.Optional[typing.Callable] = None  # 提供一个用户自定义的保存消息处理记录到某个地方例如mysql数据库的函数，函数仅仅接受一个入参，入参类型是 FunctionResultStatus，用户可以打印参数
 
     is_using_rpc_mode: bool = False  # 是否使用rpc模式，可以在发布端获取消费端的结果回调，但消耗一定性能，使用async_result.result时候会等待阻塞住当前线程。
     rpc_result_expire_seconds: int = 600  # 保存rpc结果的过期时间.
@@ -190,10 +191,11 @@ class BoosterParams(BaseJsonAbleModel):
 
     is_auto_start_consuming_message: bool = False  # 是否在定义后就自动启动消费，无需用户手动写 .consume() 来启动消息消费。
 
-    consuming_function: typing.Callable = None  # 消费函数,在@boost时候不用指定,因为装饰器知道下面的函数.
-    consuming_function_raw: typing.Callable = None
+    consuming_function: typing.Optional[typing.Callable] = None  # 消费函数,在@boost时候不用指定,因为装饰器知道下面的函数.
+    consuming_function_raw: typing.Optional[typing.Callable] = None  # 不需要传递，自动生成
+    consuming_function_name: str = '' # 不需要传递，自动生成
 
-    broker_kind: str = BrokerEnum.SQLITE_QUEUE  # 中间件选型见3.1章节 https://funboost.readthedocs.io/zh-cn/latest/articles/c3.html
+    
 
     broker_exclusive_config: dict = {}  # 加上一个不同种类中间件非通用的配置,不同中间件自身独有的配置，不是所有中间件都兼容的配置，因为框架支持30种消息队列，消息队列不仅仅是一般的先进先出queue这么简单的概念，
     # 例如kafka支持消费者组，rabbitmq也支持各种独特概念例如各种ack机制 复杂路由机制，有的中间件原生能支持消息优先级有的中间件不支持,每一种消息队列都有独特的配置参数意义，可以通过这里传递。每种中间件能传递的键值对可以看consumer类的 BROKER_EXCLUSIVE_CONFIG_DEFAULT
@@ -221,6 +223,7 @@ class BoosterParams(BaseJsonAbleModel):
 
     @root_validator(skip_on_failure=True)
     def check_values(cls, values: dict):
+       
 
         # 如果设置了qps，并且cocurrent_num是默认的50，会自动开了500并发，由于是采用的智能线程池任务少时候不会真开那么多线程而且会自动缩小线程数量。具体看ThreadPoolExecutorShrinkAble的说明
         # 由于有很好用的qps控制运行频率和智能扩大缩小的线程池，此框架建议不需要理会和设置并发数量只需要关心qps就行了，框架的并发是自适应并发数量，这一点很强很好用。
@@ -233,7 +236,8 @@ class BoosterParams(BaseJsonAbleModel):
 
         if values['concurrent_mode'] not in ConcurrentModeEnum.__dict__.values():
             raise ValueError('设置的并发模式不正确')
-        if values['broker_kind'] in [BrokerEnum.REDIS_ACK_ABLE, BrokerEnum.REDIS_STREAM, BrokerEnum.REDIS_PRIORITY, BrokerEnum.RedisBrpopLpush]:
+        if values['broker_kind'] in [BrokerEnum.REDIS_ACK_ABLE, BrokerEnum.REDIS_STREAM, BrokerEnum.REDIS_PRIORITY, 
+                                     BrokerEnum.RedisBrpopLpush,BrokerEnum.REDIS,BrokerEnum.REDIS_PUBSUB]:
             values['is_send_consumer_hearbeat_to_redis'] = True  # 需要心跳进程来辅助判断消息是否属于掉线或关闭的进程，需要重回队列
         # if not set(values.keys()).issubset(set(BoosterParams.__fields__.keys())):
         #     raise ValueError(f'{cls.__name__} 的字段包含了父类 BoosterParams 不存在的字段')
@@ -291,16 +295,16 @@ class PriorityConsumingControlConfig(BaseJsonAbleModel):
 
     function_timeout: typing.Union[float, int,None] = None
 
-    max_retry_times: int = None
+    max_retry_times: typing.Union[int,None] = None
 
-    is_print_detail_exception: bool = None
+    is_print_detail_exception: typing.Union[bool,None] = None
 
-    msg_expire_senconds: int = None
+    msg_expire_senconds: typing.Union[float, int,None] = None
 
-    is_using_rpc_mode: bool = None
+    is_using_rpc_mode: typing.Union[bool,None] = None
 
-    countdown: typing.Union[float, int] = None
-    eta: typing.Union[datetime.datetime, str] = None  # 时间对象， 或 %Y-%m-%d %H:%M:%S 字符串。
+    countdown: typing.Union[float, int,None] = None
+    eta: typing.Union[datetime.datetime, str,None] = None  # 时间对象， 或 %Y-%m-%d %H:%M:%S 字符串。
     misfire_grace_time: typing.Union[int, None] = None
 
     other_extra_params: dict = None  # 其他参数, 例如消息优先级 , priority_control_config=PriorityConsumingControlConfig(other_extra_params={'priroty': priorityxx})，
@@ -324,8 +328,8 @@ class PublisherParams(BaseJsonAbleModel):
     logger_name: str = ''  # 队列消费者发布者的日志命名空间.
     log_filename: typing.Optional[str] = None
     clear_queue_within_init: bool = False  # with 语法发布时候,先清空消息队列
-    consuming_function: typing.Callable = None  # consuming_function 作用是 inspect 模块获取函数的入参信息
-    broker_kind: str = None
+    consuming_function: typing.Optional[typing.Callable] = None  # consuming_function 作用是 inspect 模块获取函数的入参信息
+    broker_kind: typing.Optional[str] = None
     broker_exclusive_config: dict = {}
     should_check_publish_func_params: bool = True  # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字,  如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
     publisher_override_cls: typing.Optional[typing.Type] = None

funboost/core/loggers.py

@@ -53,3 +53,4 @@ if __name__ == '__main__':
     logger1 = get_funboost_file_logger('name1')
     logger1.info('啦啦啦啦啦啦啦')
     logger1.error('错错错')
+    

funboost/core/msg_result_getter.py

@@ -52,6 +52,7 @@ class AsyncResult(RedisMixin):
     @property
     def status_and_result(self):
         if not self._has_pop:
+            # print(f'{self.task_id} 正在等待结果')
             redis_value = self.redis_db_filter_and_rpc_result.blpop(self.task_id, self.timeout)
             self._has_pop = True
             if redis_value is not None:

funboost/funboost_config_deafult.py

@@ -42,7 +42,7 @@ class BrokerConnConfig(DataClassBase):
     REDIS_USERNAME = ''
     REDIS_PASSWORD = ''
     REDIS_PORT = 6379
-    REDIS_DB = 7  # redis消息队列所在db，请不要在这个db放太多其他键值对，框架里面有的功能会scan扫描unacked的键名，使用单独的db。
+    REDIS_DB = 7  # redis消息队列所在db，请不要在这个db放太多其他键值对，以及方便你自己可视化查看你的redis db，框架里面有的功能会scan扫描unacked的键名，使用单独的db。
     REDIS_DB_FILTER_AND_RPC_RESULT = 8  # 如果函数做任务参数过滤 或者使用rpc获取结果，使用这个db，因为这个db的键值对多，和redis消息队列db分开
     REDIS_URL = f'redis://{REDIS_USERNAME}:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'