funboost 23.5__py3-none-any.whl → 23.7__py3-none-any.whl

funboost/__init__.py

@@ -2,7 +2,7 @@
 from funboost.utils.dependency_packages_in_pythonpath import add_to_pythonpath # 这是把 dependency_packages_in_pythonpath 添加到 PYTHONPATH了。
 
 from funboost.utils import monkey_patches
-from funboost.utils import show_funboost_flag
+from funboost.core import show_funboost_flag
 
 # noinspection PyUnresolvedReferences
 import nb_log

funboost/assist/celery_helper.py

@@ -15,7 +15,11 @@ celery_app = celery.Celery(main='funboost_celery', broker=funboost_config_deaful
                            task_routes={}, timezone=funboost_config_deafult.TIMEZONE, enable_utc=False, )
 
 celery_app.conf.task_acks_late = True
-celery_app.conf.worker_redirect_stdouts = False
+celery_app.conf.update({
+    'worker_redirect_stdouts': False,
+    'worker_concurrency': 200
+}
+)
 
 logger = get_logger('funboost.CeleryHelper')
 
@@ -30,7 +34,7 @@ class CeleryHelper:
     def update_celery_app_conf(celery_app_conf: dict):
         """
         更新celery app的配置，celery app配置大全见 https://docs.celeryq.dev/en/stable/userguide/configuration.html
-        :param celery_app_conf:
+        :param celery_app_conf: celery app 配置，字典
         :return:
         """
         celery_app.conf.update(celery_app_conf)
@@ -42,7 +46,7 @@ class CeleryHelper:
         for k, v in celery_app.conf.items():
             conf_dict_json_able[k] = str(v)
             # print(k, ' : ', v)
-        print('celery app 的配置是：',json.dumps(conf_dict_json_able,ensure_ascii=False,indent=4))
+        print('celery app 的配置是：', json.dumps(conf_dict_json_able, ensure_ascii=False, indent=4))
 
     @staticmethod
     def celery_start_beat(beat_schedule: dict):
@@ -68,7 +72,7 @@ class CeleryHelper:
         threading.Thread(target=_f).start()
 
     @classmethod
-    def realy_start_celery_worker(cls, worker_name=None):
+    def realy_start_celery_worker(cls, worker_name=None, loglevel='INFO'):
         if len(cls.to_be_start_work_celery_queue_name_set) == 0:
             raise Exception('celery worker 没有需要运行的queue')
         queue_names_str = ','.join(list(cls.to_be_start_work_celery_queue_name_set))
@@ -80,9 +84,16 @@ class CeleryHelper:
             pool_name = 'gevent'
         if cls.concurrent_mode == ConcurrentModeEnum.EVENTLET:
             pool_name = 'eventlet'
-        argv = ['worker', f'--pool={pool_name}', '--concurrency=200',
-                '-n', f'worker_funboost_{worker_name}@%h', f'--loglevel=INFO',
-                f'--queues={queue_names_str}',
+        '''
+        并发数量在app配置中已经制定了。自己用 update_celery_app_conf 方法更新就好了。
+        celery_app.conf.update({
+             'worker_redirect_stdouts': False,
+             'worker_concurrency': 200
+         }
+        '''
+        argv = ['worker', f'--pool={pool_name}',
+                '-n', f'worker_funboost_{worker_name}@%h', f'--loglevel={loglevel}',
+                f'--queues={queue_names_str}',         # 并发数量是
                 ]
         logger.info(f'celery 启动work参数 {argv}')
         celery_app.worker_main(argv)

funboost/consumers/base_consumer.py

@@ -373,6 +373,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         self._consuming_function_cost_time_total_every_unit_time = 0
         self._last_execute_task_time = time.time()  # 最近一次执行任务的时间。
 
+        self._last_show_remaining_execution_time = 0
+        self._show_remaining_execution_time_interval = 300
+
         self._msg_num_in_broker = 0
         self._last_timestamp_when_has_task_in_queue = 0
         self._last_timestamp_print_msg_num = 0
@@ -451,13 +454,13 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
 
     ''' 日志跳转代码行不正确，不用这种方式'''
 
-    # def _log_error(self, msg, exc_info=None):
-    #     self.logger.error(msg=f'{msg} \n', exc_info=exc_info)
-    #     self.error_file_logger.error(msg=f'{msg} \n', exc_info=exc_info)
-    #
-    # def _log_critical(self, msg, exc_info=None):
-    #     self.logger.critical(msg=f'{msg} \n', exc_info=exc_info)
-    #     self.error_file_logger.critical(msg=f'{msg} \n', exc_info=exc_info)
+    def _log_error(self, msg, exc_info=None):
+        self.logger.error(msg=f'{msg} \n', exc_info=exc_info, extra={'sys_getframe_n': 3})
+        self.error_file_logger.error(msg=f'{msg} \n', exc_info=exc_info, extra={'sys_getframe_n': 3})
+
+    def _log_critical(self, msg, exc_info=None):
+        self.logger.critical(msg=f'{msg} \n', exc_info=exc_info, extra={'sys_getframe_n': 3})
+        self.error_file_logger.critical(msg=f'{msg} \n', exc_info=exc_info, extra={'sys_getframe_n': 3})
 
     @property
     @decorators.synchronized
@@ -492,8 +495,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                         except BaseException as e:
                             log_msg = func.__name__ + '   运行出错\n ' + traceback.format_exc(
                                 limit=10) if is_display_detail_exception else str(e)
-                            self.logger.error(msg=f'{log_msg} \n', exc_info=True)
-                            self.error_file_logger.error(msg=f'{log_msg} \n', exc_info=True)
+                            # self.logger.error(msg=f'{log_msg} \n', exc_info=True)
+                            # self.error_file_logger.error(msg=f'{log_msg} \n', exc_info=True)
+                            self._log_error(msg=log_msg, exc_info=True)
                         finally:
                             time.sleep(time_sleep)
 
@@ -524,9 +528,10 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             self._distributed_consumer_statistics.run()
             self.logger.warning(f'启动了分布式环境 使用 redis 的键 hearbeat:{self._queue_name} 统计活跃消费者 ，当前消费者唯一标识为 {self.consumer_identification}')
 
-        self.keep_circulating(10, block=False)(self.check_heartbeat_and_message_count)()  # 间隔时间最好比self._unit_time_for_count小整数倍，不然日志不准。
+        self.keep_circulating(60, block=False)(self.check_heartbeat_and_message_count)()  # 间隔时间最好比self._unit_time_for_count小整数倍，不然日志不准。
         if self._is_support_remote_kill_task:
-            kill_remote_task.RemoteTaskKiller(self.queue_name, None).start_cycle_kill_task()
+            self.keep_circulating(10, block=False)(kill_remote_task.RemoteTaskKiller(self.queue_name, None).start_cycle_kill_task)()
+            self._is_show_message_get_from_broker = True  # 方便用户看到从消息队列取出来的消息的task_id,然后使用task_id杀死运行中的消息。
         if self._do_task_filtering:
             self._redis_filter.delete_expire_filter_task_cycle()  # 这个默认是RedisFilter类，是个pass不运行。所以用别的消息中间件模式，不需要安装和配置redis。
         if self._schedule_tasks_on_main_thread:
@@ -739,8 +744,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 if self._is_push_to_dlx_queue_when_retry_max_times:
                     log_msg += f'  。发送到死信队列 {self._dlx_queue_name} 中'
                     self.publisher_of_dlx_queue.publish(kw['body'])
-                self.logger.critical(msg=f'{log_msg} \n', )
-                self.error_file_logger.critical(msg=f'{log_msg} \n')
+                # self.logger.critical(msg=f'{log_msg} \n', )
+                # self.error_file_logger.critical(msg=f'{log_msg} \n')
+                self._log_critical(msg=log_msg)
 
             if self._get_priority_conf(kw, 'is_using_rpc_mode'):
                 # print(function_result_status.get_status_dict(without_datetime_obj=
@@ -760,23 +766,26 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 if time.time() - self._current_time_for_execute_task_times_every_unit_time > self._unit_time_for_count:
                     avarage_function_spend_time = round(self._consuming_function_cost_time_total_every_unit_time / self._execute_task_times_every_unit_time, 4)
                     msg = f'{self._unit_time_for_count} 秒内执行了 {self._execute_task_times_every_unit_time} 次函数 [ {self.consuming_function.__name__} ] ,' \
-                          f'函数平均运行耗时 {avarage_function_spend_time} 秒'
-                    if self._msg_num_in_broker != -1:  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
+                          f'函数平均运行耗时 {avarage_function_spend_time} 秒。 '
+                    self.logger.debug(msg)
+                    if self._msg_num_in_broker != -1 and time.time() - self._last_show_remaining_execution_time > self._show_remaining_execution_time_interval:  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
                         # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
                         need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /
                                                                             self._distributed_consumer_statistics.active_consumer_num)
-                        msg += f''' ，预计还需要 {need_time}''' + \
-                               f''' 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
-                    self.logger.info(msg)
+                        msg += f''' 预计还需要 {need_time} 时间 才能执行完成 队列 {self.queue_name} 中的 {self._msg_num_in_broker} 个剩余任务'''
+                        self.logger.info(msg)
+                        self._last_show_remaining_execution_time = time.time()
                     self._current_time_for_execute_task_times_every_unit_time = time.time()
                     self._consuming_function_cost_time_total_every_unit_time = 0
                     self._execute_task_times_every_unit_time = 0
+
             if self._user_custom_record_process_info_func:
                 self._user_custom_record_process_info_func(current_function_result_status)
         except BaseException as e:
             log_msg = f' error 严重错误 {type(e)} {e} '
-            self.logger.critical(msg=f'{log_msg} \n', exc_info=True)
-            self.error_file_logger.critical(msg=f'{log_msg} \n', exc_info=True)
+            # self.logger.critical(msg=f'{log_msg} \n', exc_info=True)
+            # self.error_file_logger.critical(msg=f'{log_msg} \n', exc_info=True)
+            self._log_critical(msg=log_msg, exc_info=True)
 
     # noinspection PyProtectedMember
     def _run_consuming_function_with_confirm_and_retry(self, kw: dict, current_retry_times,
@@ -795,12 +804,13 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                     function_result_status._has_kill_task = True
                     self.logger.warning(f'取消运行 {task_id} {function_only_params}')
                     return function_result_status
-                function_run = kill_remote_task.kill_fun_deco(task_id)(function_run)  # 用杀死装饰器包装起来在另一个线程运行函数
+                function_run = kill_remote_task.kill_fun_deco(task_id)(function_run)  # 用杀死装饰器包装起来在另一个线程运行函数,以便等待远程杀死。
             function_result_status.result = function_run(**function_only_params)
             if asyncio.iscoroutine(function_result_status.result):
                 log_msg = f'''异步的协程消费函数必须使用 async 并发模式并发,请设置消费函数 {self.consuming_function.__name__} 的concurrent_mode 为 ConcurrentModeEnum.ASYNC 或 4'''
-                self.logger.critical(msg=f'{log_msg} \n')
-                self.error_file_logger.critical(msg=f'{log_msg} \n')
+                # self.logger.critical(msg=f'{log_msg} \n')
+                # self.error_file_logger.critical(msg=f'{log_msg} \n')
+                self._log_critical(msg=log_msg)
                 # noinspection PyProtectedMember,PyUnresolvedReferences
 
                 os._exit(4)
@@ -813,8 +823,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         except BaseException as e:
             if isinstance(e, (ExceptionForRequeue,)):  # mongo经常维护备份时候插入不了或挂了，或者自己主动抛出一个ExceptionForRequeue类型的错误会重新入队，不受指定重试次数逇约束。
                 log_msg = f'函数 [{self.consuming_function.__name__}] 中发生错误 {type(e)}  {e} 。消息重新放入当前队列 {self._queue_name}'
-                self.logger.critical(msg=f'{log_msg} \n')
-                self.error_file_logger.critical(msg=f'{log_msg} \n')
+                # self.logger.critical(msg=f'{log_msg} \n')
+                # self.error_file_logger.critical(msg=f'{log_msg} \n')
+                self._log_critical(msg=log_msg)
                 time.sleep(0.1)  # 防止快速无限出错入队出队，导致cpu和中间件忙
                 # 重回队列如果不修改task_id,insert插入函数消费状态结果到mongo会主键重复。要么保存函数消费状态使用replace，要么需要修改taskikd
                 # kw_new = copy.deepcopy(kw)
@@ -825,21 +836,25 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 function_result_status._has_requeue = True
             if isinstance(e, ExceptionForPushToDlxqueue):
                 log_msg = f'函数 [{self.consuming_function.__name__}] 中发生错误 {type(e)}  {e}，消息放入死信队列 {self._dlx_queue_name}'
-                self.logger.critical(msg=f'{log_msg} \n')
-                self.error_file_logger.critical(msg=f'{log_msg} \n')
+                # self.logger.critical(msg=f'{log_msg} \n')
+                # self.error_file_logger.critical(msg=f'{log_msg} \n')
+                self._log_critical(msg=log_msg)
                 self.publisher_of_dlx_queue.publish(kw['body'])  # 发布到死信队列，不重回当前队列
                 function_result_status._has_to_dlx_queue = True
             if isinstance(e, kill_remote_task.TaskHasKilledError):
                 log_msg = f'task_id 为 {task_id} , 函数 [{self.consuming_function.__name__}] 运行入参 {function_only_params}   ，已被远程指令杀死 {type(e)}  {e}'
-                self.logger.critical(msg=f'{log_msg} ')
-                self.error_file_logger.critical(msg=f'{log_msg} ')
+                # self.logger.critical(msg=f'{log_msg} ')
+                # self.error_file_logger.critical(msg=f'{log_msg} ')
+                self._log_critical(msg=log_msg)
                 function_result_status._has_kill_task = True
             if isinstance(e, (ExceptionForRequeue, ExceptionForPushToDlxqueue, kill_remote_task.TaskHasKilledError)):
                 return function_result_status
             log_msg = f'''函数 {self.consuming_function.__name__}  第{current_retry_times + 1}次运行发生错误，
-                              函数运行时间是 {round(time.time() - t_start, 4)} 秒,\n  入参是  {function_only_params}    \n 原因是 {type(e)} {e} '''
-            self.logger.error(msg=f'{log_msg} \n', exc_info=self._get_priority_conf(kw, 'is_print_detail_exception'))
-            self.error_file_logger.error(msg=f'{log_msg} \n', exc_info=self._get_priority_conf(kw, 'is_print_detail_exception'))
+                          函数运行时间是 {round(time.time() - t_start, 4)} 秒,  入参是  {function_only_params}    
+                          {type(e)} {e} '''
+            # self.logger.error(msg=f'{log_msg} \n', exc_info=self._get_priority_conf(kw, 'is_print_detail_exception'))
+            # self.error_file_logger.error(msg=f'{log_msg} \n', exc_info=self._get_priority_conf(kw, 'is_print_detail_exception'))
+            self._log_error(msg=log_msg, exc_info=self._get_priority_conf(kw, 'is_print_detail_exception'))
             # traceback.print_exc()
             function_result_status.exception = f'{e.__class__.__name__}    {str(e)}'
             function_result_status.result = FunctionResultStatus.FUNC_RUN_ERROR
@@ -876,8 +891,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 if self._is_push_to_dlx_queue_when_retry_max_times:
                     log_msg += f'  。发送到死信队列 {self._dlx_queue_name} 中'
                     await simple_run_in_executor(self.publisher_of_dlx_queue.publish, kw['body'])
-                self.logger.critical(msg=f'{log_msg} \n', )
-                self.error_file_logger.critical(msg=f'{log_msg} \n')
+                # self.logger.critical(msg=f'{log_msg} \n', )
+                # self.error_file_logger.critical(msg=f'{log_msg} \n')
+                self._log_critical(msg=log_msg)
 
                 # self._confirm_consume(kw)  # 错得超过指定的次数了，就确认消费了。
             if self._get_priority_conf(kw, 'is_using_rpc_mode'):
@@ -898,24 +914,26 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             if time.time() - self._current_time_for_execute_task_times_every_unit_time > self._unit_time_for_count:
                 avarage_function_spend_time = round(self._consuming_function_cost_time_total_every_unit_time / self._execute_task_times_every_unit_time, 4)
                 msg = f'{self._unit_time_for_count} 秒内执行了 {self._execute_task_times_every_unit_time} 次函数 [ {self.consuming_function.__name__} ] ,' \
-                      f'函数平均运行耗时 {avarage_function_spend_time} 秒'
-                if self._msg_num_in_broker != -1:
-                    if self._msg_num_in_broker != -1:  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
-                        # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
-                        need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /
-                                                                            self._distributed_consumer_statistics.active_consumer_num)
-                        msg += f''' ，预计还需要 {need_time}''' + \
-                               f''' 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
-                self.logger.info(msg)
+                      f'函数平均运行耗时 {avarage_function_spend_time} 秒。 '
+                self.logger.debug(msg)
+                if self._msg_num_in_broker != -1 and time.time() - self._last_show_remaining_execution_time > self._show_remaining_execution_time_interval:  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
+                    # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
+                    need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /
+                                                                        self._distributed_consumer_statistics.active_consumer_num)
+                    msg += f''' 预计还需要 {need_time} 时间 才能执行完成 队列 {self.queue_name} 中的 {self._msg_num_in_broker} 个剩余任务'''
+                    self.logger.info(msg)
+                    self._last_show_remaining_execution_time = time.time()
                 self._current_time_for_execute_task_times_every_unit_time = time.time()
                 self._consuming_function_cost_time_total_every_unit_time = 0
                 self._execute_task_times_every_unit_time = 0
+
             if self._user_custom_record_process_info_func:
                 await self._user_custom_record_process_info_func(current_function_result_status)
         except BaseException as e:
             log_msg = f' error 严重错误 {type(e)} {e} '
-            self.logger.critical(msg=f'{log_msg} \n', exc_info=True)
-            self.error_file_logger.critical(msg=f'{log_msg} \n', exc_info=True)
+            # self.logger.critical(msg=f'{log_msg} \n', exc_info=True)
+            # self.error_file_logger.critical(msg=f'{log_msg} \n', exc_info=True)
+            self._log_critical(msg=log_msg, exc_info=True)
 
     # noinspection PyProtectedMember
     async def _async_run_consuming_function_with_confirm_and_retry(self, kw: dict, current_retry_times,
@@ -930,8 +948,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             corotinue_obj = self.consuming_function(**function_only_params)
             if not asyncio.iscoroutine(corotinue_obj):
                 log_msg = f'''当前设置的并发模式为 async 并发模式，但消费函数不是异步协程函数，请不要把消费函数 {self.consuming_function.__name__} 的 concurrent_mode 设置为 4'''
-                self.logger.critical(msg=f'{log_msg} \n')
-                self.error_file_logger.critical(msg=f'{log_msg} \n')
+                # self.logger.critical(msg=f'{log_msg} \n')
+                # self.error_file_logger.critical(msg=f'{log_msg} \n')
+                self._log_critical(msg=log_msg)
                 # noinspection PyProtectedMember,PyUnresolvedReferences
                 os._exit(444)
             if self._function_timeout == 0:
@@ -949,8 +968,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         except BaseException as e:
             if isinstance(e, (ExceptionForRequeue,)):  # mongo经常维护备份时候插入不了或挂了，或者自己主动抛出一个ExceptionForRequeue类型的错误会重新入队，不受指定重试次数逇约束。
                 log_msg = f'函数 [{self.consuming_function.__name__}] 中发生错误 {type(e)}  {e} 。 消息重新放入当前队列 {self._queue_name}'
-                self.logger.critical(msg=f'{log_msg} \n')
-                self.error_file_logger.critical(msg=f'{log_msg} \n')
+                # self.logger.critical(msg=f'{log_msg} \n')
+                # self.error_file_logger.critical(msg=f'{log_msg} \n')
+                self._log_critical(msg=log_msg)
                 # time.sleep(1)  # 防止快速无限出错入队出队，导致cpu和中间件忙
                 await asyncio.sleep(0.1)
                 # return self._requeue(kw)
@@ -958,16 +978,19 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 function_result_status._has_requeue = True
             if isinstance(e, ExceptionForPushToDlxqueue):
                 log_msg = f'函数 [{self.consuming_function.__name__}] 中发生错误 {type(e)}  {e}，消息放入死信队列 {self._dlx_queue_name}'
-                self.logger.critical(msg=f'{log_msg} \n')
-                self.error_file_logger.critical(msg=f'{log_msg} \n')
+                # self.logger.critical(msg=f'{log_msg} \n')
+                # self.error_file_logger.critical(msg=f'{log_msg} \n')
+                self._log_critical(msg=log_msg)
                 await simple_run_in_executor(self.publisher_of_dlx_queue.publish, kw['body'])  # 发布到死信队列，不重回当前队列
                 function_result_status._has_to_dlx_queue = True
             if isinstance(e, (ExceptionForRequeue, ExceptionForPushToDlxqueue)):
                 return function_result_status
             log_msg = f'''函数 {self.consuming_function.__name__}  第{current_retry_times + 1}次运行发生错误，
-                              函数运行时间是 {round(time.time() - t_start, 4)} 秒,\n  入参是  {function_only_params}    \n 原因是 {type(e)} {e} '''
-            self.logger.error(msg=f'{log_msg} \n', exc_info=self._get_priority_conf(kw, 'is_print_detail_exception'))
-            self.error_file_logger.error(msg=f'{log_msg} \n', exc_info=self._get_priority_conf(kw, 'is_print_detail_exception'))
+                          函数运行时间是 {round(time.time() - t_start, 4)} 秒,  入参是  {function_only_params}     
+                          原因是 {type(e)} {e} '''
+            # self.logger.error(msg=f'{log_msg} \n', exc_info=self._get_priority_conf(kw, 'is_print_detail_exception'))
+            # self.error_file_logger.error(msg=f'{log_msg} \n', exc_info=self._get_priority_conf(kw, 'is_print_detail_exception'))
+            self._log_error(msg=log_msg, exc_info=self._get_priority_conf(kw, 'is_print_detail_exception'))
             function_result_status.exception = f'{e.__class__.__name__}    {str(e)}'
             function_result_status.result = FunctionResultStatus.FUNC_RUN_ERROR
         return function_result_status
@@ -1004,8 +1027,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         misfire_grace_time = self._get_priority_conf(event.function_kwargs["kw"], 'misfire_grace_time')
         log_msg = f''' 现在时间是 {time_util.DatetimeConverter().datetime_str} ,比此任务规定的本应该的运行时间 {event.scheduled_run_time} 相比 超过了指定的 {misfire_grace_time} 秒,放弃执行此任务 
                              {event.function_kwargs["kw"]["body"]} '''
-        self.logger.critical(msg=f'{log_msg} \n')
-        self.error_file_logger.critical(msg=f'{log_msg} \n')
+        # self.logger.critical(msg=f'{log_msg} \n')
+        # self.error_file_logger.critical(msg=f'{log_msg} \n')
+        self._log_critical(msg=log_msg)
         self._confirm_consume(event.function_kwargs["kw"])
 
         '''

funboost/consumers/celery_consumer.py

@@ -156,13 +156,29 @@ class CeleryConsumer(AbstractConsumer):
     '''
 
     def custom_init(self):
-        # 这就是核心，@boost时候回注册任务路由到celery_app
+        # 这就是核心，@boost时候会 @ celery app.task装饰器
         @celery_app.task(name=self.queue_name, rate_limit=f'{self._qps}/s', soft_time_limit=self._function_timeout,
                          max_retries=self._max_retry_times,
-                         **self.broker_exclusive_config['celery_task_config'])
-        def f(*args, **kwargs):
+                         **self.broker_exclusive_config['celery_task_config'],
+                         bind=True)
+        def f(this, *args, **kwargs):
             self.logger.debug(f' 这条消息是 celery 从 {self.queue_name} 队列中取出 ,是由 celery 框架调度 {self.consuming_function.__name__} 函数处理: args:  {args} ,  kwargs: {kwargs}')
-            return self.consuming_function(*args, **kwargs)
+            # return self.consuming_function(*args, **kwargs) # 如果没有声明 autoretry_for ，那么消费函数出错了就不会自动重试了。
+            try:
+                return self.consuming_function(*args, **kwargs)
+            except BaseException as exc:  # 改成自动重试。
+                # print(this.request.__dict__,dir(this))
+
+                if this.request.retries != self._max_retry_times:
+                    log_msg = f'fun: {self.consuming_function}  args: {args} , kwargs: {kwargs} 消息第{this.request.retries}次运行出错,  {exc} \n'
+                    self._log_error(log_msg, exc_info=self._is_print_detail_exception)
+                else:
+                    log_msg = f'fun: {self.consuming_function}  args: {args} , kwargs: {kwargs} 消息达到最大重试次数{this.request.retries}次仍然出错,  {exc} \n'
+                    self._log_critical(log_msg, exc_info=self._is_print_detail_exception)
+                # 发生异常，尝试重试任务,countdown 是多少秒后重试
+                raise this.retry(exc=exc, countdown=5)
+
+        celery_app.conf.task_routes.update({self.queue_name: {"queue": self.queue_name}})  # 自动配置celery每个函数使用不同的队列名。
         self.celery_task = f
         CeleryHelper.concurrent_mode = self._concurrent_mode
 

funboost/consumers/rabbitmq_amqpstorm_consumer.py

@@ -29,7 +29,7 @@ class RabbitmqConsumerAmqpStorm(AbstractConsumer):
         rp = RabbitmqPublisherUsingAmqpStorm(self.queue_name,broker_exclusive_config=self.broker_exclusive_config)
         rp.init_broker()
         rp.channel_wrapper_by_ampqstormbaic.qos(self._concurrent_num)
-        rp.channel_wrapper_by_ampqstormbaic.consume(callback=callback, queue=self.queue_name, no_ack=False)
+        rp.channel_wrapper_by_ampqstormbaic.consume(callback=callback, queue=self.queue_name, no_ack=False,)
         rp.channel.start_consuming(auto_decode=True)
 
     def _confirm_consume(self, kw):

funboost/consumers/redis_consumer_ack_able.py

@@ -156,7 +156,7 @@ class RedisConsumerAckAble(ConsumerConfirmMixinWithTheHelpOfRedisByHearbeat, Abs
                     kw = {'body': task_dict, 'task_str': task_str}
                     self._submit_task(kw)
             else:
-                time.sleep(0.5)
+                time.sleep(0.2)
 
     def _requeue(self, kw):
         self.redis_db_frame.rpush(self._queue_name, json.dumps(kw['body']))

funboost/consumers/redis_consumer_priority.py

@@ -10,6 +10,9 @@
 """
 import json
 import time
+
+import redis3
+
 from funboost.consumers.redis_consumer_ack_able import RedisConsumerAckAble
 
 
@@ -49,7 +52,7 @@ class RedisPriorityConsumer(RedisConsumerAckAble):
 
     BROKER_EXCLUSIVE_CONFIG_DEFAULT = {'x-max-priority': None}  # x-max-priority 是 rabbitmq的优先级队列配置，必须为整数，强烈建议要小于5。为None就代表队列不支持优先级。
 
-    def _shedual_task(self):
+    def _shedual_task0000(self):
 
         while True:
             # task_str_list = script(keys=[queues_str, self._unack_zset_name], args=[time.time()])
@@ -64,5 +67,31 @@ class RedisPriorityConsumer(RedisConsumerAckAble):
                 kw = {'body': task_dict, 'task_str': msg}
                 self._submit_task(kw)
 
+    def _shedual_task(self):
+        """https://redis.readthedocs.io/en/latest/advanced_features.html#default-pipelines """
+        while True:
+            # task_str_list = script(keys=[queues_str, self._unack_zset_name], args=[time.time()])
+            while True:
+                try:
+                    with self.redis_db_frame_version3.pipeline() as p:
+                        p.watch(self._unack_zset_name, *self.publisher_of_same_queue.queue_list, )
+                        task_tuple = p.blpop(keys=self.publisher_of_same_queue.queue_list, timeout=60)  # 监听了多个键名，所以间接实现了优先级，和kombu的redis 支持优先级的设计思路不谋而合。
+                        # print(task_tuple)
+                        if task_tuple:
+                            msg = task_tuple[1]
+                            p.zadd(self._unack_zset_name, {msg: time.time()})
+                            # self.logger.debug(task_tuple)
+                            p.unwatch()
+                            p.execute()
+                            break
+                except redis3.WatchError:
+                    continue
+            if task_tuple:
+                self._print_message_get_from_broker('redis', msg)
+                # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：  {task_str_list}  ')
+                task_dict = json.loads(msg)
+                kw = {'body': task_dict, 'task_str': msg}
+                self._submit_task(kw)
+
     def _requeue(self, kw):
         self.publisher_of_same_queue.publish(kw['body'])

funboost/core/show_funboost_flag.py

@@ -0,0 +1,48 @@
+import nb_log
+
+logger = nb_log.get_logger('funboost.show_funboost_flag')
+
+funboost_flag_str = '''
+
+                                                                                                                                                                                                          
+FFFFFFFFFFFFFFFFFFFFFF     UUUUUUUU     UUUUUUUU     NNNNNNNN        NNNNNNNN     BBBBBBBBBBBBBBBBB             OOOOOOOOO               OOOOOOOOO             SSSSSSSSSSSSSSS      TTTTTTTTTTTTTTTTTTTTTTT
+F::::::::::::::::::::F     U::::::U     U::::::U     N:::::::N       N::::::N     B::::::::::::::::B          OO:::::::::OO           OO:::::::::OO         SS:::::::::::::::S     T:::::::::::::::::::::T
+F::::::::::::::::::::F     U::::::U     U::::::U     N::::::::N      N::::::N     B::::::BBBBBB:::::B       OO:::::::::::::OO       OO:::::::::::::OO      S:::::SSSSSS::::::S     T:::::::::::::::::::::T
+FF::::::FFFFFFFFF::::F     UU:::::U     U:::::UU     N:::::::::N     N::::::N     BB:::::B     B:::::B     O:::::::OOO:::::::O     O:::::::OOO:::::::O     S:::::S     SSSSSSS     T:::::TT:::::::TT:::::T
+  F:::::F       FFFFFF      U:::::U     U:::::U      N::::::::::N    N::::::N       B::::B     B:::::B     O::::::O   O::::::O     O::::::O   O::::::O     S:::::S                 TTTTTT  T:::::T  TTTTTT
+  F:::::F                   U:::::D     D:::::U      N:::::::::::N   N::::::N       B::::B     B:::::B     O:::::O     O:::::O     O:::::O     O:::::O     S:::::S                         T:::::T        
+  F::::::FFFFFFFFFF         U:::::D     D:::::U      N:::::::N::::N  N::::::N       B::::BBBBBB:::::B      O:::::O     O:::::O     O:::::O     O:::::O      S::::SSSS                      T:::::T        
+  F:::::::::::::::F         U:::::D     D:::::U      N::::::N N::::N N::::::N       B:::::::::::::BB       O:::::O     O:::::O     O:::::O     O:::::O       SS::::::SSSSS                 T:::::T        
+  F:::::::::::::::F         U:::::D     D:::::U      N::::::N  N::::N:::::::N       B::::BBBBBB:::::B      O:::::O     O:::::O     O:::::O     O:::::O         SSS::::::::SS               T:::::T        
+  F::::::FFFFFFFFFF         U:::::D     D:::::U      N::::::N   N:::::::::::N       B::::B     B:::::B     O:::::O     O:::::O     O:::::O     O:::::O            SSSSSS::::S              T:::::T        
+  F:::::F                   U:::::D     D:::::U      N::::::N    N::::::::::N       B::::B     B:::::B     O:::::O     O:::::O     O:::::O     O:::::O                 S:::::S             T:::::T        
+  F:::::F                   U::::::U   U::::::U      N::::::N     N:::::::::N       B::::B     B:::::B     O::::::O   O::::::O     O::::::O   O::::::O                 S:::::S             T:::::T        
+FF:::::::FF                 U:::::::UUU:::::::U      N::::::N      N::::::::N     BB:::::BBBBBB::::::B     O:::::::OOO:::::::O     O:::::::OOO:::::::O     SSSSSSS     S:::::S           TT:::::::TT      
+F::::::::FF                  UU:::::::::::::UU       N::::::N       N:::::::N     B:::::::::::::::::B       OO:::::::::::::OO       OO:::::::::::::OO      S::::::SSSSSS:::::S           T:::::::::T      
+F::::::::FF                    UU:::::::::UU         N::::::N        N::::::N     B::::::::::::::::B          OO:::::::::OO           OO:::::::::OO        S:::::::::::::::SS            T:::::::::T      
+FFFFFFFFFFF                      UUUUUUUUU           NNNNNNNN         NNNNNNN     BBBBBBBBBBBBBBBBB             OOOOOOOOO               OOOOOOOOO           SSSSSSSSSSSSSSS              TTTTTTTTTTT      
+   
+
+'''
+
+funboost_flag_str2 = r'''
+
+      ___                  ___                    ___                                           ___                    ___                    ___                          
+     /  /\                /__/\                  /__/\                  _____                  /  /\                  /  /\                  /  /\                   ___   
+    /  /:/_               \  \:\                 \  \:\                /  /::\                /  /::\                /  /::\                /  /:/_                 /  /\  
+   /  /:/ /\               \  \:\                 \  \:\              /  /:/\:\              /  /:/\:\              /  /:/\:\              /  /:/ /\               /  /:/  
+  /  /:/ /:/           ___  \  \:\            _____\__\:\            /  /:/~/::\            /  /:/  \:\            /  /:/  \:\            /  /:/ /::\             /  /:/   
+ /__/:/ /:/           /__/\  \__\:\          /__/::::::::\          /__/:/ /:/\:|          /__/:/ \__\:\          /__/:/ \__\:\          /__/:/ /:/\:\           /  /::\   
+ \  \:\/:/            \  \:\ /  /:/          \  \:\~~\~~\/          \  \:\/:/~/:/          \  \:\ /  /:/          \  \:\ /  /:/          \  \:\/:/~/:/          /__/:/\:\  
+  \  \::/              \  \:\  /:/            \  \:\  ~~~            \  \::/ /:/            \  \:\  /:/            \  \:\  /:/            \  \::/ /:/           \__\/  \:\ 
+   \  \:\               \  \:\/:/              \  \:\                 \  \:\/:/              \  \:\/:/              \  \:\/:/              \__\/ /:/                 \  \:\
+    \  \:\               \  \::/                \  \:\                 \  \::/                \  \::/                \  \::/                 /__/:/                   \__\/
+     \__\/                \__\/                  \__\/                  \__\/                  \__\/                  \__\/                  \__\/                         
+
+
+
+'''
+
+logger.debug('\033[0m' + funboost_flag_str2 + '\033[0m')
+
+logger.info(f'''分布式函数调度框架funboost文档地址：  \033[0m https://funboost.readthedocs.io/zh/latest/ \033[0m ''')

funboost/funboost_config_deafult.py

@@ -27,7 +27,7 @@ from funboost.utils.simple_data_class import DataClassBase
 # $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$  以下是中间件连接配置    $$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$
 
 
-MONGO_CONNECT_URL = f'mongodb://127.0.0.1:27017'  # 如果有密码连接 'mongodb://myUserAdmin:8mwTdy1klnSYepNo@192.168.199.202:27016/MONGO_CONNECT_URL = 'mongodb://root:123456@192.168.64.151:27017?authSource=admin''
+MONGO_CONNECT_URL = f'mongodb://127.0.0.1:27017'  # 如果有密码连接 'mongodb://myUserAdmin:8mwTdy1klnSYepNo@192.168.199.202:27016/'   authSource 指定鉴权db，MONGO_CONNECT_URL = 'mongodb://root:123456@192.168.64.151:27017?authSource=admin'
 
 RABBITMQ_USER = 'rabbitmq_user'
 RABBITMQ_PASS = 'rabbitmq_pass'

funboost/publishers/base_publisher.py

@@ -199,7 +199,7 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
 
     def _convert_msg(self, msg: typing.Union[str, dict], task_id=None,
                      priority_control_config: PriorityConsumingControlConfig = None) -> (typing.Dict, typing.Dict, typing.Dict):
-        if isinstance(msg, (str,bytes)):
+        if isinstance(msg, (str, bytes)):
             msg = json.loads(msg)
         msg_function_kw = copy.deepcopy(msg)
         raw_extra = {}
@@ -215,7 +215,7 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             extra_params.update(priority_control_config.to_dict())
         extra_params.update(raw_extra)
         msg['extra'] = extra_params
-        return msg, msg_function_kw, extra_params,task_id
+        return msg, msg_function_kw, extra_params, task_id
 
     def publish(self, msg: typing.Union[str, dict], task_id=None,
                 priority_control_config: PriorityConsumingControlConfig = None):
@@ -226,7 +226,7 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         :param priority_control_config:优先级配置，消息可以携带优先级配置，覆盖boost的配置。
         :return:
         """
-        msg, msg_function_kw, extra_params,task_id = self._convert_msg(msg, task_id, priority_control_config)
+        msg, msg_function_kw, extra_params, task_id = self._convert_msg(msg, task_id, priority_control_config)
         t_start = time.time()
         decorators.handle_exception(retry_times=10, is_throw_error=True, time_sleep=0.1)(
             self.concrete_realization_of_publish)(json.dumps(msg, ensure_ascii=False))

funboost/publishers/celery_publisher.py

@@ -3,11 +3,7 @@
 # @Time    : 2022/8/8 0008 12:12
 import os
 import sys
-import uuid
-import copy
 import time
-import threading
-import json
 import celery
 import celery.result
 import typing
@@ -20,24 +16,6 @@ class CeleryPublisher(AbstractPublisher, ):
     """
     使用celery作为中间件
     """
-    celery_conf_lock = threading.Lock()
-
-    # noinspection PyAttributeOutsideInit
-    def custom_init(self):
-        # self.broker_exclusive_config['task_routes'] = {self.queue_name: {"queue": self.queue_name}}
-        # celery_app.config_from_object(self.broker_exclusive_config)
-        pass
-
-        # celery_app.conf.task_routes.update({self.queue_name: {"queue": self.queue_name}})
-        #
-        # @celery_app.task(name=self.queue_name)
-        # def f(*args, **kwargs):
-        #     pass
-        #
-        # self._celery_app = celery_app
-        # self._celery_fun = f
-
-        celery_app.conf.task_routes.update({self.queue_name: {"queue": self.queue_name}})
 
     def publish(self, msg: typing.Union[str, dict], task_id=None,
                 priority_control_config: PriorityConsumingControlConfig = None) -> celery.result.AsyncResult:

funboost/publishers/redis_publisher.py

@@ -9,10 +9,11 @@ from threading import Lock
 
 from funboost import funboost_config_deafult
 from funboost.publishers.base_publisher import AbstractPublisher
+from funboost.publishers.redis_queue_flush import FlushRedisQueueMixin
 from funboost.utils import RedisMixin, decorators
 
 
-class RedisPublisher(AbstractPublisher, RedisMixin):
+class RedisPublisher(FlushRedisQueueMixin, AbstractPublisher, RedisMixin, ):
     """
     使用redis作为中间件,这个是大幅优化了发布速度的方式。简单的发布是 redis_publisher_0000.py 中的代码方式。
 
@@ -42,7 +43,7 @@ class RedisPublisher(AbstractPublisher, RedisMixin):
         # print(getattr(frame_config,'has_start_a_consumer_flag',0))
         # 这里的 has_start_a_consumer_flag 是一个标志，借用此模块设置的一个标识变量而已，框架运行时候自动设定的，不要把这个变量写到模块里面。
         # if getattr(funboost_config_deafult, 'has_start_a_consumer_flag', 0) == 0:  # 加快速度推送，否则每秒只能推送4000次。如果是独立脚本推送，使用批量推送，如果是消费者中发布任务，为了保持原子性，用原来的单个推送。
-        if self.broker_exclusive_config.get('redis_bulk_push',0) == 1:   # RedisConsumer传了,  RedisAckAble  没传
+        if self.broker_exclusive_config.get('redis_bulk_push', 0) == 1:  # RedisConsumer传了,  RedisAckAble  没传
             # self._temp_msg_queue.put(msg)
             with self._lock_for_bulk_push:
                 self._temp_msg_list.append(msg)
@@ -53,17 +54,6 @@ class RedisPublisher(AbstractPublisher, RedisMixin):
             getattr(self.redis_db_frame, self._push_method)(self._queue_name, msg)
             # print(msg)
 
-        # self.redis_db_frame.rpush(self._queue_name, msg)
-    def clear(self):
-        self.redis_db_frame.delete(self._queue_name)
-        # self.redis_db_frame.delete(f'{self._queue_name}__unack')
-        unack_queue_name_list = self.redis_db_frame.scan(match=f'{self._queue_name}__unack_id_*', count=10000)[1] + \
-                                self.redis_db_frame.scan(match=f'unack_{self._queue_name}_*', count=10000)[1]  # noqa
-        self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功')
-        if unack_queue_name_list:
-            self.redis_db_frame.delete(*unack_queue_name_list)
-            self.logger.warning(f'清除 {unack_queue_name_list} 队列中的消息成功')
-
     def get_message_count(self):
         # print(self.redis_db7,self._queue_name)
         return self.redis_db_frame.llen(self._queue_name)

funboost/publishers/redis_publisher_priority.py

@@ -3,10 +3,11 @@
 # @Time    : 2022/8/8 0008 12:12
 
 from funboost.publishers.base_publisher import AbstractPublisher
+from funboost.publishers.redis_queue_flush import FlushRedisQueueMixin
 from funboost.utils.redis_manager import RedisMixin
 
 
-class RedisPriorityPublisher(AbstractPublisher, RedisMixin):
+class RedisPriorityPublisher(FlushRedisQueueMixin,AbstractPublisher, RedisMixin,):
     """
     redis队列，支持任务优先级。
     """
@@ -39,10 +40,6 @@ class RedisPriorityPublisher(AbstractPublisher, RedisMixin):
         self.logger.debug([queue_name, msg])
         self.redis_db_frame.rpush(queue_name, msg)
 
-    def clear(self):
-        self.redis_db_frame.delete(*self.queue_list)
-        self.redis_db_frame.delete(f'{self._queue_name}__unack')
-        self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功')
 
     def get_message_count(self):
         count = 0

funboost/publishers/redis_publisher_simple.py

@@ -2,10 +2,11 @@
 # @Author  : ydf
 # @Time    : 2023/8/8 0008 12:12
 from funboost.publishers.base_publisher import AbstractPublisher
+from funboost.publishers.redis_queue_flush import FlushRedisQueueMixin
 from funboost.utils import RedisMixin
 
 
-class RedisPublisher(AbstractPublisher, RedisMixin):
+class RedisPublisher(FlushRedisQueueMixin, AbstractPublisher, RedisMixin, ):
     """
     使用redis作为中间件
     """
@@ -13,11 +14,6 @@ class RedisPublisher(AbstractPublisher, RedisMixin):
     def concrete_realization_of_publish(self, msg):
         self.redis_db_frame.rpush(self._queue_name, msg)
 
-    def clear(self):
-        self.redis_db_frame.delete(self._queue_name)
-        self.redis_db_frame.delete(f'{self._queue_name}__unack')
-        self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功')
-
     def get_message_count(self):
         # nb_print(self.redis_db7,self._queue_name)
         return self.redis_db_frame.llen(self._queue_name)

funboost/publishers/redis_pubsub_publisher.py

@@ -5,7 +5,7 @@ from funboost.publishers.base_publisher import AbstractPublisher
 from funboost.utils import RedisMixin
 
 
-class RedisPubSubPublisher(AbstractPublisher, RedisMixin):
+class RedisPubSubPublisher(AbstractPublisher, RedisMixin, ):
     """
     使用redis作为中间件
     """

funboost/publishers/redis_queue_flush.py

@@ -0,0 +1,11 @@
+class FlushRedisQueueMixin:
+    # noinspection PyUnresolvedReferences
+    def clear(self):
+        self.redis_db_frame.delete(self._queue_name)
+        self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功')
+        # self.redis_db_frame.delete(f'{self._queue_name}__unack')
+        unack_queue_name_list = self.redis_db_frame.scan(match=f'{self._queue_name}__unack_id_*', count=100000)[1] + \
+                                self.redis_db_frame.scan(match=f'unack_{self._queue_name}_*', count=100000)[1]  # noqa
+        if unack_queue_name_list:
+            self.redis_db_frame.delete(*unack_queue_name_list)
+            self.logger.warning(f'清除 {unack_queue_name_list} 队列中的消息成功')

funboost/utils/redis_manager.py

@@ -14,7 +14,7 @@ class RedisManager(object):
         if (host, port, db, password) not in self.__class__._pool_dict:
             # print ('创建一个连接池')
             self.__class__._pool_dict[(host, port, db, password)] = redis.ConnectionPool(host=host, port=port, db=db,
-                                                                                         password=password)
+                                                                                         password=password,max_connections=100)
         self._r = redis.Redis(connection_pool=self._pool_dict[(host, port, db, password)])
         self._ping()
 
@@ -41,7 +41,7 @@ class RedisMixin(object):
     @decorators.cached_method_result
     def redis_db0(self):
         return RedisManager(host=funboost_config_deafult.REDIS_HOST, port=funboost_config_deafult.REDIS_PORT,
-                            password=funboost_config_deafult.REDIS_PASSWORD, db=0).get_redis()
+                            password=funboost_config_deafult.REDIS_PASSWORD, db=0,).get_redis()
 
     @property
     @decorators.cached_method_result

{funboost-23.5.dist-info → funboost-23.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 23.5
+Version: 23.7
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，python函数加速器，框架包罗万象，一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是 方便 快速 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs
@@ -50,7 +50,7 @@ Requires-Dist: pikav1
 Requires-Dist: redis2
 Requires-Dist: redis3
 Requires-Dist: redis
-Requires-Dist: nb-log (>=8.5)
+Requires-Dist: nb-log (>=9.1)
 Requires-Dist: rocketmq
 Requires-Dist: zmq
 Requires-Dist: pyzmq
@@ -159,9 +159,11 @@ funboost 是 function_scheduling_distributed_framework的包名更新版本
 
 ## 1.1 安装方式
 
+```
 pip install funboost --upgrade
 
-
+或pip install funboost[extra_brokers]  一次性安装所有小众三方中间件
+```
 
 ## 1.2 框架功能介绍
 
@@ -331,6 +333,12 @@ python通用分布式函数调度框架。适用场景范围广泛， 框架非
      暂停消费：
         支持从python解释器外部/远程机器 ，控制暂停消息消费和继续消费。
 
+     优先级队列：
+         支持队列优先级消息。
+
+     远程杀死任务：
+         支持在发布端杀死正在运行的消息，发送杀死命令时候对还未取出的消息则放弃运行消息。
+
 </pre>
 
 

{funboost-23.5.dist-info → funboost-23.7.dist-info}/RECORD

@@ -1,9 +1,9 @@
-funboost/__init__.py,sha256=eBTjiQBcy0DoDZiOdcRJpgb5RHwv8d1ekgVwDkhoHoc,2402
+funboost/__init__.py,sha256=YKM0VVSXRcZJnQbuqkIjS3PnLJxxIjw1-1JjI13Nn3A,2401
 funboost/__init__old.py,sha256=cSzw-ZQqtAAp5_-7eONXQT5fwz_JbZlRjDQPASDLUHk,20378
 funboost/constant.py,sha256=tSdHQ8nrZH63jOKKZFtOfTo2melCsiMrk-O1lFcsIKA,6489
-funboost/funboost_config_deafult.py,sha256=xpmiZBCVylJe4pC4fG2J-LZZ0c0QJsYXOM1YxQMcMtw,7399
+funboost/funboost_config_deafult.py,sha256=mh9o91CgMC26glIWkw7489GxmJ7bibi4_oBMnv3SCNU,7430
 funboost/set_frame_config.py,sha256=hz_38C-IXEulYpHQdr1fhsMcdEDCHIsF2la8MkHiIjA,9149
-funboost/assist/celery_helper.py,sha256=YvKP21xUzgazZHeFGq4FTgtd0A1g3BGNWzKLQ3gNygc,4186
+funboost/assist/celery_helper.py,sha256=ZJ8jNSC-D8fE6pjxMVGAAA4j9i0gA1kVH14_BbTkXaQ,4569
 funboost/assist/dramatiq_helper.py,sha256=lRNouO8MyCB_Qj2ppYG4FbMpf-2Aok8QhtGZLH1zWkg,2089
 funboost/assist/huey_helper.py,sha256=VEzMdQDVJJ-ujcOXKw7chrTgvieLz4si3hrjt8gIK38,1760
 funboost/assist/rq_helper.py,sha256=-tUZ00FzkczwBAkbbISPHF-8J0K7HLSZsue39WiF-cM,1509
@@ -26,8 +26,8 @@ funboost/concurrent_pool/backup/async_pool_executor0223.py,sha256=iTxxJFk2lu1P9Z
 funboost/concurrent_pool/backup/async_pool_executor_back.py,sha256=vIgUUyF4Zb0jIRPWgNPqyO09YEkQP32kkpGBldqm4qA,9568
 funboost/concurrent_pool/backup/async_pool_executor_janus.py,sha256=OHMWJ9l3EYTpPpcrPrGGKd4K0tmQ2PN8HiX0Dta0EOo,5728
 funboost/consumers/__init__.py,sha256=ZXY_6Kut1VYNQiF5aWEgIWobsW1ht9YUP0TdRZRWFqI,126
-funboost/consumers/base_consumer.py,sha256=2wZW7m4i63xf9zfXvsGIp26khTAjMod9-K9828LvbC8,85044
-funboost/consumers/celery_consumer.py,sha256=6CqorZH5pbxIjVwn5gNzcxSbos5YWT8eYqxYyYjgUcY,7574
+funboost/consumers/base_consumer.py,sha256=8goq0tx2db0ErfMeuKuNe62K5SOI-4Io58_7-xs2-X8,86846
+funboost/consumers/celery_consumer.py,sha256=l0UQbO_KmvhCLtMtUgoHxHtEqyBCIABykyVKj04lZPM,8815
 funboost/consumers/celery_consumer000.py,sha256=8SF8ppHIMH-BIAHO0NyJYnSQxe_PcT6hv05e7DySG54,4574
 funboost/consumers/confirm_mixin.py,sha256=oVBQ1RayIBFOzGNTJ69HdBR_kLd46xd0VfCOKfqDpLA,6033
 funboost/consumers/dramatiq_consumer.py,sha256=kiHM1wpSZykYDomtSGZ2PlMInCDn_8GOGEHqHUD9m0o,2144
@@ -47,14 +47,14 @@ funboost/consumers/nsq_consumer.py,sha256=PaMRsP8oeRg0H_tq4FL7YhNUdOWAqJkjrwZWoY
 funboost/consumers/peewee_conusmer.py,sha256=fbspeWbv6F3EsIIBAjkM_FNh2vMyrDsydju23WAVHvE,1238
 funboost/consumers/persist_queue_consumer.py,sha256=8HzRcMFE6OKiF_CL3atC42Ien_yf5daG3LU38YBKWi0,1005
 funboost/consumers/pulsar_consumer.py,sha256=2DuklBV5dSY-_gW2EpRWz8QSy-VjZclj0gJUvLTyJHA,2414
-funboost/consumers/rabbitmq_amqpstorm_consumer.py,sha256=Qg1Uv4Tk6Tbu9szys6P4U_5BpsOgZuO_kdiR6lb1MlQ,2048
+funboost/consumers/rabbitmq_amqpstorm_consumer.py,sha256=JJnSvewyWJDjAZmBi23r9PYUkt01knHfpRFjpmasjVw,2049
 funboost/consumers/rabbitmq_pika_consumer.py,sha256=9L7CA2wYT-RNpaVfLKrFk6UJtONEIlfuDZYCjxHDOtc,5433
 funboost/consumers/rabbitmq_pika_consumerv0.py,sha256=Yr-GJlgtkYB4qx8hKZZRx8PyCGwKAlSvRFKEf-SBXnM,4674
 funboost/consumers/rabbitmq_rabbitpy_consumer.py,sha256=q-DXy2MHgsFfBssVIlqgziFw2jPkxAT4TyeDnlE0zwI,1260
 funboost/consumers/redis_brpoplpush_consumer.py,sha256=rVdlmODLEQ8_k-gXFnaMFK8LaiKu3EiZMG2q5jmG8qk,3031
 funboost/consumers/redis_consumer.py,sha256=f9nASEQUR3VHk5JH5Plf3RqKXmxogfrshu63RaFnfDM,2829
-funboost/consumers/redis_consumer_ack_able.py,sha256=DlZd76FTPo-AUiTreQd7cSiT5QAM5K9rFItAwLt2n5Y,7545
-funboost/consumers/redis_consumer_priority.py,sha256=avxU89EZuGk-wQeWEpASBU7ustxAHp3KzQcq34wb2Fg,4282
+funboost/consumers/redis_consumer_ack_able.py,sha256=7bdG2282POZ_nbtMK2aAgYi43jNNXbw56uG0eUIII7M,7545
+funboost/consumers/redis_consumer_priority.py,sha256=2BnNvsbNPZCRHdRxrnh1C3mJlqTgb1ql7xRn4-cHULU,5827
 funboost/consumers/redis_consumer_simple.py,sha256=9D3uvLw_9WOmLmwys1K39DK3gj1ex_q6NXadXwyGwrs,922
 funboost/consumers/redis_filter.py,sha256=TVyT2i9JhmhsJFyQZDx98phTiwBccNTl9fcErEDGXTM,7135
 funboost/consumers/redis_pubsub_consumer.py,sha256=eSy5QBMPaouiQbeGQ3ZaLVpU1BF8g3B_4CAJHFqhmmI,1206
@@ -82,6 +82,7 @@ funboost/core/helper_funs.py,sha256=SKSMKytJTOFQJsuMKWJ-_mQg6e_Bgpq1ANDzjjBAcio,
 funboost/core/kill_remote_task.py,sha256=Rj4nrz13CKka8he1T8-k6CYN_yjvcaYZaZ7yurKcKzo,8159
 funboost/core/msg_result_getter.py,sha256=a2ffSE8Rvz1t1KVEt4UxIPsWgcriaHE_Armkac-JrJY,7782
 funboost/core/muliti_process_enhance.py,sha256=zNodv3Ww5yCmwO6xCh7k8HntFHjIYYJ9EPaGinvPV9Q,3814
+funboost/core/show_funboost_flag.py,sha256=EouUN_ioNXWaguA9qKQJKw1xYJqa1aLNxtd9SvL6g8A,5725
 funboost/factories/__init__.py,sha256=s7kKKjR1HU5eMjPD6r5b-SXTVMo1zBp2JjOAtkyt5Yo,178
 funboost/factories/broker_kind__publsiher_consumer_type_map.py,sha256=Ig1Kze8keS04_vXMNenZSDxibBSjxVrcMkDHviPsxnQ,8976
 funboost/factories/consumer_factory.py,sha256=KFvYkx1a7egtSiTLuVsoRMKLUAotx-QJX1jAI8Xzo3s,946
@@ -102,8 +103,8 @@ funboost/function_result_web/static/js/jquery-1.11.0.min.js,sha256=ryQZ3RXgnqkTz
 funboost/function_result_web/templates/index.html,sha256=dWe-JFQhsDpoNjSsBF4P6SJWp_KvHX8EP_yECS5r7_o,19501
 funboost/function_result_web/templates/login.html,sha256=q37dj7O0LeyiV38Zd5P1Qn_qmhjdFomuYTRY1Yk48Bo,2007
 funboost/publishers/__init__.py,sha256=xqBHlvsJQVPfbdvP84G0LHmVB7-pFBS7vDnX1Uo9pVY,131
-funboost/publishers/base_publisher.py,sha256=CU0xjCpt1tsQRcy6Orhuxd8bwsmIJ80wmMZw0JDHevY,15860
-funboost/publishers/celery_publisher.py,sha256=XCR-xvkHEQhbJSluuhrlMPv-V4RTlpA9Ot10mfe4a3E,3081
+funboost/publishers/base_publisher.py,sha256=uuchEFgHMUlY2fBtGwZFLAzeKiWa7_34UTocGADTt2c,15863
+funboost/publishers/celery_publisher.py,sha256=ELT-JR8RtmmCpsxYqVTuuFeKMFZMuK9ROKXWHkGVEjE,2334
 funboost/publishers/celery_publisher000.py,sha256=ag2s7MzPPrnNdza3u8vcbDJqtiqbQw4lJJzAVuJ6GF0,3897
 funboost/publishers/confluent_kafka_publisher.py,sha256=cpbyWvZ0T_kM62LWeBKRUuEuMkJAKOof97UUMSz6-Dk,3541
 funboost/publishers/dramatiq_publisher.py,sha256=RoZzfvkS5H-XXcmHGBomuvkQRQBlVyiaCdWpgy0LL9o,1410
@@ -124,11 +125,12 @@ funboost/publishers/pulsar_publisher.py,sha256=3BqDtywExvTIw1KZWG4kT1uz029uw2Ykn
 funboost/publishers/rabbitmq_amqpstorm_publisher.py,sha256=C32sQZpjv1iJtFDD6g_q0wI7Arw9i0RDsRVVOBRKzi4,3137
 funboost/publishers/rabbitmq_pika_publisher.py,sha256=jZZw3DUiZ4VqJ6FqZGdv7qo3F_ltzHuKsy_5-j4jkCs,2343
 funboost/publishers/rabbitmq_rabbitpy_publisher.py,sha256=GGXPKxE6-mAjqMIKqwvR9d7L-zuJQcQoU9uRsQLNGas,1953
-funboost/publishers/redis_publisher.py,sha256=mUHxfdHYrUn_Dw59NifN-mFsnj53hOhZDoEFlWs-W5M,3982
+funboost/publishers/redis_publisher.py,sha256=FUxvvYvf73J2Im33MRJd-5dmjnccjAFHaTMd4rLvdNM,3358
 funboost/publishers/redis_publisher_lpush.py,sha256=xEWuCTtbDcKFLTxGrECrbIVapFfUwqX2-GHenMClu-Q,278
-funboost/publishers/redis_publisher_priority.py,sha256=zGhSAvpFVBXfAmfGQdWL9cixdpLhLQ6yR3XvBLnbdF4,2104
-funboost/publishers/redis_publisher_simple.py,sha256=hLVWiDjy9ObGTmv7Vtrz21d18Fxkb52IfO5ZzaXjzMQ,872
-funboost/publishers/redis_pubsub_publisher.py,sha256=_6s7sbcGOSXxN2ZkBSDk9ILskmbj9cZTQyYHLQTYOuI,721
+funboost/publishers/redis_publisher_priority.py,sha256=YJCIMdiY0Sdttc7Xv85V-2P1R7ZoX5hqWz7waoibioQ,1972
+funboost/publishers/redis_publisher_simple.py,sha256=fzEurzFQxrTO6PlCONz4m1jQljwuN66spbj4tIJ3Y6c,740
+funboost/publishers/redis_pubsub_publisher.py,sha256=BxVn31I-Rt6gHFSTqntpsajmsl0ZftgMJMtiZd1LMyA,723
+funboost/publishers/redis_queue_flush.py,sha256=d-AgJiCuX25mOwP2gMorGbXCaLuFpqWO4jAJXJDBfk0,732
 funboost/publishers/redis_stream_publisher.py,sha256=DLxFcTlze0IdYFoaRmTcY8GCOaRwbj4aBNbylkt9gRA,2037
 funboost/publishers/rocketmq_publisher.py,sha256=vY82WgutDPsS9px6rukU1d3AexmsT_tqSS2dmX-Pw-c,2343
 funboost/publishers/rq_publisher.py,sha256=ANHrYnPTEj7KF_D_ov2L7PAT9swl82kjDazFqN1ULBY,893
@@ -158,7 +160,7 @@ funboost/utils/monkey_patches.py,sha256=Q0_jKxOfFrSgrIDSuSZFrgNh6w_LRGaKAITghrIp
 funboost/utils/mqtt_util.py,sha256=BfCmyYwI-B8VL9499_IuYlJDCbv6ZhwyWThMf8dANOU,3199
 funboost/utils/paramiko_util.py,sha256=pu67zkgptqNSV9m2Lznezb3zF1AFYvkWJp_6DVKFSPU,4901
 funboost/utils/rabbitmq_factory.py,sha256=NPzTwG-INZG9aJgkzp-QVk3TgV0rjiuTVT5lmRT77zg,2963
-funboost/utils/redis_manager.py,sha256=2Hg9UT-X6nS5TkZ-YrTRxRK-WPHZaLOsuvTxYuD1cCc,4628
+funboost/utils/redis_manager.py,sha256=0mkPxBFE4pBv22Zi9O0YN-txvlRHdpjpl0maW5M0QvM,4649
 funboost/utils/resource_monitoring.py,sha256=vf1htYa3a4wlMfQqksvIINMw8laiXwG5N8NXU2Zm3qQ,5532
 funboost/utils/restart_python.py,sha256=bFbV0_24ajL8hBwVRLxWe9v9kTwiX1fGLhXRroNnmgQ,1418
 funboost/utils/show_funboost_flag.py,sha256=YPXtykSkRBJZ4ulOIAXiTAfpOmo9IsjePz9G9AEE6cg,2985
@@ -213,8 +215,8 @@ funboost/utils/pysnooper_ydf/utils.py,sha256=evSmGi_Oul7vSP47AJ0DLjFwoCYCfunJZ1m
 funboost/utils/pysnooper_ydf/variables.py,sha256=QejRDESBA06KG9OH4sBT4J1M55eaU29EIHg8K_igaXo,3693
 funboost/utils/times/__init__.py,sha256=z-KQTxXapfu2ScJxISGe7QGffASuyJYL6JGsLXxD6O0,2332
 funboost/utils/times/version.py,sha256=JiZLGTB-HYyBOV4xS0rnx2K2OqVMTebUj30sZRbrleE,16
-funboost-23.5.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-funboost-23.5.dist-info/METADATA,sha256=vP7DG7GGXW7ktzISGDk4Vsgo-GkvlLpCatSrtc9z9uE,26615
-funboost-23.5.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-funboost-23.5.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
-funboost-23.5.dist-info/RECORD,,
+funboost-23.7.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+funboost-23.7.dist-info/METADATA,sha256=5MXcCOlQJKBFQo_e0-J6t0Wl3TeNhn2-yFfGl2AR4-I,26928
+funboost-23.7.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+funboost-23.7.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
+funboost-23.7.dist-info/RECORD,,