funboost 44.2__tar.gz → 44.4__tar.gz

{funboost-44.2/funboost.egg-info → funboost-44.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 44.2
+Version: 44.4
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs
@@ -54,7 +54,7 @@ pip install funboost ,python全功能分布式函数调度框架,。  用法例
 python函数加速器，框架包罗万象，一统编程思维，兼容50% python编程业务场景，适用范围广。
 只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是 方便 快速 强大。
 python万能分布式函数调度框架，支持5种并发模式，30+种消息队列中间件(或任务队列框架)，
-20种任务控制功能。给任意python函数赋能。
+30种任务控制功能。给任意python函数赋能。
 用途概念就是常规经典的 生产者 + 消息队列中间件 + 消费者 编程思想。
 框架只需要学习@boost这一个装饰器的入参就可以，所有用法几乎和1.3例子一摸一样，非常简化简单。
 </pre>
@@ -136,7 +136,7 @@ pip install funboost --upgrade
 
 ## 1.2 框架功能介绍
 
-分布式函数调度框架，支持5种并发模式，20+种消息中间件，20种任务控制功能。<br>
+分布式函数调度框架，支持5种并发模式，20+种消息中间件，30种任务控制功能。<br>
 用途概念就是常规经典的 生产者 + 消息队列中间件 + 消费者 编程思想。
 
 有了这个框架，用户再也无需亲自手写操作进程、线程、协程的并发的代码了。
@@ -426,6 +426,7 @@ if __name__ == "__main__":
 
 <a href="https://imgse.com/i/pkFkCUe"><img src="https://s21.ax1x.com/2024/04/29/pkFkCUe.png" alt="pkFkCUe.png" border="0" /></a>
 
+<a href="https://imgse.com/i/pkE6IYR"><img src="https://s21.ax1x.com/2024/05/07/pkE6IYR.png" alt="pkE6IYR.png" border="0" /></a>
 
 ## 1.4  python分布式函数执行为什么重要？
 

{funboost-44.2 → funboost-44.4}/README.md

@@ -22,7 +22,7 @@ pip install funboost ,python全功能分布式函数调度框架,。  用法例
 python函数加速器，框架包罗万象，一统编程思维，兼容50% python编程业务场景，适用范围广。
 只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是 方便 快速 强大。
 python万能分布式函数调度框架，支持5种并发模式，30+种消息队列中间件(或任务队列框架)，
-20种任务控制功能。给任意python函数赋能。
+30种任务控制功能。给任意python函数赋能。
 用途概念就是常规经典的 生产者 + 消息队列中间件 + 消费者 编程思想。
 框架只需要学习@boost这一个装饰器的入参就可以，所有用法几乎和1.3例子一摸一样，非常简化简单。
 </pre>
@@ -104,7 +104,7 @@ pip install funboost --upgrade
 
 ## 1.2 框架功能介绍
 
-分布式函数调度框架，支持5种并发模式，20+种消息中间件，20种任务控制功能。<br>
+分布式函数调度框架，支持5种并发模式，20+种消息中间件，30种任务控制功能。<br>
 用途概念就是常规经典的 生产者 + 消息队列中间件 + 消费者 编程思想。
 
 有了这个框架，用户再也无需亲自手写操作进程、线程、协程的并发的代码了。
@@ -394,6 +394,7 @@ if __name__ == "__main__":
 
 <a href="https://imgse.com/i/pkFkCUe"><img src="https://s21.ax1x.com/2024/04/29/pkFkCUe.png" alt="pkFkCUe.png" border="0" /></a>
 
+<a href="https://imgse.com/i/pkE6IYR"><img src="https://s21.ax1x.com/2024/05/07/pkE6IYR.png" alt="pkE6IYR.png" border="0" /></a>
 
 ## 1.4  python分布式函数执行为什么重要？
 

{funboost-44.2 → funboost-44.4}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "44.2"
+__version__ = "44.4"
 
 from funboost.set_frame_config import show_frame_config
 

{funboost-44.2 → funboost-44.4}/funboost/concurrent_pool/async_helper.py

@@ -2,9 +2,11 @@ from functools import partial
 import asyncio
 from concurrent.futures import Executor
 from funboost.concurrent_pool.custom_threadpool_executor import ThreadPoolExecutorShrinkAble
+# from funboost.concurrent_pool.flexible_thread_pool import FlexibleThreadPool
 
 # 没有使用内置的concurrent.futures.ThreadpoolExecutor线程池，而是使用智能伸缩线程池。
-async_executor_default = ThreadPoolExecutorShrinkAble()
+async_executor_default = ThreadPoolExecutorShrinkAble(500)
+# async_executor_default = FlexibleThreadPool(50)  # 这个不支持future特性
 
 
 async def simple_run_in_executor(f, *args, async_executor: Executor = None, async_loop=None, **kwargs):

{funboost-44.2 → funboost-44.4}/funboost/consumers/base_consumer.py

@@ -30,13 +30,12 @@ from threading import Lock
 import asyncio
 
 import nb_log
-from funboost.core.current_task import funboost_current_task, get_current_taskid
+from funboost.core.current_task import funboost_current_task
 from funboost.core.loggers import develop_logger
 
 from funboost.core.func_params_model import BoosterParams, PublisherParams, BaseJsonAbleModel
 from funboost.core.task_id_logger import TaskIdLogger
-from nb_log import (get_logger, LoggerLevelSetterMixin, LogManager, CompatibleLogger,
-                    LoggerMixinDefaultWithFileHandler, stdout_write, is_main_process,
+from nb_log import (get_logger, LoggerLevelSetterMixin, LogManager,  is_main_process,
                     nb_log_config_default)
 from funboost.core.loggers import FunboostFileLoggerMixin, logger_prompt
 
@@ -71,7 +70,7 @@ from funboost.core import kill_remote_task
 from funboost.core.exceptions import ExceptionForRequeue, ExceptionForPushToDlxqueue
 
 # from funboost.core.booster import BoostersManager  互相导入
-from funboost.core.lazy_impoter import lazy_impoter
+from funboost.core.lazy_impoter import funboost_lazy_impoter
 
 
 # patch_apscheduler_run_job()
@@ -174,6 +173,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         broker_exclusive_config_merge = dict()
         broker_exclusive_config_merge.update(self.BROKER_EXCLUSIVE_CONFIG_DEFAULT)
         broker_exclusive_config_merge.update(self.consumer_params.broker_exclusive_config)
+        # print(broker_exclusive_config_merge)
         self.consumer_params.broker_exclusive_config = broker_exclusive_config_merge
 
         self._stop_flag = None
@@ -222,7 +222,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         #                                         log_filename=consumer_params.log_filename,
         #                                         logger_name=consumer_params.logger_name,
         #                                         broker_exclusive_config=self.consumer_params.broker_exclusive_config)
-        self.publisher_params = BaseJsonAbleModel.init_by_another_model(PublisherParams, consumer_params)
+        self.publisher_params = BaseJsonAbleModel.init_by_another_model(PublisherParams, self.consumer_params)
         # print(self.publisher_params)
         if is_main_process:
             self.logger.info(f'{self.queue_name} consumer 的消费者配置:\n {self.consumer_params.json_str_value()}')
@@ -322,16 +322,16 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         # ConsumersManager.show_all_consumer_info()
         # noinspection PyBroadException
         pid_queue_name_tuple = (os.getpid(), self.queue_name)
-        if pid_queue_name_tuple in lazy_impoter.BoostersManager.pid_queue_name__has_start_consume_set:
+        if pid_queue_name_tuple in funboost_lazy_impoter.BoostersManager.pid_queue_name__has_start_consume_set:
             self.logger.warning(f'{pid_queue_name_tuple} 已启动消费,不要一直去启动消费,funboost框架自动阻止.')  # 有的人乱写代码,无数次在函数内部或for循环里面执行 f.consume(),一个队列只需要启动一次消费,不然每启动一次性能消耗很大,直到程序崩溃
             return
         else:
-            lazy_impoter.BoostersManager.pid_queue_name__has_start_consume_set.add(pid_queue_name_tuple)
+            funboost_lazy_impoter.BoostersManager.pid_queue_name__has_start_consume_set.add(pid_queue_name_tuple)
         GlobalVars.has_start_a_consumer_flag = True
         try:
             self._concurrent_mode_dispatcher.check_all_concurrent_mode()
             self._check_monkey_patch()
-        except BaseException:
+        except BaseException:  # noqa
             traceback.print_exc()
             os._exit(4444)  # noqa
         self.logger.info(f'开始消费 {self._queue_name} 中的消息')
@@ -373,7 +373,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         key = 'apscheduler.redisjobstore_runonce'
         if RedisMixin().redis_db_frame.sadd(key, runonce_uuid):  # 这样可以阻止多次启动同队列名消费者 redis jobstore多次运行函数.
             cls.logger_apscheduler.debug(f'延时任务用普通消息重新发布到普通队列 {msg}')
-            lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
+            funboost_lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
 
     @abc.abstractmethod
     def _shedual_task(self):
@@ -383,10 +383,10 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         """
         raise NotImplementedError
 
-    def convert_msg_before_run(self, msg: dict):
+    def convert_msg_before_run(self, msg: typing.Union[str, dict]) -> dict:
         """
         转换消息,消息没有使用funboost来发送,并且没有extra相关字段时候
-        用户也可以按照4.21文档,继承任意Consumer类,并实现这个方法 convert_msg_before_run,先转换消息.
+        用户也可以按照4.21文档,继承任意Consumer类,并实现这个方法 convert_msg_before_run,先转换不规范的消息.
         """
         """ 一般消息至少包含这样
         {
@@ -404,6 +404,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         extra_params = {'task_id': task_id, 'publish_time': round(time.time(), 4),
                         'publish_time_format': time.strftime('%Y-%m-%d %H:%M:%S')}
         """
+        if isinstance(msg, str):
+            msg = json.loads(msg)
+        # 以下是清洗补全字段.
         if 'extra' not in msg:
             msg['extra'] = {'is_auto_fill_extra': True}
         extra = msg['extra']
@@ -413,6 +416,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             extra['publish_time'] = MsgGenerater.generate_publish_time()
         if 'publish_time_format':
             extra['publish_time_format'] = MsgGenerater.generate_publish_time_format()
+        return msg
 
     def _submit_task(self, kw):
         while 1:  # 这一块的代码为支持暂停消费。
@@ -424,12 +428,12 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                     self._last_show_pause_log_time = time.time()
             else:
                 break
-
+        self._print_message_get_from_broker(kw['body'])
+        kw['body'] = self.convert_msg_before_run(kw['body'])
         if self._judge_is_daylight():
             self._requeue(kw)
             time.sleep(self.time_interval_for_check_do_not_run_time)
             return
-        self.convert_msg_before_run(kw['body'])
         function_only_params = delete_keys_and_return_new_dict(kw['body'], )
         if self._get_priority_conf(kw, 'do_task_filtering') and self._redis_filter.check_value_exists(
                 function_only_params):  # 对函数的参数进行检查，过滤已经执行过并且成功的任务。
@@ -524,12 +528,13 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             if self._has_execute_times_in_recent_second >= qpsx:
                 time.sleep((1 - (time.time() - self._last_start_count_qps_timestamp)) * 1)
 
-    def _print_message_get_from_broker(self, broker_name, msg):
+    def _print_message_get_from_broker(self, msg, broker_name=None):
         # print(999)
         if self.consumer_params.is_show_message_get_from_broker:
             if isinstance(msg, (dict, list)):
                 msg = json.dumps(msg, ensure_ascii=False)
-            self.logger.debug(f'从 {broker_name} 中间件 的 {self._queue_name} 中取出的消息是 {msg}')
+            # self.logger.debug(f'从 {broker_name} 中间件 的 {self._queue_name} 中取出的消息是 {msg}')
+            self.logger.debug(f'从 {broker_name or self.consumer_params.broker_kind} 中间件 的 {self._queue_name} 中取出的消息是 {msg}')
 
     def _get_priority_conf(self, kw: dict, broker_task_config_key: str):
         broker_task_config = kw['body'].get('extra', {}).get(broker_task_config_key, None)
@@ -617,13 +622,15 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                     msg = f'{self._unit_time_for_count} 秒内执行了 {self._execute_task_times_every_unit_time} 次函数 [ {self.consuming_function.__name__} ] ,' \
                           f'函数平均运行耗时 {avarage_function_spend_time} 秒。 '
                     self.logger.info(msg)
-                    if self._msg_num_in_broker != -1 and time.time() - self._last_show_remaining_execution_time > self._show_remaining_execution_time_interval:  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
-                        # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
-                        need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /
-                                                                            self._distributed_consumer_statistics.active_consumer_num)
-                        msg += f''' 预计还需要 {need_time} 时间 才能执行完成 队列 {self.queue_name} 中的 {self._msg_num_in_broker} 个剩余任务'''
-                        self.logger.info(msg)
-                        self._last_show_remaining_execution_time = time.time()
+                    if  time.time() - self._last_show_remaining_execution_time > self._show_remaining_execution_time_interval:
+                        self._msg_num_in_broker = self.publisher_of_same_queue.get_message_count()
+                        if self._msg_num_in_broker != -1 :  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
+                            # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
+                            need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /
+                                                                                self._distributed_consumer_statistics.active_consumer_num)
+                            msg += f''' 预计还需要 {need_time} 时间 才能执行完成 队列 {self.queue_name} 中的 {self._msg_num_in_broker} 个剩余任务'''
+                            self.logger.info(msg)
+                            self._last_show_remaining_execution_time = time.time()
                     self._current_time_for_execute_task_times_every_unit_time = time.time()
                     self._consuming_function_cost_time_total_every_unit_time = 0
                     self._execute_task_times_every_unit_time = 0
@@ -823,7 +830,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 self.logger.critical(msg=log_msg)
                 # noinspection PyProtectedMember,PyUnresolvedReferences
                 os._exit(444)
-            if not self.consumer_params.function_timeout :
+            if not self.consumer_params.function_timeout:
                 rs = await corotinue_obj
                 # rs = await asyncio.wait_for(corotinue_obj, timeout=4)
             else:

{funboost-44.2 → funboost-44.4}/funboost/consumers/http_consumer.py

@@ -49,7 +49,7 @@ class HTTPConsumer(AbstractConsumer, ):
         async def recv_msg(request: AioHttpImporter().Request):
             data = await request.post()
             msg = data['msg']
-            kw = {'body': json.loads(msg)}
+            kw = {'body': msg}
             self._submit_task(kw)
             return AioHttpImporter().web.Response(text="finish")
 

{funboost-44.2 → funboost-44.4}/funboost/consumers/http_consumer000.py

@@ -60,8 +60,7 @@ class HttpHandler(BaseHTTPRequestHandler):
         if self.path == '/queue':
             msg = form['msg'].value
             # print(msg)
-            self.consumer._print_message_get_from_broker('http ', msg)
-            kw = {'body': json.loads(msg)}
+            kw = {'body': msg}
             self.consumer._submit_task(kw)
 
         # 开始回复

{funboost-44.2 → funboost-44.4}/funboost/consumers/httpsqs_consumer.py

@@ -25,7 +25,7 @@ class HttpsqsConsumer(AbstractConsumer):
             if text == 'HTTPSQS_GET_END':
                 time.sleep(0.5)
             else:
-                kw = {'body': json.loads(text)}
+                kw = {'body': text}
                 self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.2 → funboost-44.4}/funboost/consumers/kafka_consumer.py

@@ -61,7 +61,7 @@ class KafkaConsumer(AbstractConsumer):
             if self.consumer_params.is_show_message_get_from_broker:
                 self.logger.debug(
                     f'从kafka的 [{message.topic}] 主题,分区 {message.partition} 中 取出的消息是：  {message.value.decode()}')
-            kw = {'consumer': consumer, 'message': message, 'body': json.loads(message.value)}
+            kw = {'consumer': consumer, 'message': message, 'body': message.value}
             self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.2 → funboost-44.4}/funboost/consumers/kafka_consumer_manually_commit.py

@@ -66,7 +66,7 @@ class KafkaConsumerManuallyCommit(AbstractConsumer):
             # value()  offset() partition()
             # print('Received message: {}'.format(msg.value().decode('utf-8'))) # noqa
             self._partion__offset_consume_status_map[msg.partition()][msg.offset()] = 0
-            kw = {'partition': msg.partition(), 'offset': msg.offset(), 'body': json.loads(msg.value())}  # noqa
+            kw = {'partition': msg.partition(), 'offset': msg.offset(), 'body': msg.value()}  # noqa
             if self.consumer_params.is_show_message_get_from_broker:
                 self.logger.debug(
                     f'从kafka的 [{self._queue_name}] 主题,分区 {msg.partition()} 中 的 offset {msg.offset()} 取出的消息是：  {msg.value()}')  # noqa
@@ -165,7 +165,7 @@ class SaslPlainKafkaConsumer(KafkaConsumerManuallyCommit):
             # print('Received message: {}'.format(msg.value().decode('utf-8'))) # noqa
             self._partion__offset_consume_status_map[msg.partition(
             )][msg.offset()] = 0
-            kw = {'partition': msg.partition(), 'offset': msg.offset(), 'body': json.loads(msg.value())}  # noqa
+            kw = {'partition': msg.partition(), 'offset': msg.offset(), 'body': msg.value()}  # noqa
             if self.consumer_params.is_show_message_get_from_broker:
                 self.logger.debug(
                     f'从kafka的 [{self._queue_name}] 主题,分区 {msg.partition()} 中 的 offset {msg.offset()} 取出的消息是：  {msg.value()}')  # noqa

{funboost-44.2 → funboost-44.4}/funboost/consumers/kombu_consumer.py

@@ -194,7 +194,6 @@ Transport Options
 
         def callback(body: dict, message: Message):
             # print(type(body),body,type(message),message)
-            self._print_message_get_from_broker('kombu', body)
             # self.logger.debug(f""" 从 kombu {self._middware_name} 中取出的消息是 {body}""")
             kw = {'body': body, 'message': message, }
             self._submit_task(kw)

{funboost-44.2 → funboost-44.4}/funboost/consumers/local_python_queue_consumer.py

@@ -20,9 +20,8 @@ class LocalPythonQueueConsumer(AbstractConsumer):
     def _shedual_task(self):
         while True:
             task = self.local_python_queue.get()
-            if isinstance(task, str):
-                task = json.loads(task)
-            self._print_message_get_from_broker('当前python解释器内部', task)
+            if isinstance(task, dict):
+                task = json.dumps(task)
             # self.logger.debug(f'从当前python解释器内部的 [{self._queue_name}] 队列中 取出的消息是：  {json.dumps(task)}  ')
             kw = {'body': task}
             self._submit_task(kw)

{funboost-44.2 → funboost-44.4}/funboost/consumers/memory_deque_consumer.py

@@ -21,9 +21,6 @@ class LocalPythonQueueConsumer(AbstractConsumer):
     def _shedual_task(self):
         while True:
             task = self.local_python_queue.popleft()
-            if isinstance(task, str):
-                task = json.loads(task)
-            self._print_message_get_from_broker('当前python解释器内部', task)
             # self.logger.debug(f'从当前python解释器内部的 [{self._queue_name}] 队列中 取出的消息是：  {json.dumps(task)}  ')
             kw = {'body': task}
             self._submit_task(kw)

{funboost-44.2 → funboost-44.4}/funboost/consumers/mongomq_consumer.py

@@ -19,7 +19,6 @@ class MongoMqConsumer(AbstractConsumer, MongoMixin):
             job = mp.queue.next()
             if job is not None:
                 # self.logger.debug(f'从mongo的 [{self._queue_name}] 队列中 取出的消息是：   消息是：  {job.payload}  ')
-                self._print_message_get_from_broker('mongo', job.payload)
                 kw = {'body': job.payload, 'job': job}
                 self._submit_task(kw)
             else:

{funboost-44.2 → funboost-44.4}/funboost/consumers/mqtt_consumer.py

@@ -48,7 +48,7 @@ class MqttConsumer(AbstractConsumer):
     # noinspection PyUnusedLocal
     def _on_message(self, client, userdata, msg):
         # print(msg.topic + " " + str(msg.payload))
-        kw = {'body': json.loads(msg.payload)}
+        kw = {'body': msg.payload}
         self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.2 → funboost-44.4}/funboost/consumers/nats_consumer.py

@@ -21,7 +21,7 @@ class NatsConsumer(AbstractConsumer):
             # print(type(msg))
             # print(msg.reply)
             # print(f"Received a message with subject {msg.subject}: {msg.payload}")
-            kw = {'body': json.loads(msg.payload)}
+            kw = {'body': msg.payload}
             self._submit_task(kw)
 
         nats_client.subscribe(subject=self.queue_name, callback=callback)

{funboost-44.2 → funboost-44.4}/funboost/consumers/nsq_consumer.py

@@ -27,10 +27,9 @@ class NsqConsumer(AbstractConsumer):
         @consumer.on_message.connect
         def handler(consumerx: GnsqImporter().Consumer, message: GnsqImporter().Message):
             # 第一条消息不能并发，第一条消息之后可以并发。
-            self._print_message_get_from_broker('nsq', message.body.decode())
             # self.logger.debug(f'从nsq的 [{self._queue_name}] 主题中 取出的消息是：  {message.body.decode()}')
             message.enable_async()
-            kw = {'consumer': consumerx, 'message': message, 'body': json.loads(message.body)}
+            kw = {'consumer': consumerx, 'message': message, 'body': message.body}
             self._submit_task(kw)
 
         consumer.start()

{funboost-44.2 → funboost-44.4}/funboost/consumers/peewee_conusmer.py

@@ -20,8 +20,7 @@ class PeeweeConsumer(AbstractConsumer):
             task_dict = self.queue.get()
             # print(task_dict)
             # self.logger.debug(f'从数据库 {frame_config.SQLACHEMY_ENGINE_URL[:25]}。。 的 [{self._queue_name}] 队列中 取出的消息是：   消息是：  {sqla_task_dict}')
-            self._print_message_get_from_broker(f'从数据库 {BrokerConnConfig.SQLACHEMY_ENGINE_URL[:25]}',task_dict)
-            kw = {'body':json.loads(task_dict['body']), 'job_id': task_dict['job_id']}
+            kw = {'body':task_dict['body'], 'job_id': task_dict['job_id']}
             self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.2 → funboost-44.4}/funboost/consumers/persist_queue_consumer.py

@@ -17,8 +17,7 @@ class PersistQueueConsumer(AbstractConsumer):
         while True:
             item = pub.queue.get()
             # self.logger.debug(f'从本地持久化sqlite的 [{self._queue_name}] 队列中 取出的消息是：   {item}  ')
-            self._print_message_get_from_broker('本地持久化sqlite', item)
-            kw = {'body': json.loads(item), 'q': pub.queue, 'item': item}
+            kw = {'body': item, 'q': pub.queue, 'item': item}
             self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.2 → funboost-44.4}/funboost/consumers/pulsar_consumer.py

@@ -53,8 +53,7 @@ class PulsarConsumer(AbstractConsumer, ):
         while True:
             msg = self._consumer.receive()
             if msg:
-                self._print_message_get_from_broker('pulsar', msg.data())
-                kw = {'body': json.loads(msg.data()), 'msg': msg}
+                kw = {'body': msg.data(), 'msg': msg}
                 self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.2 → funboost-44.4}/funboost/consumers/rabbitmq_amqpstorm_consumer.py

@@ -21,8 +21,6 @@ class RabbitmqConsumerAmqpStorm(AbstractConsumer):
         def callback(amqpstorm_message: amqpstorm.Message):
             body = amqpstorm_message.body
             # self.logger.debug(f'从rabbitmq的 [{self._queue_name}] 队列中 取出的消息是：  {body}')
-            self._print_message_get_from_broker('rabbitmq', body)
-            body = json.loads(body)
             kw = {'amqpstorm_message': amqpstorm_message, 'body': body}
             self._submit_task(kw)
 

{funboost-44.2 → funboost-44.4}/funboost/consumers/rabbitmq_pika_consumer.py

@@ -37,8 +37,6 @@ class RabbitmqConsumer(AbstractConsumer):
         def callback(ch, method, properties, body):
             body = body.decode()
             # self.logger.debug(f'从rabbitmq的 [{self._queue_name}] 队列中 取出的消息是：  {body}')
-            self._print_message_get_from_broker('rabbitmq', body)
-            body = json.loads(body)
             kw = {'ch': ch, 'method': method, 'properties': properties, 'body': body}
             self._submit_task(kw)
 

{funboost-44.2 → funboost-44.4}/funboost/consumers/rabbitmq_pika_consumerv0.py

@@ -34,7 +34,6 @@ class RabbitmqConsumer(AbstractConsumer):
         def callback(ch, method, properties, body):
             body = body.decode()
             self.logger.debug(f'从rabbitmq的 [{self._queue_name}] 队列中 取出的消息是：  {body}')
-            body = json.loads(body)
             kw = {'ch': ch, 'method': method, 'properties': properties, 'body': body}
             self._submit_task(kw)
 

{funboost-44.2 → funboost-44.4}/funboost/consumers/rabbitmq_rabbitpy_consumer.py

@@ -23,8 +23,7 @@ class RabbitmqConsumerRabbitpy(AbstractConsumer):
         for message in channel.basic_consume(self._queue_name, no_ack=False):
             body = message.body.decode()
             # self.logger.debug(f'从rabbitmq {self._queue_name} 队列中 取出的消息是：  {body}')
-            self._print_message_get_from_broker('rabbitmq', body)
-            kw = {'message': message, 'body': json.loads(message.body.decode())}
+            kw = {'message': message, 'body': body}
             self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.2 → funboost-44.4}/funboost/consumers/redis_brpoplpush_consumer.py

@@ -27,9 +27,7 @@ class RedisBrpopLpushConsumer(AbstractConsumer, RedisMixin):
         while True:
             msg = self.redis_db_frame.brpoplpush(self._queue_name, unack_list_name, timeout=60)
             if msg:
-                self._print_message_get_from_broker('redis', msg)
-                task_dict = json.loads(msg)
-                kw = {'body': task_dict, 'raw_msg': msg}
+                kw = {'body': msg, 'raw_msg': msg}
                 self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.2 → funboost-44.4}/funboost/consumers/redis_consumer.py

@@ -34,17 +34,15 @@ class RedisConsumer(AbstractConsumer, RedisMixin):
                 task_str_list = p.execute()[0]
             if task_str_list:
                 # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：  {task_str_list}  ')
-                self._print_message_get_from_broker('redis', task_str_list)
+                self._print_message_get_from_broker( task_str_list)
                 for task_str in task_str_list:
-                    kw = {'body': json.loads(task_str)}
+                    kw = {'body': task_str}
                     self._submit_task(kw)
             else:
                 result = self.redis_db_frame.brpop(self._queue_name, timeout=60)
                 if result:
                     # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：  {result[1].decode()}  ')
-                    self._print_message_get_from_broker('redis', result[1])
-                    task_dict = json.loads(result[1])
-                    kw = {'body': task_dict}
+                    kw = {'body': result[1]}
                     self._submit_task(kw)
 
     def _shedual_task00(self):
@@ -52,9 +50,7 @@ class RedisConsumer(AbstractConsumer, RedisMixin):
             result = self.redis_db_frame.blpop(self._queue_name, timeout=60)
             if result:
                 # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：  {result[1].decode()}  ')
-                self._print_message_get_from_broker('redis', result[1])
-                task_dict = json.loads(result[1])
-                kw = {'body': task_dict}
+                kw = {'body': result[1]}
                 self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.2 → funboost-44.4}/funboost/consumers/redis_consumer_ack_able.py

@@ -32,9 +32,7 @@ class RedisConsumerAckAble000(ConsumerConfirmMixinWithTheHelpOfRedis, AbstractCo
                 # 如果运行了第20行，但没运行下面这一行，仍然有极小概率会丢失1个任务。但比不做控制随意关停，丢失几百个线程你的redis任务强多了。
                 self._add_task_str_to_unack_zset(task_str, )
                 # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：     {task_str}  ')
-                self._print_message_get_from_broker('reids', task_str)
-                task_dict = json.loads(task_str)
-                kw = {'body': task_dict, 'task_str': task_str}
+                kw = {'body': task_str, 'task_str': task_str}
                 self._submit_task(kw)
 
     def _requeue(self, kw):
@@ -74,8 +72,7 @@ class RedisConsumerAckAble111(ConsumerConfirmMixinWithTheHelpOfRedis, AbstractCo
             if return_v:
                 task_str = return_v
                 self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：     {task_str}  ')
-                task_dict = json.loads(task_str)
-                kw = {'body': task_dict, 'task_str': task_str}
+                kw = {'body': task_str, 'task_str': task_str}
                 self._submit_task(kw)
             else:
                 # print('xiuxi')
@@ -118,8 +115,7 @@ class RedisConsumerAckAble(ConsumerConfirmMixinWithTheHelpOfRedisByHearbeat, Abs
             if return_v:
                 task_str = return_v
                 self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：     {task_str}  ')
-                task_dict = json.loads(task_str)
-                kw = {'body': task_dict, 'task_str': task_str}
+                kw = {'body': task_str, 'task_str': task_str}
                 self._submit_task(kw)
             else:
                 # print('xiuxi')
@@ -149,11 +145,10 @@ class RedisConsumerAckAble(ConsumerConfirmMixinWithTheHelpOfRedisByHearbeat, Abs
         while True:
             task_str_list = script(keys=[self._queue_name, self._unack_zset_name], args=[time.time()])
             if task_str_list:
-                self._print_message_get_from_broker('redis', task_str_list)
+                self._print_message_get_from_broker( task_str_list)
                 # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：  {task_str_list}  ')
                 for task_str in task_str_list:
-                    task_dict = json.loads(task_str)
-                    kw = {'body': task_dict, 'task_str': task_str}
+                    kw = {'body': task_str, 'task_str': task_str}
                     self._submit_task(kw)
             else:
                 time.sleep(0.2)

{funboost-44.2 → funboost-44.4}/funboost/consumers/redis_consumer_priority.py

@@ -60,11 +60,9 @@ class RedisPriorityConsumer(RedisConsumerAckAble):
             if task_tuple:
                 msg = task_tuple[1]
                 self.redis_db_frame.zadd(self._unack_zset_name, {msg: time.time()})
-                self.logger.debug(task_tuple)
-                self._print_message_get_from_broker('redis', msg)
+                # self.logger.debug(task_tuple)
                 # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：  {task_str_list}  ')
-                task_dict = json.loads(msg)
-                kw = {'body': task_dict, 'task_str': msg}
+                kw = {'body': msg, 'task_str': msg}
                 self._submit_task(kw)
 
     def _shedual_task(self):
@@ -87,10 +85,8 @@ class RedisPriorityConsumer(RedisConsumerAckAble):
                 except redis5.WatchError:
                     continue
             if task_tuple:
-                self._print_message_get_from_broker('redis', msg)
                 # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：  {task_str_list}  ')
-                task_dict = json.loads(msg)
-                kw = {'body': task_dict, 'task_str': msg}
+                kw = {'body': msg, 'task_str': msg}
                 self._submit_task(kw)
 
     def _requeue(self, kw):

{funboost-44.2 → funboost-44.4}/funboost/consumers/redis_consumer_simple.py

@@ -17,9 +17,8 @@ class RedisConsumer(AbstractConsumer, RedisMixin):
         while True:
             result = self.redis_db_frame.blpop(self._queue_name,timeout=60)
             if result:
-                self._print_message_get_from_broker('redis',result[1])
-                task_dict = json.loads(result[1])
-                kw = {'body': task_dict}
+
+                kw = {'body': result[1]}
                 self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.2 → funboost-44.4}/funboost/consumers/redis_pubsub_consumer.py

@@ -18,8 +18,8 @@ class RedisPbSubConsumer(AbstractConsumer, RedisMixin):
         pub.subscribe(self.queue_name)
         for item in pub.listen():
             if item['type'] == 'message':
-                self._print_message_get_from_broker('reids', item['data'])
-                kw = {'body': json.loads(item['data'])}
+
+                kw = {'body': item['data']}
                 self._submit_task(kw)
 
     def _shedual_task(self):
@@ -29,7 +29,7 @@ class RedisPbSubConsumer(AbstractConsumer, RedisMixin):
         while True:  # 无限循环
             msg_list = pub.parse_response(timeout=60)  # 得到消息内容
             if msg_list:
-                kw = {'body': json.loads(msg_list[2])}
+                kw = {'body': msg_list[2]}
                 self._submit_task(kw)