funboost 44.1__tar.gz → 44.3__tar.gz

{funboost-44.1/funboost.egg-info → funboost-44.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 44.1
+Version: 44.3
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs
@@ -27,6 +27,7 @@ Classifier: Topic :: Software Development :: Libraries
 Description-Content-Type: text/markdown
 Provides-Extra: all
 Provides-Extra: extra_brokers
+Provides-Extra: flask
 License-File: LICENSE
 
 

{funboost-44.1 → funboost-44.3}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "44.1"
+__version__ = "44.3"
 
 from funboost.set_frame_config import show_frame_config
 

{funboost-44.1 → funboost-44.3}/funboost/consumers/base_consumer.py

@@ -49,7 +49,7 @@ from funboost.concurrent_pool.single_thread_executor import SoloExecutor
 
 from funboost.core.function_result_status_saver import ResultPersistenceHelper, FunctionResultStatus, RunStatus
 
-from funboost.core.helper_funs import delete_keys_and_return_new_dict, get_publish_time, generate_task_id
+from funboost.core.helper_funs import delete_keys_and_return_new_dict, get_publish_time, MsgGenerater
 
 from funboost.concurrent_pool.async_helper import simple_run_in_executor
 from funboost.concurrent_pool.async_pool_executor import AsyncPoolExecutor
@@ -71,7 +71,7 @@ from funboost.core import kill_remote_task
 from funboost.core.exceptions import ExceptionForRequeue, ExceptionForPushToDlxqueue
 
 # from funboost.core.booster import BoostersManager  互相导入
-from funboost.core.lazy_impoter import lazy_impoter
+from funboost.core.lazy_impoter import funboost_lazy_impoter
 
 
 # patch_apscheduler_run_job()
@@ -174,6 +174,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         broker_exclusive_config_merge = dict()
         broker_exclusive_config_merge.update(self.BROKER_EXCLUSIVE_CONFIG_DEFAULT)
         broker_exclusive_config_merge.update(self.consumer_params.broker_exclusive_config)
+        # print(broker_exclusive_config_merge)
         self.consumer_params.broker_exclusive_config = broker_exclusive_config_merge
 
         self._stop_flag = None
@@ -222,7 +223,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         #                                         log_filename=consumer_params.log_filename,
         #                                         logger_name=consumer_params.logger_name,
         #                                         broker_exclusive_config=self.consumer_params.broker_exclusive_config)
-        self.publisher_params = BaseJsonAbleModel.init_by_another_model(PublisherParams, consumer_params)
+        self.publisher_params = BaseJsonAbleModel.init_by_another_model(PublisherParams, self.consumer_params)
         # print(self.publisher_params)
         if is_main_process:
             self.logger.info(f'{self.queue_name} consumer 的消费者配置:\n {self.consumer_params.json_str_value()}')
@@ -322,11 +323,11 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         # ConsumersManager.show_all_consumer_info()
         # noinspection PyBroadException
         pid_queue_name_tuple = (os.getpid(), self.queue_name)
-        if pid_queue_name_tuple in lazy_impoter.BoostersManager.pid_queue_name__has_start_consume_set:
+        if pid_queue_name_tuple in funboost_lazy_impoter.BoostersManager.pid_queue_name__has_start_consume_set:
             self.logger.warning(f'{pid_queue_name_tuple} 已启动消费,不要一直去启动消费,funboost框架自动阻止.')  # 有的人乱写代码,无数次在函数内部或for循环里面执行 f.consume(),一个队列只需要启动一次消费,不然每启动一次性能消耗很大,直到程序崩溃
             return
         else:
-            lazy_impoter.BoostersManager.pid_queue_name__has_start_consume_set.add(pid_queue_name_tuple)
+            funboost_lazy_impoter.BoostersManager.pid_queue_name__has_start_consume_set.add(pid_queue_name_tuple)
         GlobalVars.has_start_a_consumer_flag = True
         try:
             self._concurrent_mode_dispatcher.check_all_concurrent_mode()
@@ -373,7 +374,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         key = 'apscheduler.redisjobstore_runonce'
         if RedisMixin().redis_db_frame.sadd(key, runonce_uuid):  # 这样可以阻止多次启动同队列名消费者 redis jobstore多次运行函数.
             cls.logger_apscheduler.debug(f'延时任务用普通消息重新发布到普通队列 {msg}')
-            lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
+            funboost_lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
 
     @abc.abstractmethod
     def _shedual_task(self):
@@ -383,8 +384,11 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         """
         raise NotImplementedError
 
-    def _auto_fill_msg(self, msg: dict):
-        """填充消息,消息没有使用funboost来发送,并且没有extra相关字段时候"""
+    def convert_msg_before_run(self, msg: typing.Union[str,dict]):
+        """
+        转换消息,消息没有使用funboost来发送,并且没有extra相关字段时候
+        用户也可以按照4.21文档,继承任意Consumer类,并实现这个方法 convert_msg_before_run,先转换消息.
+        """
         """ 一般消息至少包含这样
         {
           "a": 42,
@@ -401,15 +405,19 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         extra_params = {'task_id': task_id, 'publish_time': round(time.time(), 4),
                         'publish_time_format': time.strftime('%Y-%m-%d %H:%M:%S')}
         """
+        if isinstance(msg,str):
+            msg = json.loads(msg)
+        # 以下是清洗补全字段.
         if 'extra' not in msg:
             msg['extra'] = {'is_auto_fill_extra': True}
         extra = msg['extra']
         if 'task_id' not in extra:
-            extra['task_id'] = generate_task_id(self._queue_name)
+            extra['task_id'] = MsgGenerater.generate_task_id(self._queue_name)
         if 'publish_time' not in extra:
-            extra['publish_time'] = round(time.time(), 4)
+            extra['publish_time'] = MsgGenerater.generate_publish_time()
         if 'publish_time_format':
-            extra['publish_time_format'] = time.strftime('%Y-%m-%d %H:%M:%S')
+            extra['publish_time_format'] = MsgGenerater.generate_publish_time_format()
+        return msg
 
     def _submit_task(self, kw):
         while 1:  # 这一块的代码为支持暂停消费。
@@ -421,12 +429,13 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                     self._last_show_pause_log_time = time.time()
             else:
                 break
-
+        msg = kw['body']
+        self._print_message_get_from_broker(msg)
+        kw['body'] = self.convert_msg_before_run(msg)
         if self._judge_is_daylight():
             self._requeue(kw)
             time.sleep(self.time_interval_for_check_do_not_run_time)
             return
-        self._auto_fill_msg(kw['body'])
         function_only_params = delete_keys_and_return_new_dict(kw['body'], )
         if self._get_priority_conf(kw, 'do_task_filtering') and self._redis_filter.check_value_exists(
                 function_only_params):  # 对函数的参数进行检查，过滤已经执行过并且成功的任务。
@@ -521,12 +530,13 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             if self._has_execute_times_in_recent_second >= qpsx:
                 time.sleep((1 - (time.time() - self._last_start_count_qps_timestamp)) * 1)
 
-    def _print_message_get_from_broker(self, broker_name, msg):
+    def _print_message_get_from_broker(self, msg,broker_name=None):
         # print(999)
         if self.consumer_params.is_show_message_get_from_broker:
             if isinstance(msg, (dict, list)):
                 msg = json.dumps(msg, ensure_ascii=False)
-            self.logger.debug(f'从 {broker_name} 中间件 的 {self._queue_name} 中取出的消息是 {msg}')
+            # self.logger.debug(f'从 {broker_name} 中间件 的 {self._queue_name} 中取出的消息是 {msg}')
+            self.logger.debug(f'从 {broker_name or self.consumer_params.broker_kind} 中间件 的 {self._queue_name} 中取出的消息是 {msg}')
 
     def _get_priority_conf(self, kw: dict, broker_task_config_key: str):
         broker_task_config = kw['body'].get('extra', {}).get(broker_task_config_key, None)

{funboost-44.1 → funboost-44.3}/funboost/consumers/http_consumer.py

@@ -4,11 +4,11 @@
 import asyncio
 import json
 
-from aiohttp import web
-from aiohttp.web_request import Request
+# from aiohttp import web
+# from aiohttp.web_request import Request
 
 from funboost.consumers.base_consumer import AbstractConsumer
-
+from funboost.core.lazy_impoter import AioHttpImporter
 
 class HTTPConsumer(AbstractConsumer, ):
     """
@@ -38,26 +38,26 @@ class HTTPConsumer(AbstractConsumer, ):
         #
         # flask_app.run('0.0.0.0', port=self._port,debug=False)
 
-        routes = web.RouteTableDef()
+        routes = AioHttpImporter().web.RouteTableDef()
 
         # noinspection PyUnusedLocal
         @routes.get('/')
         async def hello(request):
-            return web.Response(text="Hello, from funboost")
+            return AioHttpImporter().web.Response(text="Hello, from funboost")
 
         @routes.post('/queue')
-        async def recv_msg(request: Request):
+        async def recv_msg(request: AioHttpImporter().Request):
             data = await request.post()
             msg = data['msg']
-            kw = {'body': json.loads(msg)}
+            kw = {'body': msg}
             self._submit_task(kw)
-            return web.Response(text="finish")
+            return AioHttpImporter().web.Response(text="finish")
 
-        app = web.Application()
+        app = AioHttpImporter().web.Application()
         app.add_routes(routes)
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
-        web.run_app(app, host='0.0.0.0', port=self._port, )
+        AioHttpImporter().web.run_app(app, host='0.0.0.0', port=self._port, )
 
     def _confirm_consume(self, kw):
         pass  # 没有确认消费的功能。

{funboost-44.1 → funboost-44.3}/funboost/consumers/http_consumer000.py

@@ -60,8 +60,7 @@ class HttpHandler(BaseHTTPRequestHandler):
         if self.path == '/queue':
             msg = form['msg'].value
             # print(msg)
-            self.consumer._print_message_get_from_broker('http ', msg)
-            kw = {'body': json.loads(msg)}
+            kw = {'body': msg}
             self.consumer._submit_task(kw)
 
         # 开始回复

{funboost-44.1 → funboost-44.3}/funboost/consumers/httpsqs_consumer.py

@@ -25,7 +25,7 @@ class HttpsqsConsumer(AbstractConsumer):
             if text == 'HTTPSQS_GET_END':
                 time.sleep(0.5)
             else:
-                kw = {'body': json.loads(text)}
+                kw = {'body': text}
                 self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.1 → funboost-44.3}/funboost/consumers/kafka_consumer.py

@@ -3,16 +3,14 @@
 # @Time    : 2022/8/8 0008 13:32
 import json
 # noinspection PyPackageRequirements
-from kafka import KafkaConsumer as OfficialKafkaConsumer, KafkaProducer, KafkaAdminClient
-# noinspection PyPackageRequirements
-from kafka.admin import NewTopic
-# noinspection PyPackageRequirements
-from kafka.errors import TopicAlreadyExistsError
+
 from funboost.constant import BrokerEnum
 from funboost.consumers.base_consumer import AbstractConsumer
+from funboost.core.lazy_impoter import KafkaPythonImporter
 from funboost.funboost_config_deafult import BrokerConnConfig
 # from nb_log import get_logger
 from funboost.core.loggers import get_funboost_file_logger
+
 # LogManager('kafka').get_logger_and_add_handlers(30)
 get_funboost_file_logger('kafka', log_level_int=30)
 
@@ -36,18 +34,18 @@ class KafkaConsumer(AbstractConsumer):
 
     def _shedual_task(self):
         try:
-            admin_client = KafkaAdminClient(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
-            admin_client.create_topics([NewTopic(self._queue_name, 10, 1)])
+            admin_client = KafkaPythonImporter().KafkaAdminClient(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
+            admin_client.create_topics([KafkaPythonImporter().NewTopic(self._queue_name, 10, 1)])
             # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
-        except TopicAlreadyExistsError:
+        except KafkaPythonImporter().TopicAlreadyExistsError:
             pass
 
-        self._producer = KafkaProducer(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
-        consumer = OfficialKafkaConsumer(self._queue_name, bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS,
-                                         group_id=self.consumer_params.broker_exclusive_config["group_id"],
-                                         enable_auto_commit=True,
-                                         auto_offset_reset=self.consumer_params.broker_exclusive_config["auto_offset_reset"],
-                                         )
+        self._producer = KafkaPythonImporter().KafkaProducer(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
+        consumer = KafkaPythonImporter().OfficialKafkaConsumer(self._queue_name, bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS,
+                                                               group_id=self.consumer_params.broker_exclusive_config["group_id"],
+                                                               enable_auto_commit=True,
+                                                               auto_offset_reset=self.consumer_params.broker_exclusive_config["auto_offset_reset"],
+                                                               )
         #  auto_offset_reset (str): A policy for resetting offsets on
         #             OffsetOutOfRange errors: 'earliest' will move to the oldest
         #             available message, 'latest' will move to the most recent. Any
@@ -63,7 +61,7 @@ class KafkaConsumer(AbstractConsumer):
             if self.consumer_params.is_show_message_get_from_broker:
                 self.logger.debug(
                     f'从kafka的 [{message.topic}] 主题,分区 {message.partition} 中 取出的消息是：  {message.value.decode()}')
-            kw = {'consumer': consumer, 'message': message, 'body': json.loads(message.value)}
+            kw = {'consumer': consumer, 'message': message, 'body': message.value}
             self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.1 → funboost-44.3}/funboost/consumers/kafka_consumer_manually_commit.py

@@ -10,13 +10,10 @@ from collections import defaultdict, OrderedDict
 import time
 
 # noinspection PyPackageRequirements
-from kafka import KafkaProducer, KafkaAdminClient
+# pip install kafka-python==2.0.2
 
-# noinspection PyPackageRequirements
-from kafka.admin import NewTopic
-# noinspection PyPackageRequirements
-from kafka.errors import TopicAlreadyExistsError
 from funboost.consumers.base_consumer import AbstractConsumer
+from funboost.core.lazy_impoter import KafkaPythonImporter
 from funboost.funboost_config_deafult import BrokerConnConfig
 from confluent_kafka.cimpl import TopicPartition
 from confluent_kafka import Consumer as ConfluentConsumer  # 这个包在win下不好安装，用户用这个中间件的时候自己再想办法安装。win用户需要安装c++ 14.0以上环境。
@@ -39,13 +36,13 @@ class KafkaConsumerManuallyCommit(AbstractConsumer):
     def _shedual_task(self):
 
         try:
-            admin_client = KafkaAdminClient(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
-            admin_client.create_topics([NewTopic(self._queue_name, 10, 1)])
+            admin_client = KafkaPythonImporter().KafkaAdminClient(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
+            admin_client.create_topics([KafkaPythonImporter().NewTopic(self._queue_name, 10, 1)])
             # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
-        except TopicAlreadyExistsError:
+        except KafkaPythonImporter().TopicAlreadyExistsError:
             pass
 
-        self._producer = KafkaProducer(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
+        self._producer = KafkaPythonImporter().KafkaProducer(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
         # consumer 配置 https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
         self._confluent_consumer = ConfluentConsumer({
             'bootstrap.servers': ','.join(BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS),
@@ -69,7 +66,7 @@ class KafkaConsumerManuallyCommit(AbstractConsumer):
             # value()  offset() partition()
             # print('Received message: {}'.format(msg.value().decode('utf-8'))) # noqa
             self._partion__offset_consume_status_map[msg.partition()][msg.offset()] = 0
-            kw = {'partition': msg.partition(), 'offset': msg.offset(), 'body': json.loads(msg.value())}  # noqa
+            kw = {'partition': msg.partition(), 'offset': msg.offset(), 'body': msg.value()}  # noqa
             if self.consumer_params.is_show_message_get_from_broker:
                 self.logger.debug(
                     f'从kafka的 [{self._queue_name}] 主题,分区 {msg.partition()} 中 的 offset {msg.offset()} 取出的消息是：  {msg.value()}')  # noqa
@@ -130,14 +127,14 @@ class SaslPlainKafkaConsumer(KafkaConsumerManuallyCommit):
     def _shedual_task(self):
 
         try:
-            admin_client = KafkaAdminClient(
+            admin_client = KafkaPythonImporter().KafkaAdminClient(
                 **BrokerConnConfig.KFFKA_SASL_CONFIG)
-            admin_client.create_topics([NewTopic(self._queue_name, 10, 1)])
+            admin_client.create_topics([KafkaPythonImporter().NewTopic(self._queue_name, 10, 1)])
             # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
-        except TopicAlreadyExistsError:
+        except KafkaPythonImporter().TopicAlreadyExistsError:
             pass
 
-        self._producer = KafkaProducer(
+        self._producer = KafkaPythonImporter().KafkaProducer(
             **BrokerConnConfig.KFFKA_SASL_CONFIG)
         # consumer 配置 https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
         self._confluent_consumer = ConfluentConsumer({
@@ -168,7 +165,7 @@ class SaslPlainKafkaConsumer(KafkaConsumerManuallyCommit):
             # print('Received message: {}'.format(msg.value().decode('utf-8'))) # noqa
             self._partion__offset_consume_status_map[msg.partition(
             )][msg.offset()] = 0
-            kw = {'partition': msg.partition(), 'offset': msg.offset(), 'body': json.loads(msg.value())}  # noqa
+            kw = {'partition': msg.partition(), 'offset': msg.offset(), 'body': msg.value()}  # noqa
             if self.consumer_params.is_show_message_get_from_broker:
                 self.logger.debug(
                     f'从kafka的 [{self._queue_name}] 主题,分区 {msg.partition()} 中 的 offset {msg.offset()} 取出的消息是：  {msg.value()}')  # noqa

{funboost-44.1 → funboost-44.3}/funboost/consumers/kombu_consumer.py

@@ -194,7 +194,6 @@ Transport Options
 
         def callback(body: dict, message: Message):
             # print(type(body),body,type(message),message)
-            self._print_message_get_from_broker('kombu', body)
             # self.logger.debug(f""" 从 kombu {self._middware_name} 中取出的消息是 {body}""")
             kw = {'body': body, 'message': message, }
             self._submit_task(kw)

{funboost-44.1 → funboost-44.3}/funboost/consumers/local_python_queue_consumer.py

@@ -20,9 +20,8 @@ class LocalPythonQueueConsumer(AbstractConsumer):
     def _shedual_task(self):
         while True:
             task = self.local_python_queue.get()
-            if isinstance(task, str):
-                task = json.loads(task)
-            self._print_message_get_from_broker('当前python解释器内部', task)
+            if isinstance(task, dict):
+                task = json.dumps(task)
             # self.logger.debug(f'从当前python解释器内部的 [{self._queue_name}] 队列中 取出的消息是：  {json.dumps(task)}  ')
             kw = {'body': task}
             self._submit_task(kw)

{funboost-44.1 → funboost-44.3}/funboost/consumers/memory_deque_consumer.py

@@ -21,9 +21,6 @@ class LocalPythonQueueConsumer(AbstractConsumer):
     def _shedual_task(self):
         while True:
             task = self.local_python_queue.popleft()
-            if isinstance(task, str):
-                task = json.loads(task)
-            self._print_message_get_from_broker('当前python解释器内部', task)
             # self.logger.debug(f'从当前python解释器内部的 [{self._queue_name}] 队列中 取出的消息是：  {json.dumps(task)}  ')
             kw = {'body': task}
             self._submit_task(kw)

{funboost-44.1 → funboost-44.3}/funboost/consumers/mongomq_consumer.py

@@ -19,7 +19,6 @@ class MongoMqConsumer(AbstractConsumer, MongoMixin):
             job = mp.queue.next()
             if job is not None:
                 # self.logger.debug(f'从mongo的 [{self._queue_name}] 队列中 取出的消息是：   消息是：  {job.payload}  ')
-                self._print_message_get_from_broker('mongo', job.payload)
                 kw = {'body': job.payload, 'job': job}
                 self._submit_task(kw)
             else:

{funboost-44.1 → funboost-44.3}/funboost/consumers/mqtt_consumer.py

@@ -5,8 +5,9 @@ import json
 # import time
 from funboost.constant import BrokerEnum
 from funboost.consumers.base_consumer import AbstractConsumer
+from funboost.core.lazy_impoter import PahoMqttImporter
 from funboost.funboost_config_deafult import BrokerConnConfig
-import paho.mqtt.client as mqtt
+# import paho.mqtt.client as mqtt
 
 
 class MqttConsumer(AbstractConsumer):
@@ -23,7 +24,7 @@ class MqttConsumer(AbstractConsumer):
 
     # noinspection DuplicatedCode
     def _shedual_task(self):
-        client = mqtt.Client()
+        client = PahoMqttImporter().mqtt.Client()
         # client.username_pw_set('admin', password='public')
         client.on_connect = self._on_connect
         client.on_message = self._on_message
@@ -47,7 +48,7 @@ class MqttConsumer(AbstractConsumer):
     # noinspection PyUnusedLocal
     def _on_message(self, client, userdata, msg):
         # print(msg.topic + " " + str(msg.payload))
-        kw = {'body': json.loads(msg.payload)}
+        kw = {'body': msg.payload}
         self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.1 → funboost-44.3}/funboost/consumers/nats_consumer.py

@@ -1,7 +1,8 @@
 import json
-from pynats import NATSClient, NATSMessage  # noqa
-from funboost.constant import BrokerEnum
+# from pynats import NATSClient, NATSMessage  # noqa
+
 from funboost.consumers.base_consumer import AbstractConsumer
+from funboost.core.lazy_impoter import NatsImporter
 from funboost.funboost_config_deafult import BrokerConnConfig
 
 
@@ -13,14 +14,14 @@ class NatsConsumer(AbstractConsumer):
 
     def _shedual_task(self):
         # print(88888888888888)
-        nats_client = NATSClient(BrokerConnConfig.NATS_URL, socket_timeout=600, socket_keepalive=True)
+        nats_client = NatsImporter().NATSClient(BrokerConnConfig.NATS_URL, socket_timeout=600, socket_keepalive=True)
         nats_client.connect()
 
-        def callback(msg: NATSMessage):
+        def callback(msg: NatsImporter().NATSMessage):
             # print(type(msg))
             # print(msg.reply)
             # print(f"Received a message with subject {msg.subject}: {msg.payload}")
-            kw = {'body': json.loads(msg.payload)}
+            kw = {'body': msg.payload}
             self._submit_task(kw)
 
         nats_client.subscribe(subject=self.queue_name, callback=callback)

{funboost-44.1 → funboost-44.3}/funboost/consumers/nsq_consumer.py

@@ -2,8 +2,10 @@
 # @Author  : ydf
 # @Time    : 2022/8/8 0008 13:32
 import json
-from gnsq import Consumer, Message
-from funboost.constant import BrokerEnum
+
+from funboost.core.lazy_impoter import GnsqImporter
+# from gnsq import Consumer, Message
+
 from funboost.funboost_config_deafult import BrokerConnConfig
 from funboost.consumers.base_consumer import AbstractConsumer
 # from nb_log import LogManager
@@ -19,16 +21,15 @@ class NsqConsumer(AbstractConsumer):
 
 
     def _shedual_task(self):
-        consumer = Consumer(self._queue_name, 'frame_channel', BrokerConnConfig.NSQD_TCP_ADDRESSES,
+        consumer = GnsqImporter().Consumer(self._queue_name, 'frame_channel', BrokerConnConfig.NSQD_TCP_ADDRESSES,
                             max_in_flight=self.consumer_params.concurrent_num, heartbeat_interval=60, timeout=600, )  # heartbeat_interval 不能设置为600
 
         @consumer.on_message.connect
-        def handler(consumerx: Consumer, message: Message):
+        def handler(consumerx: GnsqImporter().Consumer, message: GnsqImporter().Message):
             # 第一条消息不能并发，第一条消息之后可以并发。
-            self._print_message_get_from_broker('nsq', message.body.decode())
             # self.logger.debug(f'从nsq的 [{self._queue_name}] 主题中 取出的消息是：  {message.body.decode()}')
             message.enable_async()
-            kw = {'consumer': consumerx, 'message': message, 'body': json.loads(message.body)}
+            kw = {'consumer': consumerx, 'message': message, 'body': message.body}
             self._submit_task(kw)
 
         consumer.start()

{funboost-44.1 → funboost-44.3}/funboost/consumers/peewee_conusmer.py

@@ -20,8 +20,7 @@ class PeeweeConsumer(AbstractConsumer):
             task_dict = self.queue.get()
             # print(task_dict)
             # self.logger.debug(f'从数据库 {frame_config.SQLACHEMY_ENGINE_URL[:25]}。。 的 [{self._queue_name}] 队列中 取出的消息是：   消息是：  {sqla_task_dict}')
-            self._print_message_get_from_broker(f'从数据库 {BrokerConnConfig.SQLACHEMY_ENGINE_URL[:25]}',task_dict)
-            kw = {'body':json.loads(task_dict['body']), 'job_id': task_dict['job_id']}
+            kw = {'body':task_dict['body'], 'job_id': task_dict['job_id']}
             self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.1 → funboost-44.3}/funboost/consumers/persist_queue_consumer.py

@@ -17,8 +17,7 @@ class PersistQueueConsumer(AbstractConsumer):
         while True:
             item = pub.queue.get()
             # self.logger.debug(f'从本地持久化sqlite的 [{self._queue_name}] 队列中 取出的消息是：   {item}  ')
-            self._print_message_get_from_broker('本地持久化sqlite', item)
-            kw = {'body': json.loads(item), 'q': pub.queue, 'item': item}
+            kw = {'body': item, 'q': pub.queue, 'item': item}
             self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.1 → funboost-44.3}/funboost/consumers/pulsar_consumer.py

@@ -53,8 +53,7 @@ class PulsarConsumer(AbstractConsumer, ):
         while True:
             msg = self._consumer.receive()
             if msg:
-                self._print_message_get_from_broker('pulsar', msg.data())
-                kw = {'body': json.loads(msg.data()), 'msg': msg}
+                kw = {'body': msg.data(), 'msg': msg}
                 self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.1 → funboost-44.3}/funboost/consumers/rabbitmq_amqpstorm_consumer.py

@@ -21,8 +21,6 @@ class RabbitmqConsumerAmqpStorm(AbstractConsumer):
         def callback(amqpstorm_message: amqpstorm.Message):
             body = amqpstorm_message.body
             # self.logger.debug(f'从rabbitmq的 [{self._queue_name}] 队列中 取出的消息是：  {body}')
-            self._print_message_get_from_broker('rabbitmq', body)
-            body = json.loads(body)
             kw = {'amqpstorm_message': amqpstorm_message, 'body': body}
             self._submit_task(kw)
 

{funboost-44.1 → funboost-44.3}/funboost/consumers/rabbitmq_pika_consumer.py

@@ -37,8 +37,6 @@ class RabbitmqConsumer(AbstractConsumer):
         def callback(ch, method, properties, body):
             body = body.decode()
             # self.logger.debug(f'从rabbitmq的 [{self._queue_name}] 队列中 取出的消息是：  {body}')
-            self._print_message_get_from_broker('rabbitmq', body)
-            body = json.loads(body)
             kw = {'ch': ch, 'method': method, 'properties': properties, 'body': body}
             self._submit_task(kw)
 

{funboost-44.1 → funboost-44.3}/funboost/consumers/rabbitmq_pika_consumerv0.py

@@ -34,7 +34,6 @@ class RabbitmqConsumer(AbstractConsumer):
         def callback(ch, method, properties, body):
             body = body.decode()
             self.logger.debug(f'从rabbitmq的 [{self._queue_name}] 队列中 取出的消息是：  {body}')
-            body = json.loads(body)
             kw = {'ch': ch, 'method': method, 'properties': properties, 'body': body}
             self._submit_task(kw)
 

{funboost-44.1 → funboost-44.3}/funboost/consumers/rabbitmq_rabbitpy_consumer.py

@@ -23,8 +23,7 @@ class RabbitmqConsumerRabbitpy(AbstractConsumer):
         for message in channel.basic_consume(self._queue_name, no_ack=False):
             body = message.body.decode()
             # self.logger.debug(f'从rabbitmq {self._queue_name} 队列中 取出的消息是：  {body}')
-            self._print_message_get_from_broker('rabbitmq', body)
-            kw = {'message': message, 'body': json.loads(message.body.decode())}
+            kw = {'message': message, 'body': body}
             self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.1 → funboost-44.3}/funboost/consumers/redis_brpoplpush_consumer.py

@@ -27,9 +27,7 @@ class RedisBrpopLpushConsumer(AbstractConsumer, RedisMixin):
         while True:
             msg = self.redis_db_frame.brpoplpush(self._queue_name, unack_list_name, timeout=60)
             if msg:
-                self._print_message_get_from_broker('redis', msg)
-                task_dict = json.loads(msg)
-                kw = {'body': task_dict, 'raw_msg': msg}
+                kw = {'body': msg, 'raw_msg': msg}
                 self._submit_task(kw)
 
     def _confirm_consume(self, kw):

{funboost-44.1 → funboost-44.3}/funboost/consumers/redis_consumer.py

@@ -34,17 +34,15 @@ class RedisConsumer(AbstractConsumer, RedisMixin):
                 task_str_list = p.execute()[0]
             if task_str_list:
                 # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：  {task_str_list}  ')
-                self._print_message_get_from_broker('redis', task_str_list)
+                self._print_message_get_from_broker( task_str_list)
                 for task_str in task_str_list:
-                    kw = {'body': json.loads(task_str)}
+                    kw = {'body': task_str}
                     self._submit_task(kw)
             else:
                 result = self.redis_db_frame.brpop(self._queue_name, timeout=60)
                 if result:
                     # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：  {result[1].decode()}  ')
-                    self._print_message_get_from_broker('redis', result[1])
-                    task_dict = json.loads(result[1])
-                    kw = {'body': task_dict}
+                    kw = {'body': result[1]}
                     self._submit_task(kw)
 
     def _shedual_task00(self):
@@ -52,9 +50,7 @@ class RedisConsumer(AbstractConsumer, RedisMixin):
             result = self.redis_db_frame.blpop(self._queue_name, timeout=60)
             if result:
                 # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：  {result[1].decode()}  ')
-                self._print_message_get_from_broker('redis', result[1])
-                task_dict = json.loads(result[1])
-                kw = {'body': task_dict}
+                kw = {'body': result[1]}
                 self._submit_task(kw)
 
     def _confirm_consume(self, kw):