funboost 45.3__tar.gz → 45.4__tar.gz

{funboost-45.3/funboost.egg-info → funboost-45.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 45.3
+Version: 45.4
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-45.3 → funboost-45.4}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "45.3"
+__version__ = "45.4"
 
 from funboost.set_frame_config import show_frame_config
 
@@ -65,5 +65,5 @@ from funboost.core.current_task import funboost_current_task
 # set_interrupt_signal_handler()
 
 # 有的包默认没加handlers，原始的日志不漂亮且不可跳转不知道哪里发生的。这里把warnning级别以上的日志默认加上handlers。
-# nb_log.get_logger(name='', log_level_int=30, _log_filename='pywarning.log')
+# nb_log.get_logger(name='', log_level_int=30, log_filename='pywarning.log')
 

funboost-45.4/funboost/assist/faststream_helper.py

@@ -0,0 +1,18 @@
+import asyncio
+
+from faststream import FastStream
+from faststream.rabbit import RabbitBroker
+from funboost.funboost_config_deafult import BrokerConnConfig
+from funboost.core.loggers import get_funboost_file_logger
+
+get_funboost_file_logger('faststream.access.rabbit')
+
+broker = RabbitBroker(BrokerConnConfig.RABBITMQ_URL, max_consumers=20)
+app = FastStream(broker)
+
+# asyncio.get_event_loop().run_until_complete(broker.connect())
+#
+# asyncio.get_event_loop().run_until_complete(broker.start())
+
+def get_broker(max_consumers=None):
+    return RabbitBroker(BrokerConnConfig.RABBITMQ_URL, max_consumers=max_consumers)

funboost-45.4/funboost/assist/taskiq_helper.py

@@ -0,0 +1 @@
+

{funboost-45.3 → funboost-45.4}/funboost/consumers/base_consumer.py

@@ -35,9 +35,10 @@ from funboost.core.current_task import funboost_current_task, FctContext
 from funboost.core.loggers import develop_logger
 
 from funboost.core.func_params_model import BoosterParams, PublisherParams, BaseJsonAbleModel
+from funboost.core.serialization import Serialization
 from funboost.core.task_id_logger import TaskIdLogger
 from funboost.constant import FunctionKind
-from funboost.utils.json_helper import JsonUtils
+
 from nb_libs.path_helper import PathHelper
 from nb_log import (get_logger, LoggerLevelSetterMixin, LogManager, is_main_process,
                     nb_log_config_default)
@@ -139,7 +140,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         if consumer_params.consuming_function is None:
             raise ValueError('必须传 consuming_function 参数')
 
-        self._msg_schedule_time_intercal = 0 if consumer_params.qps is None else 1.0 / consumer_params.qps
+        self._msg_schedule_time_intercal = 0 if consumer_params.qps in (None,0) else 1.0 / consumer_params.qps
 
         self._concurrent_mode_dispatcher = ConcurrentModeDispatcher(self)
         if consumer_params.concurrent_mode == ConcurrentModeEnum.ASYNC:
@@ -408,8 +409,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         extra_params = {'task_id': task_id, 'publish_time': round(time.time(), 4),
                         'publish_time_format': time.strftime('%Y-%m-%d %H:%M:%S')}
         """
-        if isinstance(msg, str):
-            msg = json.loads(msg)
+        msg = Serialization.to_dict(msg)
         # 以下是清洗补全字段.
         if 'extra' not in msg:
             msg['extra'] = {'is_auto_fill_extra': True}
@@ -536,7 +536,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         # print(999)
         if self.consumer_params.is_show_message_get_from_broker:
             # self.logger.debug(f'从 {broker_name} 中间件 的 {self._queue_name} 中取出的消息是 {msg}')
-            self.logger.debug(f'从 {broker_name or self.consumer_params.broker_kind} 中间件 的 {self._queue_name} 中取出的消息是 {JsonUtils.to_json_str(msg)}')
+            self.logger.debug(f'从 {broker_name or self.consumer_params.broker_kind} 中间件 的 {self._queue_name} 中取出的消息是 {Serialization.to_json_str(msg)}')
 
     def _get_priority_conf(self, kw: dict, broker_task_config_key: str):
         broker_task_config = kw['body'].get('extra', {}).get(broker_task_config_key, None)
@@ -585,7 +585,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             # method_cls = getattr(sys.modules[self.consumer_params.consuming_function_class_module],
             #                      self.consumer_params.consuming_function_class_name)
             if self.publisher_params.consuming_function_kind == FunctionKind.CLASS_METHOD:
-                method_cls = getattr(PathHelper(method_first_param_value[ConstStrForClassMethod.CLS_FILE]).import_as_module(),method_first_param_value[ConstStrForClassMethod.CLS_NAME])
+                method_cls = getattr(PathHelper(method_first_param_value[ConstStrForClassMethod.CLS_FILE]).import_as_module(), method_first_param_value[ConstStrForClassMethod.CLS_NAME])
                 real_function_only_params[method_first_param_name] = method_cls
             elif self.publisher_params.consuming_function_kind == FunctionKind.INSTANCE_METHOD:
                 method_cls = getattr(PathHelper(method_first_param_value[ConstStrForClassMethod.CLS_FILE]).import_as_module(), method_first_param_value[ConstStrForClassMethod.CLS_NAME])
@@ -642,7 +642,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                         # RedisMixin().redis_db_frame.expire(kw['body']['extra']['task_id'], 600)
                         current_function_result_status.rpc_result_expire_seconds = self.consumer_params.rpc_result_expire_seconds
                         p.lpush(kw['body']['extra']['task_id'],
-                                json.dumps(current_function_result_status.get_status_dict(without_datetime_obj=True)))
+                                Serialization.to_json_str(current_function_result_status.get_status_dict(without_datetime_obj=True)))
                         p.expire(kw['body']['extra']['task_id'], self.consumer_params.rpc_result_expire_seconds)
                         p.execute()
 
@@ -812,7 +812,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                     with RedisMixin().redis_db_filter_and_rpc_result.pipeline() as p:
                         current_function_result_status.rpc_result_expire_seconds = self.consumer_params.rpc_result_expire_seconds
                         p.lpush(kw['body']['extra']['task_id'],
-                                json.dumps(current_function_result_status.get_status_dict(without_datetime_obj=True)))
+                                Serialization.to_json_str(current_function_result_status.get_status_dict(without_datetime_obj=True)))
                         p.expire(kw['body']['extra']['task_id'], self.consumer_params.rpc_result_expire_seconds)
                         p.execute()
 
@@ -852,7 +852,6 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         fct = funboost_current_task()
         fct.set_fct_context(None)
 
-
     # noinspection PyProtectedMember
     async def _async_run_consuming_function_with_confirm_and_retry(self, kw: dict, current_retry_times,
                                                                    function_result_status: FunctionResultStatus, ):
@@ -1161,14 +1160,14 @@ class DistributedConsumerStatistics(RedisMixin, FunboostFileLoggerMixin):
         results = self.redis_db_frame.smembers(redis_key)
         with self.redis_db_frame.pipeline() as p:
             for result in results:
-                result_dict = json.loads(result)
+                result_dict = Serialization.to_dict(result)
                 if self.timestamp() - result_dict['hearbeat_timestamp'] > 15 \
                         or self._consumer_identification_map['consumer_uuid'] == result_dict['consumer_uuid']:
                     # 因为这个是10秒钟运行一次，15秒还没更新，那肯定是掉线了。如果消费者本身是自己也先删除。
                     p.srem(redis_key, result)
             self._consumer_identification_map['hearbeat_datetime_str'] = time_util.DatetimeConverter().datetime_str
             self._consumer_identification_map['hearbeat_timestamp'] = self.timestamp()
-            value = json.dumps(self._consumer_identification_map, sort_keys=True)
+            value = Serialization.to_json_str(self._consumer_identification_map, )
             p.sadd(redis_key, value)
             p.execute()
 

{funboost-45.3 → funboost-45.4}/funboost/consumers/confirm_mixin.py

@@ -5,7 +5,7 @@ import json
 import time
 from funboost.utils.redis_manager import RedisMixin
 from funboost.utils import decorators
-
+from funboost.core.serialization import Serialization
 """
 此模块是依赖redis的确认消费，所以比较复杂。
 """
@@ -46,7 +46,7 @@ class ConsumerConfirmMixinWithTheHelpOfRedis(RedisMixin):
                 time_max = time.time() - self.UNCONFIRMED_TIMEOUT
                 for value in self.redis_db_frame.zrangebyscore(self._unack_zset_name, 0, time_max):
                     self.logger.warning(f'向 {self._queue_name} 重新放入未消费确认的任务 {value}')
-                    self._requeue({'body': json.loads(value)})
+                    self._requeue({'body': Serialization.to_dict(value)})
                     self.redis_db_frame.zrem(self._unack_zset_name, value)
                 self.logger.info(f'{self._unack_zset_name} 中有待确认消费任务的数量是'
                                  f' {self.redis_db_frame.zcard(self._unack_zset_name)}')

funboost-45.4/funboost/consumers/faststream_consumer.py

@@ -0,0 +1,54 @@
+import asyncio
+import json
+import threading
+import time
+
+from funboost import EmptyConsumer
+from funboost.assist.faststream_helper import broker,app,get_broker
+from faststream import FastStream,Context
+from faststream.annotations import Logger
+
+from funboost.concurrent_pool.async_helper import simple_run_in_executor
+from funboost.core.helper_funs import delete_keys_and_return_new_dict
+
+
+class FastStreamConsumer(EmptyConsumer):
+    def custom_init(self):
+        self.broker = get_broker(max_consumers=self.consumer_params.concurrent_num)
+        subc = self.broker.subscriber(self.queue_name)
+        # @broker.subscriber(self.queue_name)
+        async def f(msg:str, logger: Logger,message=Context(),broker=Context(),context=Context(),):
+            self.logger.debug(f' 这条消息是 faststream 从 {self.queue_name} 队列中取出 ,是由 faststream 框架调度 {self.consuming_function.__name__} 函数处理,msg:{message} {context}')
+            # print(logger.name)
+            # return self.consuming_function(*args, **kwargs) # 如果没有声明 autoretry_for ，那么消费函数出错了就不会自动重试了。
+            # print(msg)
+            function_only_params = delete_keys_and_return_new_dict(json.loads(msg))
+            if self._consuming_function_is_asyncio:
+                result = await self.consuming_function(**function_only_params)
+            else:
+                result = await simple_run_in_executor(self.consuming_function,**function_only_params)
+            # print(result)
+            return result
+        subc(f)
+        self.faststream_subscriber = subc
+
+    def _shedual_task(self):
+        """ 完全由faststream框架接管控制消费，不使用funboost的AbstractConsumer的_run"""
+        while 1:
+            time.sleep(100)
+
+
+    def _confirm_consume(self, kw):
+        pass
+
+    def start_consuming_message(self):
+        def _f():
+            loop = asyncio.new_event_loop()
+            loop.run_until_complete(self.broker.connect())
+            loop.run_until_complete(self.faststream_subscriber.start())
+            loop.run_forever()
+        self.keep_circulating(10, block=False)(_f)()
+        # threading.Thread(target=_f).start()
+
+    def _requeue(self, kw):
+        pass

{funboost-45.3 → funboost-45.4}/funboost/consumers/redis_consumer.py

@@ -10,7 +10,7 @@ import time
 from funboost.constant import BrokerEnum
 from funboost.consumers.base_consumer import AbstractConsumer
 from funboost.utils.redis_manager import RedisMixin
-
+from funboost.core.serialization import Serialization
 
 class RedisConsumer(AbstractConsumer, RedisMixin):
     """
@@ -57,4 +57,4 @@ class RedisConsumer(AbstractConsumer, RedisMixin):
         pass  # redis没有确认消费的功能。
 
     def _requeue(self, kw):
-        self.redis_db_frame.rpush(self._queue_name, json.dumps(kw['body']))
+        self.redis_db_frame.rpush(self._queue_name,Serialization.to_json_str(kw['body']))

{funboost-45.3 → funboost-45.4}/funboost/consumers/redis_consumer_ack_using_timeout.py

@@ -4,8 +4,8 @@
 import json
 import time
 from funboost.consumers.base_consumer import AbstractConsumer
+from funboost.core.serialization import Serialization
 from funboost.utils.decorators import RedisDistributedLockContextManager
-from funboost.utils.json_helper import JsonUtils
 from funboost.utils.redis_manager import RedisMixin
 
 
@@ -40,7 +40,7 @@ class RedisConsumerAckUsingTimeout(AbstractConsumer, RedisMixin):
         self.redis_db_frame.zrem(self._unack_zset_name, kw['task_str'])
 
     def _requeue(self, kw):
-        self.redis_db_frame.rpush(self._queue_name, JsonUtils.to_json_str(kw['body']))
+        self.redis_db_frame.rpush(self._queue_name, Serialization.to_json_str(kw['body']))
 
     def _shedual_task(self):
         lua = '''

{funboost-45.3 → funboost-45.4}/funboost/consumers/redis_consumer_simple.py

@@ -4,6 +4,7 @@
 import json
 from funboost.constant import BrokerEnum
 from funboost.consumers.base_consumer import AbstractConsumer
+from funboost.core.serialization import Serialization
 from funboost.utils.redis_manager import RedisMixin
 
 
@@ -24,6 +25,6 @@ class RedisConsumer(AbstractConsumer, RedisMixin):
         pass  # redis没有确认消费的功能。
 
     def _requeue(self, kw):
-        self.redis_db_frame.rpush(self._queue_name, json.dumps(kw['body']))
+        self.redis_db_frame.rpush(self._queue_name, Serialization.to_json_str(kw['body']))
 
 

{funboost-45.3 → funboost-45.4}/funboost/consumers/redis_filter.py

@@ -11,6 +11,7 @@ import time
 from collections import OrderedDict
 import typing
 
+from funboost.core.serialization import Serialization
 from funboost.utils import  decorators
 from funboost.core.loggers import FunboostFileLoggerMixin
 
@@ -34,8 +35,7 @@ class RedisFilter(RedisMixin, FunboostFileLoggerMixin):
     @staticmethod
     def _get_ordered_str(value):
         """对json的键值对在redis中进行过滤，需要先把键值对排序，否则过滤会不准确如 {"a":1,"b":2} 和 {"b":2,"a":1}"""
-        if isinstance(value, str):
-            value = json.loads(value)
+        value = Serialization.to_dict(value)
         ordered_dict = OrderedDict()
         for k in sorted(value):
             ordered_dict[k] = value[k]

{funboost-45.3 → funboost-45.4}/funboost/core/function_result_status_saver.py

@@ -15,6 +15,7 @@ from pymongo import IndexModel, ReplaceOne
 
 from funboost.core.func_params_model import FunctionResultStatusPersistanceConfig
 from funboost.core.helper_funs import get_publish_time, delete_keys_and_return_new_dict
+from funboost.core.serialization import Serialization
 from funboost.utils import time_util, decorators
 from funboost.utils.mongo_util import MongoMixin
 # from nb_log import LoggerMixin
@@ -46,7 +47,7 @@ class FunctionResultStatus():
             self.publish_time_str = time_util.DatetimeConverter(publish_time).datetime_str
         function_params = delete_keys_and_return_new_dict(msg_dict, )
         self.params = function_params
-        self.params_str = json.dumps(function_params, ensure_ascii=False)
+        self.params_str = Serialization.to_json_str(function_params)
         self.result = None
         self.run_times = 0
         self.exception = None
@@ -78,7 +79,8 @@ class FunctionResultStatus():
         # item.pop('time_start')
         datetime_str = time_util.DatetimeConverter().datetime_str
         try:
-            json.dumps(item['result'])  # 不希望存不可json序列化的复杂类型。麻烦。存这种类型的结果是伪需求。
+            Serialization.to_json_str(item['result'])
+            # json.dumps(item['result'])  # 不希望存不可json序列化的复杂类型。麻烦。存这种类型的结果是伪需求。
         except TypeError:
             item['result'] = str(item['result'])[:1000]
         item.update({'insert_time_str': datetime_str,
@@ -98,7 +100,7 @@ class FunctionResultStatus():
         return item
 
     def __str__(self):
-        return f'''{self.__class__}   {json.dumps(self.get_status_dict(), ensure_ascii=False)}'''
+        return f'''{self.__class__}   {Serialization.to_json_str(self.get_status_dict())}'''
 
 
 class ResultPersistenceHelper(MongoMixin, FunboostFileLoggerMixin):

{funboost-45.3 → funboost-45.4}/funboost/core/msg_result_getter.py

@@ -10,7 +10,7 @@ from funboost.concurrent_pool import CustomThreadPoolExecutor
 from funboost.concurrent_pool.flexible_thread_pool import FlexibleThreadPoolMinWorkers0
 from funboost.utils.redis_manager import RedisMixin
 from funboost.utils.redis_manager import AioRedisMixin
-
+from funboost.core.serialization import Serialization
 
 class HasNotAsyncResult(Exception):
     pass
@@ -42,7 +42,7 @@ class AsyncResult(RedisMixin):
             self._has_pop = True
             if redis_value is not None:
                 status_and_result_str = redis_value[1]
-                self._status_and_result = json.loads(status_and_result_str)
+                self._status_and_result = Serialization.to_dict(status_and_result_str)
                 self.redis_db_filter_and_rpc_result.lpush(self.task_id, status_and_result_str)
                 self.redis_db_filter_and_rpc_result.expire(self.task_id, self._status_and_result['rpc_result_expire_seconds'])
                 return self._status_and_result
@@ -147,7 +147,7 @@ if __name__ == '__main__':
             self._has_pop = True
             if redis_value is not None:
                 status_and_result_str = redis_value[1]
-                self._status_and_result = json.loads(status_and_result_str)
+                self._status_and_result = Serialization.to_dict(status_and_result_str)
                 await self.aioredis_db_filter_and_rpc_result.lpush(self.task_id, status_and_result_str)
                 await self.aioredis_db_filter_and_rpc_result.expire(self.task_id, self._status_and_result['rpc_result_expire_seconds'])
                 return self._status_and_result

funboost-45.4/funboost/core/serialization.py

@@ -0,0 +1,16 @@
+import typing
+
+import orjson
+class Serialization:
+    @staticmethod
+    def to_json_str(dic:typing.Union[dict,str]):
+        if isinstance(dic,str):
+            return dic
+        str1 =orjson.dumps(dic)
+        return str1.decode('utf8')
+
+    @staticmethod
+    def to_dict(strx:typing.Union[str,dict]):
+        if isinstance(strx,dict):
+            return strx
+        return orjson.loads(strx)

{funboost-45.3 → funboost-45.4}/funboost/factories/broker_kind__publsiher_consumer_type_map.py

@@ -16,9 +16,9 @@ from funboost.publishers.local_python_queue_publisher import LocalPythonQueuePub
 from funboost.publishers.mongomq_publisher import MongoMqPublisher
 
 from funboost.publishers.persist_queue_publisher import PersistQueuePublisher
-from funboost.publishers.rabbitmq_amqpstorm_publisher import RabbitmqPublisherUsingAmqpStorm
+
 from funboost.publishers.rabbitmq_pika_publisher import RabbitmqPublisher
-from funboost.publishers.rabbitmq_rabbitpy_publisher import RabbitmqPublisherUsingRabbitpy
+
 from funboost.publishers.redis_publisher import RedisPublisher
 from funboost.publishers.rocketmq_publisher import RocketmqPublisher
 from funboost.publishers.redis_stream_publisher import RedisStreamPublisher
@@ -36,9 +36,8 @@ from funboost.consumers.nats_consumer import NatsConsumer
 
 from funboost.consumers.peewee_conusmer import PeeweeConsumer
 from funboost.consumers.persist_queue_consumer import PersistQueueConsumer
-from funboost.consumers.rabbitmq_amqpstorm_consumer import RabbitmqConsumerAmqpStorm
 from funboost.consumers.rabbitmq_pika_consumer import RabbitmqConsumer
-from funboost.consumers.rabbitmq_rabbitpy_consumer import RabbitmqConsumerRabbitpy
+
 from funboost.consumers.redis_brpoplpush_consumer import RedisBrpopLpushConsumer
 from funboost.consumers.redis_consumer import RedisConsumer
 from funboost.consumers.redis_consumer_ack_able import RedisConsumerAckAble
@@ -57,8 +56,8 @@ from funboost.consumers.base_consumer import AbstractConsumer
 from funboost.constant import BrokerEnum
 
 broker_kind__publsiher_consumer_type_map = {
-    BrokerEnum.RABBITMQ_AMQPSTORM: (RabbitmqPublisherUsingAmqpStorm, RabbitmqConsumerAmqpStorm),
-    BrokerEnum.RABBITMQ_RABBITPY: (RabbitmqPublisherUsingRabbitpy, RabbitmqConsumerRabbitpy),
+
+
     BrokerEnum.REDIS: (RedisPublisher, RedisConsumer),
     BrokerEnum.MEMORY_QUEUE: (LocalPythonQueuePublisher, LocalPythonQueueConsumer),
     BrokerEnum.RABBITMQ_PIKA: (RabbitmqPublisher, RabbitmqConsumer),
@@ -111,6 +110,15 @@ def regist_to_funboost(broker_kind: str):
     这样当用户需要使用某些三方包中间件作为消息队列时候，按照import报错信息，用户自己去pip安装好。或者 pip install funboost[all] 一次性安装所有中间件。
     建议按照 https://github.com/ydf0509/funboost/blob/master/setup.py 中的 extra_brokers 和 install_requires 里面的版本号来安装三方包版本.
     """
+    if broker_kind == BrokerEnum.RABBITMQ_AMQPSTORM:
+        from funboost.publishers.rabbitmq_amqpstorm_publisher import RabbitmqPublisherUsingAmqpStorm
+        from funboost.consumers.rabbitmq_amqpstorm_consumer import RabbitmqConsumerAmqpStorm
+        register_custom_broker(BrokerEnum.RABBITMQ_AMQPSTORM, RabbitmqPublisherUsingAmqpStorm, RabbitmqConsumerAmqpStorm)
+
+    if broker_kind == BrokerEnum.RABBITMQ_RABBITPY:
+        from funboost.publishers.rabbitmq_rabbitpy_publisher import RabbitmqPublisherUsingRabbitpy
+        from funboost.consumers.rabbitmq_rabbitpy_consumer import RabbitmqConsumerRabbitpy
+        register_custom_broker(BrokerEnum.RABBITMQ_RABBITPY, RabbitmqPublisherUsingRabbitpy, RabbitmqConsumerRabbitpy)
 
     if broker_kind == BrokerEnum.PULSAR:
         from funboost.consumers.pulsar_consumer import PulsarConsumer

{funboost-45.3 → funboost-45.4}/funboost/publishers/base_publisher.py

@@ -16,7 +16,7 @@ import time
 import typing
 from functools import wraps
 from threading import Lock
-import amqpstorm
+
 
 import nb_log
 from funboost.constant import ConstStrForClassMethod, FunctionKind
@@ -24,11 +24,12 @@ from funboost.core.func_params_model import PublisherParams, PriorityConsumingCo
 from funboost.core.helper_funs import MsgGenerater
 from funboost.core.loggers import develop_logger
 
-from pikav1.exceptions import AMQPError as PikaAMQPError
+
 
 # from nb_log import LoggerLevelSetterMixin, LoggerMixin
 from funboost.core.loggers import LoggerLevelSetterMixin, FunboostFileLoggerMixin, get_logger
 from funboost.core.msg_result_getter import AsyncResult, AioAsyncResult
+from funboost.core.serialization import Serialization
 from funboost.core.task_id_logger import TaskIdLogger
 from funboost.utils import decorators
 from funboost.funboost_config_deafult import BrokerConnConfig, FunboostCommonConfig
@@ -179,13 +180,13 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
 
     @staticmethod
     def _get_from_other_extra_params(k: str, msg):
-        msg_dict = json.loads(msg) if isinstance(msg, str) else msg
+        # msg_dict = json.loads(msg) if isinstance(msg, str) else msg
+        msg_dict = Serialization.to_dict(msg)
         return msg_dict['extra'].get('other_extra_params', {}).get(k, None)
 
     def _convert_msg(self, msg: typing.Union[str, dict], task_id=None,
-                     priority_control_config: PriorityConsumingControlConfig = None) -> (typing.Dict, typing.Dict, typing.Dict):
-        if isinstance(msg, (str, bytes)):
-            msg = json.loads(msg)
+                     priority_control_config: PriorityConsumingControlConfig = None) -> (typing.Dict, typing.Dict, typing.Dict,str):
+        msg = Serialization.to_dict(msg)
         msg_function_kw = copy.deepcopy(msg)
         raw_extra = {}
         if 'extra' in msg:
@@ -214,7 +215,7 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         msg, msg_function_kw, extra_params, task_id = self._convert_msg(msg, task_id, priority_control_config)
         t_start = time.time()
         decorators.handle_exception(retry_times=10, is_throw_error=True, time_sleep=0.1)(
-            self.concrete_realization_of_publish)(json.dumps(msg, ensure_ascii=False))
+            self.concrete_realization_of_publish)(Serialization.to_json_str(msg))
 
         self.logger.debug(f'向{self._queue_name} 队列，推送消息 耗时{round(time.time() - t_start, 4)}秒  {msg_function_kw}', extra={'task_id': task_id})  # 显示msg太长了。
         with self._lock_for_count:
@@ -228,10 +229,16 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
 
     def send_msg(self, msg: typing.Union[dict, str]):
         """直接发送任意消息内容到消息队列,不生成辅助参数,无视函数入参名字,不校验入参个数和键名"""
-        if isinstance(msg, dict):
-            msg = json.dumps(msg, ensure_ascii=False)
         decorators.handle_exception(retry_times=10, is_throw_error=True, time_sleep=0.1)(
-            self.concrete_realization_of_publish)(msg)
+            self.concrete_realization_of_publish)(Serialization.to_json_str(msg))
+
+    @staticmethod
+    def __get_cls_file(cls: type):
+        if cls.__module__ == '__main__':
+            cls_file = Path(sys.argv[0]).resolve().as_posix()
+        else:
+            cls_file = Path(sys.modules[cls.__module__].__file__).resolve().as_posix()
+        return cls_file
 
     def push(self, *func_args, **func_kwargs):
         """
@@ -260,9 +267,10 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             #                       )
             cls = func_args_list[0]
             # print(cls,cls.__name__, sys.modules[cls.__module__].__file__)
+
             func_args_list[0] = {ConstStrForClassMethod.FIRST_PARAM_NAME: self.publish_params_checker.all_arg_name[0],
                                  ConstStrForClassMethod.CLS_NAME: cls.__name__,
-                                 ConstStrForClassMethod.CLS_FILE: Path(sys.modules[cls.__module__].__file__).resolve().as_posix(),
+                                 ConstStrForClassMethod.CLS_FILE: self.__get_cls_file(cls),
                                  }
         elif self.publisher_params.consuming_function_kind == FunctionKind.INSTANCE_METHOD:
             obj = func_args[0]
@@ -271,7 +279,7 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 raise ValueError(f'消费函数 {self.publisher_params.consuming_function} 是实例方法，实例必须有 {ConstStrForClassMethod.OBJ_INIT_PARAMS} 属性')
             func_args_list[0] = {ConstStrForClassMethod.FIRST_PARAM_NAME: self.publish_params_checker.all_arg_name[0],
                                  ConstStrForClassMethod.CLS_NAME: cls.__name__,
-                                 ConstStrForClassMethod.CLS_FILE: Path(sys.modules[cls.__module__].__file__).resolve().as_posix(),
+                                 ConstStrForClassMethod.CLS_FILE: self.__get_cls_file(cls),
                                  ConstStrForClassMethod.OBJ_INIT_PARAMS: getattr(obj, ConstStrForClassMethod.OBJ_INIT_PARAMS),
 
                                  }
@@ -330,11 +338,14 @@ def deco_mq_conn_error(f):
             # noinspection PyBroadException
             try:
                 return f(self, *args, **kwargs)
-
-            except (PikaAMQPError, amqpstorm.AMQPError,) as e:  # except BaseException as e:   # 现在装饰器用到了绝大多出地方，单个异常类型不行。ex
-                self.logger.error(f'中间件链接出错   ,方法 {f.__name__}  出错 ，{e}')
-                self.init_broker()
-                return f(self, *args, **kwargs)
+            except Exception as e:
+                import amqpstorm
+                from pikav1.exceptions import AMQPError as PikaAMQPError
+                if isinstance(e,(PikaAMQPError, amqpstorm.AMQPError)):
+                # except (PikaAMQPError, amqpstorm.AMQPError,) as e:  # except BaseException as e:   # 现在装饰器用到了绝大多出地方，单个异常类型不行。ex
+                    self.logger.error(f'中间件链接出错   ,方法 {f.__name__}  出错 ，{e}')
+                    self.init_broker()
+                    return f(self, *args, **kwargs)
             except BaseException as e:
                 self.logger.critical(e, exc_info=True)
 

funboost-45.4/funboost/publishers/faststream_publisher.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+# @Author  : ydf
+# @Time    : 2023/8/8 0008 12:12
+
+import abc
+import asyncio
+import json
+import time
+import typing
+
+from funboost import PriorityConsumingControlConfig
+from funboost.core.serialization import Serialization
+from funboost.publishers.base_publisher import AbstractPublisher
+from funboost.assist.faststream_helper import app,get_broker
+from faststream import FastStream,Context
+from faststream.annotations import Logger
+
+class FastStreamPublisher(AbstractPublisher, metaclass=abc.ABCMeta):
+    """
+    空的发布者，空的实现，需要搭配 boost入参的 consumer_override_cls 和 publisher_override_cls使用，或者被继承。
+    """
+    def custom_init(self):
+        pass
+        # asyncio.get_event_loop().run_until_complete(broker.start())
+        self.broker = get_broker()
+        asyncio.get_event_loop().run_until_complete(self.broker.connect())
+
+    def publish(self, msg: typing.Union[str, dict], task_id=None,
+                priority_control_config: PriorityConsumingControlConfig = None) :
+        msg, msg_function_kw, extra_params, task_id = self._convert_msg(msg, task_id, priority_control_config)
+        t_start = time.time()
+        faststream_result =  asyncio.get_event_loop().run_until_complete(self.broker.publish(Serialization.to_json_str(msg), self.queue_name))
+        self.logger.debug(f'向{self._queue_name} 队列，推送消息 耗时{round(time.time() - t_start, 4)}秒  {msg_function_kw}')  # 显示msg太长了。
+        with self._lock_for_count:
+            self.count_per_minute += 1
+            self.publish_msg_num_total += 1
+            if time.time() - self._current_time > 10:
+                self.logger.info(
+                    f'10秒内推送了 {self.count_per_minute} 条消息,累计推送了 {self.publish_msg_num_total} 条消息到 {self._queue_name} 队列中')
+                self._init_count()
+        # return AsyncResult(task_id)
+        return faststream_result  #
+
+    def concrete_realization_of_publish(self, msg):
+        pass
+
+
+    def clear(self):
+        pass
+
+
+    def get_message_count(self):
+        return -1
+
+
+    def close(self):
+        pass

{funboost-45.3 → funboost-45.4}/funboost/publishers/local_python_queue_publisher.py

@@ -22,6 +22,7 @@ class LocalPythonQueuePublisher(AbstractPublisher):
 
     def concrete_realization_of_publish(self, msg):
         # noinspection PyTypeChecker
+        pass
         self.queue.put(msg)
 
     def clear(self):

{funboost-45.3 → funboost-45.4}/funboost/utils/json_helper.py

@@ -45,20 +45,20 @@ def monkey_patch_json():
     json.dumps = _dumps
 
 
-class JsonUtils:
-    @staticmethod
-    def to_dict(obj:typing.Union[str,dict,list]):
-        if isinstance(obj,str):
-            return json.loads(obj)
-        else:
-            return obj
-
-    @staticmethod
-    def to_json_str(obj:typing.Union[str,dict,list]):
-        if isinstance(obj,str):
-            return obj
-        else:
-            return json.dumps(obj,ensure_ascii=False)
+# class JsonUtils:
+#     @staticmethod
+#     def to_dict(obj:typing.Union[str,dict,list]):
+#         if isinstance(obj,str):
+#             return json.loads(obj)
+#         else:
+#             return obj
+#
+#     @staticmethod
+#     def to_json_str(obj:typing.Union[str,dict,list]):
+#         if isinstance(obj,str):
+#             return obj
+#         else:
+#             return json.dumps(obj,ensure_ascii=False)
 
 if __name__ == '__main__':
     pass

{funboost-45.3 → funboost-45.4/funboost.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 45.3
+Version: 45.4
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs