funboost 44.5__tar.gz → 44.6__tar.gz

{funboost-44.5/funboost.egg-info → funboost-44.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 44.5
+Version: 44.6
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs
@@ -492,3 +492,4 @@ funboost通过支持celery作为broker_kind,使celer框架变成了funboost的
 ![](https://visitor-badge.glitch.me/badge?page_id=distributed_framework)
 
 <div> </div>
+

{funboost-44.5 → funboost-44.6}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "44.5"
+__version__ = "44.6"
 
 from funboost.set_frame_config import show_frame_config
 
@@ -31,12 +31,13 @@ from funboost.utils.paramiko_util import ParamikoFolderUploader
 
 from funboost.consumers.base_consumer import (wait_for_possible_has_finish_all_tasks_by_conusmer_list,
                                               FunctionResultStatus, AbstractConsumer)
-
+from funboost.consumers.empty_consumer import EmptyConsumer
 from funboost.core.exceptions import ExceptionForRetry, ExceptionForRequeue, ExceptionForPushToDlxqueue
 from funboost.core.active_cousumer_info_getter import ActiveCousumerProcessInfoGetter
 from funboost.core.msg_result_getter import HasNotAsyncResult, ResultFromMongo
 from funboost.publishers.base_publisher import (PriorityConsumingControlConfig,
                                                 AbstractPublisher, AsyncResult, AioAsyncResult)
+from funboost.publishers.empty_publisher import EmptyPublisher
 from funboost.factories.broker_kind__publsiher_consumer_type_map import register_custom_broker
 from funboost.factories.publisher_factotry import get_publisher
 from funboost.factories.consumer_factory import get_consumer

{funboost-44.5 → funboost-44.6}/funboost/constant.py

@@ -1,5 +1,7 @@
 # coding= utf-8
 class BrokerEnum:
+    EMPTY = 'empty' # 空的实现，需要搭配 boost入参的 consumer_override_cls 和 publisher_override_cls使用，或者被继承。
+
     RABBITMQ_AMQPSTORM = 'RABBITMQ_AMQPSTORM'  # 使用 amqpstorm 包操作rabbitmq  作为 分布式消息队列，支持消费确认.强烈推荐这个作为funboost中间件。
     RABBITMQ = RABBITMQ_AMQPSTORM
 

{funboost-44.5 → funboost-44.6}/funboost/consumers/base_consumer.py

@@ -562,6 +562,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         """
         self._do_not_delete_extra_from_msg = True
 
+    def user_custom_record_process_info_func(self,current_function_result_status:FunctionResultStatus):  # 这个可以继承
+        pass
+
     # noinspection PyProtectedMember
     def _run(self, kw: dict, ):
         # print(kw)
@@ -633,7 +636,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                     self._current_time_for_execute_task_times_every_unit_time = time.time()
                     self._consuming_function_cost_time_total_every_unit_time = 0
                     self._execute_task_times_every_unit_time = 0
-
+            self.user_custom_record_process_info_func(current_function_result_status) # 两种方式都可以自定义,记录结果.
             if self.consumer_params.user_custom_record_process_info_func:
                 self.consumer_params.user_custom_record_process_info_func(current_function_result_status)
         except BaseException as e:

funboost-44.6/funboost/consumers/empty_consumer.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# @Author  : ydf
+# @Time    : 2023/8/8 0008 13:32
+
+import abc
+from funboost.consumers.base_consumer import AbstractConsumer
+
+
+class EmptyConsumer(AbstractConsumer, metaclass=abc.ABCMeta):
+    """
+    空的消费者没实现，空的实现，需要搭配 boost入参的 consumer_override_cls 和 publisher_override_cls使用，或者被继承。
+    """
+    def custom_init(self):
+        pass
+
+    @abc.abstractmethod
+    def _shedual_task(self):
+        raise NotImplemented('not realization')
+
+    @abc.abstractmethod
+    def _confirm_consume(self, kw):
+        raise NotImplemented('not realization')
+
+    @abc.abstractmethod
+    def _requeue(self, kw):
+        raise NotImplemented('not realization')

{funboost-44.5 → funboost-44.6}/funboost/consumers/redis_consumer.py

@@ -17,20 +17,20 @@ class RedisConsumer(AbstractConsumer, RedisMixin):
     redis作为中间件实现的，使用redis list 结构实现的。
     这个如果消费脚本在运行时候随意反复重启或者非正常关闭或者消费宕机，会丢失大批任务。高可靠需要用rabbitmq或者redis_ack_able或者redis_stream的中间件方式。
 
-    这个是复杂版，一次性拉取100个，简单版在 funboost/consumers/redis_consumer_simple.py
+    这个是复杂版，一次性拉取100个,减少和redis的交互，简单版在 funboost/consumers/redis_consumer_simple.py
     """
 
-    BROKER_EXCLUSIVE_CONFIG_DEFAULT = {'redis_bulk_push':1}   #redis_bulk_push 是否redis批量推送
+    BROKER_EXCLUSIVE_CONFIG_DEFAULT = {'redis_bulk_push':1,'pull_msg_batch_size':100}   #redis_bulk_push 是否redis批量推送
 
     # noinspection DuplicatedCode
     def _shedual_task(self):
+        pull_msg_batch_size =  self.consumer_params.broker_exclusive_config['pull_msg_batch_size']
         while True:
             # if False:
             #     pass
             with self.redis_db_frame.pipeline() as p:
-                get_msg_batch_size = 100
-                p.lrange(self._queue_name, 0, get_msg_batch_size - 1)
-                p.ltrim(self._queue_name, get_msg_batch_size, -1)
+                p.lrange(self._queue_name, 0, pull_msg_batch_size- 1)
+                p.ltrim(self._queue_name, pull_msg_batch_size, -1)
                 task_str_list = p.execute()[0]
             if task_str_list:
                 # self.logger.debug(f'从redis的 [{self._queue_name}] 队列中 取出的消息是：  {task_str_list}  ')

{funboost-44.5 → funboost-44.6}/funboost/consumers/redis_consumer_ack_able.py

@@ -99,6 +99,7 @@ class RedisConsumerAckAble(ConsumerConfirmMixinWithTheHelpOfRedisByHearbeat, Abs
     # print(script_4(keys=["text_pipelien1","text_pipelien1b"]))
     """
 
+    BROKER_EXCLUSIVE_CONFIG_DEFAULT = { 'pull_msg_batch_size': 100}
 
     def _shedual_task000(self):
         # 可以采用lua脚本，也可以采用redis的watch配合pipeline使用。比代码分两行pop和zadd比还能减少一次io交互，还能防止丢失小概率一个任务。
@@ -122,8 +123,9 @@ class RedisConsumerAckAble(ConsumerConfirmMixinWithTheHelpOfRedisByHearbeat, Abs
                 time.sleep(0.5)
 
     def _shedual_task(self):
-        lua = '''
-                     local task_list = redis.call("lrange", KEYS[1],0,99)
+        pull_msg_batch_size = self.consumer_params.broker_exclusive_config['pull_msg_batch_size']
+        lua = f'''
+                     local task_list = redis.call("lrange", KEYS[1],0,{pull_msg_batch_size-1})
                      redis.call("ltrim", KEYS[1],100,-1)
                      if (#task_list > 0) then
                         for task_index,task_value in ipairs(task_list)

{funboost-44.5 → funboost-44.6}/funboost/consumers/redis_stream_consumer.py

@@ -14,7 +14,7 @@ class RedisStreamConsumer(AbstractConsumer, RedisMixin):
     redis 的 stream 结构 作为中间件实现的。需要redis 5.0以上，redis stream结构 是redis的消息队列，概念类似kafka，功能远超 list结构。
     """
     GROUP = 'funboost_group'
-    BROKER_EXCLUSIVE_CONFIG_DEFAULT = {'group': 'funboost_group'}
+    BROKER_EXCLUSIVE_CONFIG_DEFAULT = {'group': 'funboost_group','pull_msg_batch_size': 100}
 
     def custom_init(self):
         self.group = self.consumer_params.broker_exclusive_config['group'] or self.GROUP
@@ -33,6 +33,7 @@ class RedisStreamConsumer(AbstractConsumer, RedisMixin):
         self.keep_circulating(60, block=False)(self._requeue_tasks_which_unconfirmed)()
 
     def _shedual_task(self):
+        pull_msg_batch_size = self.consumer_params.broker_exclusive_config['pull_msg_batch_size']
 
         try:
             self.redis_db_frame.xgroup_create(self._queue_name,self.group , id=0, mkstream=True)
@@ -41,7 +42,7 @@ class RedisStreamConsumer(AbstractConsumer, RedisMixin):
         while True:
             # redis服务端必须是5.0以上，并且确保这个键的类型是stream不能是list数据结构。
             results = self.redis_db_frame.xreadgroup(self.group, self.consumer_identification,
-                                                              {self.queue_name: ">"}, count=100, block=60 * 1000)
+                                                              {self.queue_name: ">"}, count=pull_msg_batch_size, block=60 * 1000)
             if results:
                 # self.logger.debug(f'从redis的 [{self._queue_name}] stream 中 取出的消息是：  {results}  ')
                 self._print_message_get_from_broker( results)

funboost-44.6/funboost/core/funboost_current_task_context_thread.py

@@ -0,0 +1,12 @@
+# import threading
+#
+# from funboost.core.current_task import thread_current_task
+#
+#
+# class FctThread(threading.Thread):
+#     thread_current_task__dict_key = 'thread_current_task__dict'
+#
+#
+#     def run(self):
+#         thread_current_task._fct_local_data.__dict__.update(getattr(self,self.thread_current_task__dict_key))  # 把funboost的消费线程上下文需要传递到线程上下文里面来.
+#         super().run()

{funboost-44.5 → funboost-44.6}/funboost/core/func_params_model.py

@@ -178,13 +178,16 @@ class BoosterParams(BaseJsonAbleModel):
 
     consuming_function: typing.Callable = None  # 消费函数,在@boost时候不用指定,因为装饰器知道下面的函数.
 
-    broker_kind: str = BrokerEnum.PERSISTQUEUE  # 中间件选型见3.1章节 https://funboost.readthedocs.io/zh/latest/articles/c3.html
+    broker_kind: str = BrokerEnum.SQLITE_QUEUE  # 中间件选型见3.1章节 https://funboost.readthedocs.io/zh/latest/articles/c3.html
 
     broker_exclusive_config: dict = {}  # 加上一个不同种类中间件非通用的配置,不同中间件自身独有的配置，不是所有中间件都兼容的配置，因为框架支持30种消息队列，消息队列不仅仅是一般的先进先出queue这么简单的概念，
     # 例如kafka支持消费者组，rabbitmq也支持各种独特概念例如各种ack机制 复杂路由机制，有的中间件原生能支持消息优先级有的中间件不支持,每一种消息队列都有独特的配置参数意义，可以通过这里传递。每种中间件能传递的键值对可以看consumer类的 BROKER_EXCLUSIVE_CONFIG_DEFAULT
 
     should_check_publish_func_params: bool = True  # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字,  如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
 
+    consumer_override_cls: typing.Optional[typing.Type] = None  # 使用 consumer_override_cls 和 publisher_override_cls 来自定义重写或新增消费者 发布者,见文档4.21b介绍，
+    publisher_override_cls: typing.Optional[typing.Type] = None
+
     auto_generate_info: dict = {}  # 自动生成的信息,不需要用户主动传参.
 
     @root_validator(skip_on_failure=True)
@@ -266,6 +269,7 @@ class PublisherParams(BaseJsonAbleModel):
     broker_kind: str = None
     broker_exclusive_config: dict = {}
     should_check_publish_func_params: bool = True  # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字,  如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
+    publisher_override_cls: typing.Optional[typing.Type] = None
 
 
 if __name__ == '__main__':
@@ -277,4 +281,4 @@ if __name__ == '__main__':
     # print(PriorityConsumingControlConfig().get_str_dict())
 
     print(BoosterParams(queue_name='3213', specify_concurrent_pool=FlexibleThreadPool(100)).json_pre())
-    print(PublisherParams.schema_json())
+    print(PublisherParams.schema_json())

{funboost-44.5 → funboost-44.6}/funboost/factories/broker_kind__publsiher_consumer_type_map.py

@@ -1,6 +1,6 @@
 import typing
 
-
+from funboost.publishers.empty_publisher import EmptyPublisher
 from funboost.publishers.http_publisher import HTTPPublisher
 from funboost.publishers.nats_publisher import NatsPublisher
 from funboost.publishers.peewee_publisher import PeeweePublisher
@@ -25,6 +25,7 @@ from funboost.publishers.redis_stream_publisher import RedisStreamPublisher
 from funboost.publishers.mqtt_publisher import MqttPublisher
 from funboost.publishers.httpsqs_publisher import HttpsqsPublisher
 
+from funboost.consumers.empty_consumer import EmptyConsumer
 from funboost.consumers.redis_consumer_priority import RedisPriorityConsumer
 from funboost.consumers.redis_pubsub_consumer import RedisPbSubConsumer
 from funboost.consumers.http_consumer import HTTPConsumer
@@ -80,6 +81,7 @@ broker_kind__publsiher_consumer_type_map = {
     BrokerEnum.PEEWEE: (PeeweePublisher, PeeweeConsumer),
     BrokerEnum.REDIS_PUBSUB: (RedisPubSubPublisher, RedisPbSubConsumer),
     BrokerEnum.REIDS_ACK_USING_TIMEOUT: (RedisPublisher, RedisConsumerAckUsingTimeout),
+    BrokerEnum.EMPTY:(EmptyPublisher,EmptyConsumer),
 
 }
 

{funboost-44.5 → funboost-44.6}/funboost/factories/consumer_factory.py

@@ -20,4 +20,12 @@ def get_consumer(boost_params: BoosterParams) -> AbstractConsumer:
 
     if boost_params.broker_kind not in broker_kind__publsiher_consumer_type_map:
         raise ValueError(f'设置的中间件种类数字不正确,你设置的值是 {boost_params.broker_kind} ')
-    return broker_kind__publsiher_consumer_type_map[boost_params.broker_kind][1](boost_params)
+    consumer_cls = broker_kind__publsiher_consumer_type_map[boost_params.broker_kind][1]
+    if not boost_params.consumer_override_cls:
+        return consumer_cls(boost_params)
+    else:
+        ConsumerClsOverride = type(f'{consumer_cls.__name__}__{boost_params.consumer_override_cls.__name__}', (boost_params.consumer_override_cls, consumer_cls, AbstractConsumer), {})
+        # class ConsumerClsOverride(boost_params.consumer_override_cls, consumer_cls, AbstractConsumer):
+        #     pass
+
+        return ConsumerClsOverride(boost_params)

{funboost-44.5 → funboost-44.6}/funboost/factories/publisher_factotry.py

@@ -32,4 +32,13 @@ def get_publisher(publisher_params: PublisherParams) -> AbstractPublisher:
     regist_to_funboost(broker_kind)  # 动态注册中间件到框架是为了延迟导入，用户没安装不需要的第三方包不报错。
     if broker_kind not in broker_kind__publsiher_consumer_type_map:
         raise ValueError(f'设置的中间件种类数字不正确,你设置的值是 {broker_kind} ')
-    return broker_kind__publsiher_consumer_type_map[broker_kind][0](publisher_params)
+    publisher_cls = broker_kind__publsiher_consumer_type_map[broker_kind][0]
+    if not publisher_params.publisher_override_cls:
+        return publisher_cls(publisher_params)
+    else:
+        PublsiherClsOverride = type(f'{publisher_cls.__name__}__{publisher_params.publisher_override_cls.__name__}', (publisher_params.publisher_override_cls, publisher_cls, AbstractPublisher), {})
+        # class PublsiherClsOverride(publisher_params.publisher_override_cls, publisher_cls, AbstractPublisher):
+        #     pass
+
+        return PublsiherClsOverride(publisher_params)
+

funboost-44.6/funboost/publishers/empty_publisher.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# @Author  : ydf
+# @Time    : 2023/8/8 0008 12:12
+
+import abc
+from funboost.publishers.base_publisher import AbstractPublisher
+
+
+class EmptyPublisher(AbstractPublisher, metaclass=abc.ABCMeta):
+    """
+    空的发布者，空的实现，需要搭配 boost入参的 consumer_override_cls 和 publisher_override_cls使用，或者被继承。
+    """
+
+    def custom_init(self):
+        pass
+
+    @abc.abstractmethod
+    def concrete_realization_of_publish(self, msg: str):
+        raise NotImplemented('not realization')
+
+    @abc.abstractmethod
+    def clear(self):
+        raise NotImplemented('not realization')
+
+    @abc.abstractmethod
+    def get_message_count(self):
+        raise NotImplemented('not realization')
+
+    @abc.abstractmethod
+    def close(self):
+        raise NotImplemented('not realization')

{funboost-44.5 → funboost-44.6}/funboost/set_frame_config.py

@@ -98,7 +98,7 @@ def show_frame_config():
         #             only_print_on_main_process(f'{var_name}:                {var_value[0]}{"*" * (len(var_value) - 2)}{var_value[-1]}')
         #         else:
         #             only_print_on_main_process(f'{var_name}:                {var_value}')
-        logger_prompt.debug(f'''读取的 BrokerConnConfig 配置是:\n {funboost_config_deafult.BrokerConnConfig().get_json(indent=None)} ''')
+        logger_prompt.debug(f'''读取的 BrokerConnConfig 配置是:\n {funboost_config_deafult.BrokerConnConfig().get_pwd_enc_json(indent=4)} ''')
 
         logger_prompt.debug(f'''读取的 FunboostCommonConfig 配置是:\n  {funboost_config_deafult.FunboostCommonConfig().get_json(indent=None)} ''')
 

funboost-44.6/funboost/utils/class_utils.py

@@ -0,0 +1,51 @@
+#
+#
+# # def get_child_custom_attr(child_cls:type,):
+# #     __dict = child_cls.__dict__
+#
+# def merge_cls(cls1:type,cls2:type):
+#     class Cls(cls2):
+#         pass
+#     for k,v in cls1.__dict__.items():
+#         if k.startswith('__') and k.endswith('__'):
+#             continue
+#         print(k,v)
+#         setattr(Cls,k,v)
+#     return Cls
+#
+#
+# if __name__ == '__main__':
+#     class Parent:
+#         attr1=1
+#         def method_from_parent(self):
+#             print('method_from_parent')
+#
+#         def method_from_parent2(self):
+#             print('method_from_parent2')
+#
+#
+#     class Child(Parent):
+#         attr1=2
+#         attr2=22
+#
+#         def method_from_parent2(self):
+#             print('method_from_parent2chile')
+#         def method_from_child(self:Parent):
+#             print('method_from_child')
+#
+#     class Child2(Parent,Parent):
+#         attr1 = 3
+#
+#     class Child2b(Child2):
+#         attr1 = '2b'
+#
+#
+#     class Child2New(Child2b,Child):
+#         pass
+#
+#
+#     print(Child2().method_from_parent2())
+#     print(Child2New().method_from_parent2())
+#
+#
+#

{funboost-44.5 → funboost-44.6}/funboost/utils/decorators.py

@@ -509,7 +509,7 @@ def cached_method_result_for_instance(fun):
 
 
 class FunctionResultCacher:
-    logger = LogManager('FunctionResultChche').get_logger_and_add_handlers()
+    logger = LogManager('FunctionResultChche').get_logger_and_add_handlers(log_level_int=20)
     func_result_dict = {}
     """
     {

{funboost-44.5 → funboost-44.6}/funboost/utils/dependency_packages_in_pythonpath/aioredis/__init__.py

@@ -1,59 +1,59 @@
-from aioredis.client import Redis, StrictRedis
-from aioredis.connection import (
-    BlockingConnectionPool,
-    Connection,
-    ConnectionPool,
-    SSLConnection,
-    UnixDomainSocketConnection,
-)
-from aioredis.exceptions import (
-    AuthenticationError,
-    AuthenticationWrongNumberOfArgsError,
-    BusyLoadingError,
-    ChildDeadlockedError,
-    ConnectionError,
-    DataError,
-    InvalidResponse,
-    PubSubError,
-    ReadOnlyError,
-    RedisError,
-    ResponseError,
-    TimeoutError,
-    WatchError,
-)
-from aioredis.utils import from_url
-
-
-def int_or_str(value):
-    try:
-        return int(value)
-    except ValueError:
-        return value
-
-
-__version__ = "2.0.1"
-VERSION = tuple(map(int_or_str, __version__.split(".")))
-
-__all__ = [
-    "AuthenticationError",
-    "AuthenticationWrongNumberOfArgsError",
-    "BlockingConnectionPool",
-    "BusyLoadingError",
-    "ChildDeadlockedError",
-    "Connection",
-    "ConnectionError",
-    "ConnectionPool",
-    "DataError",
-    "from_url",
-    "InvalidResponse",
-    "PubSubError",
-    "ReadOnlyError",
-    "Redis",
-    "RedisError",
-    "ResponseError",
-    "SSLConnection",
-    "StrictRedis",
-    "TimeoutError",
-    "UnixDomainSocketConnection",
-    "WatchError",
-]
+from aioredis.client import Redis, StrictRedis
+from aioredis.connection import (
+    BlockingConnectionPool,
+    Connection,
+    ConnectionPool,
+    SSLConnection,
+    UnixDomainSocketConnection,
+)
+from aioredis.exceptions import (
+    AuthenticationError,
+    AuthenticationWrongNumberOfArgsError,
+    BusyLoadingError,
+    ChildDeadlockedError,
+    ConnectionError,
+    DataError,
+    InvalidResponse,
+    PubSubError,
+    ReadOnlyError,
+    RedisError,
+    ResponseError,
+    TimeoutError,
+    WatchError,
+)
+from aioredis.utils import from_url
+
+
+def int_or_str(value):
+    try:
+        return int(value)
+    except ValueError:
+        return value
+
+
+__version__ = "2.0.1"
+VERSION = tuple(map(int_or_str, __version__.split(".")))
+
+__all__ = [
+    "AuthenticationError",
+    "AuthenticationWrongNumberOfArgsError",
+    "BlockingConnectionPool",
+    "BusyLoadingError",
+    "ChildDeadlockedError",
+    "Connection",
+    "ConnectionError",
+    "ConnectionPool",
+    "DataError",
+    "from_url",
+    "InvalidResponse",
+    "PubSubError",
+    "ReadOnlyError",
+    "Redis",
+    "RedisError",
+    "ResponseError",
+    "SSLConnection",
+    "StrictRedis",
+    "TimeoutError",
+    "UnixDomainSocketConnection",
+    "WatchError",
+]