funboost 41.1__tar.gz → 41.2__tar.gz

{funboost-41.1/funboost.egg-info → funboost-41.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 41.1
+Version: 41.2
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持celery框架整体作为funboost中间件，python函数加速器，框架包罗万象，一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是 方便 快速 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-41.1 → funboost-41.2}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "41.1"
+__version__ = "41.2"
 
 from funboost.set_frame_config import show_frame_config
 

{funboost-41.1 → funboost-41.2}/funboost/concurrent_pool/__init__.py

@@ -16,4 +16,5 @@ from .async_pool_executor import AsyncPoolExecutor
 # from .custom_gevent_pool_executor import GeventPoolExecutor
 from .bounded_threadpoolexcutor import BoundedThreadPoolExecutor
 from .custom_threadpool_executor import CustomThreadPoolExecutor
-from .flexible_thread_pool import FlexibleThreadPool
+from .flexible_thread_pool import FlexibleThreadPool
+from .pool_commons import ConcurrentPoolBuilder

{funboost-41.1 → funboost-41.2}/funboost/concurrent_pool/flexible_thread_pool.py

@@ -10,14 +10,16 @@
 
 import asyncio
 import inspect
+import os
 import queue
 import threading
 from functools import wraps
 
 from funboost.concurrent_pool import FunboostBaseConcurrentPool
-from funboost.core.loggers import FunboostFileLoggerMixin,LoggerLevelSetterMixin,FunboostMetaTypeFileLogger
+from funboost.core.loggers import FunboostFileLoggerMixin, LoggerLevelSetterMixin, FunboostMetaTypeFileLogger
 
-class FlexibleThreadPool(FunboostFileLoggerMixin, LoggerLevelSetterMixin,FunboostBaseConcurrentPool):
+
+class FlexibleThreadPool(FunboostFileLoggerMixin, LoggerLevelSetterMixin, FunboostBaseConcurrentPool):
     KEEP_ALIVE_TIME = 10
     MIN_WORKERS = 2
 
@@ -51,6 +53,7 @@ class FlexibleThreadPool(FunboostFileLoggerMixin, LoggerLevelSetterMixin,Funboos
 class FlexibleThreadPoolMinWorkers0(FlexibleThreadPool):
     MIN_WORKERS = 0
 
+
 def run_sync_or_async_fun000(func, *args, **kwargs):
     """这种方式造成电脑很卡,不行"""
     fun_is_asyncio = inspect.iscoroutinefunction(func)
@@ -63,12 +66,16 @@ def run_sync_or_async_fun000(func, *args, **kwargs):
     else:
         return func(*args, **kwargs)
 
+
 tl = threading.local()
+
+
 def _get_thread_local_loop() -> asyncio.AbstractEventLoop:
-    if not hasattr(tl,'asyncio_loop'):
+    if not hasattr(tl, 'asyncio_loop'):
         tl.asyncio_loop = asyncio.new_event_loop()
     return tl.asyncio_loop
 
+
 def run_sync_or_async_fun(func, *args, **kwargs):
     fun_is_asyncio = inspect.iscoroutinefunction(func)
     if fun_is_asyncio:
@@ -122,6 +129,9 @@ class _KeepAliveTimeThread(threading.Thread, metaclass=FunboostMetaTypeFileLogge
             self.pool._change_threads_free_count(1)
 
 
+
+
+
 if __name__ == '__main__':
     import time
     from concurrent.futures import ThreadPoolExecutor

funboost-41.2/funboost/concurrent_pool/pool_commons.py

@@ -0,0 +1,27 @@
+import functools
+import threading
+import typing
+import os
+
+from funboost.concurrent_pool.flexible_thread_pool import FlexibleThreadPool
+from funboost.concurrent_pool.base_pool_type import FunboostBaseConcurrentPool
+
+
+class ConcurrentPoolBuilder:
+    _pid__pool_map = {}
+    _lock = threading.Lock()
+
+    @classmethod
+    def get_pool(cls, pool_type: typing.Type[FunboostBaseConcurrentPool], max_workers: int = None):
+        key = (os.getpid(), pool_type)
+        with cls._lock:
+            if key not in cls._pid__pool_map:
+                pool = pool_type(max_workers)  # noqa
+                cls._pid__pool_map[key] = pool
+            return cls._pid__pool_map[key]
+
+
+if __name__ == '__main__':
+    for i in range(10):
+        pool = functools.partial(ConcurrentPoolBuilder.get_pool, FlexibleThreadPool, 200)()
+        print(id(pool))

{funboost-41.1 → funboost-41.2}/funboost/consumers/base_consumer.py

@@ -74,6 +74,7 @@ class GlobalVars:
     global_concurrent_mode = None
     has_start_a_consumer_flag = False
 
+
 # noinspection DuplicatedCode
 class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
     time_interval_for_check_do_not_run_time = 60
@@ -108,7 +109,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         :return:
         """
         # ConsumersManager.join_all_consumer_shedual_task_thread()
-        if GlobalVars.has_start_a_consumer_flag :
+        if GlobalVars.has_start_a_consumer_flag:
             while 1:
                 time.sleep(10)
 
@@ -134,7 +135,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         self._concurrent_mode_dispatcher = ConcurrentModeDispatcher(self)
         if consumer_params.concurrent_mode == ConcurrentModeEnum.ASYNC:
             self._run = self._async_run  # 这里做了自动转化，使用async_run代替run
-        self.logger:logging.Logger
+        self.logger: logging.Logger
         self._build_logger()
         # stdout_write(f'''{time.strftime("%H:%M:%S")} "{self.consumer_params.auto_generate_info['where_to_instantiate']}"  \033[0;37;44m此行 实例化队列名 {self.queue_name} 的消费者, 类型为 {self.__class__}\033[0m\n''')
         print(f'''\033[0m
@@ -185,6 +186,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
 
         self.consumer_identification = f'{nb_log_config_default.computer_name}_{nb_log_config_default.computer_ip}_' \
                                        f'{time_util.DatetimeConverter().datetime_str.replace(":", "-")}_{os.getpid()}_{id(self)}'
+        # noinspection PyUnresolvedReferences
         self.consumer_identification_map = {'queue_name': self.queue_name,
                                             'computer_name': nb_log_config_default.computer_name,
                                             'computer_ip': nb_log_config_default.computer_ip,
@@ -222,10 +224,11 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         logger_name = f'funboost.{logger_prefix}{self.__class__.__name__}--{self.queue_name}'
         log_filename = self.consumer_params.log_filename or f'funboost.{self.queue_name}.log'
         self.logger = get_logger(logger_name,
-            log_level_int=self.consumer_params.log_level,
+                                 log_level_int=self.consumer_params.log_level,
                                  log_filename=log_filename if self.consumer_params.create_logger_file else None,
-                                 error_log_filename= nb_log.generate_error_file_name(log_filename),
-            formatter_template=FunboostCommonConfig.NB_LOG_FORMATER_INDEX_FOR_CONSUMER_AND_PUBLISHER, )
+                                 error_log_filename=nb_log.generate_error_file_name(log_filename),
+                                 formatter_template=FunboostCommonConfig.NB_LOG_FORMATER_INDEX_FOR_CONSUMER_AND_PUBLISHER, )
+        self.logger.info(f'队列 {self.queue_name} 的日志写入到 {nb_log_config_default.LOG_PATH} 文件夹的 {log_filename} 和 {nb_log.generate_error_file_name(log_filename)} 文件中')
 
     def _check_broker_exclusive_config(self):
         broker_exclusive_config_keys = self.consumer_params.broker_exclusive_config.keys()
@@ -305,7 +308,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         # ConsumersManager.show_all_consumer_info()
         # noinspection PyBroadException
 
-        GlobalVars.has_start_a_consumer_flag =True
+        GlobalVars.has_start_a_consumer_flag = True
         try:
             self._concurrent_mode_dispatcher.check_all_concurrent_mode()
             self._check_monkey_patch()
@@ -546,9 +549,10 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                     with RedisMixin().redis_db_filter_and_rpc_result.pipeline() as p:
                         # RedisMixin().redis_db_frame.lpush(kw['body']['extra']['task_id'], json.dumps(function_result_status.get_status_dict(without_datetime_obj=True)))
                         # RedisMixin().redis_db_frame.expire(kw['body']['extra']['task_id'], 600)
+                        current_function_result_status.rpc_result_expire_seconds = self.consumer_params.rpc_result_expire_seconds
                         p.lpush(kw['body']['extra']['task_id'],
                                 json.dumps(current_function_result_status.get_status_dict(without_datetime_obj=True)))
-                        p.expire(kw['body']['extra']['task_id'], 600)
+                        p.expire(kw['body']['extra']['task_id'], self.consumer_params.rpc_result_expire_seconds)
                         p.execute()
 
             with self._lock_for_count_execute_task_times_every_unit_time:
@@ -695,9 +699,10 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             if self._get_priority_conf(kw, 'is_using_rpc_mode'):
                 def push_result():
                     with RedisMixin().redis_db_filter_and_rpc_result.pipeline() as p:
+                        current_function_result_status.rpc_result_expire_seconds = self.consumer_params.rpc_result_expire_seconds
                         p.lpush(kw['body']['extra']['task_id'],
                                 json.dumps(current_function_result_status.get_status_dict(without_datetime_obj=True)))
-                        p.expire(kw['body']['extra']['task_id'], 600)
+                        p.expire(kw['body']['extra']['task_id'], self.consumer_params.rpc_result_expire_seconds)
                         p.execute()
 
                 if (current_function_result_status.success is False and current_retry_times == max_retry_times) or current_function_result_status.success is True:
@@ -911,8 +916,8 @@ class ConcurrentModeDispatcher(FunboostFileLoggerMixin):
                 self.consumer.consumer_params.concurrent_mode != GlobalVars.global_concurrent_mode:
             # print({self.consumer._concurrent_mode, ConsumersManager.global_concurrent_mode})
             if not {self.consumer.consumer_params.concurrent_mode, GlobalVars.global_concurrent_mode}.issubset({ConcurrentModeEnum.THREADING,
-                                                                                                                                 ConcurrentModeEnum.ASYNC,
-                                                                                                                                 ConcurrentModeEnum.SINGLE_THREAD}):
+                                                                                                                ConcurrentModeEnum.ASYNC,
+                                                                                                                ConcurrentModeEnum.SINGLE_THREAD}):
                 # threding、asyncio、solo 这几种模式可以共存。但同一个解释器不能同时选择 gevent + 其它并发模式，也不能 eventlet + 其它并发模式。
                 raise ValueError('''由于猴子补丁的原因，同一解释器中不可以设置两种并发类型,请查看显示的所有消费者的信息，
                                  搜索 concurrent_mode 关键字，确保当前解释器内的所有消费者的并发模式只有一种(或可以共存),

{funboost-41.1 → funboost-41.2}/funboost/core/booster.py

@@ -12,7 +12,8 @@ from funboost.core.exceptions import BoostDecoParamsIsOldVersion
 from funboost.core.func_params_model import BoosterParams, FunctionResultStatusPersistanceConfig, PriorityConsumingControlConfig
 
 from funboost.factories.consumer_factory import get_consumer
-
+if typing.TYPE_CHECKING:
+    from funboost.core.msg_result_getter import AsyncResult
 
 class Booster:
     """
@@ -107,13 +108,13 @@ class Booster:
         else:
             return self.consuming_function(*args, **kwargs)
 
-    def _safe_push(self, *func_args, **func_kwargs):
+    def _safe_push(self, *func_args, **func_kwargs) -> AsyncResult:
         """ 多进程安全的,在fork多进程(非spawn多进程)情况下,有的包多进程不能共用一个连接,例如kafka"""
         consumer = BoostersManager.get_or_create_booster_by_queue_name(self.queue_name).consumer
         return consumer.publisher_of_same_queue.push(*func_args, **func_kwargs)
 
     def _safe_publish(self, msg: typing.Union[str, dict], task_id=None,
-                      priority_control_config: PriorityConsumingControlConfig = None):
+                      priority_control_config: PriorityConsumingControlConfig = None) -> AsyncResult:
         """ 多进程安全的,在fork多进程(非spawn多进程)情况下,很多包跨线程/进程不能共享中间件连接,"""
         consumer = BoostersManager.get_or_create_booster_by_queue_name(self.queue_name).consumer
         return  consumer.publisher_of_same_queue.publish(msg=msg, task_id=task_id, priority_control_config=priority_control_config)

{funboost-41.1 → funboost-41.2}/funboost/core/exceptions.py

@@ -21,7 +21,7 @@ def f(x):
     
 
 用户需要做的改变如下:
-@boost(boost_paams=BoosterParams(queue_name='queue_name_xx',qps=3))
+@boost(BoosterParams(queue_name='queue_name_xx',qps=3))
 def f(x):
     pass
 

{funboost-41.1 → funboost-41.2}/funboost/core/func_params_model.py

@@ -1,13 +1,14 @@
 import asyncio
 import datetime
+import functools
 import json
 import logging
 import typing
 from collections import OrderedDict
 
-from funboost.concurrent_pool import FunboostBaseConcurrentPool
+from funboost.concurrent_pool import FunboostBaseConcurrentPool, FlexibleThreadPool, ConcurrentPoolBuilder
 from funboost.constant import ConcurrentModeEnum, BrokerEnum
-from pydantic import BaseModel, validator, root_validator, BaseConfig
+from pydantic import BaseModel, validator, root_validator, BaseConfig, Field
 
 # noinspection PyUnresolvedReferences
 from funboost.core.loggers import develop_logger
@@ -158,6 +159,7 @@ class BoosterParams(BaseJsonAbleModel):
     user_custom_record_process_info_func: typing.Callable = None  # 提供一个用户自定义的保存消息处理记录到某个地方例如mysql数据库的函数，函数仅仅接受一个入参，入参类型是 FunctionResultStatus，用户可以打印参数
 
     is_using_rpc_mode: bool = False  # 是否使用rpc模式，可以在发布端获取消费端的结果回调，但消耗一定性能，使用async_result.result时候会等待阻塞住当前线程。
+    rpc_result_expire_seconds: int = 600  # 保存rpc结果的过期时间.
 
     is_support_remote_kill_task: bool = False  # 是否支持远程任务杀死功能，如果任务数量少，单个任务耗时长，确实需要远程发送命令来杀死正在运行的函数，才设置为true，否则不建议开启此功能。
 
@@ -199,19 +201,22 @@ class BoosterParams(BaseJsonAbleModel):
 
 class BoosterParamsComplete(BoosterParams):
     """
-    例如一个子类,这个子类可以作为@booot的传参,每个@boost可以少写一些这些重复的入参字段.
+    例如一个子类,这个BoosterParams的子类可以作为@booot的传参,每个@boost可以少写一些这些重复的入参字段.
 
-    支持函数消费状态 结果状态持久化
-    支持发送消费者的心跳到redis,便于统计分布式环境的活跃消费者
-    支持rpc模式
-    永远是使用 amqpstorm包 操作 rabbbitmq作为消息队列.
+    function_result_status_persistance_conf 永远支持函数消费状态 结果状态持久化
+    is_send_consumer_hearbeat_to_redis 永远支持发送消费者的心跳到redis,便于统计分布式环境的活跃消费者
+    is_using_rpc_mode  永远支持rpc模式
+    broker_kind 永远是使用 amqpstorm包 操作 rabbbitmq作为消息队列.
+    specify_concurrent_pool 同一个进程的不同booster函数,共用一个线程池,线程资源利用更高.
     """
 
     function_result_status_persistance_conf: FunctionResultStatusPersistanceConfig = FunctionResultStatusPersistanceConfig(
         is_save_result=True, is_save_status=True, expire_seconds=7 * 24 * 3600, is_use_bulk_insert=True)  # 开启函数消费状态 结果持久化到 mongo,为True用户必须要安装mongo和多浪费一丝丝性能.
     is_send_consumer_hearbeat_to_redis: bool = True  # 消费者心跳发到redis,为True那么用户必须安装reids
     is_using_rpc_mode: bool = True  # 固定支持rpc模式,不用每次指定 (不需要使用rpc模式的同学,就不要指定为True,必须安装redis和浪费一点性能)
+    rpc_result_expire_seconds: int = 1000
     broker_kind: str = BrokerEnum.RABBITMQ_AMQPSTORM  # 固定使用rabbitmq,不用每次指定
+    specify_concurrent_pool: FunboostBaseConcurrentPool = Field(default_factory=functools.partial(ConcurrentPoolBuilder.get_pool, FlexibleThreadPool, 500))  # 多个消费函数共享线程池
 
 
 class PriorityConsumingControlConfig(BaseJsonAbleModel):

{funboost-41.1 → funboost-41.2}/funboost/core/function_result_status_saver.py

@@ -55,6 +55,7 @@ class FunctionResultStatus():
         self._has_requeue = False
         self._has_to_dlx_queue = False
         self._has_kill_task = False
+        self.rpc_result_expire_seconds = None
 
     def get_status_dict(self, without_datetime_obj=False):
         self.time_end = time.time()

{funboost-41.1 → funboost-41.2}/funboost/core/msg_result_getter.py

@@ -44,7 +44,7 @@ class AsyncResult(RedisMixin):
                 status_and_result_str = redis_value[1]
                 self._status_and_result = json.loads(status_and_result_str)
                 self.redis_db_filter_and_rpc_result.lpush(self.task_id, status_and_result_str)
-                self.redis_db_filter_and_rpc_result.expire(self.task_id, 600)
+                self.redis_db_filter_and_rpc_result.expire(self.task_id, self._status_and_result['rpc_result_expire_seconds'])
                 return self._status_and_result
             return None
         return self._status_and_result
@@ -149,7 +149,7 @@ if __name__ == '__main__':
                 status_and_result_str = redis_value[1]
                 self._status_and_result = json.loads(status_and_result_str)
                 await self.aioredis_db_filter_and_rpc_result.lpush(self.task_id, status_and_result_str)
-                await self.aioredis_db_filter_and_rpc_result.expire(self.task_id, 600)
+                await self.aioredis_db_filter_and_rpc_result.expire(self.task_id, self._status_and_result['rpc_result_expire_seconds'])
                 return self._status_and_result
             return None
         return self._status_and_result

{funboost-41.1 → funboost-41.2/funboost.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 41.1
+Version: 41.2
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持celery框架整体作为funboost中间件，python函数加速器，框架包罗万象，一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是 方便 快速 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-41.1 → funboost-41.2}/funboost.egg-info/SOURCES.txt

@@ -36,6 +36,7 @@ funboost/concurrent_pool/custom_threadpool_executor.py
 funboost/concurrent_pool/custom_threadpool_executor000.py
 funboost/concurrent_pool/fixed_thread_pool.py
 funboost/concurrent_pool/flexible_thread_pool.py
+funboost/concurrent_pool/pool_commons.py
 funboost/concurrent_pool/single_thread_executor.py
 funboost/concurrent_pool/backup/__init__.py
 funboost/concurrent_pool/backup/async_pool_executor0223.py