funboost 23.9__py3-none-any.whl → 24.2__py3-none-any.whl

funboost/__init__.py

@@ -6,6 +6,8 @@ from funboost.core import show_funboost_flag
 
 # noinspection PyUnresolvedReferences
 import nb_log
+# noinspection PyUnresolvedReferences
+from nb_log import get_logger,nb_print
 from funboost.set_frame_config import patch_frame_config, show_frame_config
 from funboost.funboost_config_deafult import BoostDecoratorDefaultParams
 
@@ -29,7 +31,7 @@ from funboost.timing_job import fsdf_background_scheduler, timing_publish_deco,
 from funboost.constant import BrokerEnum, ConcurrentModeEnum
 
 from funboost.core.booster import boost, Booster
-from funboost.core.get_booster import get_booster
+from funboost.core.get_booster import get_booster,get_or_create_booster,get_boost_params_and_consuming_function
 from funboost.core.kill_remote_task import RemoteTaskKiller
 
 from funboost.core.exit_signal import set_interrupt_signal_handler

funboost/beggar_version_implementation/beggar_redis_consumer.py

@@ -58,7 +58,7 @@ def start_consuming_message(queue_name, consume_function, threads_num=50):
             redis_task = redis_db_frame.brpop(queue_name, timeout=60)
             if redis_task:
                 task_str = redis_task[1].decode()
-                print(f'从redis的 {queue_name} 队列中 取出的消息是： {task_str}')
+                # print(f'从redis的 {queue_name} 队列中 取出的消息是： {task_str}')
                 pool.submit(consume_function, **json.loads(task_str))
             else:
                 print(f'redis的 {queue_name} 队列中没有任务')

funboost/concurrent_pool/flexible_thread_pool.py

@@ -0,0 +1,140 @@
+"""
+比更简单的 ThreadPoolExecutorShrinkAble 的弹性线程池，因为 funboost的并发池永远不需要判断代码结束，所以不用 ThreadPoolExecutorShrinkAble 那么复杂来兼容判断并发池要随代码退出而结束循环
+"""
+
+import asyncio
+import inspect
+import queue
+import threading
+from functools import wraps
+
+import nb_log
+from nb_log import LoggerMixin, LoggerLevelSetterMixin
+
+
+class FlexibleThreadPool(LoggerMixin, LoggerLevelSetterMixin):
+    KEEP_ALIVE_TIME = 10
+    MIN_WORKERS = 0
+
+    def __init__(self, max_workers: int = None):
+        self.work_queue = queue.Queue(10)
+        self.max_workers = max_workers
+        self._threads_num = 0
+        self.threads_free_count = 0
+        self._lock_compute_start_thread = threading.Lock()
+        self._lock_compute_threads_free_count = threading.Lock()
+        self._lock_for_adjust_thread = threading.Lock()
+        self._lock_for_judge_threads_free_count = threading.Lock()
+        self.pool_ident = id(self)
+        self.asyncio_loop = asyncio.new_event_loop()
+
+    def _change_threads_free_count(self, change_num):
+        with self._lock_compute_threads_free_count:
+            self.threads_free_count += change_num
+
+    def _change_threads_start_count(self, change_num):
+        with self._lock_compute_start_thread:
+            self._threads_num += change_num
+
+    def submit(self, func, *args, **kwargs):
+        self.work_queue.put([func, args, kwargs])
+        with self._lock_for_adjust_thread:
+            if self.threads_free_count <= self.MIN_WORKERS and self._threads_num < self.max_workers:
+                _KeepAliveTimeThread(self).start()
+
+
+def run_sync_or_async_fun(func, *args, **kwargs):
+    t1 =time.time()
+    fun_is_asyncio = inspect.iscoroutinefunction(func)
+
+    if fun_is_asyncio:
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+        print(time.time() - t1)
+        try:
+            result =  loop.run_until_complete(func(*args, **kwargs))
+
+            return result
+        finally:
+            pass
+            # loop.close()
+    else:
+        return func(*args, **kwargs)
+
+
+def sync_or_async_fun_deco(func):
+    @wraps(func)
+    def _inner(*args, **kwargs):
+        return run_sync_or_async_fun(func, *args, **kwargs)
+
+    return _inner
+
+
+# noinspection PyProtectedMember
+class _KeepAliveTimeThread(threading.Thread):
+    logger = nb_log.get_logger('_KeepAliveTimeThread')
+
+    def __init__(self, thread_pool: FlexibleThreadPool):
+        super().__init__()
+        self.pool = thread_pool
+
+    def run(self) -> None:
+        self.logger.debug(f'新启动线程 {self.ident} ')
+        self.pool._change_threads_free_count(1)
+        self.pool._change_threads_start_count(1)
+        while 1:
+            try:
+                func, args, kwargs = self.pool.work_queue.get(block=True, timeout=self.pool.KEEP_ALIVE_TIME)
+            except queue.Empty:
+
+                with self.pool._lock_for_judge_threads_free_count:
+                    # print(self.pool.threads_free_count)
+                    if self.pool.threads_free_count > self.pool.MIN_WORKERS:
+                        self.logger.debug(f'停止线程 {self._ident}, 触发条件是 {self.pool.pool_ident} 线程池中的 {self.ident} 线程 超过 {self.pool.KEEP_ALIVE_TIME} 秒没有任务，线程池中不在工作状态中的线程数量是 {self.pool.threads_free_count}，超过了指定的最小核心数量 {self.pool.MIN_WORKERS}')  # noqa
+                        self.pool._change_threads_free_count(-1)
+                        self.pool._change_threads_start_count(-1)
+                        break  # 退出while 1，即是结束。
+                    else:
+                        continue
+            self.pool._change_threads_free_count(-1)
+            try:
+                t1 = time.time()
+                fun = sync_or_async_fun_deco(func)
+                result = fun(*args, **kwargs)
+                print(time.time()-t1)
+                # print(result)
+            except BaseException as exc:
+                self.logger.exception(f'函数 {func.__name__} 中发生错误，错误原因是 {type(exc)} {exc} ')
+            self.pool._change_threads_free_count(1)
+
+
+if __name__ == '__main__':
+    import time
+    from concurrent.futures import ThreadPoolExecutor
+    from custom_threadpool_executor import ThreadPoolExecutorShrinkAble
+
+
+    def testf(x):
+        # time.sleep(10)
+        if x % 10000 == 0:
+            print(x)
+
+
+    async def aiotestf(x):
+        # await asyncio.sleep(1)
+        # print(x)
+        return x * 2
+
+
+    pool = FlexibleThreadPool(100)
+    # pool = ThreadPoolExecutor(100)
+    # pool = ThreadPoolExecutorShrinkAble(100)
+
+    for i in range(2000):
+        # time.sleep(2)
+        pool.submit(aiotestf, i)
+
+    # for i in range(1000000):
+    #     pool.submit(testf, i)

funboost/consumers/base_consumer.py

@@ -452,8 +452,6 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         else:
             check_not_monkey()
 
-    ''' 日志跳转代码行不正确，不用这种方式'''
-
     def _log_error(self, msg, exc_info=None):
         self.logger.error(msg=f'{msg} \n', exc_info=exc_info, extra={'sys_getframe_n': 3})
         self.error_file_logger.error(msg=f'{msg} \n', exc_info=exc_info, extra={'sys_getframe_n': 3})
@@ -767,7 +765,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                     avarage_function_spend_time = round(self._consuming_function_cost_time_total_every_unit_time / self._execute_task_times_every_unit_time, 4)
                     msg = f'{self._unit_time_for_count} 秒内执行了 {self._execute_task_times_every_unit_time} 次函数 [ {self.consuming_function.__name__} ] ,' \
                           f'函数平均运行耗时 {avarage_function_spend_time} 秒。 '
-                    self.logger.debug(msg)
+                    self.logger.info(msg)
                     if self._msg_num_in_broker != -1 and time.time() - self._last_show_remaining_execution_time > self._show_remaining_execution_time_interval:  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
                         # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
                         need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /
@@ -915,7 +913,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 avarage_function_spend_time = round(self._consuming_function_cost_time_total_every_unit_time / self._execute_task_times_every_unit_time, 4)
                 msg = f'{self._unit_time_for_count} 秒内执行了 {self._execute_task_times_every_unit_time} 次函数 [ {self.consuming_function.__name__} ] ,' \
                       f'函数平均运行耗时 {avarage_function_spend_time} 秒。 '
-                self.logger.debug(msg)
+                self.logger.info(msg)
                 if self._msg_num_in_broker != -1 and time.time() - self._last_show_remaining_execution_time > self._show_remaining_execution_time_interval:  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
                     # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
                     need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /

funboost/consumers/local_python_queue_consumer.py

@@ -14,7 +14,7 @@ class LocalPythonQueueConsumer(AbstractConsumer):
     """
 
     @property
-    def local_python_queue(self) -> SimpleQueue:
+    def local_python_queue(self) -> Queue:
         return local_python_queue_publisher.local_pyhton_queue_name__local_pyhton_queue_obj_map[self._queue_name]
 
     def _shedual_task(self):
@@ -32,3 +32,4 @@ class LocalPythonQueueConsumer(AbstractConsumer):
 
     def _requeue(self, kw):
         self.local_python_queue.put(kw['body'])
+

funboost/consumers/rabbitmq_amqpstorm_consumer.py

@@ -29,7 +29,8 @@ class RabbitmqConsumerAmqpStorm(AbstractConsumer):
         rp = RabbitmqPublisherUsingAmqpStorm(self.queue_name,broker_exclusive_config=self.broker_exclusive_config)
         rp.init_broker()
         rp.channel_wrapper_by_ampqstormbaic.qos(self._concurrent_num)
-        rp.channel_wrapper_by_ampqstormbaic.consume(callback=callback, queue=self.queue_name, no_ack=False,)
+        rp.channel_wrapper_by_ampqstormbaic.consume(callback=callback, queue=self.queue_name, no_ack=False,
+                                                    )
         rp.channel.start_consuming(auto_decode=True)
 
     def _confirm_consume(self, kw):

funboost/consumers/rabbitmq_pika_consumerv0.py

@@ -25,6 +25,7 @@ class RabbitmqConsumer(AbstractConsumer):
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
         self._lock_for_pika = Lock()
+        raise Exception('不建议使用这个中间件模式，建议使用 BrokerEnum.RABBITMQ_AMQPSTORM 操作rabbitmq')
 
     def _shedual_task(self):
         # channel = RabbitMqFactory(is_use_rabbitpy=0).get_rabbit_cleint().creat_a_channel()

funboost/consumers/rabbitmq_rabbitpy_consumer.py

@@ -12,7 +12,8 @@ class RabbitmqConsumerRabbitpy(AbstractConsumer):
     """
     使用rabbitpy实现的
     """
-
+    def custom_init(self):
+        raise Exception('不建议使用这个中间件模式，建议使用 BrokerEnum.RABBITMQ_AMQPSTORM 操作rabbitmq')
 
     def _shedual_task(self):
         # noinspection PyTypeChecker

funboost/core/booster.py

@@ -21,6 +21,8 @@ class Booster:
     funboost极其重视代码能在pycharm下自动补全。元编程经常造成在pycharm下代码无法自动补全提示，要是实现代码补全难。
     这种__call__写法在pycahrm下 不仅能补全消费函数的 push consume等方法，也能补全函数本身的入参，一举两得。代码能自动补全很重要。
     一个函数fun被 boost装饰器装饰后， isinstance(fun,Booster) 为True.
+
+    Booster 是把Consumer 和 Publisher的方法集为一体。
     """
 
     def __init__(self, queue_name,

funboost/core/get_booster.py

@@ -23,3 +23,16 @@ def get_boost_params_and_consuming_function(queue_name: str) -> (dict, typing.Ca
     booster_current_pid = boost(**boost_params)(consuming_function)
     """
     return queue_name__boost_params_consuming_function_map[queue_name]
+
+
+def get_or_create_booster(queue_name, consuming_function, **boost_params, ) -> Booster:
+    """
+    当前进程获得或者创建booster对象。方便有的人需要在函数内部临时动态根据队列名创建booster,不会无数次临时生成消费者、生产者、创建消息队列连接。
+    :param boost_params: 就是 Booster的入参。
+    :return:
+    """
+    try:
+        return get_booster(queue_name)
+    except ValueError:  # 不存在就创建。
+        boost_params['queue_name'] = queue_name
+        return Booster(**boost_params)(consuming_function)

funboost/publishers/local_python_queue_publisher.py

@@ -16,7 +16,7 @@ class LocalPythonQueuePublisher(AbstractPublisher):
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
         if self._queue_name not in local_pyhton_queue_name__local_pyhton_queue_obj_map:
-            local_pyhton_queue_name__local_pyhton_queue_obj_map[self._queue_name] = SimpleQueue()
+            local_pyhton_queue_name__local_pyhton_queue_obj_map[self._queue_name] = Queue(10000)
         self.queue = local_pyhton_queue_name__local_pyhton_queue_obj_map[self._queue_name]
 
     def concrete_realization_of_publish(self, msg):

funboost/publishers/rabbitmq_amqpstorm_publisher.py

@@ -19,7 +19,7 @@ class RabbitmqPublisherUsingAmqpStorm(AbstractPublisher):
     DURABLE = True
 
     def custom_init(self):
-        arguments = {}
+        arguments = {}     #  {'x-queue-type':'classic'} classic stream lazy quorum
         if self.broker_exclusive_config['x-max-priority']:
             arguments['x-max-priority'] = self.broker_exclusive_config['x-max-priority']
         self.queue_declare_params = dict(queue=self._queue_name, durable=self.DURABLE, arguments=arguments,auto_delete=False)

funboost/utils/dependency_packages_in_pythonpath/aioredis/__init__.py

@@ -1,59 +1,59 @@
-from aioredis.client import Redis, StrictRedis
-from aioredis.connection import (
-    BlockingConnectionPool,
-    Connection,
-    ConnectionPool,
-    SSLConnection,
-    UnixDomainSocketConnection,
-)
-from aioredis.exceptions import (
-    AuthenticationError,
-    AuthenticationWrongNumberOfArgsError,
-    BusyLoadingError,
-    ChildDeadlockedError,
-    ConnectionError,
-    DataError,
-    InvalidResponse,
-    PubSubError,
-    ReadOnlyError,
-    RedisError,
-    ResponseError,
-    TimeoutError,
-    WatchError,
-)
-from aioredis.utils import from_url
-
-
-def int_or_str(value):
-    try:
-        return int(value)
-    except ValueError:
-        return value
-
-
-__version__ = "2.0.1"
-VERSION = tuple(map(int_or_str, __version__.split(".")))
-
-__all__ = [
-    "AuthenticationError",
-    "AuthenticationWrongNumberOfArgsError",
-    "BlockingConnectionPool",
-    "BusyLoadingError",
-    "ChildDeadlockedError",
-    "Connection",
-    "ConnectionError",
-    "ConnectionPool",
-    "DataError",
-    "from_url",
-    "InvalidResponse",
-    "PubSubError",
-    "ReadOnlyError",
-    "Redis",
-    "RedisError",
-    "ResponseError",
-    "SSLConnection",
-    "StrictRedis",
-    "TimeoutError",
-    "UnixDomainSocketConnection",
-    "WatchError",
-]
+from aioredis.client import Redis, StrictRedis
+from aioredis.connection import (
+    BlockingConnectionPool,
+    Connection,
+    ConnectionPool,
+    SSLConnection,
+    UnixDomainSocketConnection,
+)
+from aioredis.exceptions import (
+    AuthenticationError,
+    AuthenticationWrongNumberOfArgsError,
+    BusyLoadingError,
+    ChildDeadlockedError,
+    ConnectionError,
+    DataError,
+    InvalidResponse,
+    PubSubError,
+    ReadOnlyError,
+    RedisError,
+    ResponseError,
+    TimeoutError,
+    WatchError,
+)
+from aioredis.utils import from_url
+
+
+def int_or_str(value):
+    try:
+        return int(value)
+    except ValueError:
+        return value
+
+
+__version__ = "2.0.1"
+VERSION = tuple(map(int_or_str, __version__.split(".")))
+
+__all__ = [
+    "AuthenticationError",
+    "AuthenticationWrongNumberOfArgsError",
+    "BlockingConnectionPool",
+    "BusyLoadingError",
+    "ChildDeadlockedError",
+    "Connection",
+    "ConnectionError",
+    "ConnectionPool",
+    "DataError",
+    "from_url",
+    "InvalidResponse",
+    "PubSubError",
+    "ReadOnlyError",
+    "Redis",
+    "RedisError",
+    "ResponseError",
+    "SSLConnection",
+    "StrictRedis",
+    "TimeoutError",
+    "UnixDomainSocketConnection",
+    "WatchError",
+]