funboost 24.0__py3-none-any.whl → 24.2__py3-none-any.whl

funboost/__init__.py

@@ -6,6 +6,8 @@ from funboost.core import show_funboost_flag
 
 # noinspection PyUnresolvedReferences
 import nb_log
+# noinspection PyUnresolvedReferences
+from nb_log import get_logger,nb_print
 from funboost.set_frame_config import patch_frame_config, show_frame_config
 from funboost.funboost_config_deafult import BoostDecoratorDefaultParams
 

funboost/concurrent_pool/flexible_thread_pool.py

@@ -0,0 +1,140 @@
+"""
+比更简单的 ThreadPoolExecutorShrinkAble 的弹性线程池，因为 funboost的并发池永远不需要判断代码结束，所以不用 ThreadPoolExecutorShrinkAble 那么复杂来兼容判断并发池要随代码退出而结束循环
+"""
+
+import asyncio
+import inspect
+import queue
+import threading
+from functools import wraps
+
+import nb_log
+from nb_log import LoggerMixin, LoggerLevelSetterMixin
+
+
+class FlexibleThreadPool(LoggerMixin, LoggerLevelSetterMixin):
+    KEEP_ALIVE_TIME = 10
+    MIN_WORKERS = 0
+
+    def __init__(self, max_workers: int = None):
+        self.work_queue = queue.Queue(10)
+        self.max_workers = max_workers
+        self._threads_num = 0
+        self.threads_free_count = 0
+        self._lock_compute_start_thread = threading.Lock()
+        self._lock_compute_threads_free_count = threading.Lock()
+        self._lock_for_adjust_thread = threading.Lock()
+        self._lock_for_judge_threads_free_count = threading.Lock()
+        self.pool_ident = id(self)
+        self.asyncio_loop = asyncio.new_event_loop()
+
+    def _change_threads_free_count(self, change_num):
+        with self._lock_compute_threads_free_count:
+            self.threads_free_count += change_num
+
+    def _change_threads_start_count(self, change_num):
+        with self._lock_compute_start_thread:
+            self._threads_num += change_num
+
+    def submit(self, func, *args, **kwargs):
+        self.work_queue.put([func, args, kwargs])
+        with self._lock_for_adjust_thread:
+            if self.threads_free_count <= self.MIN_WORKERS and self._threads_num < self.max_workers:
+                _KeepAliveTimeThread(self).start()
+
+
+def run_sync_or_async_fun(func, *args, **kwargs):
+    t1 =time.time()
+    fun_is_asyncio = inspect.iscoroutinefunction(func)
+
+    if fun_is_asyncio:
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+        print(time.time() - t1)
+        try:
+            result =  loop.run_until_complete(func(*args, **kwargs))
+
+            return result
+        finally:
+            pass
+            # loop.close()
+    else:
+        return func(*args, **kwargs)
+
+
+def sync_or_async_fun_deco(func):
+    @wraps(func)
+    def _inner(*args, **kwargs):
+        return run_sync_or_async_fun(func, *args, **kwargs)
+
+    return _inner
+
+
+# noinspection PyProtectedMember
+class _KeepAliveTimeThread(threading.Thread):
+    logger = nb_log.get_logger('_KeepAliveTimeThread')
+
+    def __init__(self, thread_pool: FlexibleThreadPool):
+        super().__init__()
+        self.pool = thread_pool
+
+    def run(self) -> None:
+        self.logger.debug(f'新启动线程 {self.ident} ')
+        self.pool._change_threads_free_count(1)
+        self.pool._change_threads_start_count(1)
+        while 1:
+            try:
+                func, args, kwargs = self.pool.work_queue.get(block=True, timeout=self.pool.KEEP_ALIVE_TIME)
+            except queue.Empty:
+
+                with self.pool._lock_for_judge_threads_free_count:
+                    # print(self.pool.threads_free_count)
+                    if self.pool.threads_free_count > self.pool.MIN_WORKERS:
+                        self.logger.debug(f'停止线程 {self._ident}, 触发条件是 {self.pool.pool_ident} 线程池中的 {self.ident} 线程 超过 {self.pool.KEEP_ALIVE_TIME} 秒没有任务，线程池中不在工作状态中的线程数量是 {self.pool.threads_free_count}，超过了指定的最小核心数量 {self.pool.MIN_WORKERS}')  # noqa
+                        self.pool._change_threads_free_count(-1)
+                        self.pool._change_threads_start_count(-1)
+                        break  # 退出while 1，即是结束。
+                    else:
+                        continue
+            self.pool._change_threads_free_count(-1)
+            try:
+                t1 = time.time()
+                fun = sync_or_async_fun_deco(func)
+                result = fun(*args, **kwargs)
+                print(time.time()-t1)
+                # print(result)
+            except BaseException as exc:
+                self.logger.exception(f'函数 {func.__name__} 中发生错误，错误原因是 {type(exc)} {exc} ')
+            self.pool._change_threads_free_count(1)
+
+
+if __name__ == '__main__':
+    import time
+    from concurrent.futures import ThreadPoolExecutor
+    from custom_threadpool_executor import ThreadPoolExecutorShrinkAble
+
+
+    def testf(x):
+        # time.sleep(10)
+        if x % 10000 == 0:
+            print(x)
+
+
+    async def aiotestf(x):
+        # await asyncio.sleep(1)
+        # print(x)
+        return x * 2
+
+
+    pool = FlexibleThreadPool(100)
+    # pool = ThreadPoolExecutor(100)
+    # pool = ThreadPoolExecutorShrinkAble(100)
+
+    for i in range(2000):
+        # time.sleep(2)
+        pool.submit(aiotestf, i)
+
+    # for i in range(1000000):
+    #     pool.submit(testf, i)

funboost/consumers/base_consumer.py

@@ -452,8 +452,6 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         else:
             check_not_monkey()
 
-    ''' 日志跳转代码行不正确，不用这种方式'''
-
     def _log_error(self, msg, exc_info=None):
         self.logger.error(msg=f'{msg} \n', exc_info=exc_info, extra={'sys_getframe_n': 3})
         self.error_file_logger.error(msg=f'{msg} \n', exc_info=exc_info, extra={'sys_getframe_n': 3})
@@ -767,7 +765,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                     avarage_function_spend_time = round(self._consuming_function_cost_time_total_every_unit_time / self._execute_task_times_every_unit_time, 4)
                     msg = f'{self._unit_time_for_count} 秒内执行了 {self._execute_task_times_every_unit_time} 次函数 [ {self.consuming_function.__name__} ] ,' \
                           f'函数平均运行耗时 {avarage_function_spend_time} 秒。 '
-                    self.logger.debug(msg)
+                    self.logger.info(msg)
                     if self._msg_num_in_broker != -1 and time.time() - self._last_show_remaining_execution_time > self._show_remaining_execution_time_interval:  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
                         # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
                         need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /
@@ -915,7 +913,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 avarage_function_spend_time = round(self._consuming_function_cost_time_total_every_unit_time / self._execute_task_times_every_unit_time, 4)
                 msg = f'{self._unit_time_for_count} 秒内执行了 {self._execute_task_times_every_unit_time} 次函数 [ {self.consuming_function.__name__} ] ,' \
                       f'函数平均运行耗时 {avarage_function_spend_time} 秒。 '
-                self.logger.debug(msg)
+                self.logger.info(msg)
                 if self._msg_num_in_broker != -1 and time.time() - self._last_show_remaining_execution_time > self._show_remaining_execution_time_interval:  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
                     # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
                     need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /

funboost/consumers/local_python_queue_consumer.py

@@ -14,7 +14,7 @@ class LocalPythonQueueConsumer(AbstractConsumer):
     """
 
     @property
-    def local_python_queue(self) -> SimpleQueue:
+    def local_python_queue(self) -> Queue:
         return local_python_queue_publisher.local_pyhton_queue_name__local_pyhton_queue_obj_map[self._queue_name]
 
     def _shedual_task(self):
@@ -32,3 +32,4 @@ class LocalPythonQueueConsumer(AbstractConsumer):
 
     def _requeue(self, kw):
         self.local_python_queue.put(kw['body'])
+

funboost/core/booster.py

@@ -21,6 +21,8 @@ class Booster:
     funboost极其重视代码能在pycharm下自动补全。元编程经常造成在pycharm下代码无法自动补全提示，要是实现代码补全难。
     这种__call__写法在pycahrm下 不仅能补全消费函数的 push consume等方法，也能补全函数本身的入参，一举两得。代码能自动补全很重要。
     一个函数fun被 boost装饰器装饰后， isinstance(fun,Booster) 为True.
+
+    Booster 是把Consumer 和 Publisher的方法集为一体。
     """
 
     def __init__(self, queue_name,

funboost/core/get_booster.py

@@ -25,9 +25,9 @@ def get_boost_params_and_consuming_function(queue_name: str) -> (dict, typing.Ca
     return queue_name__boost_params_consuming_function_map[queue_name]
 
 
-def get_or_create_booster(queue_name, consuming_function, **boost_params, ):
+def get_or_create_booster(queue_name, consuming_function, **boost_params, ) -> Booster:
     """
-    当前进程获得或者创建booster对象。方便有的人需要在函数内部临时动态根据队列名创建booster,不会无数次临时生成消费者 生产者创建消息队列连接。
+    当前进程获得或者创建booster对象。方便有的人需要在函数内部临时动态根据队列名创建booster,不会无数次临时生成消费者、生产者、创建消息队列连接。
     :param boost_params: 就是 Booster的入参。
     :return:
     """

funboost/publishers/local_python_queue_publisher.py

@@ -16,7 +16,7 @@ class LocalPythonQueuePublisher(AbstractPublisher):
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
         if self._queue_name not in local_pyhton_queue_name__local_pyhton_queue_obj_map:
-            local_pyhton_queue_name__local_pyhton_queue_obj_map[self._queue_name] = SimpleQueue()
+            local_pyhton_queue_name__local_pyhton_queue_obj_map[self._queue_name] = Queue(10000)
         self.queue = local_pyhton_queue_name__local_pyhton_queue_obj_map[self._queue_name]
 
     def concrete_realization_of_publish(self, msg):

{funboost-24.0.dist-info → funboost-24.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 24.0
+Version: 24.2
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持celery框架整体作为funboost中间件，python函数加速器，框架包罗万象，一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是 方便 快速 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-24.0.dist-info → funboost-24.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-funboost/__init__.py,sha256=9Khpiu4k86MTbo_dltppDd1a6L-zH-HdBFhOizg2O7k,2465
+funboost/__init__.py,sha256=JZZsKhrVUpcLzWOPkZA0SDdnKNLPXLv66WGLXSx8FbM,2544
 funboost/__init__old.py,sha256=cSzw-ZQqtAAp5_-7eONXQT5fwz_JbZlRjDQPASDLUHk,20378
 funboost/constant.py,sha256=tSdHQ8nrZH63jOKKZFtOfTo2melCsiMrk-O1lFcsIKA,6489
 funboost/funboost_config_deafult.py,sha256=mh9o91CgMC26glIWkw7489GxmJ7bibi4_oBMnv3SCNU,7430
@@ -20,13 +20,14 @@ funboost/concurrent_pool/custom_evenlet_pool_executor.py,sha256=f8ubB21J6TZr0qaW
 funboost/concurrent_pool/custom_gevent_pool_executor.py,sha256=qCkWB836UuBm6yVdI6hhWOaI5vFlog0N1y46ByzO0ro,4421
 funboost/concurrent_pool/custom_threadpool_executor.py,sha256=Ydqpx2bJ4mnbWCcQj247xepSvNN72ZfekaYA_vBMfyM,11745
 funboost/concurrent_pool/custom_threadpool_executor000.py,sha256=jJLXy3h-bELap6nZA6yLtdozzTWcvCtZ7IY6MTqLEAM,9317
+funboost/concurrent_pool/flexible_thread_pool.py,sha256=kDSydmWI8M7qtnN-dNDzAqeVabx4ADsF-qO2_CHYDZA,5022
 funboost/concurrent_pool/single_thread_executor.py,sha256=NDWOegh8Nxpb-Bp-lUlj-DONWvepSmA9qepL1yNgdQI,373
 funboost/concurrent_pool/backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 funboost/concurrent_pool/backup/async_pool_executor0223.py,sha256=iTxxJFk2lu1P9ZAIkBip3euq3oEQ4_qTODy3xUaOecY,9548
 funboost/concurrent_pool/backup/async_pool_executor_back.py,sha256=vIgUUyF4Zb0jIRPWgNPqyO09YEkQP32kkpGBldqm4qA,9568
 funboost/concurrent_pool/backup/async_pool_executor_janus.py,sha256=OHMWJ9l3EYTpPpcrPrGGKd4K0tmQ2PN8HiX0Dta0EOo,5728
 funboost/consumers/__init__.py,sha256=ZXY_6Kut1VYNQiF5aWEgIWobsW1ht9YUP0TdRZRWFqI,126
-funboost/consumers/base_consumer.py,sha256=CdBohmk2DIgT4oiyX-REuqhWh3GwBjyPV4SKmXm04xA,86845
+funboost/consumers/base_consumer.py,sha256=1yt-Exk1x2vCUR8zm0UVPJtZpp5B8gQIXkRL0E-7hTc,86777
 funboost/consumers/celery_consumer.py,sha256=Jt7m7k53dXzdZFdbjl3ECwxlAq3ojdjhPqJLPG2jW4M,9088
 funboost/consumers/celery_consumer000.py,sha256=8SF8ppHIMH-BIAHO0NyJYnSQxe_PcT6hv05e7DySG54,4574
 funboost/consumers/confirm_mixin.py,sha256=oVBQ1RayIBFOzGNTJ69HdBR_kLd46xd0VfCOKfqDpLA,6033
@@ -38,7 +39,7 @@ funboost/consumers/huey_consumer.py,sha256=_Z2lpfAzvs1HqkSouncN7eH1VfyQJPYNZNgv1
 funboost/consumers/kafka_consumer.py,sha256=2BZT5UQIGqjS1emsX--V1J8gWFOhMjCMMiSnWy_Hto0,4217
 funboost/consumers/kafka_consumer_manually_commit.py,sha256=L-GYQrZ91TWCeI-GIDMXYg3LXFcq0mQ1ACu0gp4bkTI,6947
 funboost/consumers/kombu_consumer.py,sha256=VlU5LR69G0we6K5UCzWOiz-Ul_LJArWznMvTUCBwyR8,10186
-funboost/consumers/local_python_queue_consumer.py,sha256=Sc6uPDzpVZRRNCWWoNDHCOwVaixFoXWsLeNEOl2TKuU,1309
+funboost/consumers/local_python_queue_consumer.py,sha256=jNYohp7IW1d8BT8weHE_6R11fvMFnMOFd8HE7kW8_yg,1305
 funboost/consumers/mongomq_consumer.py,sha256=W5d4QnYun1OnnCn8K41ZmXuGdpAXmnzAa4EI7H7KQO4,1090
 funboost/consumers/mqtt_consumer.py,sha256=StlfPUeQ6o0HiZBpt7TlC_r2DjzPHBZBF2xLSxQW13A,2228
 funboost/consumers/nameko_consumer.py,sha256=IVwUxBixUx2m3F_q8Xn-UsnJWXayMpnOIZjICCd6mcU,2183
@@ -71,12 +72,12 @@ funboost/contrib/queue2queue.py,sha256=I6e1d2s_TQ_29rbwSs3KuA7Tuj9B905EcJruT287C
 funboost/contrib/redis_consume_latest_msg_broker.py,sha256=V-8-pOyotOXNbs2olS7vig-yN1PxRdw_MeLOtpUufJc,1817
 funboost/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 funboost/core/active_cousumer_info_getter.py,sha256=VwVRxsApdKqnoBdg1PsHEkyWiOFJMm0_qWIdCFbPOWg,4984
-funboost/core/booster.py,sha256=9jqNczG36-bEs9x708PHWqsugk5Y0gjOZcPO_zq3kmU,16688
+funboost/core/booster.py,sha256=vc0mIztI2YeztvW2zt0sbbmGhCJk8bylamex6XgszaI,16756
 funboost/core/exit_signal.py,sha256=GDP3si_hNm7-iNZeeRaON62He47j_jX5weOOsDYW68Y,576
 funboost/core/fabric_deploy_helper.py,sha256=yIdWhlJAR1rCpzvcSme-KrU7RwKH8B7tdwHG-q6QH0U,8875
 funboost/core/function_result_status_config.py,sha256=CE7xaGoL7vT2VsLTHvv-EnJE7K83XVWivBimHVmVfnQ,1570
 funboost/core/function_result_status_saver.py,sha256=tNYRvUfb2T59mda9fLWDftbPbLvyKQg7Sph6BTBy_J4,8886
-funboost/core/get_booster.py,sha256=GcHmPyk0Rezw_fjVgj2bw7NW6_koXF_ttruX6dqTv30,1746
+funboost/core/get_booster.py,sha256=G47Gukmy-2DTBng8ayTPfK-blQ7gxT83CJlryrhqpOo,1762
 funboost/core/global_boosters.py,sha256=CDrnKhxEEb-GxTL7JhrDTD1tyhLj7ciAtO1Cy2iyi1Q,340
 funboost/core/helper_funs.py,sha256=SKSMKytJTOFQJsuMKWJ-_mQg6e_Bgpq1ANDzjjBAcio,1192
 funboost/core/kill_remote_task.py,sha256=Rj4nrz13CKka8he1T8-k6CYN_yjvcaYZaZ7yurKcKzo,8159
@@ -115,7 +116,7 @@ funboost/publishers/httpsqs_publisher.py,sha256=7cf5ijwrbp4smq6ofndrKisruAqG0Wzf
 funboost/publishers/huey_publisher.py,sha256=z1CF18YZkQY6lqeoc6tiJkaYsRjRicbhYYeoHSvdD-w,1101
 funboost/publishers/kafka_publisher.py,sha256=cmlJ0mvwq0Ajlth4VQqwnoe6v_bZ4eIz49GgkiJr-ZU,2160
 funboost/publishers/kombu_publisher.py,sha256=TJt24M_Y4x8wjXHFl2hp6z3Dat6rjuA_xGiqA8RbRLg,5321
-funboost/publishers/local_python_queue_publisher.py,sha256=oETalt6ICcWbvvALG3dulgTQxOS4Ikq3BwcYl-tEX2Q,1383
+funboost/publishers/local_python_queue_publisher.py,sha256=3Xl29I0ul2lWO-cIVyeDdtAYDGK2Jn_Q3qacHu2Ulpg,1382
 funboost/publishers/mongomq_publisher.py,sha256=xQr3KMQEKksX4OEvzPlCl8v1VeBHaoZtYw2QujOUyGo,1874
 funboost/publishers/mqtt_publisher.py,sha256=NKVDE5R12QL99IXgRjJtF4phyW8QaXKxHkqW5p_kXr4,3050
 funboost/publishers/nameko_publisher.py,sha256=2sqRe08mTjBlvnDe-mE55wFMxllzHCPKA4WtMABnKZ0,1670
@@ -242,8 +243,8 @@ funboost/utils/pysnooper_ydf/utils.py,sha256=evSmGi_Oul7vSP47AJ0DLjFwoCYCfunJZ1m
 funboost/utils/pysnooper_ydf/variables.py,sha256=QejRDESBA06KG9OH4sBT4J1M55eaU29EIHg8K_igaXo,3693
 funboost/utils/times/__init__.py,sha256=Y4bQD3SIA_E7W2YvHq2Qdi0dGM4H2DxyFNdDOuFOq1w,2417
 funboost/utils/times/version.py,sha256=11XfnZVVzOgIhXXdeN_mYfdXThfrsbQHpA0wCjz-hpg,17
-funboost-24.0.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-funboost-24.0.dist-info/METADATA,sha256=BS1P5aE66v6zQpatiFyJw1ORCVqC2t4DiFZQMmVWn5E,26979
-funboost-24.0.dist-info/WHEEL,sha256=D1Wh14kWDxPnrM-5t_6UCB-UuQNrEODtRa3vF4OsvQY,97
-funboost-24.0.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
-funboost-24.0.dist-info/RECORD,,
+funboost-24.2.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+funboost-24.2.dist-info/METADATA,sha256=0-Yhyjd8dSqlYiv80xxV7wdA_YyjLsYLW60bjmuSQrk,26979
+funboost-24.2.dist-info/WHEEL,sha256=D1Wh14kWDxPnrM-5t_6UCB-UuQNrEODtRa3vF4OsvQY,97
+funboost-24.2.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
+funboost-24.2.dist-info/RECORD,,