funboost 43.3__tar.gz → 43.5__tar.gz

{funboost-43.3/funboost.egg-info → funboost-43.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 43.3
+Version: 43.5
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-43.3 → funboost-43.5}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "43.3"
+__version__ = "43.5"
 
 from funboost.set_frame_config import show_frame_config
 

{funboost-43.3 → funboost-43.5}/funboost/assist/celery_helper.py

@@ -40,6 +40,7 @@ class CeleryHelper:
         :return:
         """
         celery_app.conf.update(celery_app_conf)
+        # 例如  celery_app.conf.update({'broker_transport_options':{'visibility_timeout': 18000,'max_retries':5}})
 
     @staticmethod
     def show_celery_app_conf():

{funboost-43.3 → funboost-43.5}/funboost/concurrent_pool/__init__.py

@@ -12,8 +12,6 @@ gevent协程
 """
 from .base_pool_type import FunboostBaseConcurrentPool
 from .async_pool_executor import AsyncPoolExecutor
-# from .custom_evenlet_pool_executor import CustomEventletPoolExecutor
-# from .custom_gevent_pool_executor import GeventPoolExecutor
 from .bounded_threadpoolexcutor import BoundedThreadPoolExecutor
 from .custom_threadpool_executor import CustomThreadPoolExecutor
 from .flexible_thread_pool import FlexibleThreadPool

{funboost-43.3 → funboost-43.5}/funboost/concurrent_pool/custom_evenlet_pool_executor.py

@@ -4,19 +4,26 @@
 import atexit
 import time
 import warnings
-from eventlet import greenpool, monkey_patch, patcher, Timeout
+# from eventlet import greenpool, monkey_patch, patcher, Timeout
 
 from funboost.concurrent_pool import FunboostBaseConcurrentPool
 from funboost.core.loggers import get_funboost_file_logger
 # print('eventlet 导入')
+from funboost.core.lazy_impoter import EventletImporter
+
 
 def check_evenlet_monkey_patch(raise_exc=True):
-    if not patcher.is_monkey_patched('socket'):  # 随便选一个检测标志
+    try:
+        if not EventletImporter().patcher.is_monkey_patched('socket'):  # 随便选一个检测标志
+            if raise_exc:
+                warnings.warn(f'检测到没有打 evenlet 包的猴子补丁 ,请在起始脚本文件首行加上     import eventlet;eventlet.monkey_patch(all=True) ')
+                raise Exception('检测到没有打 evenlet 包的猴子补丁 ,请在起始脚本文件首行加上    import eventlet;eventlet.monkey_patch(all=True)')
+        else:
+            return 1
+    except ModuleNotFoundError:
         if raise_exc:
             warnings.warn(f'检测到没有打 evenlet 包的猴子补丁 ,请在起始脚本文件首行加上     import eventlet;eventlet.monkey_patch(all=True) ')
             raise Exception('检测到没有打 evenlet 包的猴子补丁 ,请在起始脚本文件首行加上    import eventlet;eventlet.monkey_patch(all=True)')
-    else:
-        return 1
 
 
 logger_evenlet_timeout_deco = get_funboost_file_logger('evenlet_timeout_deco')
@@ -25,12 +32,12 @@ logger_evenlet_timeout_deco = get_funboost_file_logger('evenlet_timeout_deco')
 def evenlet_timeout_deco(timeout_t):
     def _evenlet_timeout_deco(f):
         def __evenlet_timeout_deco(*args, **kwargs):
-            timeout = Timeout(timeout_t, )
+            timeout = EventletImporter().Timeout(timeout_t, )
             # timeout.start()  # 与gevent不一样,直接start了。
             result = None
             try:
                 result = f(*args, **kwargs)
-            except Timeout as t:
+            except EventletImporter().Timeout as t:
                 logger_evenlet_timeout_deco.error(f'函数 {f} 运行超过了 {timeout_t} 秒')
                 if t is not timeout:
                     print(t)
@@ -44,24 +51,27 @@ def evenlet_timeout_deco(timeout_t):
     return _evenlet_timeout_deco
 
 
-class CustomEventletPoolExecutor(greenpool.GreenPool,FunboostBaseConcurrentPool):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        check_evenlet_monkey_patch()  # basecomer.py中检查。
-        atexit.register(self.shutdown)
+def get_eventlet_pool_executor(*args2, **kwargs2):
+    class CustomEventletPoolExecutor(EventletImporter().greenpool.GreenPool, FunboostBaseConcurrentPool):
+        def __init__(self, *args, **kwargs):
+            super().__init__(*args, **kwargs)
+            check_evenlet_monkey_patch()  # basecomer.py中检查。
+            atexit.register(self.shutdown)
+
+        def submit(self, *args, **kwargs):  # 保持为一直的公有用法。
+            # nb_print(args)
+            self.spawn_n(*args, **kwargs)
+            # self.spawn_n(*args, **kwargs)
 
-    def submit(self, *args, **kwargs):  # 保持为一直的公有用法。
-        # nb_print(args)
-        self.spawn_n(*args, **kwargs)
-        # self.spawn_n(*args, **kwargs)
+        def shutdown(self):
+            self.waitall()
 
-    def shutdown(self):
-        self.waitall()
+    return CustomEventletPoolExecutor(*args2, **kwargs2)
 
 
 if __name__ == '__main__':
     # greenpool.GreenPool.waitall()
-    monkey_patch(all=True)
+    EventletImporter().monkey_patch(all=True)
 
 
     def f2(x):
@@ -70,7 +80,7 @@ if __name__ == '__main__':
         print(x)
 
 
-    pool = CustomEventletPoolExecutor(4)
+    pool = get_eventlet_pool_executor(4)
 
     for i in range(15):
         print(f'放入{i}')

{funboost-43.3 → funboost-43.5}/funboost/concurrent_pool/custom_gevent_pool_executor.py

@@ -7,23 +7,27 @@ import warnings
 # from collections import Callable
 from typing import Callable
 import threading
-import gevent
-from gevent import pool as gevent_pool
-from gevent import monkey
-from gevent.queue import JoinableQueue
+from funboost.core.lazy_impoter import GeventImporter
 
 # from nb_log import LoggerMixin, nb_print, LogManager
 from funboost.concurrent_pool import FunboostBaseConcurrentPool
-from funboost.core.loggers import get_funboost_file_logger,FunboostFileLoggerMixin
+from funboost.core.loggers import get_funboost_file_logger, FunboostFileLoggerMixin
+
+
 # print('gevent 导入')
 
 def check_gevent_monkey_patch(raise_exc=True):
-    if not monkey.is_module_patched('socket'):  # 随便选一个检测标志
+    try:
+        if not GeventImporter().monkey.is_module_patched('socket'):  # 随便选一个检测标志
+            if raise_exc:
+                warnings.warn(f'检测到 你还没有打gevent包的猴子补丁，请在所运行的起始脚本第一行写上  【import gevent.monkey;gevent.monkey.patch_all()】  这句话。')
+                raise Exception(f'检测到 你还没有打gevent包的猴子补丁，请在所运行的起始脚本第一行写上  【import gevent.monkey;gevent.monkey.patch_all()】  这句话。')
+        else:
+            return 1
+    except ModuleNotFoundError:
         if raise_exc:
             warnings.warn(f'检测到 你还没有打gevent包的猴子补丁，请在所运行的起始脚本第一行写上  【import gevent.monkey;gevent.monkey.patch_all()】  这句话。')
             raise Exception(f'检测到 你还没有打gevent包的猴子补丁，请在所运行的起始脚本第一行写上  【import gevent.monkey;gevent.monkey.patch_all()】  这句话。')
-    else:
-        return 1
 
 
 logger_gevent_timeout_deco = get_funboost_file_logger('gevent_timeout_deco')
@@ -32,12 +36,12 @@ logger_gevent_timeout_deco = get_funboost_file_logger('gevent_timeout_deco')
 def gevent_timeout_deco(timeout_t):
     def _gevent_timeout_deco(f):
         def __gevent_timeout_deceo(*args, **kwargs):
-            timeout = gevent.Timeout(timeout_t, )
+            timeout = GeventImporter().gevent.Timeout(timeout_t, )
             timeout.start()
             result = None
             try:
                 result = f(*args, **kwargs)
-            except gevent.Timeout as t:
+            except GeventImporter().gevent.Timeout as t:
                 logger_gevent_timeout_deco.error(f'函数 {f} 运行超过了 {timeout_t} 秒')
                 if t is not timeout:
                     print(t)
@@ -51,25 +55,28 @@ def gevent_timeout_deco(timeout_t):
     return _gevent_timeout_deco
 
 
-class GeventPoolExecutor(gevent_pool.Pool,FunboostBaseConcurrentPool):
-    def __init__(self, size=None, greenlet_class=None):
-        check_gevent_monkey_patch() # basecomer.py中检查。
-        super().__init__(size, greenlet_class)
-        atexit.register(self.shutdown)
+def get_gevent_pool_executor(size=None, greenlet_class=None):
+    class GeventPoolExecutor(GeventImporter().gevent_pool.Pool, FunboostBaseConcurrentPool):
+        def __init__(self, size2=None, greenlet_class2=None):
+            check_gevent_monkey_patch()  # basecomer.py中检查。
+            super().__init__(size2, greenlet_class2)
+            atexit.register(self.shutdown)
+
+        def submit(self, *args, **kwargs):
+            self.spawn(*args, **kwargs)
 
-    def submit(self, *args, **kwargs):
-        self.spawn(*args, **kwargs)
+        def shutdown(self):
+            self.join()
 
-    def shutdown(self):
-        self.join()
+    return GeventPoolExecutor(size, greenlet_class)
 
 
-class GeventPoolExecutor2(FunboostFileLoggerMixin,FunboostBaseConcurrentPool):
+class GeventPoolExecutor2(FunboostFileLoggerMixin, FunboostBaseConcurrentPool):
     def __init__(self, max_works, ):
-        self._q = JoinableQueue(maxsize=max_works)
+        self._q = GeventImporter().JoinableQueue(maxsize=max_works)
         # self._q = Queue(maxsize=max_works)
         for _ in range(max_works):
-            gevent.spawn(self.__worker)
+            GeventImporter().gevent.spawn(self.__worker)
         # atexit.register(self.__atexit)
         self._q.join(timeout=100)
 
@@ -92,12 +99,12 @@ class GeventPoolExecutor2(FunboostFileLoggerMixin,FunboostBaseConcurrentPool):
         self._q.join()
 
 
-class GeventPoolExecutor3(FunboostFileLoggerMixin,FunboostBaseConcurrentPool):
+class GeventPoolExecutor3(FunboostFileLoggerMixin, FunboostBaseConcurrentPool):
     def __init__(self, max_works, ):
-        self._q = gevent.queue.Queue(max_works)
+        self._q = GeventImporter().gevent.queue.Queue(max_works)
         self.g_list = []
         for _ in range(max_works):
-            self.g_list.append(gevent.spawn(self.__worker))
+            self.g_list.append(GeventImporter().gevent.spawn(self.__worker))
         atexit.register(self.__atexit)
 
     def __worker(self):
@@ -112,7 +119,7 @@ class GeventPoolExecutor3(FunboostFileLoggerMixin,FunboostBaseConcurrentPool):
         self._q.put((fn, args, kwargs))
 
     def joinall(self):
-        gevent.joinall(self.g_list)
+        GeventImporter().gevent.joinall(self.g_list)
 
     def joinall_in_new_thread(self):
         threading.Thread(target=self.joinall)
@@ -123,7 +130,7 @@ class GeventPoolExecutor3(FunboostFileLoggerMixin,FunboostBaseConcurrentPool):
 
 
 if __name__ == '__main__':
-    monkey.patch_all(thread=False)
+    GeventImporter().monkey.patch_all(thread=False)
 
 
     def f2(x):

{funboost-43.3 → funboost-43.5}/funboost/consumers/base_consumer.py

@@ -972,11 +972,11 @@ class ConcurrentModeDispatcher(FunboostFileLoggerMixin):
             # pool_type = BoundedThreadPoolExecutor
             pool_type = FlexibleThreadPool
         elif self._concurrent_mode == ConcurrentModeEnum.GEVENT:
-            from funboost.concurrent_pool.custom_gevent_pool_executor import GeventPoolExecutor
-            pool_type = GeventPoolExecutor
+            from funboost.concurrent_pool.custom_gevent_pool_executor import get_gevent_pool_executor
+            pool_type = get_gevent_pool_executor
         elif self._concurrent_mode == ConcurrentModeEnum.EVENTLET:
-            from funboost.concurrent_pool.custom_evenlet_pool_executor import CustomEventletPoolExecutor
-            pool_type = CustomEventletPoolExecutor
+            from funboost.concurrent_pool.custom_evenlet_pool_executor import get_eventlet_pool_executor
+            pool_type = get_eventlet_pool_executor
         elif self._concurrent_mode == ConcurrentModeEnum.ASYNC:
             pool_type = AsyncPoolExecutor
         elif self._concurrent_mode == ConcurrentModeEnum.SINGLE_THREAD:

{funboost-43.3 → funboost-43.5}/funboost/contrib/api_publish_msg.py

@@ -1,4 +1,5 @@
 import traceback
+import typing
 
 from funboost import AioAsyncResult, AsyncResult
 
@@ -18,7 +19,7 @@ class MsgItem(BaseModel):
 class PublishResponse(BaseModel):
     succ: bool
     msg: str
-    status_and_result: dict = None  # 消费函数的消费状态和结果.
+    status_and_result: typing.Optional[dict] = None  # 消费函数的消费状态和结果.
 
 
 # 创建 FastAPI 应用实例

{funboost-43.3 → funboost-43.5}/funboost/core/fabric_deploy_helper.py

@@ -3,7 +3,8 @@ import re
 import sys
 import threading
 import time
-
+import os
+from pathlib import Path
 from fabric2 import Connection
 
 # from funboost.core.booster import Booster
@@ -12,11 +13,12 @@ from funboost.utils.paramiko_util import ParamikoFolderUploader
 # import nb_log
 from funboost.core.loggers import get_funboost_file_logger
 from funboost.core.booster import Booster
+
 logger = get_funboost_file_logger(__name__)
 
 
 # noinspection PyDefaultArgument
-def fabric_deploy(booster:Booster, host, port, user, password,
+def fabric_deploy(booster: Booster, host, port, user, password,
                   path_pattern_exluded_tuple=('/.git/', '/.idea/', '/dist/', '/build/'),
                   file_suffix_tuple_exluded=('.pyc', '.log', '.gz'),
                   only_upload_within_the_last_modify_time=3650 * 24 * 60 * 60,
@@ -69,9 +71,13 @@ def fabric_deploy(booster:Booster, host, port, user, password,
     python_proj_dir_short = python_proj_dir.split('/')[-2]
     # 获取被调用函数所在模块文件名
     file_name = sys._getframe(2).f_code.co_filename.replace('\\', '/')  # noqa\
-    # print(file_name)
-    relative_file_name = re.sub(f'^{python_proj_dir}', '', file_name)
+    # print(file_name,python_proj_dir)
+    # print(Path(file_name).relative_to(Path(python_proj_dir)))
+    # print(8888)
+    # relative_file_name = re.sub(f'^{python_proj_dir}', '', file_name)
+    relative_file_name = Path(file_name).relative_to(Path(python_proj_dir)).as_posix()
     relative_module = relative_file_name.replace('/', '.')[:-3]  # -3是去掉.py
+    print(relative_module)
     if user == 'root':  # 文件夹会被自动创建，无需用户创建。
         remote_dir = f'/codes/{python_proj_dir_short}'
     else:
@@ -95,7 +101,7 @@ def fabric_deploy(booster:Booster, host, port, user, password,
         kill_shell = f'''ps -aux|grep {process_mark}|grep -v grep|awk '{{print $2}}' |xargs kill -9'''
         logger.warning(f'{kill_shell} 命令杀死 {process_mark} 标识的进程')
         # uploader.ssh.exec_command(kill_shell)
-        conn.run(kill_shell, encoding='utf-8',warn=True)  # 不想提示，免得烦扰用户以为有什么异常了。所以用上面的paramiko包的ssh.exec_command
+        conn.run(kill_shell, encoding='utf-8', warn=True)  # 不想提示，免得烦扰用户以为有什么异常了。所以用上面的paramiko包的ssh.exec_command
 
         python_exec_str = f'''export is_funboost_remote_run=1;export PYTHONPATH={remote_dir}:$PYTHONPATH ;{python_interpreter} -c "from {relative_module} import {func_name};{func_name}.multi_process_consume({process_num})"  -funboostmark {process_mark} '''
         shell_str = f'''cd {remote_dir}; {python_exec_str}'''

funboost-43.5/funboost/core/lazy_impoter.py

@@ -0,0 +1,78 @@
+import abc
+
+from funboost.utils.decorators import cached_method_result, singleton, SingletonBaseNew, SingletonBaseCustomInit
+
+
+# @singleton   # 不方便代码补全
+
+class LazyImpoter(SingletonBaseNew):
+    """
+    延迟导入,避免需要互相导入.
+    """
+
+    @property
+    @cached_method_result
+    def BoostersManager(self):
+        from funboost.core.booster import BoostersManager
+        return BoostersManager
+
+    # @property
+    # @cached_method_result
+    # def get_current_taskid(self):
+    #     from funboost.core.current_task import get_current_taskid
+    #     return get_current_taskid
+
+
+lazy_impoter = LazyImpoter()
+
+
+class GeventImporter(SingletonBaseNew):
+    """
+    import gevent
+    from gevent import pool as gevent_pool
+    from gevent import monkey
+    from gevent.queue import JoinableQueue
+    """
+
+    @property
+    @cached_method_result
+    def gevent(self):
+        import gevent
+        return gevent
+
+    @property
+    @cached_method_result
+    def gevent_pool(self):
+        from gevent import pool as gevent_pool
+        return gevent_pool
+
+    @property
+    @cached_method_result
+    def monkey(self):
+        from gevent import monkey
+        return monkey
+
+    @property
+    @cached_method_result
+    def JoinableQueue(self):
+        from gevent.queue import JoinableQueue
+        return JoinableQueue
+
+
+class EventletImporter(SingletonBaseCustomInit):
+    """
+    from eventlet import greenpool, monkey_patch, patcher, Timeout
+    """
+
+    def _custom_init(self, ):
+        from eventlet import greenpool, monkey_patch, patcher, Timeout
+        self.greenpool = greenpool
+        self.monkey_patch = monkey_patch
+        self.patcher = patcher
+        self.Timeout = Timeout
+
+
+if __name__ == '__main__':
+    for i in range(10000):
+        # lazy_impoter.BoostersManager
+        EventletImporter().greenpool

{funboost-43.3 → funboost-43.5}/funboost/utils/decorators.py

@@ -1,6 +1,7 @@
 # coding=utf-8
 import base64
 import copy
+import abc
 import logging
 import random
 import uuid
@@ -165,6 +166,60 @@ def singleton(cls):
     return _singleton
 
 
+class SingletonMeta(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super().__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+
+class SingletonBaseCall(metaclass=SingletonMeta):
+    """
+    单例基类。任何继承自这个基类的子类都会自动成为单例。
+
+    示例：
+    class MyClass(SingletonBase):
+        pass
+
+    instance1 = MyClass()
+    instance2 = MyClass()
+
+    assert instance1 is instance2  # 实例1和实例2实际上是同一个对象
+    """
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        # 可以在此处添加对子类的额外处理，比如检查其是否符合单例要求等
+
+
+class SingletonBaseNew:
+    _instance = None
+
+    def __new__(cls, *args, **kwargs):
+        if not cls._instance:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        # 可以在此处添加对子类的额外处理，比如检查其是否符合单例要求等
+
+
+class SingletonBaseCustomInit(metaclass=abc.ABCMeta):
+    _instance = None
+
+    def __new__(cls, *args, **kwargs):
+        if not cls._instance:
+            cls._instance = super().__new__(cls)
+            cls._instance._custom_init(*args, **kwargs)
+        return cls._instance
+
+    def _custom_init(self, *args, **kwargs):
+        raise NotImplemented
+
+
 def flyweight(cls):
     _instance = {}
 
@@ -291,6 +346,7 @@ class RedisDistributedLockContextManager(LoggerMixin, LoggerLevelSetterMixin):
         else:
             return False
 
+
 """
 @contextmanager
         def some_generator(<arguments>):
@@ -536,8 +592,6 @@ def timeout(seconds):
 
     def timeout_decorator(func):
 
-
-
         def _(*args, **kwargs):
             def _new_func(oldfunc, result, oldfunc_args, oldfunc_kwargs):
                 result.append(oldfunc(*oldfunc_args, **oldfunc_kwargs))
@@ -738,8 +792,6 @@ class _Test(unittest.TestCase):
         f(5)
 
 
-
-
 if __name__ == '__main__':
     pass
     unittest.main()

{funboost-43.3 → funboost-43.5/funboost.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 43.3
+Version: 43.5
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-43.3 → funboost-43.5}/funboost.egg-info/requires.txt

@@ -1,6 +1,4 @@
 nb_log>=12.6
-eventlet==0.33.3
-gevent==22.10.2
 pymongo==4.3.3
 AMQPStorm==2.10.6
 rabbitpy==2.0.1

{funboost-43.3 → funboost-43.5}/setup.py

@@ -63,8 +63,8 @@ setup(
     ],
     install_requires=[
         'nb_log>=12.6',
-        'eventlet==0.33.3',
-        'gevent==22.10.2',
+        # 'eventlet==0.33.3',
+        # 'gevent==22.10.2',
         'pymongo==4.3.3',  # 3.5.1  -> 4.0.2
         'AMQPStorm==2.10.6',
         'rabbitpy==2.0.1',

funboost-43.3/funboost/core/lazy_impoter.py

@@ -1,26 +0,0 @@
-from funboost.utils.decorators import cached_method_result
-
-
-class LazyImpoter:
-    """
-    延迟导入,避免需要互相导入.
-    """
-
-    @property
-    @cached_method_result
-    def BoostersManager(self):
-        from funboost.core.booster import BoostersManager
-        return BoostersManager
-
-    # @property
-    # @cached_method_result
-    # def get_current_taskid(self):
-    #     from funboost.core.current_task import get_current_taskid
-    #     return get_current_taskid
-
-lazy_impoter = LazyImpoter()
-
-if __name__ == '__main__':
-    for i in range(10000):
-        lazy_impoter.BoostersManager
-        lazy_impoter.BoostersManager