funboost 47.7__tar.gz → 47.8__tar.gz

{funboost-47.7/funboost.egg-info → funboost-47.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 47.7
+Version: 47.8
 Summary: pip install funboost，python全功能分布式函数调度框架,funboost的功能是全面性重量级，用户能想得到的功能99%全都有;funboost的使用方式是轻量级，只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-47.7 → funboost-47.8}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "47.7"
+__version__ = "47.8"
 
 from funboost.set_frame_config import show_frame_config
 

{funboost-47.7 → funboost-47.8}/funboost/consumers/base_consumer.py

@@ -65,7 +65,7 @@ from funboost.concurrent_pool.bounded_threadpoolexcutor import \
     BoundedThreadPoolExecutor
 from funboost.utils.redis_manager import RedisMixin
 # from func_timeout import func_set_timeout  # noqa
-from funboost.utils.func_timeout.dafunc import func_set_timeout
+from funboost.utils.func_timeout import dafunc
 
 from funboost.concurrent_pool.custom_threadpool_executor import check_not_monkey
 from funboost.concurrent_pool.flexible_thread_pool import FlexibleThreadPool, sync_or_async_fun_deco
@@ -1058,7 +1058,7 @@ class ConcurrentModeDispatcher(FunboostFileLoggerMixin):
         self.timeout_deco = None
         if self._concurrent_mode in (ConcurrentModeEnum.THREADING, ConcurrentModeEnum.SINGLE_THREAD):
             # self.timeout_deco = decorators.timeout
-            self.timeout_deco = func_set_timeout  # 这个超时装饰器性能好很多。
+            self.timeout_deco = dafunc.func_set_timeout  # 这个超时装饰器性能好很多。
         elif self._concurrent_mode == ConcurrentModeEnum.GEVENT:
             from funboost.concurrent_pool.custom_gevent_pool_executor import gevent_timeout_deco
             self.timeout_deco = gevent_timeout_deco

funboost-47.8/funboost/utils/func_timeout/StoppableThread.py

@@ -0,0 +1,134 @@
+'''
+    Copyright (c) 2016, 2017, 2019 Timothy Savannah All Rights Reserved.
+
+    Licensed under the Lesser GNU Public License Version 3, LGPLv3. You should have recieved a copy of this with the source distribution as
+    LICENSE, otherwise it is available at https://github.com/kata198/func_timeout/LICENSE
+'''
+
+import os
+import ctypes
+import threading
+
+from funboost.core.current_task import FctContextThread
+__all__ = ('StoppableThread', 'JoinThread')
+
+class StoppableThread(FctContextThread): # 这里重要，继承的是FctContextThread，而不是原生 threading.Thread
+    '''
+        StoppableThread - A thread that can be stopped by forcing an exception in the execution context.
+
+          This works both to interrupt code that is in C or in python code, at either the next call to a python function,
+           or the next line in python code.
+
+        It is recommended that if you call stop ( @see StoppableThread.stop ) that you use an exception that inherits BaseException, to ensure it likely isn't caught.
+
+         Also, beware unmarked exception handlers in your code. Code like this:
+
+            while True:
+                try:
+                    doSomething()
+                except:
+                    continue
+
+        will never be able to abort, because the exception you raise is immediately caught.
+
+        The exception is raised over and over, with a specifed delay (default 2.0 seconds)
+    '''
+
+
+    def _stopThread(self, exception, raiseEvery=2.0):
+        '''
+            _stopThread - @see StoppableThread.stop
+        '''
+        if self.is_alive() is False:
+            return True
+
+        self._stderr = open(os.devnull, 'w')
+
+        # Create "joining" thread which will raise the provided exception
+        #  on a repeat, until the thread stops.
+        joinThread = JoinThread(self, exception, repeatEvery=raiseEvery)
+
+        # Try to prevent spurrious prints
+        joinThread._stderr = self._stderr
+        joinThread.start()
+        joinThread._stderr = self._stderr
+
+
+    def stop(self, exception, raiseEvery=2.0):
+        '''
+            Stops the thread by raising a given exception.
+
+            @param exception <Exception type> - Exception to throw. Likely, you want to use something
+
+              that inherits from BaseException (so except BaseException as e: continue; isn't a problem)
+
+              This should be a class/type, NOT an instance, i.e.  MyExceptionType   not  MyExceptionType()
+
+
+            @param raiseEvery <float> Default 2.0 - We will keep raising this exception every #raiseEvery seconds,
+
+                until the thread terminates.
+
+                If your code traps a specific exception type, this will allow you #raiseEvery seconds to cleanup before exit.
+
+                If you're calling third-party code you can't control, which catches BaseException, set this to a low number
+
+                  to break out of their exception handler.
+
+
+             @return <None>
+        '''
+        return self._stopThread(exception, raiseEvery)
+
+
+class JoinThread(FctContextThread):
+    '''
+        JoinThread - The workhouse that stops the StoppableThread.
+
+            Takes an exception, and upon being started immediately raises that exception in the current context
+              of the thread's execution (so next line of python gets it, or next call to a python api function in C code ).
+
+            @see StoppableThread for more details
+    '''
+
+    def __init__(self, otherThread, exception, repeatEvery=2.0):
+        '''
+            __init__ - Create a JoinThread (don't forget to call .start() ! )
+
+                @param otherThread <threading.Thread> - A thread
+
+                @param exception <BaseException> - An exception. Should be a BaseException, to prevent "catch Exception as e: continue" type code
+                  from never being terminated. If such code is unavoidable, you can try setting #repeatEvery to a very low number, like .00001,
+                  and it will hopefully raise within the context of the catch, and be able to break free.
+
+                @param repeatEvery <float> Default 2.0 - After starting, the given exception is immediately raised. Then, every #repeatEvery seconds,
+                  it is raised again, until the thread terminates.
+        '''
+        threading.Thread.__init__(self)
+        self.otherThread = otherThread
+        self.exception = exception
+        self.repeatEvery = repeatEvery
+        self.daemon = True
+
+    def run(self):
+        '''
+            run - The thread main. Will attempt to stop and join the attached thread.
+        '''
+
+        # Try to silence default exception printing.
+        self.otherThread._Thread__stderr = self._stderr
+        if hasattr(self.otherThread, '_Thread__stop'):
+            # If py2, call this first to start thread termination cleanly.
+            #   Python3 does not need such ( nor does it provide.. )
+            self.otherThread._Thread__stop()
+        while self.otherThread.is_alive():
+            # We loop raising exception incase it's caught hopefully this breaks us far out.
+            ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(self.otherThread.ident), ctypes.py_object(self.exception))
+            self.otherThread.join(self.repeatEvery)
+
+        try:
+            self._stderr.close()
+        except:
+            pass
+
+# vim: set ts=4 sw=4 expandtab :

funboost-47.8/funboost/utils/func_timeout/dafunc.py

@@ -0,0 +1,244 @@
+
+# vim: set ts=4 sw=4 expandtab :
+
+'''
+    Copyright (c) 2016, 2017 Tim Savannah All Rights Reserved.
+
+    Licensed under the Lesser GNU Public License Version 3, LGPLv3. You should have recieved a copy of this with the source distribution as
+    LICENSE, otherwise it is available at https://github.com/kata198/func_timeout/LICENSE
+'''
+
+import copy
+import functools
+import inspect
+import threading
+import time
+import types
+import sys
+
+
+from .exceptions import FunctionTimedOut
+from .StoppableThread import StoppableThread
+
+
+from .py3_raise import raise_exception
+
+
+from functools import wraps
+
+__all__ = ('func_timeout', 'func_set_timeout')
+
+
+
+
+def func_timeout(timeout, func, args=(), kwargs=None):
+    '''
+        func_timeout - Runs the given function for up to #timeout# seconds.
+
+        Raises any exceptions #func# would raise, returns what #func# would return (unless timeout is exceeded), in which case it raises FunctionTimedOut
+
+        @param timeout <float> - Maximum number of seconds to run #func# before terminating
+
+        @param func <function> - The function to call
+
+        @param args    <tuple> - Any ordered arguments to pass to the function
+
+        @param kwargs  <dict/None> - Keyword arguments to pass to the function.
+
+
+        @raises - FunctionTimedOut if #timeout# is exceeded, otherwise anything #func# could raise will be raised
+
+        If the timeout is exceeded, FunctionTimedOut will be raised within the context of the called function every two seconds until it terminates,
+        but will not block the calling thread (a new thread will be created to perform the join). If possible, you should try/except FunctionTimedOut
+        to return cleanly, but in most cases it will 'just work'.
+
+        @return - The return value that #func# gives
+    '''
+
+    if not kwargs:
+        kwargs = {}
+    if not args:
+        args = ()
+
+    ret = []
+    exception = []
+    isStopped = False
+
+    from funboost.core.current_task import thread_current_task
+
+    def funcwrap(args2, kwargs2,):
+        # fct = thread_current_task
+        # fct.set_fct_context(fct_context)  # 把funboost的消费线程上下文需要传递到超时线程上下文里面来.
+        try:
+            ret.append( func(*args2, **kwargs2) )
+        except FunctionTimedOut:
+            # Don't print traceback to stderr if we time out
+            pass
+        except BaseException as e:
+            exc_info = sys.exc_info()
+            if isStopped is False:
+                # Assemble the alternate traceback, excluding this function
+                #  from the trace (by going to next frame)
+                # Pytohn3 reads native from __traceback__,
+                # python2 has a different form for "raise"
+                e.__traceback__ = exc_info[2].tb_next
+                exception.append( e )
+
+
+    # fct = funboost_current_task()
+    thread = StoppableThread(target=funcwrap, args=(args, kwargs,))
+    thread.daemon = True
+
+    thread.start()
+    thread.join(timeout)
+
+    stopException = None
+    if thread.is_alive():
+        isStopped = True
+
+        class FunctionTimedOutTempType(FunctionTimedOut):
+            def __init__(self):
+                return FunctionTimedOut.__init__(self, '', timeout, func, args, kwargs)
+
+        FunctionTimedOutTemp = type('FunctionTimedOut' + str( hash( "%d_%d_%d_%d" %(id(timeout), id(func), id(args), id(kwargs))) ), FunctionTimedOutTempType.__bases__, dict(FunctionTimedOutTempType.__dict__))
+
+        stopException = FunctionTimedOutTemp
+        # raise FunctionTimedOut('', timeout, func, args, kwargs)
+        thread._stopThread(stopException)
+        thread.join(min(.1, timeout / 50.0))
+        raise FunctionTimedOut('', timeout, func, args, kwargs)
+    else:
+        # We can still cleanup the thread here..
+        # Still give a timeout... just... cuz..
+        thread.join(.5)
+
+    if exception:
+        raise_exception(exception)
+
+    if ret:
+        return ret[0]
+
+
+def func_set_timeout(timeout, allowOverride=False):
+    '''
+        func_set_timeout - Decorator to run a function with a given/calculated timeout (max execution time).
+            Optionally (if #allowOverride is True), adds a paramater, "forceTimeout", to the
+            function which, if provided, will override the default timeout for that invocation.
+
+            If #timeout is provided as a lambda/function, it will be called
+              prior to each invocation of the decorated function to calculate the timeout to be used
+              for that call, based on the arguments passed to the decorated function.
+
+              For example, you may have a "processData" function whose execution time
+              depends on the number of "data" elements, so you may want a million elements to have a
+              much higher timeout than seven elements.)
+
+            If #allowOverride is True AND a kwarg of "forceTimeout" is passed to the wrapped function, that timeout
+             will be used for that single call.
+
+        @param timeout <float OR lambda/function> -
+
+            **If float:**
+                Default number of seconds max to allow function to execute
+                  before throwing FunctionTimedOut
+
+            **If lambda/function:
+
+                 If a function/lambda is provided, it will be called for every
+                  invocation of the decorated function (unless #allowOverride=True and "forceTimeout" was passed)
+                  to determine the timeout to use based on the arguments to the decorated function.
+
+                    The arguments as passed into the decorated function will be passed to this function.
+                     They either must match exactly to what the decorated function has, OR
+                      if you prefer to get the *args (list of ordered args) and **kwargs ( key : value  keyword args form),
+                      define your calculate function like:
+
+                        def calculateTimeout(*args, **kwargs):
+                            ...
+
+                      or lambda like:
+
+                        calculateTimeout = lambda *args, **kwargs : ...
+
+                    otherwise the args to your calculate function should match exactly the decorated function.
+
+
+        @param allowOverride <bool> Default False, if True adds a keyword argument to the decorated function,
+            "forceTimeout" which, if provided, will override the #timeout. If #timeout was provided as a lambda / function, it
+             will not be called.
+
+        @throws FunctionTimedOut If time alloted passes without function returning naturally
+
+        @see func_timeout
+    '''
+    # Try to be as efficent as possible... don't compare the args more than once
+
+    #  Helps closure issue on some versions of python
+    defaultTimeout = copy.copy(timeout)
+
+    isTimeoutAFunction = bool( issubclass(timeout.__class__, (types.FunctionType, types.MethodType, types.LambdaType, types.BuiltinFunctionType, types.BuiltinMethodType) ) )
+
+    if not isTimeoutAFunction:
+        if not issubclass(timeout.__class__, (float, int)):
+            try:
+                timeout = float(timeout)
+            except:
+                raise ValueError('timeout argument must be a float/int for number of seconds, or a function/lambda which gets passed the function arguments and returns a calculated timeout (as float or int). Passed type: < %s > is not of any of these, and cannot be converted to a float.' %( timeout.__class__.__name__, ))
+
+
+    if not allowOverride and not isTimeoutAFunction:
+        # Only defaultTimeout provided. Simple function wrapper
+        def _function_decorator(func):
+
+            return wraps(func)(lambda *args, **kwargs : func_timeout(defaultTimeout, func, args=args, kwargs=kwargs))
+
+#            def _function_wrapper(*args, **kwargs):
+#                return func_timeout(defaultTimeout, func, args=args, kwargs=kwargs)
+#            return _function_wrapper
+        return _function_decorator
+
+    if not isTimeoutAFunction:
+        # allowOverride is True and timeout is not a function. Simple conditional on every call
+        def _function_decorator(func):
+            def _function_wrapper(*args, **kwargs):
+                if 'forceTimeout' in kwargs:
+                    useTimeout = kwargs.pop('forceTimeout')
+                else:
+                    useTimeout = defaultTimeout
+
+                return func_timeout(useTimeout, func, args=args, kwargs=kwargs)
+
+            return wraps(func)(_function_wrapper)
+        return _function_decorator
+
+
+    # At this point, timeout IS known to be a function.
+    timeoutFunction = timeout
+
+    if allowOverride:
+        # Could use a lambda here... but want traceback to highlight the calculate function,
+        #  and not the invoked function
+        def _function_decorator(func):
+            def _function_wrapper(*args, **kwargs):
+                if 'forceTimeout' in kwargs:
+                    useTimeout = kwargs.pop('forceTimeout')
+                else:
+                    useTimeout = timeoutFunction(*args, **kwargs)
+
+                return func_timeout(useTimeout, func, args=args, kwargs=kwargs)
+
+            return wraps(func)(_function_wrapper)
+        return _function_decorator
+
+    # Cannot override, and calculate timeout function
+    def _function_decorator(func):
+        def _function_wrapper(*args, **kwargs):
+            useTimeout = timeoutFunction(*args, **kwargs)
+
+            return func_timeout(useTimeout, func, args=args, kwargs=kwargs)
+
+        return wraps(func)(_function_wrapper)
+    return _function_decorator
+
+
+# vim: set ts=4 sw=4 expandtab :

funboost-47.8/funboost/utils/func_timeout/exceptions.py

@@ -0,0 +1,98 @@
+'''
+    Copyright (c) 2016 Tim Savannah All Rights Reserved.
+
+    Licensed under the Lesser GNU Public License Version 3, LGPLv3. You should have recieved a copy of this with the source distribution as
+    LICENSE, otherwise it is available at https://github.com/kata198/func_timeout/LICENSE
+'''
+
+__all__ = ('FunctionTimedOut', 'RETRY_SAME_TIMEOUT')
+
+RETRY_SAME_TIMEOUT = 'RETRY_SAME_TIMEOUT'
+
+class FunctionTimedOut(BaseException):
+    '''
+        FunctionTimedOut - Exception raised when a function times out
+
+        @property timedOutAfter - Number of seconds before timeout was triggered
+
+        @property timedOutFunction - Function called which timed out
+        @property timedOutArgs - Ordered args to function
+        @property timedOutKwargs - Keyword args to function
+
+        @method retry - Retries the function with same arguments, with option to run with original timeout, no timeout, or a different
+          explicit timeout. @see FunctionTimedOut.retry
+    '''
+
+
+    def __init__(self, msg='', timedOutAfter=None, timedOutFunction=None, timedOutArgs=None, timedOutKwargs=None):
+        '''
+            __init__ - Create this exception type.
+
+                You should not need to do this outside of testing, it will be created by the func_timeout API
+
+                    @param msg <str> - A predefined message, otherwise we will attempt to generate one from the other arguments.
+
+                    @param timedOutAfter <None/float> - Number of seconds before timing-out. Filled-in by API, None will produce "Unknown"
+
+                    @param timedOutFunction <None/function> - Reference to the function that timed-out. Filled-in by API." None will produce "Unknown Function"
+
+                    @param timedOutArgs <None/tuple/list> - List of fixed-order arguments ( *args ), or None for no args.
+
+                    @param timedOutKwargs <None/dict> - Dict of keyword arg ( **kwargs ) names to values, or None for no kwargs.
+
+        '''
+
+        self.timedOutAfter = timedOutAfter
+
+        self.timedOutFunction = timedOutFunction
+        self.timedOutArgs = timedOutArgs
+        self.timedOutKwargs = timedOutKwargs
+
+        if not msg:
+            msg = self.getMsg()
+
+        BaseException.__init__(self, msg)
+
+        self.msg = msg
+
+
+    def getMsg(self):
+        '''
+            getMsg - Generate a default message based on parameters to FunctionTimedOut exception'
+
+            @return <str> - Message
+        '''
+        # Try to gather the function name, if available.
+        # If it is not, default to an "unknown" string to allow default instantiation
+        if self.timedOutFunction is not None:
+            timedOutFuncName = self.timedOutFunction.__name__
+        else:
+            timedOutFuncName = 'Unknown Function'
+        if self.timedOutAfter is not None:
+            timedOutAfterStr = "%f" %(self.timedOutAfter, )
+        else:
+            timedOutAfterStr = "Unknown"
+
+        return 'Function %s (args=%s) (kwargs=%s) timed out after %s seconds.\n' %(timedOutFuncName, repr(self.timedOutArgs), repr(self.timedOutKwargs), timedOutAfterStr)
+
+    def retry(self, timeout=RETRY_SAME_TIMEOUT):
+        '''
+            retry - Retry the timed-out function with same arguments.
+
+            @param timeout <float/RETRY_SAME_TIMEOUT/None> Default RETRY_SAME_TIMEOUT
+
+                If RETRY_SAME_TIMEOUT : Will retry the function same args, same timeout
+                If a float/int : Will retry the function same args with provided timeout
+                If None : Will retry function same args no timeout
+
+            @return - Returnval from function
+        '''
+        if timeout is None:
+            return self.timedOutFunction(*(self.timedOutArgs), **self.timedOutKwargs)
+
+        from .dafunc import func_timeout
+
+        if timeout == RETRY_SAME_TIMEOUT:
+            timeout = self.timedOutAfter
+
+        return func_timeout(timeout, self.timedOutFunction, args=self.timedOutArgs, kwargs=self.timedOutKwargs)

funboost-47.8/funboost/utils/func_timeout/py2_raise.py

@@ -0,0 +1,7 @@
+
+
+# Python2 allows specifying an alternate traceback.
+def raise_exception(exception):
+    '''
+    raise exception[0] , None , exception[0].__traceback__  # noqa
+    '''

funboost-47.8/funboost/utils/func_timeout/py3_raise.py

@@ -0,0 +1,7 @@
+
+# PEP 409 - Raise with the chained exception context disabled
+#  This, in effect, prevents the "funcwrap" wrapper ( chained
+#   in context to an exception raised here, due to scope )
+# Only available in python3.3+
+def raise_exception(exception):
+    raise exception[0] from None

{funboost-47.7 → funboost-47.8/funboost.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 47.7
+Version: 47.8
 Summary: pip install funboost，python全功能分布式函数调度框架,funboost的功能是全面性重量级，用户能想得到的功能99%全都有;funboost的使用方式是轻量级，只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-47.7 → funboost-47.8}/funboost.egg-info/SOURCES.txt

@@ -280,6 +280,12 @@ funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/except
 funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/py3_raise.cpython-311.pyc
 funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/py3_raise.cpython-37.pyc
 funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/py3_raise.cpython-39.pyc
+funboost/utils/func_timeout/StoppableThread.py
+funboost/utils/func_timeout/__init__.py
+funboost/utils/func_timeout/dafunc.py
+funboost/utils/func_timeout/exceptions.py
+funboost/utils/func_timeout/py2_raise.py
+funboost/utils/func_timeout/py3_raise.py
 funboost/utils/pysnooper_ydf/__init__.py
 funboost/utils/pysnooper_ydf/pycompat.py
 funboost/utils/pysnooper_ydf/tracer.py