funboost 46.7__tar.gz → 46.8__tar.gz

{funboost-46.7/funboost.egg-info → funboost-46.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 46.7
+Version: 46.8
 Summary: pip install funboost，python全功能分布式函数调度框架,funboost的功能是全面性重量级，用户能想得到的功能99%全都有;funboost的使用方式是轻量级，只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-46.7 → funboost-46.8}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "46.7"
+__version__ = "46.8"
 
 from funboost.set_frame_config import show_frame_config
 

{funboost-46.7 → funboost-46.8}/funboost/consumers/base_consumer.py

@@ -369,7 +369,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
 
     def _start_delay_task_scheduler(self):
         from funboost.timing_job import FsdfBackgroundScheduler
-        from funboost.timing_job import FunboostBackgroundSchedulerProcessJobsWithinRedisLock
+        from funboost.timing_job.apscheduler_use_redis_store import FunboostBackgroundSchedulerProcessJobsWithinRedisLock
         # print(self.consumer_params.delay_task_apsscheduler_jobstores_kind )
         if self.consumer_params.delay_task_apscheduler_jobstores_kind == 'redis':
             jobstores = {
@@ -392,7 +392,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             raise Exception(f'delay_task_apsscheduler_jobstores_kind is error: {self.consumer_params.delay_task_apscheduler_jobstores_kind}')
 
 
-        self._delay_task_scheduler.add_executor(ApschedulerThreadPoolExecutor(2))  # 只是运行submit任务到并发池，不需要很多线程。
+        # self._delay_task_scheduler.add_executor(ApschedulerThreadPoolExecutor(2))  # 只是运行submit任务到并发池，不需要很多线程。
         # self._delay_task_scheduler.add_listener(self._apscheduler_job_miss, EVENT_JOB_MISSED)
         self._delay_task_scheduler.start()
 

{funboost-46.7 → funboost-46.8}/funboost/timing_job/__init__.py

@@ -8,7 +8,9 @@ import importlib
 import pickle
 
 import time
-from funboost.utils.decorators import RedisDistributedLockContextManager
+from apscheduler.executors.pool import BasePoolExecutor
+
+from funboost.utils.decorators import RedisDistributedLockContextManager,RedisDistributedBlockLockContextManager
 from typing import Union
 import threading
 
@@ -26,7 +28,8 @@ from funboost.consumers.base_consumer import AbstractConsumer
 from funboost.core.booster import BoostersManager, Booster
 from funboost.publishers.base_publisher import AbstractPublisher
 from funboost import BoosterParams
-
+from funboost.concurrent_pool.flexible_thread_pool import FlexibleThreadPool
+from funboost.concurrent_pool.custom_threadpool_executor import ThreadPoolExecutorShrinkAble
 
 @deprecated.deprecated(reason='以后不要再使用这种方式，对于job_store为数据库时候需要序列化不好。使用内存和数据库都兼容的添加任务方式: add_push_job')
 def timing_publish_deco(consuming_func_decorated_or_consumer: Union[callable, AbstractConsumer]):
@@ -54,6 +57,20 @@ def push_fun_params_to_broker(queue_name: str, *args, runonce_uuid=None, **kwarg
     else:
         BoostersManager.get_or_create_booster_by_queue_name(queue_name).push(*args, **kwargs)
 
+class ThreadPoolExecutorForAps(BasePoolExecutor):
+    """
+    An executor that runs jobs in a concurrent.futures thread pool.
+
+    Plugin alias: ``threadpool``
+
+    :param max_workers: the maximum number of spawned threads.
+    :param pool_kwargs: dict of keyword arguments to pass to the underlying
+        ThreadPoolExecutor constructor
+    """
+
+    def __init__(self, max_workers=10, ):
+        pool = ThreadPoolExecutorShrinkAble(int(max_workers),)
+        super().__init__(pool)
 
 class FunboostBackgroundScheduler(BackgroundScheduler):
     """
@@ -150,7 +167,7 @@ class FunboostBackgroundScheduler(BackgroundScheduler):
         或者下一个需要运行的任务的wait_seconds是3600秒后，此时新加了一个动态任务需要3600秒后，
         现在最多只需要1秒就能扫描到动态新增的定时任务了。
         """
-        MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS = 1.5
+        MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS = 0.5
         wait_seconds = None
         while self.state == STATE_RUNNING:
             if wait_seconds is None:
@@ -163,24 +180,8 @@ class FunboostBackgroundScheduler(BackgroundScheduler):
             time.sleep(self._last_wait_seconds)  # 这个要取最小值，不然例如定时间隔0.1秒运行，不取最小值，不会每隔0.1秒运行。
             wait_seconds = self._process_jobs()
 
-
-class FunboostBackgroundSchedulerProcessJobsWithinRedisLock(FunboostBackgroundScheduler):
-    process_jobs_redis_lock_key = f'funboost.BackgroundSchedulerProcessJobsWithinRedisLock'
-
-    def set_process_jobs_redis_lock_key(self,lock_key):
-        self.process_jobs_redis_lock_key = lock_key
-
-    def _process_jobs(self):
-        for i in range(10) :
-            with RedisDistributedLockContextManager(RedisMixin().redis_db_frame, self.process_jobs_redis_lock_key, ) as lock:
-                if lock.has_aquire_lock:
-                    wait_seconds = super()._process_jobs()
-                    return wait_seconds
-                else:
-                    time.sleep(0.1)
-        return 0.1
-
-
+    def _create_default_executor(self):
+        return ThreadPoolExecutorForAps()  # 必须是apscheduler pool的子类
 
 
 FsdfBackgroundScheduler = FunboostBackgroundScheduler  # 兼容一下名字，fsdf是 function-scheduling-distributed-framework 老框架名字的缩写

funboost-46.8/funboost/timing_job/apscheduler_use_redis_store.py

@@ -0,0 +1,61 @@
+from apscheduler.jobstores.redis import RedisJobStore
+from funboost.utils.redis_manager import RedisMixin
+
+from funboost.timing_job import FunboostBackgroundScheduler
+from funboost.funboost_config_deafult import BrokerConnConfig, FunboostCommonConfig
+from funboost.utils.decorators import RedisDistributedBlockLockContextManager
+
+"""
+这个是使用redis作为定时任务持久化，支持跨机器好跨进程，外部远程 动态修改/添加/删除定时任务
+"""
+
+
+class FunboostBackgroundSchedulerProcessJobsWithinRedisLock(FunboostBackgroundScheduler):
+    """
+    分布式或多进程都启动某个apscheduler实例，如果都使用的同一个数据库类型的jobstores ，_process_jobs有很大概率会造成报错， 因为_process_jobs使用的是线程锁，管不了其他进程和分布式机器。
+
+    https://groups.google.com/g/apscheduler/c/Gjc_JQMPePc 问题也提到了这个bug
+
+    继承 Custom schedulers https://apscheduler.readthedocs.io/en/3.x/extending.html   可以重写 _create_lock
+    """
+
+    process_jobs_redis_lock_key = f'funboost.BackgroundSchedulerProcessJobsWithinRedisLock'
+
+    def set_process_jobs_redis_lock_key(self, lock_key):
+        self.process_jobs_redis_lock_key = lock_key
+
+    # def  _create_lock(self):
+    #     return RedisDistributedBlockLockContextManager(RedisMixin().redis_db_frame,self.process_jobs_redis_lock_key,) 这个类的写法不适合固定的单例，
+    #     RedisDistributedBlockLockContextManager的写法不适合 永远用一个 对象，所以还是放到 def  _process_jobs 里面运行
+
+    # def _process_jobs(self):
+    #     for i in range(10) :
+    #         with RedisDistributedLockContextManager(RedisMixin().redis_db_frame, self.process_jobs_redis_lock_key, ) as lock:
+    #             if lock.has_aquire_lock:
+    #                 wait_seconds = super()._process_jobs()
+    #                 return wait_seconds
+    #             else:
+    #                 time.sleep(0.1)
+    #     return 0.1
+
+    def _process_jobs(self):
+        with RedisDistributedBlockLockContextManager(RedisMixin().redis_db_frame, self.process_jobs_redis_lock_key, ):
+            return super()._process_jobs()
+
+
+jobstores = {
+    "default": RedisJobStore(db=BrokerConnConfig.REDIS_DB, host=BrokerConnConfig.REDIS_HOST,
+                             port=BrokerConnConfig.REDIS_PORT, password=BrokerConnConfig.REDIS_PASSWORD,
+                             username=BrokerConnConfig.REDIS_USERNAME, jobs_key='funboost.apscheduler.jobs')
+}
+
+funboost_background_scheduler_redis_store = FunboostBackgroundSchedulerProcessJobsWithinRedisLock(timezone=FunboostCommonConfig.TIMEZONE, daemon=False, jobstores=jobstores)
+
+"""
+
+跨python解释器 跨机器动态修改定时任务配置的例子在
+
+test_frame/test_apschedual/test_aps_redis_store.py
+test_frame/test_apschedual/test_change_aps_conf.py
+
+"""

{funboost-46.7 → funboost-46.8}/funboost/utils/decorators.py

@@ -361,6 +361,30 @@ class RedisDistributedLockContextManager(LoggerMixin, LoggerLevelSetterMixin):
             return False
 
 
+class RedisDistributedBlockLockContextManager(RedisDistributedLockContextManager):
+    def __init__(self, redis_client, redis_lock_key, expire_seconds=30, check_interval=0.1):
+        super().__init__(redis_client,redis_lock_key,expire_seconds)
+        self.check_interval = check_interval
+        # self.logger.setLevel(logging.DEBUG)
+
+    def __enter__(self):
+        while True:
+            self._line = sys._getframe().f_back.f_lineno  # 调用此方法的代码的函数
+            self._file_name = sys._getframe(1).f_code.co_filename  # 哪个文件调了用此方法
+            ret = self.redis_client.set(self.redis_lock_key, value=self.identifier, ex=self._expire_seconds, nx=True)
+            has_aquire_lock = False if ret is None else True
+            if has_aquire_lock:
+                log_msg = f'\n"{self._file_name}:{self._line}" 这行代码获得了redis锁 {self.redis_lock_key}'
+            else:
+                log_msg = f'\n"{self._file_name}:{self._line}" 这行代码此次没有获得redis锁 {self.redis_lock_key}'
+            # print(self.logger.level,log_msg)
+            self.logger.debug(log_msg)
+            if has_aquire_lock:
+                break
+            else:
+                time.sleep(self.check_interval)
+
+
 """
 @contextmanager
         def some_generator(<arguments>):
@@ -372,6 +396,9 @@ class RedisDistributedLockContextManager(LoggerMixin, LoggerLevelSetterMixin):
 """
 
 
+
+
+
 class ExceptionContextManager:
     """
     用上下文管理器捕获异常，可对代码片段进行错误捕捉，比装饰器更细腻

{funboost-46.7 → funboost-46.8/funboost.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 46.7
+Version: 46.8
 Summary: pip install funboost，python全功能分布式函数调度框架,funboost的功能是全面性重量级，用户能想得到的功能99%全都有;funboost的使用方式是轻量级，只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

funboost-46.7/funboost/timing_job/apscheduler_use_redis_store.py

@@ -1,30 +0,0 @@
-from apscheduler.jobstores.redis import RedisJobStore
-
-from funboost.funboost_config_deafult import BrokerConnConfig,FunboostCommonConfig
-
-from funboost.timing_job import FunboostBackgroundSchedulerProcessJobsWithinRedisLock
-
-"""
-这个是使用redis作为定时任务持久化，支持动态修改/添加/删除定时任务
-"""
-
-jobstores = {
-    "default": RedisJobStore(db=BrokerConnConfig.REDIS_DB, host=BrokerConnConfig.REDIS_HOST,
-                             port=BrokerConnConfig.REDIS_PORT, password=BrokerConnConfig.REDIS_PASSWORD,
-                             username=BrokerConnConfig.REDIS_USERNAME,jobs_key='funboost.apscheduler.jobs')
-}
-
-funboost_background_scheduler_redis_store = FunboostBackgroundSchedulerProcessJobsWithinRedisLock(timezone=FunboostCommonConfig.TIMEZONE, daemon=False, jobstores=jobstores)
-
-
-
-
-
-"""
-
-跨python解释器 跨机器动态修改定时任务配置的例子在
-
-test_frame/test_apschedual/test_aps_redis_store.py
-test_frame/test_apschedual/test_change_aps_conf.py
-
-"""