funboost 46.7__py3-none-any.whl → 46.8__py3-none-any.whl

funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "46.7"
+__version__ = "46.8"
 
 from funboost.set_frame_config import show_frame_config
 

funboost/consumers/base_consumer.py

@@ -369,7 +369,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
 
     def _start_delay_task_scheduler(self):
         from funboost.timing_job import FsdfBackgroundScheduler
-        from funboost.timing_job import FunboostBackgroundSchedulerProcessJobsWithinRedisLock
+        from funboost.timing_job.apscheduler_use_redis_store import FunboostBackgroundSchedulerProcessJobsWithinRedisLock
         # print(self.consumer_params.delay_task_apsscheduler_jobstores_kind )
         if self.consumer_params.delay_task_apscheduler_jobstores_kind == 'redis':
             jobstores = {
@@ -392,7 +392,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             raise Exception(f'delay_task_apsscheduler_jobstores_kind is error: {self.consumer_params.delay_task_apscheduler_jobstores_kind}')
 
 
-        self._delay_task_scheduler.add_executor(ApschedulerThreadPoolExecutor(2))  # 只是运行submit任务到并发池，不需要很多线程。
+        # self._delay_task_scheduler.add_executor(ApschedulerThreadPoolExecutor(2))  # 只是运行submit任务到并发池，不需要很多线程。
         # self._delay_task_scheduler.add_listener(self._apscheduler_job_miss, EVENT_JOB_MISSED)
         self._delay_task_scheduler.start()
 

funboost/timing_job/__init__.py

@@ -8,7 +8,9 @@ import importlib
 import pickle
 
 import time
-from funboost.utils.decorators import RedisDistributedLockContextManager
+from apscheduler.executors.pool import BasePoolExecutor
+
+from funboost.utils.decorators import RedisDistributedLockContextManager,RedisDistributedBlockLockContextManager
 from typing import Union
 import threading
 
@@ -26,7 +28,8 @@ from funboost.consumers.base_consumer import AbstractConsumer
 from funboost.core.booster import BoostersManager, Booster
 from funboost.publishers.base_publisher import AbstractPublisher
 from funboost import BoosterParams
-
+from funboost.concurrent_pool.flexible_thread_pool import FlexibleThreadPool
+from funboost.concurrent_pool.custom_threadpool_executor import ThreadPoolExecutorShrinkAble
 
 @deprecated.deprecated(reason='以后不要再使用这种方式，对于job_store为数据库时候需要序列化不好。使用内存和数据库都兼容的添加任务方式: add_push_job')
 def timing_publish_deco(consuming_func_decorated_or_consumer: Union[callable, AbstractConsumer]):
@@ -54,6 +57,20 @@ def push_fun_params_to_broker(queue_name: str, *args, runonce_uuid=None, **kwarg
     else:
         BoostersManager.get_or_create_booster_by_queue_name(queue_name).push(*args, **kwargs)
 
+class ThreadPoolExecutorForAps(BasePoolExecutor):
+    """
+    An executor that runs jobs in a concurrent.futures thread pool.
+
+    Plugin alias: ``threadpool``
+
+    :param max_workers: the maximum number of spawned threads.
+    :param pool_kwargs: dict of keyword arguments to pass to the underlying
+        ThreadPoolExecutor constructor
+    """
+
+    def __init__(self, max_workers=10, ):
+        pool = ThreadPoolExecutorShrinkAble(int(max_workers),)
+        super().__init__(pool)
 
 class FunboostBackgroundScheduler(BackgroundScheduler):
     """
@@ -150,7 +167,7 @@ class FunboostBackgroundScheduler(BackgroundScheduler):
         或者下一个需要运行的任务的wait_seconds是3600秒后，此时新加了一个动态任务需要3600秒后，
         现在最多只需要1秒就能扫描到动态新增的定时任务了。
         """
-        MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS = 1.5
+        MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS = 0.5
         wait_seconds = None
         while self.state == STATE_RUNNING:
             if wait_seconds is None:
@@ -163,24 +180,8 @@ class FunboostBackgroundScheduler(BackgroundScheduler):
             time.sleep(self._last_wait_seconds)  # 这个要取最小值，不然例如定时间隔0.1秒运行，不取最小值，不会每隔0.1秒运行。
             wait_seconds = self._process_jobs()
 
-
-class FunboostBackgroundSchedulerProcessJobsWithinRedisLock(FunboostBackgroundScheduler):
-    process_jobs_redis_lock_key = f'funboost.BackgroundSchedulerProcessJobsWithinRedisLock'
-
-    def set_process_jobs_redis_lock_key(self,lock_key):
-        self.process_jobs_redis_lock_key = lock_key
-
-    def _process_jobs(self):
-        for i in range(10) :
-            with RedisDistributedLockContextManager(RedisMixin().redis_db_frame, self.process_jobs_redis_lock_key, ) as lock:
-                if lock.has_aquire_lock:
-                    wait_seconds = super()._process_jobs()
-                    return wait_seconds
-                else:
-                    time.sleep(0.1)
-        return 0.1
-
-
+    def _create_default_executor(self):
+        return ThreadPoolExecutorForAps()  # 必须是apscheduler pool的子类
 
 
 FsdfBackgroundScheduler = FunboostBackgroundScheduler  # 兼容一下名字，fsdf是 function-scheduling-distributed-framework 老框架名字的缩写

funboost/timing_job/apscheduler_use_redis_store.py

@@ -1,25 +1,56 @@
 from apscheduler.jobstores.redis import RedisJobStore
+from funboost.utils.redis_manager import RedisMixin
 
-from funboost.funboost_config_deafult import BrokerConnConfig,FunboostCommonConfig
-
-from funboost.timing_job import FunboostBackgroundSchedulerProcessJobsWithinRedisLock
+from funboost.timing_job import FunboostBackgroundScheduler
+from funboost.funboost_config_deafult import BrokerConnConfig, FunboostCommonConfig
+from funboost.utils.decorators import RedisDistributedBlockLockContextManager
 
 """
-这个是使用redis作为定时任务持久化，支持动态修改/添加/删除定时任务
+这个是使用redis作为定时任务持久化，支持跨机器好跨进程，外部远程 动态修改/添加/删除定时任务
 """
 
+
+class FunboostBackgroundSchedulerProcessJobsWithinRedisLock(FunboostBackgroundScheduler):
+    """
+    分布式或多进程都启动某个apscheduler实例，如果都使用的同一个数据库类型的jobstores ，_process_jobs有很大概率会造成报错， 因为_process_jobs使用的是线程锁，管不了其他进程和分布式机器。
+
+    https://groups.google.com/g/apscheduler/c/Gjc_JQMPePc 问题也提到了这个bug
+
+    继承 Custom schedulers https://apscheduler.readthedocs.io/en/3.x/extending.html   可以重写 _create_lock
+    """
+
+    process_jobs_redis_lock_key = f'funboost.BackgroundSchedulerProcessJobsWithinRedisLock'
+
+    def set_process_jobs_redis_lock_key(self, lock_key):
+        self.process_jobs_redis_lock_key = lock_key
+
+    # def  _create_lock(self):
+    #     return RedisDistributedBlockLockContextManager(RedisMixin().redis_db_frame,self.process_jobs_redis_lock_key,) 这个类的写法不适合固定的单例，
+    #     RedisDistributedBlockLockContextManager的写法不适合 永远用一个 对象，所以还是放到 def  _process_jobs 里面运行
+
+    # def _process_jobs(self):
+    #     for i in range(10) :
+    #         with RedisDistributedLockContextManager(RedisMixin().redis_db_frame, self.process_jobs_redis_lock_key, ) as lock:
+    #             if lock.has_aquire_lock:
+    #                 wait_seconds = super()._process_jobs()
+    #                 return wait_seconds
+    #             else:
+    #                 time.sleep(0.1)
+    #     return 0.1
+
+    def _process_jobs(self):
+        with RedisDistributedBlockLockContextManager(RedisMixin().redis_db_frame, self.process_jobs_redis_lock_key, ):
+            return super()._process_jobs()
+
+
 jobstores = {
     "default": RedisJobStore(db=BrokerConnConfig.REDIS_DB, host=BrokerConnConfig.REDIS_HOST,
                              port=BrokerConnConfig.REDIS_PORT, password=BrokerConnConfig.REDIS_PASSWORD,
-                             username=BrokerConnConfig.REDIS_USERNAME,jobs_key='funboost.apscheduler.jobs')
+                             username=BrokerConnConfig.REDIS_USERNAME, jobs_key='funboost.apscheduler.jobs')
 }
 
 funboost_background_scheduler_redis_store = FunboostBackgroundSchedulerProcessJobsWithinRedisLock(timezone=FunboostCommonConfig.TIMEZONE, daemon=False, jobstores=jobstores)
 
-
-
-
-
 """
 
 跨python解释器 跨机器动态修改定时任务配置的例子在
@@ -27,4 +58,4 @@ funboost_background_scheduler_redis_store = FunboostBackgroundSchedulerProcessJo
 test_frame/test_apschedual/test_aps_redis_store.py
 test_frame/test_apschedual/test_change_aps_conf.py
 
-"""
+"""

funboost/utils/decorators.py

@@ -361,6 +361,30 @@ class RedisDistributedLockContextManager(LoggerMixin, LoggerLevelSetterMixin):
             return False
 
 
+class RedisDistributedBlockLockContextManager(RedisDistributedLockContextManager):
+    def __init__(self, redis_client, redis_lock_key, expire_seconds=30, check_interval=0.1):
+        super().__init__(redis_client,redis_lock_key,expire_seconds)
+        self.check_interval = check_interval
+        # self.logger.setLevel(logging.DEBUG)
+
+    def __enter__(self):
+        while True:
+            self._line = sys._getframe().f_back.f_lineno  # 调用此方法的代码的函数
+            self._file_name = sys._getframe(1).f_code.co_filename  # 哪个文件调了用此方法
+            ret = self.redis_client.set(self.redis_lock_key, value=self.identifier, ex=self._expire_seconds, nx=True)
+            has_aquire_lock = False if ret is None else True
+            if has_aquire_lock:
+                log_msg = f'\n"{self._file_name}:{self._line}" 这行代码获得了redis锁 {self.redis_lock_key}'
+            else:
+                log_msg = f'\n"{self._file_name}:{self._line}" 这行代码此次没有获得redis锁 {self.redis_lock_key}'
+            # print(self.logger.level,log_msg)
+            self.logger.debug(log_msg)
+            if has_aquire_lock:
+                break
+            else:
+                time.sleep(self.check_interval)
+
+
 """
 @contextmanager
         def some_generator(<arguments>):
@@ -372,6 +396,9 @@ class RedisDistributedLockContextManager(LoggerMixin, LoggerLevelSetterMixin):
 """
 
 
+
+
+
 class ExceptionContextManager:
     """
     用上下文管理器捕获异常，可对代码片段进行错误捕捉，比装饰器更细腻

{funboost-46.7.dist-info → funboost-46.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 46.7
+Version: 46.8
 Summary: pip install funboost，python全功能分布式函数调度框架,funboost的功能是全面性重量级，用户能想得到的功能99%全都有;funboost的使用方式是轻量级，只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-46.7.dist-info → funboost-46.8.dist-info}/RECORD

@@ -1,4 +1,4 @@
-funboost/__init__.py,sha256=4AscHUBU9yAh_IbFd_y1pbOT34_ltO5-u-B6-IO1aHo,3956
+funboost/__init__.py,sha256=cnQABGz4I9cAvUXg57u7bUJRhrElwmZrM7LR1CR3JbE,3956
 funboost/__init__old.py,sha256=9Kv3cPLnPkbzMRnuJFVkPsuDdx1CdcSIuITkpdncZSc,20382
 funboost/__main__.py,sha256=-6Nogi666Y0LN8fVm3JmHGTOk8xEGWvotW_GDbSaZME,1065
 funboost/constant.py,sha256=STzRDZbuCC5FFV-uD_0r2beGsD1Zni4kregzR11z3Ok,8148
@@ -35,7 +35,7 @@ funboost/concurrent_pool/backup/async_pool_executor0223.py,sha256=RVUZiylUvpTm6U
 funboost/concurrent_pool/backup/async_pool_executor_back.py,sha256=KL6zEQaa1KkZOlAO85mCC1gwLm-YC5Ghn21IUom0UKM,9598
 funboost/concurrent_pool/backup/async_pool_executor_janus.py,sha256=OHMWJ9l3EYTpPpcrPrGGKd4K0tmQ2PN8HiX0Dta0EOo,5728
 funboost/consumers/__init__.py,sha256=ZXY_6Kut1VYNQiF5aWEgIWobsW1ht9YUP0TdRZRWFqI,126
-funboost/consumers/base_consumer.py,sha256=nbW-xB1o-GWqO8pe8T1MHISLDFlxe8rcfT6c0571GLY,82220
+funboost/consumers/base_consumer.py,sha256=29KgSWGuA6xoINi6_K9XKKr0aSOES4Xhb5nIX-rnPc8,82250
 funboost/consumers/celery_consumer.py,sha256=nQpSkzPBJ4bRpxn4i9ms0axrJiq9RWhb4lG2nAdCIig,9254
 funboost/consumers/confirm_mixin.py,sha256=5xC9AAQr_MY4tbSed8U-M6tOVmh69Qv9X0ld0JLT9Tk,6185
 funboost/consumers/dramatiq_consumer.py,sha256=ozmeAfeF0U-YNYHK4suQB0N264h5AZdfMH0O45Mh-8A,2229
@@ -172,9 +172,9 @@ funboost/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 funboost/queues/memory_queues_map.py,sha256=e1S_cnjnCVI4DBsA_iupF51S_eX4OvCtlefQCqS1TYA,424
 funboost/queues/peewee_queue.py,sha256=FrwegrilkHZG6Y1cGdl5Bt_UtA25f7m5TJQJMZ9YnJI,5280
 funboost/queues/sqla_queue.py,sha256=b-2QPvvuMxklm41ibZhGKehaAV9trXBQFCOHzgThx_s,11080
-funboost/timing_job/__init__.py,sha256=gZIfFUn0ZLfZX6GUR3vycIJEzf3hURWWd5G57JkaVi8,10187
+funboost/timing_job/__init__.py,sha256=R-qXGtgnBD6fGxKzTclY980iYLZSj4wv72kVXcjmYWQ,10362
 funboost/timing_job/apscheduler_use_mysql_store.py,sha256=ss92DiSLzbWuVIo19sTLgC99GessltWLOlqqOd4AIL4,471
-funboost/timing_job/apscheduler_use_redis_store.py,sha256=RspecIM8tTzcAfoS-HG9mOgUsJshvqq70hfm0_NKx3s,1043
+funboost/timing_job/apscheduler_use_redis_store.py,sha256=RNZVerUTTzpSFMFEHWkDrUOSz71B4Ml_hlcavkL1tAA,2933
 funboost/utils/__init__.py,sha256=rAyXE7lgCo_3VdMvGrIJiqsTHv2nZPTJDTj1f6s_KgE,586
 funboost/utils/apscheduler_monkey.py,sha256=CcUISbqX6nMWSxr_QjZ26IvvhUk_ojYZWRaKenpsKfE,3124
 funboost/utils/block_exit.py,sha256=BnfxNYo3lnmhk686RAEoc4u3D4RU_iEMMMgu5L8gIuI,96
@@ -182,7 +182,7 @@ funboost/utils/bulk_operation.py,sha256=B4FBxlz5f4oqlKDWqer7axn4gnDSfsYoMW2zSUCn
 funboost/utils/class_utils.py,sha256=yDsGF4-SaO7k4gtkpYLlACSeejS5FfdIHn087BE_4Cw,3579
 funboost/utils/ctrl_c_end.py,sha256=FgT9An-qsUA5gW-V-UKWqOh5shC7C_uvTFn0fS7i8GI,439
 funboost/utils/custom_pysnooper.py,sha256=7yXLKEMY_JjPRRt0Y0N-wV2CFhILlYNh40Y6uRBUaj8,5923
-funboost/utils/decorators.py,sha256=lyi9TCBg__7xkoV17AZhRItTn95vU2XMtOxfXJVV5B4,26601
+funboost/utils/decorators.py,sha256=gpwof-Nw__iFjeJjVQWx1l-scnxTivxcCI_0XqhMu6c,27885
 funboost/utils/develop_log.py,sha256=Wsx0ongGjTit5xqgk1BztYlVEkC6d0-Y7GENXLedVqY,271
 funboost/utils/expire_lock.py,sha256=AOkd1KlvZeIwQaz8ZoKxLpGxWgqQ4mfNHcFphh04o8Q,4732
 funboost/utils/json_helper.py,sha256=kwN5O-0Z3_9KS3JdBx-BltS9LI3Rfoq329vgkxMPVw4,2286
@@ -277,9 +277,9 @@ funboost/utils/pysnooper_ydf/utils.py,sha256=evSmGi_Oul7vSP47AJ0DLjFwoCYCfunJZ1m
 funboost/utils/pysnooper_ydf/variables.py,sha256=QejRDESBA06KG9OH4sBT4J1M55eaU29EIHg8K_igaXo,3693
 funboost/utils/times/__init__.py,sha256=Y4bQD3SIA_E7W2YvHq2Qdi0dGM4H2DxyFNdDOuFOq1w,2417
 funboost/utils/times/version.py,sha256=11XfnZVVzOgIhXXdeN_mYfdXThfrsbQHpA0wCjz-hpg,17
-funboost-46.7.dist-info/LICENSE,sha256=9EPP2ktG_lAPB8PjmWV-c9BiaJHc_FP6pPLcUrUwx0E,11562
-funboost-46.7.dist-info/METADATA,sha256=5Iw6_m1gcjBO2vvl2jS-kfO1Vhy9bAY-qVmOkkNynl8,32849
-funboost-46.7.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-funboost-46.7.dist-info/entry_points.txt,sha256=BQMqRALuw-QT9x2d7puWaUHriXfy3wIzvfzF61AnSSI,97
-funboost-46.7.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
-funboost-46.7.dist-info/RECORD,,
+funboost-46.8.dist-info/LICENSE,sha256=9EPP2ktG_lAPB8PjmWV-c9BiaJHc_FP6pPLcUrUwx0E,11562
+funboost-46.8.dist-info/METADATA,sha256=WoRBshgxIusIER8SK4_DC3iIzpB89_Le-VWEmkTYW_w,32849
+funboost-46.8.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+funboost-46.8.dist-info/entry_points.txt,sha256=BQMqRALuw-QT9x2d7puWaUHriXfy3wIzvfzF61AnSSI,97
+funboost-46.8.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
+funboost-46.8.dist-info/RECORD,,