funboost 46.6__tar.gz → 46.8__tar.gz

{funboost-46.6/funboost.egg-info → funboost-46.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 46.6
+Version: 46.8
 Summary: pip install funboost，python全功能分布式函数调度框架,funboost的功能是全面性重量级，用户能想得到的功能99%全都有;funboost的使用方式是轻量级，只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-46.6 → funboost-46.8}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "46.6"
+__version__ = "46.8"
 
 from funboost.set_frame_config import show_frame_config
 

{funboost-46.6 → funboost-46.8}/funboost/consumers/base_consumer.py

@@ -40,6 +40,7 @@ from funboost.core.serialization import Serialization
 from funboost.core.task_id_logger import TaskIdLogger
 from funboost.constant import FunctionKind
 
+
 from nb_libs.path_helper import PathHelper
 from nb_log import (get_logger, LoggerLevelSetterMixin, LogManager, is_main_process,
                     nb_log_config_default)
@@ -368,6 +369,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
 
     def _start_delay_task_scheduler(self):
         from funboost.timing_job import FsdfBackgroundScheduler
+        from funboost.timing_job.apscheduler_use_redis_store import FunboostBackgroundSchedulerProcessJobsWithinRedisLock
         # print(self.consumer_params.delay_task_apsscheduler_jobstores_kind )
         if self.consumer_params.delay_task_apscheduler_jobstores_kind == 'redis':
             jobstores = {
@@ -376,14 +378,21 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                                          run_times_key=f'funboost.apscheduler.{self.queue_name}.run_times',
                                          )
             }
+            self._delay_task_scheduler = FunboostBackgroundSchedulerProcessJobsWithinRedisLock(timezone=FunboostCommonConfig.TIMEZONE, daemon=False,
+                                                       jobstores=jobstores  # push 方法的序列化带thredignn.lock
+                                                       )
+            self._delay_task_scheduler.set_process_jobs_redis_lock_key(f'funboost.BackgroundSchedulerProcessJobsWithinRedisLock.{self.queue_name}')
         elif self.consumer_params.delay_task_apscheduler_jobstores_kind == 'memory':
             jobstores = {"default": MemoryJobStore()}
+            self._delay_task_scheduler = FsdfBackgroundScheduler(timezone=FunboostCommonConfig.TIMEZONE, daemon=False,
+                                                       jobstores=jobstores  # push 方法的序列化带thredignn.lock
+                                                       )
+
         else:
             raise Exception(f'delay_task_apsscheduler_jobstores_kind is error: {self.consumer_params.delay_task_apscheduler_jobstores_kind}')
-        self._delay_task_scheduler = FsdfBackgroundScheduler(timezone=FunboostCommonConfig.TIMEZONE, daemon=False,
-                                                             jobstores=jobstores  # push 方法的序列化带thredignn.lock
-                                                             )
-        self._delay_task_scheduler.add_executor(ApschedulerThreadPoolExecutor(2))  # 只是运行submit任务到并发池，不需要很多线程。
+
+
+        # self._delay_task_scheduler.add_executor(ApschedulerThreadPoolExecutor(2))  # 只是运行submit任务到并发池，不需要很多线程。
         # self._delay_task_scheduler.add_listener(self._apscheduler_job_miss, EVENT_JOB_MISSED)
         self._delay_task_scheduler.start()
 
@@ -500,7 +509,8 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             # 数据库作为apscheduler的jobstores时候， 不能用 self.pbulisher_of_same_queue.publish，self不能序列化
             self._delay_task_scheduler.add_job(self._push_for_apscheduler_use_database_store, 'date', run_date=run_date,
                                                kwargs={'queue_name': self.queue_name, 'msg': msg_no_delay, 'runonce_uuid': str(uuid.uuid4())},
-                                               misfire_grace_time=misfire_grace_time)
+                                               misfire_grace_time=misfire_grace_time,
+                                              )
             self._confirm_consume(kw)
 
         else:  # 普通任务

{funboost-46.6 → funboost-46.8}/funboost/consumers/rabbitmq_amqpstorm_consumer.py

@@ -14,7 +14,7 @@ class RabbitmqConsumerAmqpStorm(AbstractConsumer):
     使用AmqpStorm实现的，多线程安全的，不用加锁。
     funboost 强烈推荐使用这个做消息队列中间件。
     """
-    BROKER_EXCLUSIVE_CONFIG_DEFAULT = {'x-max-priority': None}  # x-max-priority 是 rabbitmq的优先级队列配置，必须为整数，强烈建议要小于5。为None就代表队列不支持优先级。
+    BROKER_EXCLUSIVE_CONFIG_DEFAULT = {'x-max-priority': None,'no_ack':False}  # x-max-priority 是 rabbitmq的优先级队列配置，必须为整数，强烈建议要小于5。为None就代表队列不支持优先级。
 
     def _shedual_task(self):
         # noinspection PyTypeChecker
@@ -28,16 +28,17 @@ class RabbitmqConsumerAmqpStorm(AbstractConsumer):
                                                                               broker_exclusive_config=self.consumer_params.broker_exclusive_config))
         rp.init_broker()
         rp.channel_wrapper_by_ampqstormbaic.qos(self.consumer_params.concurrent_num)
-        rp.channel_wrapper_by_ampqstormbaic.consume(callback=callback, queue=self.queue_name, no_ack=False,
+        rp.channel_wrapper_by_ampqstormbaic.consume(callback=callback, queue=self.queue_name, no_ack=self.consumer_params.broker_exclusive_config['no_ack'],
                                                     )
         rp.channel.start_consuming(auto_decode=True)
 
     def _confirm_consume(self, kw):
         # noinspection PyBroadException
-        try:
-            kw['amqpstorm_message'].ack()  # 确认消费
-        except BaseException as e:
-            self.logger.error(f'AmqpStorm确认消费失败  {type(e)} {e}')
+        if self.consumer_params.broker_exclusive_config['no_ack'] is False:
+            try:
+                kw['amqpstorm_message'].ack()  # 确认消费
+            except BaseException as e:
+                self.logger.error(f'AmqpStorm确认消费失败  {type(e)} {e}')
 
     def _requeue(self, kw):
         # amqpstorm.Message.delivery_tag

{funboost-46.6 → funboost-46.8}/funboost/core/booster.py

@@ -206,7 +206,7 @@ class Booster:
         fabric_deploy(self, **params)
 
 
-boost = Booster
+boost = Booster   # @boost 后消费函数.  不能自动补全方法就用 Booster就可以。 2024版本的 pycharm抽风了，@boost的消费函数不能自动补全提示 .consume  .push 这些方法。
 task_deco = boost  # 两个装饰器名字都可以。task_deco是原来名字，兼容一下。
 
 

{funboost-46.6 → funboost-46.8}/funboost/timing_job/__init__.py

@@ -8,6 +8,9 @@ import importlib
 import pickle
 
 import time
+from apscheduler.executors.pool import BasePoolExecutor
+
+from funboost.utils.decorators import RedisDistributedLockContextManager,RedisDistributedBlockLockContextManager
 from typing import Union
 import threading
 
@@ -15,7 +18,7 @@ from apscheduler.schedulers.background import BackgroundScheduler
 from apscheduler.jobstores.redis import RedisJobStore
 # noinspection PyProtectedMember
 from apscheduler.schedulers.base import STATE_STOPPED, STATE_RUNNING
-from apscheduler.util import undefined
+from apscheduler.util import undefined,TIMEOUT_MAX
 import deprecated
 from funboost.utils.redis_manager import RedisMixin
 
@@ -25,7 +28,8 @@ from funboost.consumers.base_consumer import AbstractConsumer
 from funboost.core.booster import BoostersManager, Booster
 from funboost.publishers.base_publisher import AbstractPublisher
 from funboost import BoosterParams
-
+from funboost.concurrent_pool.flexible_thread_pool import FlexibleThreadPool
+from funboost.concurrent_pool.custom_threadpool_executor import ThreadPoolExecutorShrinkAble
 
 @deprecated.deprecated(reason='以后不要再使用这种方式，对于job_store为数据库时候需要序列化不好。使用内存和数据库都兼容的添加任务方式: add_push_job')
 def timing_publish_deco(consuming_func_decorated_or_consumer: Union[callable, AbstractConsumer]):
@@ -53,6 +57,20 @@ def push_fun_params_to_broker(queue_name: str, *args, runonce_uuid=None, **kwarg
     else:
         BoostersManager.get_or_create_booster_by_queue_name(queue_name).push(*args, **kwargs)
 
+class ThreadPoolExecutorForAps(BasePoolExecutor):
+    """
+    An executor that runs jobs in a concurrent.futures thread pool.
+
+    Plugin alias: ``threadpool``
+
+    :param max_workers: the maximum number of spawned threads.
+    :param pool_kwargs: dict of keyword arguments to pass to the underlying
+        ThreadPoolExecutor constructor
+    """
+
+    def __init__(self, max_workers=10, ):
+        pool = ThreadPoolExecutorShrinkAble(int(max_workers),)
+        super().__init__(pool)
 
 class FunboostBackgroundScheduler(BackgroundScheduler):
     """
@@ -60,6 +78,9 @@ class FunboostBackgroundScheduler(BackgroundScheduler):
     通过重写 _main_loop ，使得动态修改增加删除定时任务配置更好。
     """
 
+    _last_wait_seconds = None
+    _last_has_task = False
+
     @deprecated.deprecated(reason='以后不要再使用这种方式，对于job_store为数据库时候需要序列化不好。使用内存和数据库都兼容的添加任务方式: add_push_job')
     def add_timing_publish_job(self, func, trigger=None, args=None, kwargs=None, id=None, name=None,
                                misfire_grace_time=undefined, coalesce=undefined, max_instances=undefined,
@@ -146,14 +167,22 @@ class FunboostBackgroundScheduler(BackgroundScheduler):
         或者下一个需要运行的任务的wait_seconds是3600秒后，此时新加了一个动态任务需要3600秒后，
         现在最多只需要1秒就能扫描到动态新增的定时任务了。
         """
-        MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS = 1
+        MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS = 0.5
         wait_seconds = None
         while self.state == STATE_RUNNING:
             if wait_seconds is None:
                 wait_seconds = MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS
-            time.sleep(min(wait_seconds, MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS))  # 这个要取最小值，不然例如定时间隔0.1秒运行，不取最小值，不会每隔0.1秒运行。
+            self._last_wait_seconds = min(wait_seconds, MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS)
+            if wait_seconds in (None,TIMEOUT_MAX):
+                self._last_has_task = False
+            else:
+                self._last_has_task = True
+            time.sleep(self._last_wait_seconds)  # 这个要取最小值，不然例如定时间隔0.1秒运行，不取最小值，不会每隔0.1秒运行。
             wait_seconds = self._process_jobs()
 
+    def _create_default_executor(self):
+        return ThreadPoolExecutorForAps()  # 必须是apscheduler pool的子类
+
 
 FsdfBackgroundScheduler = FunboostBackgroundScheduler  # 兼容一下名字，fsdf是 function-scheduling-distributed-framework 老框架名字的缩写
 # funboost_aps_scheduler定时配置基于内存的，不可以跨机器远程动态添加/修改/删除定时任务配置。如果需要动态增删改查定时任务，可以使用funboost_background_scheduler_redis_store

{funboost-46.6 → funboost-46.8}/funboost/timing_job/apscheduler_use_mysql_store.py

@@ -6,4 +6,5 @@ from funboost.timing_job import FsdfBackgroundScheduler
 
 """
 这个是使用mysql作为定时任务持久化，支持动态修改 添加定时任务,用户完全可以自己按照 funboost/timing_job/apscheduler_use_redis_store.py 中的代码实现，因为apscheduler支持SQLAlchemyJobStore
+只是scheduler改个jobstores类型，和funboost知识毫无关系，是apscheduler的知识。
 """

funboost-46.8/funboost/timing_job/apscheduler_use_redis_store.py

@@ -0,0 +1,61 @@
+from apscheduler.jobstores.redis import RedisJobStore
+from funboost.utils.redis_manager import RedisMixin
+
+from funboost.timing_job import FunboostBackgroundScheduler
+from funboost.funboost_config_deafult import BrokerConnConfig, FunboostCommonConfig
+from funboost.utils.decorators import RedisDistributedBlockLockContextManager
+
+"""
+这个是使用redis作为定时任务持久化，支持跨机器好跨进程，外部远程 动态修改/添加/删除定时任务
+"""
+
+
+class FunboostBackgroundSchedulerProcessJobsWithinRedisLock(FunboostBackgroundScheduler):
+    """
+    分布式或多进程都启动某个apscheduler实例，如果都使用的同一个数据库类型的jobstores ，_process_jobs有很大概率会造成报错， 因为_process_jobs使用的是线程锁，管不了其他进程和分布式机器。
+
+    https://groups.google.com/g/apscheduler/c/Gjc_JQMPePc 问题也提到了这个bug
+
+    继承 Custom schedulers https://apscheduler.readthedocs.io/en/3.x/extending.html   可以重写 _create_lock
+    """
+
+    process_jobs_redis_lock_key = f'funboost.BackgroundSchedulerProcessJobsWithinRedisLock'
+
+    def set_process_jobs_redis_lock_key(self, lock_key):
+        self.process_jobs_redis_lock_key = lock_key
+
+    # def  _create_lock(self):
+    #     return RedisDistributedBlockLockContextManager(RedisMixin().redis_db_frame,self.process_jobs_redis_lock_key,) 这个类的写法不适合固定的单例，
+    #     RedisDistributedBlockLockContextManager的写法不适合 永远用一个 对象，所以还是放到 def  _process_jobs 里面运行
+
+    # def _process_jobs(self):
+    #     for i in range(10) :
+    #         with RedisDistributedLockContextManager(RedisMixin().redis_db_frame, self.process_jobs_redis_lock_key, ) as lock:
+    #             if lock.has_aquire_lock:
+    #                 wait_seconds = super()._process_jobs()
+    #                 return wait_seconds
+    #             else:
+    #                 time.sleep(0.1)
+    #     return 0.1
+
+    def _process_jobs(self):
+        with RedisDistributedBlockLockContextManager(RedisMixin().redis_db_frame, self.process_jobs_redis_lock_key, ):
+            return super()._process_jobs()
+
+
+jobstores = {
+    "default": RedisJobStore(db=BrokerConnConfig.REDIS_DB, host=BrokerConnConfig.REDIS_HOST,
+                             port=BrokerConnConfig.REDIS_PORT, password=BrokerConnConfig.REDIS_PASSWORD,
+                             username=BrokerConnConfig.REDIS_USERNAME, jobs_key='funboost.apscheduler.jobs')
+}
+
+funboost_background_scheduler_redis_store = FunboostBackgroundSchedulerProcessJobsWithinRedisLock(timezone=FunboostCommonConfig.TIMEZONE, daemon=False, jobstores=jobstores)
+
+"""
+
+跨python解释器 跨机器动态修改定时任务配置的例子在
+
+test_frame/test_apschedual/test_aps_redis_store.py
+test_frame/test_apschedual/test_change_aps_conf.py
+
+"""

{funboost-46.6 → funboost-46.8}/funboost/utils/decorators.py

@@ -361,6 +361,30 @@ class RedisDistributedLockContextManager(LoggerMixin, LoggerLevelSetterMixin):
             return False
 
 
+class RedisDistributedBlockLockContextManager(RedisDistributedLockContextManager):
+    def __init__(self, redis_client, redis_lock_key, expire_seconds=30, check_interval=0.1):
+        super().__init__(redis_client,redis_lock_key,expire_seconds)
+        self.check_interval = check_interval
+        # self.logger.setLevel(logging.DEBUG)
+
+    def __enter__(self):
+        while True:
+            self._line = sys._getframe().f_back.f_lineno  # 调用此方法的代码的函数
+            self._file_name = sys._getframe(1).f_code.co_filename  # 哪个文件调了用此方法
+            ret = self.redis_client.set(self.redis_lock_key, value=self.identifier, ex=self._expire_seconds, nx=True)
+            has_aquire_lock = False if ret is None else True
+            if has_aquire_lock:
+                log_msg = f'\n"{self._file_name}:{self._line}" 这行代码获得了redis锁 {self.redis_lock_key}'
+            else:
+                log_msg = f'\n"{self._file_name}:{self._line}" 这行代码此次没有获得redis锁 {self.redis_lock_key}'
+            # print(self.logger.level,log_msg)
+            self.logger.debug(log_msg)
+            if has_aquire_lock:
+                break
+            else:
+                time.sleep(self.check_interval)
+
+
 """
 @contextmanager
         def some_generator(<arguments>):
@@ -372,6 +396,9 @@ class RedisDistributedLockContextManager(LoggerMixin, LoggerLevelSetterMixin):
 """
 
 
+
+
+
 class ExceptionContextManager:
     """
     用上下文管理器捕获异常，可对代码片段进行错误捕捉，比装饰器更细腻

{funboost-46.6 → funboost-46.8/funboost.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 46.6
+Version: 46.8
 Summary: pip install funboost，python全功能分布式函数调度框架,funboost的功能是全面性重量级，用户能想得到的功能99%全都有;funboost的使用方式是轻量级，只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

funboost-46.6/funboost/timing_job/apscheduler_use_redis_store.py

@@ -1,30 +0,0 @@
-from apscheduler.jobstores.redis import RedisJobStore
-
-from funboost.funboost_config_deafult import BrokerConnConfig,FunboostCommonConfig
-
-from funboost.timing_job import FunboostBackgroundScheduler
-
-"""
-这个是使用redis作为定时任务持久化，支持动态修改/添加/删除定时任务
-"""
-
-jobstores = {
-    "default": RedisJobStore(db=BrokerConnConfig.REDIS_DB, host=BrokerConnConfig.REDIS_HOST,
-                             port=BrokerConnConfig.REDIS_PORT, password=BrokerConnConfig.REDIS_PASSWORD,
-                             username=BrokerConnConfig.REDIS_USERNAME,jobs_key='funboost.apscheduler.jobs')
-}
-
-funboost_background_scheduler_redis_store = FunboostBackgroundScheduler(timezone=FunboostCommonConfig.TIMEZONE, daemon=False, jobstores=jobstores)
-
-
-
-
-
-"""
-
-跨python解释器 跨机器动态修改定时任务配置的例子在
-
-test_frame/test_apschedual/test_aps_redis_store.py
-test_frame/test_apschedual/test_change_aps_conf.py
-
-"""