funboost 47.9__tar.gz → 48.0__tar.gz

{funboost-47.9/funboost.egg-info → funboost-48.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 47.9
+Version: 48.0
 Summary: pip install funboost，python全功能分布式函数调度框架,funboost的功能是全面性重量级，用户能想得到的功能99%全都有;funboost的使用方式是轻量级，只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs
@@ -44,7 +44,7 @@ Requires-Dist: redis3
 Requires-Dist: redis5
 Requires-Dist: redis
 Requires-Dist: setuptools_rust
-Requires-Dist: fabric2==2.6.0
+Requires-Dist: fabric2>=2.6.0
 Requires-Dist: nb_filelock
 Requires-Dist: pysnooper
 Requires-Dist: deprecated

{funboost-47.9 → funboost-48.0}/README.md

@@ -1,3 +1,9 @@
+---
+noteId: "5f9681b0e9bb11efa4500f7226f4d3b6"
+tags: []
+
+---
+
 
 
 # 1.python万能分布式函数调度框架简funboost简介

{funboost-47.9 → funboost-48.0}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "47.9"
+__version__ = "48.0"
 
 from funboost.set_frame_config import show_frame_config
 
@@ -42,7 +42,7 @@ from funboost.factories.broker_kind__publsiher_consumer_type_map import register
 from funboost.factories.publisher_factotry import get_publisher
 from funboost.factories.consumer_factory import get_consumer
 
-from funboost.timing_job import fsdf_background_scheduler, timing_publish_deco, funboost_aps_scheduler
+from funboost.timing_job import fsdf_background_scheduler, timing_publish_deco, funboost_aps_scheduler,ApsJobAdder
 from funboost.constant import BrokerEnum, ConcurrentModeEnum
 
 from funboost.core.booster import boost, Booster, BoostersManager
@@ -58,7 +58,7 @@ from funboost.utils.ctrl_c_end import ctrl_c_recv
 from funboost.utils.redis_manager import RedisMixin
 from funboost.concurrent_pool.custom_threadpool_executor import show_current_threads_num
 
-from funboost.core.current_task import funboost_current_task
+from funboost.core.current_task import funboost_current_task,fct,get_current_taskid
 
 
 # atexit.register(ctrl_c_recv)  # 还是需要用户自己在代码末尾加才可以.

{funboost-47.9 → funboost-48.0}/funboost/assist/celery_helper.py

@@ -85,7 +85,7 @@ class CeleryHelper:
 
         if is_start_consume_all_queues is False:
             to_be_start_work_celery_queue_name_set_new = copy.copy(cls.to_be_start_work_celery_queue_name_set)
-            to_be_start_work_celery_queue_name_set_new.update(set(start_consume_queue_name_list))
+            to_be_start_work_celery_queue_name_set_new.update(set(start_consume_queue_name_list or []))
         else:
             from funboost import BoostersManager
             # print(BoostersManager.get_all_queues())

{funboost-47.9 → funboost-48.0}/funboost/consumers/base_consumer.py

@@ -403,13 +403,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
     logger_apscheduler = get_logger('push_for_apscheduler_use_database_store', log_filename='push_for_apscheduler_use_database_store.log')
 
     @classmethod
-    def _push_apscheduler_task_to_broker(cls, queue_name, msg, runonce_uuid):
+    def _push_apscheduler_task_to_broker(cls, queue_name, msg):
         funboost_lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
-        # key = 'apscheduler.redisjobstore_runonce'
-        # if RedisMixin().redis_db_frame.sadd(key, runonce_uuid):  # 这样可以阻止多次启动同队列名消费者 redis jobstore多次运行函数.
-        #     cls.logger_apscheduler.debug(f'延时任务用普通消息重新发布到普通队列 {msg}')
-        #     funboost_lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
-
+       
     @abc.abstractmethod
     def _shedual_task(self):
         """
@@ -512,7 +508,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             # print(msg_no_delay)
             # 数据库作为apscheduler的jobstores时候， 不能用 self.pbulisher_of_same_queue.publish，self不能序列化
             self._delay_task_scheduler.add_job(self._push_apscheduler_task_to_broker, 'date', run_date=run_date,
-                                               kwargs={'queue_name': self.queue_name, 'msg': msg_no_delay, 'runonce_uuid': str(uuid.uuid4())},
+                                               kwargs={'queue_name': self.queue_name, 'msg': msg_no_delay, },
                                                misfire_grace_time=misfire_grace_time,
                                               )
             self._confirm_consume(kw)

{funboost-47.9 → funboost-48.0}/funboost/consumers/redis_consumer_ack_able.py

@@ -126,7 +126,7 @@ class RedisConsumerAckAble(ConsumerConfirmMixinWithTheHelpOfRedisByHearbeat, Abs
         pull_msg_batch_size = self.consumer_params.broker_exclusive_config['pull_msg_batch_size']
         lua = f'''
                      local task_list = redis.call("lrange", KEYS[1],0,{pull_msg_batch_size-1})
-                     redis.call("ltrim", KEYS[1],100,-1)
+                     redis.call("ltrim", KEYS[1],{pull_msg_batch_size},-1)
                      if (#task_list > 0) then
                         for task_index,task_value in ipairs(task_list)
                         do

{funboost-47.9 → funboost-48.0}/funboost/core/current_task.py

@@ -158,6 +158,43 @@ def funboost_current_task():
     else:
         return thread_current_task
 
+class _FctProxy:
+    """后来多新增这个类了，"""
+    @property
+    def fct_context(self) ->FctContext:
+        ct = funboost_current_task()
+        return ct.get_fct_context()
+
+    @property
+    def function_params(self):
+        return self.fct_context.function_params
+
+    @property
+    def full_msg(self) -> dict:
+        return self.fct_context.full_msg
+
+    @property
+    def function_result_status(self) -> FunctionResultStatus:
+        return self.fct_context.function_result_status
+
+    @property
+    def task_id(self) -> FunctionResultStatus:
+        return self.fct_context.function_result_status.task_id
+
+    @property
+    def logger(self) -> logging.Logger:
+        return self.fct_context.logger
+
+    def __str__(self):
+        return f'<{self.__class__.__name__} [{self.function_result_status.get_status_dict()}]>'
+
+"""
+可以直接导入这个fct，不需要 手动写 fct = funboost_current_task() 了。 直接 from funboost import fct就完了，不需要先 fct = funboost_current_task()。
+funboost的fct 相当于flask的request那种对象 ，自动线程/协程 级别隔离。 多个线程不会互相干扰。
+"""
+fct = _FctProxy()
+
+
 
 def get_current_taskid():
     # return fct.function_result_status.task_id

{funboost-47.9 → funboost-48.0}/funboost/core/func_params_model.py

@@ -207,13 +207,13 @@ class BoosterParams(BaseJsonAbleModel):
     # func_params_is_pydantic_model: bool = False  # funboost 兼容支持 函数娼还是 pydantic model类型，funboost在发布之前和取出来时候自己转化。
 
     consuming_function_kind: typing.Optional[str] = None  # 自动生成的信息,不需要用户主动传参,如果自动判断失误就传递。是判断消费函数是函数还是实例方法还是类方法。如果传递了，就不自动获取函数类型。
-    ''' consuming_function_kind 可以为以下类型，
+    """ consuming_function_kind 可以为以下类型，
     class FunctionKind:
         CLASS_METHOD = 'CLASS_METHOD'
         INSTANCE_METHOD = 'INSTANCE_METHOD'
         STATIC_METHOD = 'STATIC_METHOD'
         COMMON_FUNCTION = 'COMMON_FUNCTION'
-    '''
+    """
 
     auto_generate_info: dict = {}  # 自动生成的信息,不需要用户主动传参.
 

funboost-48.0/funboost/timing_job/__init__.py

@@ -0,0 +1,6 @@
+from  funboost.timing_job.timing_job_base import (FsdfBackgroundScheduler ,
+funboost_aps_scheduler ,fsdf_background_scheduler,timing_publish_deco,FunboostBackgroundScheduler,push_fun_params_to_broker )
+
+
+
+from funboost.timing_job.timing_push import ApsJobAdder

{funboost-47.9 → funboost-48.0}/funboost/timing_job/apscheduler_use_redis_store.py

@@ -5,6 +5,7 @@ from funboost.timing_job import FunboostBackgroundScheduler
 from funboost.funboost_config_deafult import BrokerConnConfig, FunboostCommonConfig
 from funboost.utils.decorators import RedisDistributedBlockLockContextManager
 
+
 """
 这个是使用redis作为定时任务持久化，支持跨机器好跨进程，外部远程 动态修改/添加/删除定时任务
 """
@@ -51,8 +52,12 @@ jobstores = {
 
 funboost_background_scheduler_redis_store = FunboostBackgroundSchedulerProcessJobsWithinRedisLock(timezone=FunboostCommonConfig.TIMEZONE, daemon=False, jobstores=jobstores)
 
-"""
 
+
+
+
+
+"""
 跨python解释器 跨机器动态修改定时任务配置的例子在
 
 test_frame/test_apschedual/test_aps_redis_store.py

funboost-47.9/funboost/timing_job/__init__.py → funboost-48.0/funboost/timing_job/timing_job_base.py

@@ -39,19 +39,13 @@ def timing_publish_deco(consuming_func_decorated_or_consumer: Union[callable, Ab
     return _deco
 
 
-def push_fun_params_to_broker(queue_name: str, *args, runonce_uuid=None, **kwargs):
+def push_fun_params_to_broker(queue_name: str, *args, **kwargs):
     """
     queue_name 队列名字
     *args **kwargs 是消费函数的入参
-    发布消息中可以包括,runonce_uuid这个入参,确保分布式多个脚本都启动了定时器,导致每个定时器重复发布到消息队列,值你自己写  str(uuid.uuid4())
-    # 不需要传递 runonce_uuid 了，已经用专门的 FunboostBackgroundSchedulerProcessJobsWithinRedisLock 解决了。
     """
-    if runonce_uuid:   # 不需要传递 runonce_uuid 了，已经用专门的 FunboostBackgroundSchedulerProcessJobsWithinRedisLock 解决了 process_jobs的问题。
-        key = 'apscheduler.redisjobstore_runonce2'
-        if RedisMixin().redis_db_frame.sadd(key, runonce_uuid):
-            BoostersManager.get_or_create_booster_by_queue_name(queue_name).push(*args, **kwargs)
-    else:
-        BoostersManager.get_or_create_booster_by_queue_name(queue_name).push(*args, **kwargs)
+    
+    BoostersManager.get_or_create_booster_by_queue_name(queue_name).push(*args, **kwargs)
 
 
 class ThreadPoolExecutorForAps(BasePoolExecutor):
@@ -89,7 +83,7 @@ class FunboostBackgroundScheduler(BackgroundScheduler):
                             next_run_time, jobstore, executor,
                             replace_existing, **trigger_args)
 
-    def add_push_job(self, func: Booster, trigger=None, args=None, kwargs=None, runonce_uuid=None,
+    def add_push_job(self, func: Booster, trigger=None, args=None, kwargs=None, 
                      id=None, name=None,
                      misfire_grace_time=undefined, coalesce=undefined, max_instances=undefined,
                      next_run_time=undefined, jobstore='default', executor='default',
@@ -123,8 +117,6 @@ class FunboostBackgroundScheduler(BackgroundScheduler):
         args_list = list(args)
         args_list.insert(0, func.queue_name)
         args = tuple(args_list)
-        kwargs = kwargs or {}
-        kwargs['runonce_uuid'] = runonce_uuid  # 忽略，用户不需要传递runonce_uuid入参。
         return self.add_job(push_fun_params_to_broker, trigger, args, kwargs, id, name,
                             misfire_grace_time, coalesce, max_instances,
                             next_run_time, jobstore, executor,

funboost-48.0/funboost/timing_job/timing_push.py

@@ -0,0 +1,136 @@
+from funboost.utils import redis_manager
+from funboost.core.booster import BoostersManager, Booster
+
+from apscheduler.jobstores.redis import RedisJobStore
+from funboost.timing_job.timing_job_base import funboost_aps_scheduler, undefined
+from funboost.timing_job.apscheduler_use_redis_store import FunboostBackgroundSchedulerProcessJobsWithinRedisLock
+from funboost.funboost_config_deafult import FunboostCommonConfig
+
+
+class ApsJobAdder:
+    """
+    20250116新增加的统一的新增定时任务的方式，推荐这种方式。
+    用户不用像之前再去关心使用哪个apscheduler对象去添加定时任务了。
+
+    例如 add_numbers 是@boost装饰的消费函数
+    ApsJobAdder(add_numbers,job_store_kind='memory').add_push_job(
+        args=(1, 2),
+        trigger='date',  # 使用日期触发器
+        run_date='2025-01-16 18:23:50',  # 设置运行时间
+        # id='add_numbers_job'  # 任务ID
+    )
+
+    """
+
+    queue__redis_aps_map = {}
+
+    def __init__(self, booster: Booster, job_store_kind: str = 'memory'):
+        """
+        Initialize the ApsJobAdder.
+
+        :param booster: A Booster object representing the function to be scheduled.
+        :param job_store_kind: The type of job store to use. Default is 'memory'.
+                               Can be 'memory' or 'redis'.
+        """
+        self.booster = booster
+        self.job_store_kind = job_store_kind
+
+    @classmethod
+    def get_funboost_redis_apscheduler(cls, queue_name):
+        """ 
+        每个队列名字的定时任务用不同的redis jobstore的 jobs_key 和 run_times_key，防止互相干扰和取出不属于自己的任务
+        """
+        if queue_name in cls.queue__redis_aps_map:
+            return cls.queue__redis_aps_map[queue_name]
+        redis_jobstores = {
+
+            "default": RedisJobStore(**redis_manager.get_redis_conn_kwargs(),
+                                     jobs_key=f'funboost.apscheduler.{queue_name}.jobs',
+                                     run_times_key=f'funboost.apscheduler.{queue_name}.run_times',
+                                     )
+        }
+        redis_aps = FunboostBackgroundSchedulerProcessJobsWithinRedisLock(timezone=FunboostCommonConfig.TIMEZONE,
+                                                                          daemon=False, jobstores=redis_jobstores)
+        cls.queue__redis_aps_map[queue_name] = redis_aps
+        return redis_aps
+
+    @property
+    def aps_obj(self):
+        if self.job_store_kind == 'redis':
+            return self.get_funboost_redis_apscheduler(self.booster.queue_name)
+        elif self.job_store_kind == 'memory':
+            return funboost_aps_scheduler
+        else:
+            raise ValueError('Unsupported job_store_kind')
+
+    def add_push_job(self, trigger=None, args=None, kwargs=None, 
+                     id=None, name=None,
+                     misfire_grace_time=undefined, coalesce=undefined, max_instances=undefined,
+                     next_run_time=undefined, jobstore='default', executor='default',
+                     replace_existing=False, **trigger_args,):
+        """
+        这里的入参都是和apscheduler的add_job的入参一样的，funboost作者没有创造新的入参。
+        但是官方apscheduler的入参第一个入参是函数，
+        funboost的ApsJobAdder对象.add_push_job入参去掉了函数，因为类的实例化时候会把函数传进来，不需要再麻烦用户一次了。
+        """
+      
+        if not getattr(self.aps_obj, 'has_started_flag', False):
+            self.aps_obj.has_started_flag = True
+            self.aps_obj.start(paused=False)
+        return self.aps_obj.add_push_job(self.booster, trigger, args, kwargs, id, name,
+                                         misfire_grace_time, coalesce, max_instances,
+                                         next_run_time, jobstore, executor,
+                                         replace_existing, **trigger_args, )
+
+
+
+
+
+if __name__ == '__main__':
+    """
+    2025年后定时任务现在推荐使用 ApsJobAdder 写法 ，用户不需要亲自选择使用 apscheduler对象来添加定时任务
+    """
+    from funboost import boost, BrokerEnum,ctrl_c_recv,BoosterParams,ApsJobAdder
+  
+
+    # 定义任务处理函数
+    @BoosterParams(queue_name='sum_queue3', broker_kind=BrokerEnum.REDIS)
+    def sum_two_numbers(x, y):  
+        result = x + y 
+        print(f'The sum of {x} and {y} is {result}')  
+        
+    # 启动消费者
+    sum_two_numbers.consume()
+    
+    # 发布任务
+    sum_two_numbers.push(3, 5)
+    sum_two_numbers.push(10, 20)
+    
+    # 使用ApsJobAdder添加定时任务， 里面的定时语法，和apscheduler是一样的，用户需要自己熟悉知名框架apscheduler的add_job定时入参
+
+    # 方式1：指定日期执行一次
+    ApsJobAdder(sum_two_numbers, job_store_kind='redis').add_push_job(
+        trigger='date',
+        run_date='2025-01-17 23:25:40', 
+        args=(7, 8)
+    )
+
+    # 方式2：固定间隔执行
+    ApsJobAdder(sum_two_numbers, job_store_kind='memory').add_push_job(
+        trigger='interval', 
+        seconds=5, 
+        args=(4, 6)
+    )
+
+    # 方式3：使用cron表达式定时执行
+    ApsJobAdder(sum_two_numbers, job_store_kind='redis').add_push_job(
+        trigger='cron',
+        day_of_week='*', 
+        hour=23, 
+        minute=49, 
+        second=50,
+        kwargs={"x":50,"y":60},
+        replace_existing=True,
+        id='cron_job1')
+        
+    ctrl_c_recv()

{funboost-47.9 → funboost-48.0}/funboost/utils/ctrl_c_end.py

@@ -4,7 +4,7 @@ import time
 
 
 def ctrl_c_recv():
-    for i in range(4):
+    for i in range(3):
         while 1:
             try:
                 time.sleep(2)

{funboost-47.9 → funboost-48.0}/funboost/utils/dependency_packages_in_pythonpath/aioredis/readme.md

@@ -1,3 +1,9 @@
+---
+noteId: "6116b910e9bb11efa4500f7226f4d3b6"
+tags: []
+
+---
+
 
 ## 1 asioredis包将不再更新 ，ptyhon 3.11 版本 import aioredis会出错。
 

{funboost-47.9 → funboost-48.0}/funboost/utils/dependency_packages_in_pythonpath/readme.md

@@ -1,3 +1,9 @@
+---
+noteId: "6116b911e9bb11efa4500f7226f4d3b6"
+tags: []
+
+---
+
 ## 这个文件夹被添加到 sys.path中去了。
 
 funboost __init__.py 第一行就把这个添加到 sys.path了,相当于 export PYTHONPATH 了。

{funboost-47.9 → funboost-48.0/funboost.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 47.9
+Version: 48.0
 Summary: pip install funboost，python全功能分布式函数调度框架,funboost的功能是全面性重量级，用户能想得到的功能99%全都有;funboost的使用方式是轻量级，只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs
@@ -44,7 +44,7 @@ Requires-Dist: redis3
 Requires-Dist: redis5
 Requires-Dist: redis
 Requires-Dist: setuptools_rust
-Requires-Dist: fabric2==2.6.0
+Requires-Dist: fabric2>=2.6.0
 Requires-Dist: nb_filelock
 Requires-Dist: pysnooper
 Requires-Dist: deprecated

{funboost-47.9 → funboost-48.0}/funboost.egg-info/SOURCES.txt

@@ -185,6 +185,8 @@ funboost/queues/sqla_queue.py
 funboost/timing_job/__init__.py
 funboost/timing_job/apscheduler_use_mysql_store.py
 funboost/timing_job/apscheduler_use_redis_store.py
+funboost/timing_job/timing_job_base.py
+funboost/timing_job/timing_push.py
 funboost/utils/__init__.py
 funboost/utils/apscheduler_monkey.py
 funboost/utils/block_exit.py

{funboost-47.9 → funboost-48.0}/funboost.egg-info/requires.txt

@@ -15,7 +15,7 @@ redis3
 redis5
 redis
 setuptools_rust
-fabric2==2.6.0
+fabric2>=2.6.0
 nb_filelock
 pysnooper
 deprecated

{funboost-47.9 → funboost-48.0}/setup.py

@@ -95,7 +95,7 @@ setup(
         'redis5',
         'redis',
         'setuptools_rust',
-        'fabric2==2.6.0',  # 有的机器包rust错误， 这样做 curl https://sh.rustup.rs -sSf | sh
+        'fabric2>=2.6.0',  # 有的机器包rust错误， 这样做 curl https://sh.rustup.rs -sSf | sh
         'nb_filelock',
         'pysnooper',
         'deprecated',