funboost 47.8__py3-none-any.whl → 48.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of funboost might be problematic. Click here for more details.
- funboost/__init__.py +3 -3
- funboost/assist/celery_helper.py +1 -1
- funboost/consumers/base_consumer.py +3 -7
- funboost/consumers/redis_consumer_ack_able.py +1 -1
- funboost/core/current_task.py +37 -0
- funboost/core/func_params_model.py +3 -2
- funboost/timing_job/__init__.py +3 -218
- funboost/timing_job/apscheduler_use_redis_store.py +6 -1
- funboost/timing_job/timing_job_base.py +213 -0
- funboost/timing_job/timing_push.py +136 -0
- funboost/utils/class_utils2.py +94 -0
- funboost/utils/ctrl_c_end.py +1 -1
- funboost/utils/dependency_packages_in_pythonpath/aioredis/readme.md +6 -0
- funboost/utils/dependency_packages_in_pythonpath/readme.md +6 -0
- {funboost-47.8.dist-info → funboost-48.0.dist-info}/METADATA +2 -2
- {funboost-47.8.dist-info → funboost-48.0.dist-info}/RECORD +20 -17
- {funboost-47.8.dist-info → funboost-48.0.dist-info}/LICENSE +0 -0
- {funboost-47.8.dist-info → funboost-48.0.dist-info}/WHEEL +0 -0
- {funboost-47.8.dist-info → funboost-48.0.dist-info}/entry_points.txt +0 -0
- {funboost-47.8.dist-info → funboost-48.0.dist-info}/top_level.txt +0 -0
funboost/__init__.py
CHANGED
|
@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
|
|
|
13
13
|
这段注释说明和使用的用户无关,只和框架开发人员有关.
|
|
14
14
|
'''
|
|
15
15
|
|
|
16
|
-
__version__ = "
|
|
16
|
+
__version__ = "48.0"
|
|
17
17
|
|
|
18
18
|
from funboost.set_frame_config import show_frame_config
|
|
19
19
|
|
|
@@ -42,7 +42,7 @@ from funboost.factories.broker_kind__publsiher_consumer_type_map import register
|
|
|
42
42
|
from funboost.factories.publisher_factotry import get_publisher
|
|
43
43
|
from funboost.factories.consumer_factory import get_consumer
|
|
44
44
|
|
|
45
|
-
from funboost.timing_job import fsdf_background_scheduler, timing_publish_deco, funboost_aps_scheduler
|
|
45
|
+
from funboost.timing_job import fsdf_background_scheduler, timing_publish_deco, funboost_aps_scheduler,ApsJobAdder
|
|
46
46
|
from funboost.constant import BrokerEnum, ConcurrentModeEnum
|
|
47
47
|
|
|
48
48
|
from funboost.core.booster import boost, Booster, BoostersManager
|
|
@@ -58,7 +58,7 @@ from funboost.utils.ctrl_c_end import ctrl_c_recv
|
|
|
58
58
|
from funboost.utils.redis_manager import RedisMixin
|
|
59
59
|
from funboost.concurrent_pool.custom_threadpool_executor import show_current_threads_num
|
|
60
60
|
|
|
61
|
-
from funboost.core.current_task import funboost_current_task
|
|
61
|
+
from funboost.core.current_task import funboost_current_task,fct,get_current_taskid
|
|
62
62
|
|
|
63
63
|
|
|
64
64
|
# atexit.register(ctrl_c_recv) # 还是需要用户自己在代码末尾加才可以.
|
funboost/assist/celery_helper.py
CHANGED
|
@@ -85,7 +85,7 @@ class CeleryHelper:
|
|
|
85
85
|
|
|
86
86
|
if is_start_consume_all_queues is False:
|
|
87
87
|
to_be_start_work_celery_queue_name_set_new = copy.copy(cls.to_be_start_work_celery_queue_name_set)
|
|
88
|
-
to_be_start_work_celery_queue_name_set_new.update(set(start_consume_queue_name_list))
|
|
88
|
+
to_be_start_work_celery_queue_name_set_new.update(set(start_consume_queue_name_list or []))
|
|
89
89
|
else:
|
|
90
90
|
from funboost import BoostersManager
|
|
91
91
|
# print(BoostersManager.get_all_queues())
|
|
@@ -403,13 +403,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
403
403
|
logger_apscheduler = get_logger('push_for_apscheduler_use_database_store', log_filename='push_for_apscheduler_use_database_store.log')
|
|
404
404
|
|
|
405
405
|
@classmethod
|
|
406
|
-
def _push_apscheduler_task_to_broker(cls, queue_name, msg
|
|
406
|
+
def _push_apscheduler_task_to_broker(cls, queue_name, msg):
|
|
407
407
|
funboost_lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
|
|
408
|
-
|
|
409
|
-
# if RedisMixin().redis_db_frame.sadd(key, runonce_uuid): # 这样可以阻止多次启动同队列名消费者 redis jobstore多次运行函数.
|
|
410
|
-
# cls.logger_apscheduler.debug(f'延时任务用普通消息重新发布到普通队列 {msg}')
|
|
411
|
-
# funboost_lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
|
|
412
|
-
|
|
408
|
+
|
|
413
409
|
@abc.abstractmethod
|
|
414
410
|
def _shedual_task(self):
|
|
415
411
|
"""
|
|
@@ -512,7 +508,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
512
508
|
# print(msg_no_delay)
|
|
513
509
|
# 数据库作为apscheduler的jobstores时候, 不能用 self.pbulisher_of_same_queue.publish,self不能序列化
|
|
514
510
|
self._delay_task_scheduler.add_job(self._push_apscheduler_task_to_broker, 'date', run_date=run_date,
|
|
515
|
-
kwargs={'queue_name': self.queue_name, 'msg': msg_no_delay,
|
|
511
|
+
kwargs={'queue_name': self.queue_name, 'msg': msg_no_delay, },
|
|
516
512
|
misfire_grace_time=misfire_grace_time,
|
|
517
513
|
)
|
|
518
514
|
self._confirm_consume(kw)
|
|
@@ -126,7 +126,7 @@ class RedisConsumerAckAble(ConsumerConfirmMixinWithTheHelpOfRedisByHearbeat, Abs
|
|
|
126
126
|
pull_msg_batch_size = self.consumer_params.broker_exclusive_config['pull_msg_batch_size']
|
|
127
127
|
lua = f'''
|
|
128
128
|
local task_list = redis.call("lrange", KEYS[1],0,{pull_msg_batch_size-1})
|
|
129
|
-
redis.call("ltrim", KEYS[1],
|
|
129
|
+
redis.call("ltrim", KEYS[1],{pull_msg_batch_size},-1)
|
|
130
130
|
if (#task_list > 0) then
|
|
131
131
|
for task_index,task_value in ipairs(task_list)
|
|
132
132
|
do
|
funboost/core/current_task.py
CHANGED
|
@@ -158,6 +158,43 @@ def funboost_current_task():
|
|
|
158
158
|
else:
|
|
159
159
|
return thread_current_task
|
|
160
160
|
|
|
161
|
+
class _FctProxy:
|
|
162
|
+
"""后来多新增这个类了,"""
|
|
163
|
+
@property
|
|
164
|
+
def fct_context(self) ->FctContext:
|
|
165
|
+
ct = funboost_current_task()
|
|
166
|
+
return ct.get_fct_context()
|
|
167
|
+
|
|
168
|
+
@property
|
|
169
|
+
def function_params(self):
|
|
170
|
+
return self.fct_context.function_params
|
|
171
|
+
|
|
172
|
+
@property
|
|
173
|
+
def full_msg(self) -> dict:
|
|
174
|
+
return self.fct_context.full_msg
|
|
175
|
+
|
|
176
|
+
@property
|
|
177
|
+
def function_result_status(self) -> FunctionResultStatus:
|
|
178
|
+
return self.fct_context.function_result_status
|
|
179
|
+
|
|
180
|
+
@property
|
|
181
|
+
def task_id(self) -> FunctionResultStatus:
|
|
182
|
+
return self.fct_context.function_result_status.task_id
|
|
183
|
+
|
|
184
|
+
@property
|
|
185
|
+
def logger(self) -> logging.Logger:
|
|
186
|
+
return self.fct_context.logger
|
|
187
|
+
|
|
188
|
+
def __str__(self):
|
|
189
|
+
return f'<{self.__class__.__name__} [{self.function_result_status.get_status_dict()}]>'
|
|
190
|
+
|
|
191
|
+
"""
|
|
192
|
+
可以直接导入这个fct,不需要 手动写 fct = funboost_current_task() 了。 直接 from funboost import fct就完了,不需要先 fct = funboost_current_task()。
|
|
193
|
+
funboost的fct 相当于flask的request那种对象 ,自动线程/协程 级别隔离。 多个线程不会互相干扰。
|
|
194
|
+
"""
|
|
195
|
+
fct = _FctProxy()
|
|
196
|
+
|
|
197
|
+
|
|
161
198
|
|
|
162
199
|
def get_current_taskid():
|
|
163
200
|
# return fct.function_result_status.task_id
|
|
@@ -154,6 +154,7 @@ class BoosterParams(BaseJsonAbleModel):
|
|
|
154
154
|
retry_interval: typing.Union[float, int] = 0 # 函数出错后间隔多少秒再重试.
|
|
155
155
|
is_push_to_dlx_queue_when_retry_max_times: bool = False # 函数达到最大重试次数仍然没成功,是否发送到死信队列,死信队列的名字是 队列名字 + _dlx。
|
|
156
156
|
|
|
157
|
+
|
|
157
158
|
consumin_function_decorator: typing.Callable = None # 函数的装饰器。因为此框架做参数自动转指点,需要获取精准的入参名称,不支持在消费函数上叠加 @ *args **kwargs的装饰器,如果想用装饰器可以这里指定。
|
|
158
159
|
function_timeout: typing.Union[int, float,None] = None # 超时秒数,函数运行超过这个时间,则自动杀死函数。为0是不限制。 谨慎使用,非必要别去设置超时时间,设置后性能会降低(因为需要把用户函数包装到另一个线单独的程中去运行),而且突然强制超时杀死运行中函数,可能会造成死锁.(例如用户函数在获得线程锁后突然杀死函数,别的线程再也无法获得锁了)
|
|
159
160
|
|
|
@@ -206,13 +207,13 @@ class BoosterParams(BaseJsonAbleModel):
|
|
|
206
207
|
# func_params_is_pydantic_model: bool = False # funboost 兼容支持 函数娼还是 pydantic model类型,funboost在发布之前和取出来时候自己转化。
|
|
207
208
|
|
|
208
209
|
consuming_function_kind: typing.Optional[str] = None # 自动生成的信息,不需要用户主动传参,如果自动判断失误就传递。是判断消费函数是函数还是实例方法还是类方法。如果传递了,就不自动获取函数类型。
|
|
209
|
-
|
|
210
|
+
""" consuming_function_kind 可以为以下类型,
|
|
210
211
|
class FunctionKind:
|
|
211
212
|
CLASS_METHOD = 'CLASS_METHOD'
|
|
212
213
|
INSTANCE_METHOD = 'INSTANCE_METHOD'
|
|
213
214
|
STATIC_METHOD = 'STATIC_METHOD'
|
|
214
215
|
COMMON_FUNCTION = 'COMMON_FUNCTION'
|
|
215
|
-
|
|
216
|
+
"""
|
|
216
217
|
|
|
217
218
|
auto_generate_info: dict = {} # 自动生成的信息,不需要用户主动传参.
|
|
218
219
|
|
funboost/timing_job/__init__.py
CHANGED
|
@@ -1,221 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
"""
|
|
4
|
-
import atexit
|
|
1
|
+
from funboost.timing_job.timing_job_base import (FsdfBackgroundScheduler ,
|
|
2
|
+
funboost_aps_scheduler ,fsdf_background_scheduler,timing_publish_deco,FunboostBackgroundScheduler,push_fun_params_to_broker )
|
|
5
3
|
|
|
6
|
-
import time
|
|
7
|
-
from apscheduler.executors.pool import BasePoolExecutor
|
|
8
4
|
|
|
9
|
-
from typing import Union
|
|
10
|
-
import threading
|
|
11
5
|
|
|
12
|
-
from
|
|
13
|
-
# noinspection PyProtectedMember
|
|
14
|
-
from apscheduler.schedulers.base import STATE_STOPPED, STATE_RUNNING
|
|
15
|
-
from apscheduler.util import undefined
|
|
16
|
-
from threading import TIMEOUT_MAX
|
|
17
|
-
import deprecated
|
|
18
|
-
from funboost.utils.redis_manager import RedisMixin
|
|
19
|
-
|
|
20
|
-
from funboost.funboost_config_deafult import FunboostCommonConfig
|
|
21
|
-
|
|
22
|
-
from funboost.consumers.base_consumer import AbstractConsumer
|
|
23
|
-
from funboost.core.booster import BoostersManager, Booster
|
|
24
|
-
|
|
25
|
-
from funboost import BoosterParams
|
|
26
|
-
from funboost.concurrent_pool.custom_threadpool_executor import ThreadPoolExecutorShrinkAble
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
@deprecated.deprecated(reason='以后不要再使用这种方式,对于job_store为数据库时候需要序列化不好。使用内存和数据库都兼容的添加任务方式: add_push_job')
|
|
30
|
-
def timing_publish_deco(consuming_func_decorated_or_consumer: Union[callable, AbstractConsumer]):
|
|
31
|
-
def _deco(*args, **kwargs):
|
|
32
|
-
if getattr(consuming_func_decorated_or_consumer, 'is_decorated_as_consume_function', False) is True:
|
|
33
|
-
consuming_func_decorated_or_consumer.push(*args, **kwargs)
|
|
34
|
-
elif isinstance(consuming_func_decorated_or_consumer, AbstractConsumer):
|
|
35
|
-
consuming_func_decorated_or_consumer.publisher_of_same_queue.push(*args, **kwargs)
|
|
36
|
-
else:
|
|
37
|
-
raise TypeError('consuming_func_decorated_or_consumer 必须是被 boost 装饰的函数或者consumer类型')
|
|
38
|
-
|
|
39
|
-
return _deco
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
def push_fun_params_to_broker(queue_name: str, *args, runonce_uuid=None, **kwargs):
|
|
43
|
-
"""
|
|
44
|
-
queue_name 队列名字
|
|
45
|
-
*args **kwargs 是消费函数的入参
|
|
46
|
-
发布消息中可以包括,runonce_uuid这个入参,确保分布式多个脚本都启动了定时器,导致每个定时器重复发布到消息队列,值你自己写 str(uuid.uuid4())
|
|
47
|
-
# 不需要传递 runonce_uuid 了,已经用专门的 FunboostBackgroundSchedulerProcessJobsWithinRedisLock 解决了。
|
|
48
|
-
"""
|
|
49
|
-
if runonce_uuid: # 不需要传递 runonce_uuid 了,已经用专门的 FunboostBackgroundSchedulerProcessJobsWithinRedisLock 解决了 process_jobs的问题。
|
|
50
|
-
key = 'apscheduler.redisjobstore_runonce2'
|
|
51
|
-
if RedisMixin().redis_db_frame.sadd(key, runonce_uuid):
|
|
52
|
-
BoostersManager.get_or_create_booster_by_queue_name(queue_name).push(*args, **kwargs)
|
|
53
|
-
else:
|
|
54
|
-
BoostersManager.get_or_create_booster_by_queue_name(queue_name).push(*args, **kwargs)
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
class ThreadPoolExecutorForAps(BasePoolExecutor):
|
|
58
|
-
"""
|
|
59
|
-
An executor that runs jobs in a concurrent.futures thread pool.
|
|
60
|
-
|
|
61
|
-
Plugin alias: ``threadpool``
|
|
62
|
-
|
|
63
|
-
:param max_workers: the maximum number of spawned threads.
|
|
64
|
-
:param pool_kwargs: dict of keyword arguments to pass to the underlying
|
|
65
|
-
ThreadPoolExecutor constructor
|
|
66
|
-
"""
|
|
67
|
-
|
|
68
|
-
def __init__(self, max_workers=10, pool_kwargs=None):
|
|
69
|
-
pool = ThreadPoolExecutorShrinkAble(int(max_workers), )
|
|
70
|
-
super().__init__(pool)
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
class FunboostBackgroundScheduler(BackgroundScheduler):
|
|
74
|
-
"""
|
|
75
|
-
自定义的, 继承了官方BackgroundScheduler,
|
|
76
|
-
通过重写 _main_loop ,使得动态修改增加删除定时任务配置更好。
|
|
77
|
-
"""
|
|
78
|
-
|
|
79
|
-
_last_wait_seconds = None
|
|
80
|
-
_last_has_task = False
|
|
81
|
-
|
|
82
|
-
@deprecated.deprecated(reason='以后不要再使用这种方式,对于job_store为数据库时候需要序列化不好。使用内存和数据库都兼容的添加任务方式: add_push_job')
|
|
83
|
-
def add_timing_publish_job(self, func, trigger=None, args=None, kwargs=None, id=None, name=None,
|
|
84
|
-
misfire_grace_time=undefined, coalesce=undefined, max_instances=undefined,
|
|
85
|
-
next_run_time=undefined, jobstore='default', executor='default',
|
|
86
|
-
replace_existing=False, **trigger_args):
|
|
87
|
-
return self.add_job(timing_publish_deco(func), trigger, args, kwargs, id, name,
|
|
88
|
-
misfire_grace_time, coalesce, max_instances,
|
|
89
|
-
next_run_time, jobstore, executor,
|
|
90
|
-
replace_existing, **trigger_args)
|
|
91
|
-
|
|
92
|
-
def add_push_job(self, func: Booster, trigger=None, args=None, kwargs=None, runonce_uuid=None,
|
|
93
|
-
id=None, name=None,
|
|
94
|
-
misfire_grace_time=undefined, coalesce=undefined, max_instances=undefined,
|
|
95
|
-
next_run_time=undefined, jobstore='default', executor='default',
|
|
96
|
-
replace_existing=False, **trigger_args, ):
|
|
97
|
-
"""
|
|
98
|
-
:param func: 被@boost装饰器装饰的函数
|
|
99
|
-
:param trigger:
|
|
100
|
-
:param args:
|
|
101
|
-
:param kwargs:
|
|
102
|
-
:param id:
|
|
103
|
-
:param name:
|
|
104
|
-
:param misfire_grace_time:
|
|
105
|
-
:param coalesce:
|
|
106
|
-
:param max_instances:
|
|
107
|
-
:param next_run_time:
|
|
108
|
-
:param jobstore:
|
|
109
|
-
:param executor:
|
|
110
|
-
:param replace_existing:
|
|
111
|
-
:param trigger_args:
|
|
112
|
-
:return:
|
|
113
|
-
"""
|
|
114
|
-
# args = args or {}
|
|
115
|
-
# kwargs['queue_name'] = func.queue_name
|
|
116
|
-
|
|
117
|
-
"""
|
|
118
|
-
用户如果不使用funboost的 FunboostBackgroundScheduler 类型对象,而是使用原生的apscheduler类型对象,可以scheduler.add_job(push_fun_params_to_broker,args=(,),kwargs={})
|
|
119
|
-
push_fun_params_to_broker函数入参是消费函数队列的 queue_name 加上 原消费函数的入参
|
|
120
|
-
"""
|
|
121
|
-
if args is None:
|
|
122
|
-
args = tuple()
|
|
123
|
-
args_list = list(args)
|
|
124
|
-
args_list.insert(0, func.queue_name)
|
|
125
|
-
args = tuple(args_list)
|
|
126
|
-
kwargs = kwargs or {}
|
|
127
|
-
kwargs['runonce_uuid'] = runonce_uuid # 忽略,用户不需要传递runonce_uuid入参。
|
|
128
|
-
return self.add_job(push_fun_params_to_broker, trigger, args, kwargs, id, name,
|
|
129
|
-
misfire_grace_time, coalesce, max_instances,
|
|
130
|
-
next_run_time, jobstore, executor,
|
|
131
|
-
replace_existing, **trigger_args, )
|
|
132
|
-
|
|
133
|
-
def start(self, paused=False, block_exit=True):
|
|
134
|
-
# def _block_exit():
|
|
135
|
-
# while True:
|
|
136
|
-
# time.sleep(3600)
|
|
137
|
-
#
|
|
138
|
-
# threading.Thread(target=_block_exit,).start() # 既不希望用BlockingScheduler阻塞主进程也不希望定时退出。
|
|
139
|
-
# self._daemon = False
|
|
140
|
-
def _when_exit():
|
|
141
|
-
while 1:
|
|
142
|
-
# print('阻止退出')
|
|
143
|
-
time.sleep(100)
|
|
144
|
-
|
|
145
|
-
if block_exit:
|
|
146
|
-
atexit.register(_when_exit)
|
|
147
|
-
super().start(paused=paused, )
|
|
148
|
-
# _block_exit() # python3.9 判断守护线程结束必须主线程在运行。你自己在你的运行代碼的最末尾加上 while 1: time.sleep(100) ,来阻止主线程退出。
|
|
149
|
-
|
|
150
|
-
def _main_loop00000(self):
|
|
151
|
-
"""
|
|
152
|
-
原来的代码是这,动态添加任务不友好。
|
|
153
|
-
:return:
|
|
154
|
-
"""
|
|
155
|
-
wait_seconds = threading.TIMEOUT_MAX
|
|
156
|
-
while self.state != STATE_STOPPED:
|
|
157
|
-
print(6666, self._event.is_set(), wait_seconds)
|
|
158
|
-
self._event.wait(wait_seconds)
|
|
159
|
-
print(7777, self._event.is_set(), wait_seconds)
|
|
160
|
-
self._event.clear()
|
|
161
|
-
wait_seconds = self._process_jobs()
|
|
162
|
-
|
|
163
|
-
def _main_loop(self):
|
|
164
|
-
"""原来的_main_loop 删除所有任务后wait_seconds 会变成None,无限等待。
|
|
165
|
-
或者下一个需要运行的任务的wait_seconds是3600秒后,此时新加了一个动态任务需要3600秒后,
|
|
166
|
-
现在最多只需要1秒就能扫描到动态新增的定时任务了。
|
|
167
|
-
"""
|
|
168
|
-
MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS = 0.5
|
|
169
|
-
wait_seconds = None
|
|
170
|
-
while self.state == STATE_RUNNING:
|
|
171
|
-
if wait_seconds is None:
|
|
172
|
-
wait_seconds = MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS
|
|
173
|
-
self._last_wait_seconds = min(wait_seconds, MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS)
|
|
174
|
-
if wait_seconds in (None, TIMEOUT_MAX):
|
|
175
|
-
self._last_has_task = False
|
|
176
|
-
else:
|
|
177
|
-
self._last_has_task = True
|
|
178
|
-
time.sleep(self._last_wait_seconds) # 这个要取最小值,不然例如定时间隔0.1秒运行,不取最小值,不会每隔0.1秒运行。
|
|
179
|
-
wait_seconds = self._process_jobs()
|
|
180
|
-
|
|
181
|
-
def _create_default_executor(self):
|
|
182
|
-
return ThreadPoolExecutorForAps() # 必须是apscheduler pool的子类
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
FsdfBackgroundScheduler = FunboostBackgroundScheduler # 兼容一下名字,fsdf是 function-scheduling-distributed-framework 老框架名字的缩写
|
|
186
|
-
# funboost_aps_scheduler定时配置基于内存的,不可以跨机器远程动态添加/修改/删除定时任务配置。如果需要动态增删改查定时任务,可以使用funboost_background_scheduler_redis_store
|
|
187
|
-
|
|
188
|
-
funboost_aps_scheduler = FunboostBackgroundScheduler(timezone=FunboostCommonConfig.TIMEZONE, daemon=False, )
|
|
189
|
-
fsdf_background_scheduler = funboost_aps_scheduler # 兼容一下老名字。
|
|
190
|
-
|
|
191
|
-
if __name__ == '__main__':
|
|
192
|
-
# 定时运行消费演示
|
|
193
|
-
import datetime
|
|
194
|
-
from funboost import boost, BrokerEnum, fsdf_background_scheduler, timing_publish_deco, run_forever
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
@Booster(boost_params=BoosterParams(queue_name='queue_test_666', broker_kind=BrokerEnum.LOCAL_PYTHON_QUEUE))
|
|
198
|
-
def consume_func(x, y):
|
|
199
|
-
print(f'{x} + {y} = {x + y}')
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
print(consume_func, type(consume_func))
|
|
203
|
-
|
|
204
|
-
# 定时每隔3秒执行一次。
|
|
205
|
-
funboost_aps_scheduler.add_push_job(consume_func,
|
|
206
|
-
'interval', id='3_second_job', seconds=3, kwargs={"x": 5, "y": 6})
|
|
207
|
-
|
|
208
|
-
# 定时,只执行一次
|
|
209
|
-
funboost_aps_scheduler.add_push_job(consume_func,
|
|
210
|
-
'date', run_date=datetime.datetime(2020, 7, 24, 13, 53, 6), args=(5, 6,))
|
|
211
|
-
|
|
212
|
-
# 定时,每天的11点32分20秒都执行一次。
|
|
213
|
-
funboost_aps_scheduler.add_push_job(consume_func,
|
|
214
|
-
'cron', day_of_week='*', hour=18, minute=22, second=20, args=(5, 6,))
|
|
215
|
-
|
|
216
|
-
# 启动定时
|
|
217
|
-
funboost_aps_scheduler.start()
|
|
218
|
-
|
|
219
|
-
# 启动消费
|
|
220
|
-
consume_func.consume()
|
|
221
|
-
run_forever()
|
|
6
|
+
from funboost.timing_job.timing_push import ApsJobAdder
|
|
@@ -5,6 +5,7 @@ from funboost.timing_job import FunboostBackgroundScheduler
|
|
|
5
5
|
from funboost.funboost_config_deafult import BrokerConnConfig, FunboostCommonConfig
|
|
6
6
|
from funboost.utils.decorators import RedisDistributedBlockLockContextManager
|
|
7
7
|
|
|
8
|
+
|
|
8
9
|
"""
|
|
9
10
|
这个是使用redis作为定时任务持久化,支持跨机器好跨进程,外部远程 动态修改/添加/删除定时任务
|
|
10
11
|
"""
|
|
@@ -51,8 +52,12 @@ jobstores = {
|
|
|
51
52
|
|
|
52
53
|
funboost_background_scheduler_redis_store = FunboostBackgroundSchedulerProcessJobsWithinRedisLock(timezone=FunboostCommonConfig.TIMEZONE, daemon=False, jobstores=jobstores)
|
|
53
54
|
|
|
54
|
-
"""
|
|
55
55
|
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
"""
|
|
56
61
|
跨python解释器 跨机器动态修改定时任务配置的例子在
|
|
57
62
|
|
|
58
63
|
test_frame/test_apschedual/test_aps_redis_store.py
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
"""
|
|
2
|
+
集成定时任务。
|
|
3
|
+
"""
|
|
4
|
+
import atexit
|
|
5
|
+
|
|
6
|
+
import time
|
|
7
|
+
from apscheduler.executors.pool import BasePoolExecutor
|
|
8
|
+
|
|
9
|
+
from typing import Union
|
|
10
|
+
import threading
|
|
11
|
+
|
|
12
|
+
from apscheduler.schedulers.background import BackgroundScheduler
|
|
13
|
+
# noinspection PyProtectedMember
|
|
14
|
+
from apscheduler.schedulers.base import STATE_STOPPED, STATE_RUNNING
|
|
15
|
+
from apscheduler.util import undefined
|
|
16
|
+
from threading import TIMEOUT_MAX
|
|
17
|
+
import deprecated
|
|
18
|
+
from funboost.utils.redis_manager import RedisMixin
|
|
19
|
+
|
|
20
|
+
from funboost.funboost_config_deafult import FunboostCommonConfig
|
|
21
|
+
|
|
22
|
+
from funboost.consumers.base_consumer import AbstractConsumer
|
|
23
|
+
from funboost.core.booster import BoostersManager, Booster
|
|
24
|
+
|
|
25
|
+
from funboost import BoosterParams
|
|
26
|
+
from funboost.concurrent_pool.custom_threadpool_executor import ThreadPoolExecutorShrinkAble
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@deprecated.deprecated(reason='以后不要再使用这种方式,对于job_store为数据库时候需要序列化不好。使用内存和数据库都兼容的添加任务方式: add_push_job')
|
|
30
|
+
def timing_publish_deco(consuming_func_decorated_or_consumer: Union[callable, AbstractConsumer]):
|
|
31
|
+
def _deco(*args, **kwargs):
|
|
32
|
+
if getattr(consuming_func_decorated_or_consumer, 'is_decorated_as_consume_function', False) is True:
|
|
33
|
+
consuming_func_decorated_or_consumer.push(*args, **kwargs)
|
|
34
|
+
elif isinstance(consuming_func_decorated_or_consumer, AbstractConsumer):
|
|
35
|
+
consuming_func_decorated_or_consumer.publisher_of_same_queue.push(*args, **kwargs)
|
|
36
|
+
else:
|
|
37
|
+
raise TypeError('consuming_func_decorated_or_consumer 必须是被 boost 装饰的函数或者consumer类型')
|
|
38
|
+
|
|
39
|
+
return _deco
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def push_fun_params_to_broker(queue_name: str, *args, **kwargs):
|
|
43
|
+
"""
|
|
44
|
+
queue_name 队列名字
|
|
45
|
+
*args **kwargs 是消费函数的入参
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
BoostersManager.get_or_create_booster_by_queue_name(queue_name).push(*args, **kwargs)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class ThreadPoolExecutorForAps(BasePoolExecutor):
|
|
52
|
+
"""
|
|
53
|
+
An executor that runs jobs in a concurrent.futures thread pool.
|
|
54
|
+
|
|
55
|
+
Plugin alias: ``threadpool``
|
|
56
|
+
|
|
57
|
+
:param max_workers: the maximum number of spawned threads.
|
|
58
|
+
:param pool_kwargs: dict of keyword arguments to pass to the underlying
|
|
59
|
+
ThreadPoolExecutor constructor
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
def __init__(self, max_workers=10, pool_kwargs=None):
|
|
63
|
+
pool = ThreadPoolExecutorShrinkAble(int(max_workers), )
|
|
64
|
+
super().__init__(pool)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class FunboostBackgroundScheduler(BackgroundScheduler):
|
|
68
|
+
"""
|
|
69
|
+
自定义的, 继承了官方BackgroundScheduler,
|
|
70
|
+
通过重写 _main_loop ,使得动态修改增加删除定时任务配置更好。
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
_last_wait_seconds = None
|
|
74
|
+
_last_has_task = False
|
|
75
|
+
|
|
76
|
+
@deprecated.deprecated(reason='以后不要再使用这种方式,对于job_store为数据库时候需要序列化不好。使用内存和数据库都兼容的添加任务方式: add_push_job')
|
|
77
|
+
def add_timing_publish_job(self, func, trigger=None, args=None, kwargs=None, id=None, name=None,
|
|
78
|
+
misfire_grace_time=undefined, coalesce=undefined, max_instances=undefined,
|
|
79
|
+
next_run_time=undefined, jobstore='default', executor='default',
|
|
80
|
+
replace_existing=False, **trigger_args):
|
|
81
|
+
return self.add_job(timing_publish_deco(func), trigger, args, kwargs, id, name,
|
|
82
|
+
misfire_grace_time, coalesce, max_instances,
|
|
83
|
+
next_run_time, jobstore, executor,
|
|
84
|
+
replace_existing, **trigger_args)
|
|
85
|
+
|
|
86
|
+
def add_push_job(self, func: Booster, trigger=None, args=None, kwargs=None,
|
|
87
|
+
id=None, name=None,
|
|
88
|
+
misfire_grace_time=undefined, coalesce=undefined, max_instances=undefined,
|
|
89
|
+
next_run_time=undefined, jobstore='default', executor='default',
|
|
90
|
+
replace_existing=False, **trigger_args, ):
|
|
91
|
+
"""
|
|
92
|
+
:param func: 被@boost装饰器装饰的函数
|
|
93
|
+
:param trigger:
|
|
94
|
+
:param args:
|
|
95
|
+
:param kwargs:
|
|
96
|
+
:param id:
|
|
97
|
+
:param name:
|
|
98
|
+
:param misfire_grace_time:
|
|
99
|
+
:param coalesce:
|
|
100
|
+
:param max_instances:
|
|
101
|
+
:param next_run_time:
|
|
102
|
+
:param jobstore:
|
|
103
|
+
:param executor:
|
|
104
|
+
:param replace_existing:
|
|
105
|
+
:param trigger_args:
|
|
106
|
+
:return:
|
|
107
|
+
"""
|
|
108
|
+
# args = args or {}
|
|
109
|
+
# kwargs['queue_name'] = func.queue_name
|
|
110
|
+
|
|
111
|
+
"""
|
|
112
|
+
用户如果不使用funboost的 FunboostBackgroundScheduler 类型对象,而是使用原生的apscheduler类型对象,可以scheduler.add_job(push_fun_params_to_broker,args=(,),kwargs={})
|
|
113
|
+
push_fun_params_to_broker函数入参是消费函数队列的 queue_name 加上 原消费函数的入参
|
|
114
|
+
"""
|
|
115
|
+
if args is None:
|
|
116
|
+
args = tuple()
|
|
117
|
+
args_list = list(args)
|
|
118
|
+
args_list.insert(0, func.queue_name)
|
|
119
|
+
args = tuple(args_list)
|
|
120
|
+
return self.add_job(push_fun_params_to_broker, trigger, args, kwargs, id, name,
|
|
121
|
+
misfire_grace_time, coalesce, max_instances,
|
|
122
|
+
next_run_time, jobstore, executor,
|
|
123
|
+
replace_existing, **trigger_args, )
|
|
124
|
+
|
|
125
|
+
def start(self, paused=False, block_exit=True):
|
|
126
|
+
# def _block_exit():
|
|
127
|
+
# while True:
|
|
128
|
+
# time.sleep(3600)
|
|
129
|
+
#
|
|
130
|
+
# threading.Thread(target=_block_exit,).start() # 既不希望用BlockingScheduler阻塞主进程也不希望定时退出。
|
|
131
|
+
# self._daemon = False
|
|
132
|
+
def _when_exit():
|
|
133
|
+
while 1:
|
|
134
|
+
# print('阻止退出')
|
|
135
|
+
time.sleep(100)
|
|
136
|
+
|
|
137
|
+
if block_exit:
|
|
138
|
+
atexit.register(_when_exit)
|
|
139
|
+
super().start(paused=paused, )
|
|
140
|
+
# _block_exit() # python3.9 判断守护线程结束必须主线程在运行。你自己在你的运行代碼的最末尾加上 while 1: time.sleep(100) ,来阻止主线程退出。
|
|
141
|
+
|
|
142
|
+
def _main_loop00000(self):
|
|
143
|
+
"""
|
|
144
|
+
原来的代码是这,动态添加任务不友好。
|
|
145
|
+
:return:
|
|
146
|
+
"""
|
|
147
|
+
wait_seconds = threading.TIMEOUT_MAX
|
|
148
|
+
while self.state != STATE_STOPPED:
|
|
149
|
+
print(6666, self._event.is_set(), wait_seconds)
|
|
150
|
+
self._event.wait(wait_seconds)
|
|
151
|
+
print(7777, self._event.is_set(), wait_seconds)
|
|
152
|
+
self._event.clear()
|
|
153
|
+
wait_seconds = self._process_jobs()
|
|
154
|
+
|
|
155
|
+
def _main_loop(self):
|
|
156
|
+
"""原来的_main_loop 删除所有任务后wait_seconds 会变成None,无限等待。
|
|
157
|
+
或者下一个需要运行的任务的wait_seconds是3600秒后,此时新加了一个动态任务需要3600秒后,
|
|
158
|
+
现在最多只需要1秒就能扫描到动态新增的定时任务了。
|
|
159
|
+
"""
|
|
160
|
+
MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS = 0.5
|
|
161
|
+
wait_seconds = None
|
|
162
|
+
while self.state != STATE_STOPPED:
|
|
163
|
+
if wait_seconds is None:
|
|
164
|
+
wait_seconds = MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS
|
|
165
|
+
self._last_wait_seconds = min(wait_seconds, MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS)
|
|
166
|
+
if wait_seconds in (None, TIMEOUT_MAX):
|
|
167
|
+
self._last_has_task = False
|
|
168
|
+
else:
|
|
169
|
+
self._last_has_task = True
|
|
170
|
+
time.sleep(self._last_wait_seconds) # 这个要取最小值,不然例如定时间隔0.1秒运行,不取最小值,不会每隔0.1秒运行。
|
|
171
|
+
wait_seconds = self._process_jobs()
|
|
172
|
+
|
|
173
|
+
def _create_default_executor(self):
|
|
174
|
+
return ThreadPoolExecutorForAps() # 必须是apscheduler pool的子类
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
FsdfBackgroundScheduler = FunboostBackgroundScheduler # 兼容一下名字,fsdf是 function-scheduling-distributed-framework 老框架名字的缩写
|
|
178
|
+
# funboost_aps_scheduler定时配置基于内存的,不可以跨机器远程动态添加/修改/删除定时任务配置。如果需要动态增删改查定时任务,可以使用funboost_background_scheduler_redis_store
|
|
179
|
+
|
|
180
|
+
funboost_aps_scheduler = FunboostBackgroundScheduler(timezone=FunboostCommonConfig.TIMEZONE, daemon=False, )
|
|
181
|
+
fsdf_background_scheduler = funboost_aps_scheduler # 兼容一下老名字。
|
|
182
|
+
|
|
183
|
+
if __name__ == '__main__':
|
|
184
|
+
# 定时运行消费演示
|
|
185
|
+
import datetime
|
|
186
|
+
from funboost import boost, BrokerEnum, fsdf_background_scheduler, timing_publish_deco, run_forever
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@Booster(boost_params=BoosterParams(queue_name='queue_test_666', broker_kind=BrokerEnum.LOCAL_PYTHON_QUEUE))
|
|
190
|
+
def consume_func(x, y):
|
|
191
|
+
print(f'{x} + {y} = {x + y}')
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
print(consume_func, type(consume_func))
|
|
195
|
+
|
|
196
|
+
# 定时每隔3秒执行一次。
|
|
197
|
+
funboost_aps_scheduler.add_push_job(consume_func,
|
|
198
|
+
'interval', id='3_second_job', seconds=3, kwargs={"x": 5, "y": 6})
|
|
199
|
+
|
|
200
|
+
# 定时,只执行一次
|
|
201
|
+
funboost_aps_scheduler.add_push_job(consume_func,
|
|
202
|
+
'date', run_date=datetime.datetime(2020, 7, 24, 13, 53, 6), args=(5, 6,))
|
|
203
|
+
|
|
204
|
+
# 定时,每天的11点32分20秒都执行一次。
|
|
205
|
+
funboost_aps_scheduler.add_push_job(consume_func,
|
|
206
|
+
'cron', day_of_week='*', hour=18, minute=22, second=20, args=(5, 6,))
|
|
207
|
+
|
|
208
|
+
# 启动定时
|
|
209
|
+
funboost_aps_scheduler.start()
|
|
210
|
+
|
|
211
|
+
# 启动消费
|
|
212
|
+
consume_func.consume()
|
|
213
|
+
run_forever()
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
from funboost.utils import redis_manager
|
|
2
|
+
from funboost.core.booster import BoostersManager, Booster
|
|
3
|
+
|
|
4
|
+
from apscheduler.jobstores.redis import RedisJobStore
|
|
5
|
+
from funboost.timing_job.timing_job_base import funboost_aps_scheduler, undefined
|
|
6
|
+
from funboost.timing_job.apscheduler_use_redis_store import FunboostBackgroundSchedulerProcessJobsWithinRedisLock
|
|
7
|
+
from funboost.funboost_config_deafult import FunboostCommonConfig
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ApsJobAdder:
|
|
11
|
+
"""
|
|
12
|
+
20250116新增加的统一的新增定时任务的方式,推荐这种方式。
|
|
13
|
+
用户不用像之前再去关心使用哪个apscheduler对象去添加定时任务了。
|
|
14
|
+
|
|
15
|
+
例如 add_numbers 是@boost装饰的消费函数
|
|
16
|
+
ApsJobAdder(add_numbers,job_store_kind='memory').add_push_job(
|
|
17
|
+
args=(1, 2),
|
|
18
|
+
trigger='date', # 使用日期触发器
|
|
19
|
+
run_date='2025-01-16 18:23:50', # 设置运行时间
|
|
20
|
+
# id='add_numbers_job' # 任务ID
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
queue__redis_aps_map = {}
|
|
26
|
+
|
|
27
|
+
def __init__(self, booster: Booster, job_store_kind: str = 'memory'):
|
|
28
|
+
"""
|
|
29
|
+
Initialize the ApsJobAdder.
|
|
30
|
+
|
|
31
|
+
:param booster: A Booster object representing the function to be scheduled.
|
|
32
|
+
:param job_store_kind: The type of job store to use. Default is 'memory'.
|
|
33
|
+
Can be 'memory' or 'redis'.
|
|
34
|
+
"""
|
|
35
|
+
self.booster = booster
|
|
36
|
+
self.job_store_kind = job_store_kind
|
|
37
|
+
|
|
38
|
+
@classmethod
|
|
39
|
+
def get_funboost_redis_apscheduler(cls, queue_name):
|
|
40
|
+
"""
|
|
41
|
+
每个队列名字的定时任务用不同的redis jobstore的 jobs_key 和 run_times_key,防止互相干扰和取出不属于自己的任务
|
|
42
|
+
"""
|
|
43
|
+
if queue_name in cls.queue__redis_aps_map:
|
|
44
|
+
return cls.queue__redis_aps_map[queue_name]
|
|
45
|
+
redis_jobstores = {
|
|
46
|
+
|
|
47
|
+
"default": RedisJobStore(**redis_manager.get_redis_conn_kwargs(),
|
|
48
|
+
jobs_key=f'funboost.apscheduler.{queue_name}.jobs',
|
|
49
|
+
run_times_key=f'funboost.apscheduler.{queue_name}.run_times',
|
|
50
|
+
)
|
|
51
|
+
}
|
|
52
|
+
redis_aps = FunboostBackgroundSchedulerProcessJobsWithinRedisLock(timezone=FunboostCommonConfig.TIMEZONE,
|
|
53
|
+
daemon=False, jobstores=redis_jobstores)
|
|
54
|
+
cls.queue__redis_aps_map[queue_name] = redis_aps
|
|
55
|
+
return redis_aps
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def aps_obj(self):
|
|
59
|
+
if self.job_store_kind == 'redis':
|
|
60
|
+
return self.get_funboost_redis_apscheduler(self.booster.queue_name)
|
|
61
|
+
elif self.job_store_kind == 'memory':
|
|
62
|
+
return funboost_aps_scheduler
|
|
63
|
+
else:
|
|
64
|
+
raise ValueError('Unsupported job_store_kind')
|
|
65
|
+
|
|
66
|
+
def add_push_job(self, trigger=None, args=None, kwargs=None,
|
|
67
|
+
id=None, name=None,
|
|
68
|
+
misfire_grace_time=undefined, coalesce=undefined, max_instances=undefined,
|
|
69
|
+
next_run_time=undefined, jobstore='default', executor='default',
|
|
70
|
+
replace_existing=False, **trigger_args,):
|
|
71
|
+
"""
|
|
72
|
+
这里的入参都是和apscheduler的add_job的入参一样的,funboost作者没有创造新的入参。
|
|
73
|
+
但是官方apscheduler的入参第一个入参是函数,
|
|
74
|
+
funboost的ApsJobAdder对象.add_push_job入参去掉了函数,因为类的实例化时候会把函数传进来,不需要再麻烦用户一次了。
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
if not getattr(self.aps_obj, 'has_started_flag', False):
|
|
78
|
+
self.aps_obj.has_started_flag = True
|
|
79
|
+
self.aps_obj.start(paused=False)
|
|
80
|
+
return self.aps_obj.add_push_job(self.booster, trigger, args, kwargs, id, name,
|
|
81
|
+
misfire_grace_time, coalesce, max_instances,
|
|
82
|
+
next_run_time, jobstore, executor,
|
|
83
|
+
replace_existing, **trigger_args, )
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
if __name__ == '__main__':
|
|
90
|
+
"""
|
|
91
|
+
2025年后定时任务现在推荐使用 ApsJobAdder 写法 ,用户不需要亲自选择使用 apscheduler对象来添加定时任务
|
|
92
|
+
"""
|
|
93
|
+
from funboost import boost, BrokerEnum,ctrl_c_recv,BoosterParams,ApsJobAdder
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# 定义任务处理函数
|
|
97
|
+
@BoosterParams(queue_name='sum_queue3', broker_kind=BrokerEnum.REDIS)
|
|
98
|
+
def sum_two_numbers(x, y):
|
|
99
|
+
result = x + y
|
|
100
|
+
print(f'The sum of {x} and {y} is {result}')
|
|
101
|
+
|
|
102
|
+
# 启动消费者
|
|
103
|
+
sum_two_numbers.consume()
|
|
104
|
+
|
|
105
|
+
# 发布任务
|
|
106
|
+
sum_two_numbers.push(3, 5)
|
|
107
|
+
sum_two_numbers.push(10, 20)
|
|
108
|
+
|
|
109
|
+
# 使用ApsJobAdder添加定时任务, 里面的定时语法,和apscheduler是一样的,用户需要自己熟悉知名框架apscheduler的add_job定时入参
|
|
110
|
+
|
|
111
|
+
# 方式1:指定日期执行一次
|
|
112
|
+
ApsJobAdder(sum_two_numbers, job_store_kind='redis').add_push_job(
|
|
113
|
+
trigger='date',
|
|
114
|
+
run_date='2025-01-17 23:25:40',
|
|
115
|
+
args=(7, 8)
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
# 方式2:固定间隔执行
|
|
119
|
+
ApsJobAdder(sum_two_numbers, job_store_kind='memory').add_push_job(
|
|
120
|
+
trigger='interval',
|
|
121
|
+
seconds=5,
|
|
122
|
+
args=(4, 6)
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
# 方式3:使用cron表达式定时执行
|
|
126
|
+
ApsJobAdder(sum_two_numbers, job_store_kind='redis').add_push_job(
|
|
127
|
+
trigger='cron',
|
|
128
|
+
day_of_week='*',
|
|
129
|
+
hour=23,
|
|
130
|
+
minute=49,
|
|
131
|
+
second=50,
|
|
132
|
+
kwargs={"x":50,"y":60},
|
|
133
|
+
replace_existing=True,
|
|
134
|
+
id='cron_job1')
|
|
135
|
+
|
|
136
|
+
ctrl_c_recv()
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
import gc
|
|
3
|
+
import inspect
|
|
4
|
+
import re
|
|
5
|
+
import sys
|
|
6
|
+
import traceback
|
|
7
|
+
import typing
|
|
8
|
+
|
|
9
|
+
import nb_log
|
|
10
|
+
from types import MethodType, FunctionType
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ClsHelper:
|
|
16
|
+
@staticmethod
|
|
17
|
+
def is_static_method(func):
|
|
18
|
+
# 获取类名
|
|
19
|
+
class_name = func.__qualname__.split('.')[0]
|
|
20
|
+
# 使用 inspect 获取函数的原始定义
|
|
21
|
+
return isinstance(func, staticmethod) or (inspect.isfunction(func) and func.__qualname__.startswith(f'{class_name}.'))
|
|
22
|
+
|
|
23
|
+
# 判断函数是否是实例方法
|
|
24
|
+
@staticmethod
|
|
25
|
+
def is_instance_method(method):
|
|
26
|
+
# 检查方法是否是绑定到类实例上的方法
|
|
27
|
+
return inspect.ismethod(method) or (inspect.isfunction(method) and getattr(method, '__self__', None) is not None)
|
|
28
|
+
|
|
29
|
+
@staticmethod
|
|
30
|
+
def is_class_method(method):
|
|
31
|
+
# 检查方法是否是类方法
|
|
32
|
+
return isinstance(method, classmethod) or (inspect.isfunction(method) and method.__self__ is None)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def is_common_function(cls, method):
|
|
38
|
+
if cls.is_static_method(method):
|
|
39
|
+
return False
|
|
40
|
+
if cls.is_class_method(method):
|
|
41
|
+
return False
|
|
42
|
+
if cls.is_instance_method(method):
|
|
43
|
+
return False
|
|
44
|
+
if isinstance(method, FunctionType):
|
|
45
|
+
sourcelines = inspect.getsourcelines(method)
|
|
46
|
+
for line in sourcelines[0][:50]:
|
|
47
|
+
if not line.replace(' ', '').startswith('#'):
|
|
48
|
+
if not re.search('\(\s*?self\s*?,', line):
|
|
49
|
+
return True
|
|
50
|
+
|
|
51
|
+
@classmethod
|
|
52
|
+
def get_method_kind(cls, method: typing.Callable) -> str:
|
|
53
|
+
func =method
|
|
54
|
+
try:
|
|
55
|
+
if cls.is_instance_method(func):
|
|
56
|
+
return "实例方法"
|
|
57
|
+
if cls.is_static_method(func):
|
|
58
|
+
return "静态方法"
|
|
59
|
+
if cls.is_class_method(func):
|
|
60
|
+
return "类方法"
|
|
61
|
+
if inspect.isfunction(func):
|
|
62
|
+
return "模块级函数"
|
|
63
|
+
except Exception as e:
|
|
64
|
+
print(traceback.format_exc())
|
|
65
|
+
|
|
66
|
+
@staticmethod
|
|
67
|
+
def get_obj_init_params_for_funboost(obj_init_params: dict):
|
|
68
|
+
obj_init_params.pop('self')
|
|
69
|
+
return copy.deepcopy(obj_init_params)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
if __name__ == '__main__':
|
|
75
|
+
def module_function():
|
|
76
|
+
return "I am a module-level function"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class MyClass:
|
|
80
|
+
@staticmethod
|
|
81
|
+
def static_method():
|
|
82
|
+
return "I am a static method"
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def class_method(cls):
|
|
86
|
+
return "I am a class method"
|
|
87
|
+
|
|
88
|
+
def instance_method(self):
|
|
89
|
+
return "I am a instance method"
|
|
90
|
+
|
|
91
|
+
print(ClsHelper.get_method_kind(module_function)) # 输出: 模块级函数
|
|
92
|
+
print(ClsHelper.get_method_kind(MyClass.static_method)) # 输出: 静态方法
|
|
93
|
+
print(ClsHelper.get_method_kind(MyClass.class_method)) # 输出: 类方法
|
|
94
|
+
print(ClsHelper.get_method_kind(MyClass.instance_method)) # 输出: 实例方法
|
funboost/utils/ctrl_c_end.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: funboost
|
|
3
|
-
Version:
|
|
3
|
+
Version: 48.0
|
|
4
4
|
Summary: pip install funboost,python全功能分布式函数调度框架,funboost的功能是全面性重量级,用户能想得到的功能99%全都有;funboost的使用方式是轻量级,只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件,支持如 celery dramatiq等框架整体作为funboost中间件,python函数加速器,框架包罗万象,用户能想到的控制功能全都有。一统编程思维,兼容50% python业务场景,适用范围广。只需要一行代码即可分布式执行python一切函数,99%用过funboost的pythoner 感受是 简易 方便 强劲 强大,相见恨晚
|
|
5
5
|
Home-page: https://github.com/ydf0509/funboost
|
|
6
6
|
Author: bfzs
|
|
@@ -44,7 +44,7 @@ Requires-Dist: redis3
|
|
|
44
44
|
Requires-Dist: redis5
|
|
45
45
|
Requires-Dist: redis
|
|
46
46
|
Requires-Dist: setuptools_rust
|
|
47
|
-
Requires-Dist: fabric2
|
|
47
|
+
Requires-Dist: fabric2>=2.6.0
|
|
48
48
|
Requires-Dist: nb_filelock
|
|
49
49
|
Requires-Dist: pysnooper
|
|
50
50
|
Requires-Dist: deprecated
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
funboost/__init__.py,sha256=
|
|
1
|
+
funboost/__init__.py,sha256=Ctao94tU5bAybW_spUlf55MhEaGzoyff3i7t4KHF4so,3991
|
|
2
2
|
funboost/__init__old.py,sha256=9Kv3cPLnPkbzMRnuJFVkPsuDdx1CdcSIuITkpdncZSc,20382
|
|
3
3
|
funboost/__main__.py,sha256=-6Nogi666Y0LN8fVm3JmHGTOk8xEGWvotW_GDbSaZME,1065
|
|
4
4
|
funboost/constant.py,sha256=STzRDZbuCC5FFV-uD_0r2beGsD1Zni4kregzR11z3Ok,8148
|
|
5
5
|
funboost/funboost_config_deafult.py,sha256=K-kCFGEjD107wHWFspNrIWsPNSVteP2Xww1yRbXd-Wk,6651
|
|
6
6
|
funboost/set_frame_config.py,sha256=kxYo_x2pMihwfTgFWrLxNbgI0IbFYC_HpSd9YLQM1ig,14439
|
|
7
7
|
funboost/assist/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
-
funboost/assist/celery_helper.py,sha256=
|
|
8
|
+
funboost/assist/celery_helper.py,sha256=zhyAIBIbnPcS8pdp0oqVh57Jx-hIDX0njTZiYZFCLFw,6486
|
|
9
9
|
funboost/assist/dramatiq_helper.py,sha256=9mUyfBMAJXzwvB8LwOmapn3rY30a6UXt3tNOfL6OXoM,2106
|
|
10
10
|
funboost/assist/faststream_helper.py,sha256=BmBaFsvsjQK2SO71ulHRsEwO28uYZm2uMMvXkHzLi1E,631
|
|
11
11
|
funboost/assist/huey_helper.py,sha256=PuJHIzK5oRd5RzbuxLMHhWlkKO3J-ObRK0mrMs_iQWs,1770
|
|
@@ -35,7 +35,7 @@ funboost/concurrent_pool/backup/async_pool_executor0223.py,sha256=RVUZiylUvpTm6U
|
|
|
35
35
|
funboost/concurrent_pool/backup/async_pool_executor_back.py,sha256=KL6zEQaa1KkZOlAO85mCC1gwLm-YC5Ghn21IUom0UKM,9598
|
|
36
36
|
funboost/concurrent_pool/backup/async_pool_executor_janus.py,sha256=OHMWJ9l3EYTpPpcrPrGGKd4K0tmQ2PN8HiX0Dta0EOo,5728
|
|
37
37
|
funboost/consumers/__init__.py,sha256=ZXY_6Kut1VYNQiF5aWEgIWobsW1ht9YUP0TdRZRWFqI,126
|
|
38
|
-
funboost/consumers/base_consumer.py,sha256
|
|
38
|
+
funboost/consumers/base_consumer.py,sha256=-b398kkcptpc7w6GqILWpUf838IcNEfWfaqj1m7HFEY,81961
|
|
39
39
|
funboost/consumers/celery_consumer.py,sha256=nQpSkzPBJ4bRpxn4i9ms0axrJiq9RWhb4lG2nAdCIig,9254
|
|
40
40
|
funboost/consumers/confirm_mixin.py,sha256=5xC9AAQr_MY4tbSed8U-M6tOVmh69Qv9X0ld0JLT9Tk,6185
|
|
41
41
|
funboost/consumers/dramatiq_consumer.py,sha256=ozmeAfeF0U-YNYHK4suQB0N264h5AZdfMH0O45Mh-8A,2229
|
|
@@ -64,7 +64,7 @@ funboost/consumers/rabbitmq_pika_consumerv0.py,sha256=rIQToBTBqGdjnzMhg0vyZMY87N
|
|
|
64
64
|
funboost/consumers/rabbitmq_rabbitpy_consumer.py,sha256=xxINY037pmgF3_lJA-zhf9qUIUx6DdqC7tsUOQL3dL4,1330
|
|
65
65
|
funboost/consumers/redis_brpoplpush_consumer.py,sha256=HA3WYKNHuSFZJwTe5EEMFAQfTCK6E2upyFnvdQReJug,2967
|
|
66
66
|
funboost/consumers/redis_consumer.py,sha256=RdxJvWWQvQdjXWWdUebmgB7KRDWQBeF87Q9kd_DeUZk,2735
|
|
67
|
-
funboost/consumers/redis_consumer_ack_able.py,sha256=
|
|
67
|
+
funboost/consumers/redis_consumer_ack_able.py,sha256=CiPIo-ToujSUy1uvznWc1QuxBfeOyOmYbH6P77r9apI,7429
|
|
68
68
|
funboost/consumers/redis_consumer_ack_using_timeout.py,sha256=E-7Ci3yb1-THvGoqDNyHJm_zYThIcacHEuUQbPU5e-0,4228
|
|
69
69
|
funboost/consumers/redis_consumer_priority.py,sha256=C-ftnlGPPoB7YV3GvwLu9POVGDn_GKlBqO6NlsZ-hdY,5566
|
|
70
70
|
funboost/consumers/redis_consumer_simple.py,sha256=trPrMHSVU1Y_fY-xz5tFFmt1zjV-9_joPYRHqvyZnL8,874
|
|
@@ -87,12 +87,12 @@ funboost/contrib/save_result_status_to_sqldb.py,sha256=AxvD7nHs4sjr9U0kwEZzyPKrs
|
|
|
87
87
|
funboost/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
88
88
|
funboost/core/active_cousumer_info_getter.py,sha256=09fEc-BTEIRfDDfHmOvKnMjLjtOyp4edLsUlAXUR_Qs,4966
|
|
89
89
|
funboost/core/booster.py,sha256=GNQLDutqiFH8ymQZgBUi0vhzlBm9VD97usLGlAzjnJY,20287
|
|
90
|
-
funboost/core/current_task.py,sha256=
|
|
90
|
+
funboost/core/current_task.py,sha256=AuvrLjLGxfONILzQAd0Vm0nXDsDjf_7gOc3vGy_6r5I,7004
|
|
91
91
|
funboost/core/exceptions.py,sha256=pLF7BkRJAfDiWp2_xGZqencmwdPiSQI1NENbImExknY,1311
|
|
92
92
|
funboost/core/fabric_deploy_helper.py,sha256=foieeqlNySuU9axJzNF6TavPjIUSYBx9UO3syVKUiyY,9999
|
|
93
93
|
funboost/core/funboost_config_getter.py,sha256=b5nAdAmUxahskY-ohB7ptf2gKywFlDA0Fq1cWroxxbs,384
|
|
94
94
|
funboost/core/funboost_time.py,sha256=a0MacbUBfYk8mf7D3UUyCxH5QJsu8YiGVXwJqPnSQH0,1779
|
|
95
|
-
funboost/core/func_params_model.py,sha256=
|
|
95
|
+
funboost/core/func_params_model.py,sha256=X4ibmg9lmv7kRg2-ym__HicCGK5VJns0DedVxL8S9Hw,22512
|
|
96
96
|
funboost/core/function_result_status_config.py,sha256=PyjqAQOiwsLt28sRtH-eYRjiI3edPFO4Nde0ILFRReE,1764
|
|
97
97
|
funboost/core/function_result_status_saver.py,sha256=yHKZF9MjmhI-Q4Mkrka7DdweJ0wpgfLmgfAlsfkCeCk,9274
|
|
98
98
|
funboost/core/helper_funs.py,sha256=SsMa7A3iJyLek6v1qRK02kINebDp6kuAmlYkrYLXwQ0,2369
|
|
@@ -172,15 +172,18 @@ funboost/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
172
172
|
funboost/queues/memory_queues_map.py,sha256=e1S_cnjnCVI4DBsA_iupF51S_eX4OvCtlefQCqS1TYA,424
|
|
173
173
|
funboost/queues/peewee_queue.py,sha256=FrwegrilkHZG6Y1cGdl5Bt_UtA25f7m5TJQJMZ9YnJI,5280
|
|
174
174
|
funboost/queues/sqla_queue.py,sha256=b-2QPvvuMxklm41ibZhGKehaAV9trXBQFCOHzgThx_s,11080
|
|
175
|
-
funboost/timing_job/__init__.py,sha256=
|
|
175
|
+
funboost/timing_job/__init__.py,sha256=_rIiU7pMXe-IwUCeok50hSiWXoUoMBar22u6-pZLlAI,265
|
|
176
176
|
funboost/timing_job/apscheduler_use_mysql_store.py,sha256=ss92DiSLzbWuVIo19sTLgC99GessltWLOlqqOd4AIL4,471
|
|
177
|
-
funboost/timing_job/apscheduler_use_redis_store.py,sha256=
|
|
177
|
+
funboost/timing_job/apscheduler_use_redis_store.py,sha256=Iwr9x1SnGU4wU5PmMWRAoV72AoSTSuB8aLJKBGCjJs4,2943
|
|
178
|
+
funboost/timing_job/timing_job_base.py,sha256=Ho-fg1mh3uVctQaolOZ_SDMd-1gnb3UTIMuBy0iztJ8,9479
|
|
179
|
+
funboost/timing_job/timing_push.py,sha256=NrmMQ4JIlzoSUCLFaibxnPiPCkjRU7rVBNqpEKUVNbo,5630
|
|
178
180
|
funboost/utils/__init__.py,sha256=rAyXE7lgCo_3VdMvGrIJiqsTHv2nZPTJDTj1f6s_KgE,586
|
|
179
181
|
funboost/utils/apscheduler_monkey.py,sha256=CcUISbqX6nMWSxr_QjZ26IvvhUk_ojYZWRaKenpsKfE,3124
|
|
180
182
|
funboost/utils/block_exit.py,sha256=BnfxNYo3lnmhk686RAEoc4u3D4RU_iEMMMgu5L8gIuI,96
|
|
181
183
|
funboost/utils/bulk_operation.py,sha256=B4FBxlz5f4oqlKDWqer7axn4gnDSfsYoMW2zSUCnGcQ,10101
|
|
182
184
|
funboost/utils/class_utils.py,sha256=xtP9RU_5vVnWye7QXXqkloDzwVE5N3N-4_2fUZNfXlo,3591
|
|
183
|
-
funboost/utils/
|
|
185
|
+
funboost/utils/class_utils2.py,sha256=ND45cMR385xG4fOmwWDHxXFOmcEi1ZG8B0iN8_6ZAeo,3015
|
|
186
|
+
funboost/utils/ctrl_c_end.py,sha256=uSlMEbcyrbeosZ6E816AwlXxVZ7KuExwMfz6WJXlyB0,439
|
|
184
187
|
funboost/utils/custom_pysnooper.py,sha256=7yXLKEMY_JjPRRt0Y0N-wV2CFhILlYNh40Y6uRBUaj8,5923
|
|
185
188
|
funboost/utils/decorators.py,sha256=gpwof-Nw__iFjeJjVQWx1l-scnxTivxcCI_0XqhMu6c,27885
|
|
186
189
|
funboost/utils/develop_log.py,sha256=Wsx0ongGjTit5xqgk1BztYlVEkC6d0-Y7GENXLedVqY,271
|
|
@@ -209,7 +212,7 @@ funboost/utils/dependency_packages/mongomq/test.py,sha256=Tcmme3U3KXFSkdknO71bge
|
|
|
209
212
|
funboost/utils/dependency_packages/mongomq/utils.py,sha256=ljhcLhNf3yOc7IgnuRdFqLtwTGynRNd2uXZNRvStAL0,377
|
|
210
213
|
funboost/utils/dependency_packages_in_pythonpath/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
211
214
|
funboost/utils/dependency_packages_in_pythonpath/add_to_pythonpath.py,sha256=eOaK0Cr1yAmLcHhOM5-nV9XxXhQFZQkiaBECY65sFuc,341
|
|
212
|
-
funboost/utils/dependency_packages_in_pythonpath/readme.md,sha256=
|
|
215
|
+
funboost/utils/dependency_packages_in_pythonpath/readme.md,sha256=CRo6NJXeq0TwW56_gCJrW4dId1MTejWt9t-Sq8pJHC0,982
|
|
213
216
|
funboost/utils/dependency_packages_in_pythonpath/__pycache__/__init__.cpython-311.pyc,sha256=XpiufbSphxt8UXSMqXz9vOGejT4noRAKE3dnfsEGvAg,187
|
|
214
217
|
funboost/utils/dependency_packages_in_pythonpath/__pycache__/__init__.cpython-37.pyc,sha256=Oy1-_q-VLcFLQ7RRp8B-fbBkNOb2SepePLYW4L5DQ6U,165
|
|
215
218
|
funboost/utils/dependency_packages_in_pythonpath/__pycache__/__init__.cpython-39.pyc,sha256=a54jxiFA239ImQBOQalJk2t8AULxlc3yxz-QaxcWAUo,169
|
|
@@ -224,7 +227,7 @@ funboost/utils/dependency_packages_in_pythonpath/aioredis/exceptions.py,sha256=S
|
|
|
224
227
|
funboost/utils/dependency_packages_in_pythonpath/aioredis/lock.py,sha256=CnB9LpvykAEXEdQyBEcgUU7iHxwNF3xuGzX7UYuUbiQ,11651
|
|
225
228
|
funboost/utils/dependency_packages_in_pythonpath/aioredis/log.py,sha256=qTxLRo5EqoHZIGqMguzLm90mtThBRYje_FaZz-fDbhg,427
|
|
226
229
|
funboost/utils/dependency_packages_in_pythonpath/aioredis/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
227
|
-
funboost/utils/dependency_packages_in_pythonpath/aioredis/readme.md,sha256=
|
|
230
|
+
funboost/utils/dependency_packages_in_pythonpath/aioredis/readme.md,sha256=D-YD-NjTpC1AArLqLWujHH9msvlBC6zL4LuoogSk_ok,679
|
|
228
231
|
funboost/utils/dependency_packages_in_pythonpath/aioredis/sentinel.py,sha256=ldPwRIOWVskKsItBwFqbqPfraMlFsGs42_ZDXtxza5U,12536
|
|
229
232
|
funboost/utils/dependency_packages_in_pythonpath/aioredis/utils.py,sha256=90Dgj62Q9ABGVAwAPwufo3OKM2s0ws6LeQMZg8ifJb0,1284
|
|
230
233
|
funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/__init__.cpython-311.pyc,sha256=PGpc0JrGU6UjKZ7WgxlpsCL2OuhJhWpE9WrxMHREtPc,1696
|
|
@@ -283,9 +286,9 @@ funboost/utils/pysnooper_ydf/utils.py,sha256=evSmGi_Oul7vSP47AJ0DLjFwoCYCfunJZ1m
|
|
|
283
286
|
funboost/utils/pysnooper_ydf/variables.py,sha256=QejRDESBA06KG9OH4sBT4J1M55eaU29EIHg8K_igaXo,3693
|
|
284
287
|
funboost/utils/times/__init__.py,sha256=Y4bQD3SIA_E7W2YvHq2Qdi0dGM4H2DxyFNdDOuFOq1w,2417
|
|
285
288
|
funboost/utils/times/version.py,sha256=11XfnZVVzOgIhXXdeN_mYfdXThfrsbQHpA0wCjz-hpg,17
|
|
286
|
-
funboost-
|
|
287
|
-
funboost-
|
|
288
|
-
funboost-
|
|
289
|
-
funboost-
|
|
290
|
-
funboost-
|
|
291
|
-
funboost-
|
|
289
|
+
funboost-48.0.dist-info/LICENSE,sha256=9EPP2ktG_lAPB8PjmWV-c9BiaJHc_FP6pPLcUrUwx0E,11562
|
|
290
|
+
funboost-48.0.dist-info/METADATA,sha256=eOfSzKu5y7_WNG9qboIt5IuuiTgGzD-gwNQnpvjUZJ4,33382
|
|
291
|
+
funboost-48.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
|
|
292
|
+
funboost-48.0.dist-info/entry_points.txt,sha256=yMSSAGRzRAAhGyNNQHw24MooKlDZsaJ499_D6fPl58A,96
|
|
293
|
+
funboost-48.0.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
|
|
294
|
+
funboost-48.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|