funboost 46.7__py3-none-any.whl → 46.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of funboost might be problematic. Click here for more details.
- funboost/__init__.py +1 -1
- funboost/concurrent_pool/custom_threadpool_executor.py +5 -0
- funboost/consumers/base_consumer.py +13 -10
- funboost/core/func_params_model.py +2 -2
- funboost/timing_job/__init__.py +29 -31
- funboost/timing_job/apscheduler_use_redis_store.py +41 -10
- funboost/utils/decorators.py +27 -0
- funboost/utils/time_util.py +17 -16
- {funboost-46.7.dist-info → funboost-46.9.dist-info}/METADATA +3 -3
- {funboost-46.7.dist-info → funboost-46.9.dist-info}/RECORD +14 -14
- {funboost-46.7.dist-info → funboost-46.9.dist-info}/LICENSE +0 -0
- {funboost-46.7.dist-info → funboost-46.9.dist-info}/WHEEL +0 -0
- {funboost-46.7.dist-info → funboost-46.9.dist-info}/entry_points.txt +0 -0
- {funboost-46.7.dist-info → funboost-46.9.dist-info}/top_level.txt +0 -0
funboost/__init__.py
CHANGED
|
@@ -19,6 +19,8 @@
|
|
|
19
19
|
|
|
20
20
|
可以在各种地方加入 time.sleep 来验证 第1条和第2条的自动智能缩放功能。
|
|
21
21
|
"""
|
|
22
|
+
import logging
|
|
23
|
+
|
|
22
24
|
import os
|
|
23
25
|
import atexit
|
|
24
26
|
import queue
|
|
@@ -175,6 +177,9 @@ class _CustomThread(threading.Thread, FunboostFileLoggerMixin, LoggerLevelSetter
|
|
|
175
177
|
# noinspection PyProtectedMember
|
|
176
178
|
def run(self):
|
|
177
179
|
# noinspection PyUnresolvedReferences
|
|
180
|
+
# print(logging.getLogger(None).level,logging.getLogger(None).handlers)
|
|
181
|
+
# print(self.logger.level)
|
|
182
|
+
# print(self.logger.handlers)
|
|
178
183
|
self.logger.debug(f'新启动线程 {self._ident} ')
|
|
179
184
|
self._executorx._change_threads_free_count(1)
|
|
180
185
|
while True:
|
|
@@ -13,6 +13,7 @@ import typing
|
|
|
13
13
|
import abc
|
|
14
14
|
import copy
|
|
15
15
|
from apscheduler.jobstores.memory import MemoryJobStore
|
|
16
|
+
from funboost.core.funboost_time import FunboostTime
|
|
16
17
|
from pathlib import Path
|
|
17
18
|
# from multiprocessing import Process
|
|
18
19
|
import datetime
|
|
@@ -369,7 +370,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
369
370
|
|
|
370
371
|
def _start_delay_task_scheduler(self):
|
|
371
372
|
from funboost.timing_job import FsdfBackgroundScheduler
|
|
372
|
-
from funboost.timing_job import FunboostBackgroundSchedulerProcessJobsWithinRedisLock
|
|
373
|
+
from funboost.timing_job.apscheduler_use_redis_store import FunboostBackgroundSchedulerProcessJobsWithinRedisLock
|
|
373
374
|
# print(self.consumer_params.delay_task_apsscheduler_jobstores_kind )
|
|
374
375
|
if self.consumer_params.delay_task_apscheduler_jobstores_kind == 'redis':
|
|
375
376
|
jobstores = {
|
|
@@ -392,7 +393,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
392
393
|
raise Exception(f'delay_task_apsscheduler_jobstores_kind is error: {self.consumer_params.delay_task_apscheduler_jobstores_kind}')
|
|
393
394
|
|
|
394
395
|
|
|
395
|
-
self._delay_task_scheduler.add_executor(ApschedulerThreadPoolExecutor(2)) # 只是运行submit任务到并发池,不需要很多线程。
|
|
396
|
+
# self._delay_task_scheduler.add_executor(ApschedulerThreadPoolExecutor(2)) # 只是运行submit任务到并发池,不需要很多线程。
|
|
396
397
|
# self._delay_task_scheduler.add_listener(self._apscheduler_job_miss, EVENT_JOB_MISSED)
|
|
397
398
|
self._delay_task_scheduler.start()
|
|
398
399
|
|
|
@@ -401,11 +402,12 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
401
402
|
logger_apscheduler = get_logger('push_for_apscheduler_use_database_store', log_filename='push_for_apscheduler_use_database_store.log')
|
|
402
403
|
|
|
403
404
|
@classmethod
|
|
404
|
-
def
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
405
|
+
def _push_apscheduler_task_to_broker(cls, queue_name, msg, runonce_uuid):
|
|
406
|
+
funboost_lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
|
|
407
|
+
# key = 'apscheduler.redisjobstore_runonce'
|
|
408
|
+
# if RedisMixin().redis_db_frame.sadd(key, runonce_uuid): # 这样可以阻止多次启动同队列名消费者 redis jobstore多次运行函数.
|
|
409
|
+
# cls.logger_apscheduler.debug(f'延时任务用普通消息重新发布到普通队列 {msg}')
|
|
410
|
+
# funboost_lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
|
|
409
411
|
|
|
410
412
|
@abc.abstractmethod
|
|
411
413
|
def _shedual_task(self):
|
|
@@ -486,9 +488,10 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
486
488
|
run_date = None
|
|
487
489
|
# print(kw)
|
|
488
490
|
if msg_countdown:
|
|
489
|
-
run_date =
|
|
491
|
+
run_date = FunboostTime(kw['body']['extra']['publish_time']).datetime_obj + datetime.timedelta(seconds=msg_countdown)
|
|
490
492
|
if msg_eta:
|
|
491
|
-
|
|
493
|
+
|
|
494
|
+
run_date = FunboostTime(msg_eta).datetime_obj
|
|
492
495
|
# print(run_date,time_util.DatetimeConverter().datetime_obj)
|
|
493
496
|
# print(run_date.timestamp(),time_util.DatetimeConverter().datetime_obj.timestamp())
|
|
494
497
|
# print(self.concurrent_pool)
|
|
@@ -507,7 +510,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
507
510
|
self.__delete_eta_countdown(msg_no_delay)
|
|
508
511
|
# print(msg_no_delay)
|
|
509
512
|
# 数据库作为apscheduler的jobstores时候, 不能用 self.pbulisher_of_same_queue.publish,self不能序列化
|
|
510
|
-
self._delay_task_scheduler.add_job(self.
|
|
513
|
+
self._delay_task_scheduler.add_job(self._push_apscheduler_task_to_broker, 'date', run_date=run_date,
|
|
511
514
|
kwargs={'queue_name': self.queue_name, 'msg': msg_no_delay, 'runonce_uuid': str(uuid.uuid4())},
|
|
512
515
|
misfire_grace_time=misfire_grace_time,
|
|
513
516
|
)
|
|
@@ -194,7 +194,7 @@ class BoosterParams(BaseJsonAbleModel):
|
|
|
194
194
|
broker_exclusive_config: dict = {} # 加上一个不同种类中间件非通用的配置,不同中间件自身独有的配置,不是所有中间件都兼容的配置,因为框架支持30种消息队列,消息队列不仅仅是一般的先进先出queue这么简单的概念,
|
|
195
195
|
# 例如kafka支持消费者组,rabbitmq也支持各种独特概念例如各种ack机制 复杂路由机制,有的中间件原生能支持消息优先级有的中间件不支持,每一种消息队列都有独特的配置参数意义,可以通过这里传递。每种中间件能传递的键值对可以看consumer类的 BROKER_EXCLUSIVE_CONFIG_DEFAULT
|
|
196
196
|
|
|
197
|
-
should_check_publish_func_params: bool = True # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2
|
|
197
|
+
should_check_publish_func_params: bool = True # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字; 如果消费函数加了装饰器 ,你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
|
|
198
198
|
|
|
199
199
|
consumer_override_cls: typing.Optional[typing.Type] = None # 使用 consumer_override_cls 和 publisher_override_cls 来自定义重写或新增消费者 发布者,见文档4.21b介绍,
|
|
200
200
|
publisher_override_cls: typing.Optional[typing.Type] = None
|
|
@@ -283,7 +283,7 @@ class PriorityConsumingControlConfig(BaseJsonAbleModel):
|
|
|
283
283
|
is_using_rpc_mode: bool = None
|
|
284
284
|
|
|
285
285
|
countdown: typing.Union[float, int] = None
|
|
286
|
-
eta: datetime.datetime = None
|
|
286
|
+
eta: typing.Union[datetime.datetime, str] = None # 时间对象, 或 %Y-%m-%d %H:%M:%S 字符串。
|
|
287
287
|
misfire_grace_time: typing.Union[int, None] = None
|
|
288
288
|
|
|
289
289
|
other_extra_params: dict = None # 其他参数, 例如消息优先级 , priority_control_config=PriorityConsumingControlConfig(other_extra_params={'priroty': priorityxx}),
|
funboost/timing_job/__init__.py
CHANGED
|
@@ -2,30 +2,27 @@
|
|
|
2
2
|
集成定时任务。
|
|
3
3
|
"""
|
|
4
4
|
import atexit
|
|
5
|
-
import copy
|
|
6
|
-
import importlib
|
|
7
|
-
|
|
8
|
-
import pickle
|
|
9
5
|
|
|
10
6
|
import time
|
|
11
|
-
from
|
|
7
|
+
from apscheduler.executors.pool import BasePoolExecutor
|
|
8
|
+
|
|
12
9
|
from typing import Union
|
|
13
10
|
import threading
|
|
14
11
|
|
|
15
12
|
from apscheduler.schedulers.background import BackgroundScheduler
|
|
16
|
-
from apscheduler.jobstores.redis import RedisJobStore
|
|
17
13
|
# noinspection PyProtectedMember
|
|
18
14
|
from apscheduler.schedulers.base import STATE_STOPPED, STATE_RUNNING
|
|
19
|
-
from apscheduler.util import undefined,TIMEOUT_MAX
|
|
15
|
+
from apscheduler.util import undefined, TIMEOUT_MAX
|
|
20
16
|
import deprecated
|
|
21
17
|
from funboost.utils.redis_manager import RedisMixin
|
|
22
18
|
|
|
23
|
-
from funboost.funboost_config_deafult import
|
|
19
|
+
from funboost.funboost_config_deafult import FunboostCommonConfig
|
|
24
20
|
|
|
25
21
|
from funboost.consumers.base_consumer import AbstractConsumer
|
|
26
22
|
from funboost.core.booster import BoostersManager, Booster
|
|
27
|
-
|
|
23
|
+
|
|
28
24
|
from funboost import BoosterParams
|
|
25
|
+
from funboost.concurrent_pool.custom_threadpool_executor import ThreadPoolExecutorShrinkAble
|
|
29
26
|
|
|
30
27
|
|
|
31
28
|
@deprecated.deprecated(reason='以后不要再使用这种方式,对于job_store为数据库时候需要序列化不好。使用内存和数据库都兼容的添加任务方式: add_push_job')
|
|
@@ -46,8 +43,9 @@ def push_fun_params_to_broker(queue_name: str, *args, runonce_uuid=None, **kwarg
|
|
|
46
43
|
queue_name 队列名字
|
|
47
44
|
*args **kwargs 是消费函数的入参
|
|
48
45
|
发布消息中可以包括,runonce_uuid这个入参,确保分布式多个脚本都启动了定时器,导致每个定时器重复发布到消息队列,值你自己写 str(uuid.uuid4())
|
|
46
|
+
# 不需要传递 runonce_uuid 了,已经用专门的 FunboostBackgroundSchedulerProcessJobsWithinRedisLock 解决了。
|
|
49
47
|
"""
|
|
50
|
-
if runonce_uuid:
|
|
48
|
+
if runonce_uuid: # 不需要传递 runonce_uuid 了,已经用专门的 FunboostBackgroundSchedulerProcessJobsWithinRedisLock 解决了 process_jobs的问题。
|
|
51
49
|
key = 'apscheduler.redisjobstore_runonce2'
|
|
52
50
|
if RedisMixin().redis_db_frame.sadd(key, runonce_uuid):
|
|
53
51
|
BoostersManager.get_or_create_booster_by_queue_name(queue_name).push(*args, **kwargs)
|
|
@@ -55,6 +53,22 @@ def push_fun_params_to_broker(queue_name: str, *args, runonce_uuid=None, **kwarg
|
|
|
55
53
|
BoostersManager.get_or_create_booster_by_queue_name(queue_name).push(*args, **kwargs)
|
|
56
54
|
|
|
57
55
|
|
|
56
|
+
class ThreadPoolExecutorForAps(BasePoolExecutor):
|
|
57
|
+
"""
|
|
58
|
+
An executor that runs jobs in a concurrent.futures thread pool.
|
|
59
|
+
|
|
60
|
+
Plugin alias: ``threadpool``
|
|
61
|
+
|
|
62
|
+
:param max_workers: the maximum number of spawned threads.
|
|
63
|
+
:param pool_kwargs: dict of keyword arguments to pass to the underlying
|
|
64
|
+
ThreadPoolExecutor constructor
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
def __init__(self, max_workers=10, pool_kwargs=None):
|
|
68
|
+
pool = ThreadPoolExecutorShrinkAble(int(max_workers), )
|
|
69
|
+
super().__init__(pool)
|
|
70
|
+
|
|
71
|
+
|
|
58
72
|
class FunboostBackgroundScheduler(BackgroundScheduler):
|
|
59
73
|
"""
|
|
60
74
|
自定义的, 继承了官方BackgroundScheduler,
|
|
@@ -109,7 +123,7 @@ class FunboostBackgroundScheduler(BackgroundScheduler):
|
|
|
109
123
|
args_list.insert(0, func.queue_name)
|
|
110
124
|
args = tuple(args_list)
|
|
111
125
|
kwargs = kwargs or {}
|
|
112
|
-
kwargs['runonce_uuid'] = runonce_uuid
|
|
126
|
+
kwargs['runonce_uuid'] = runonce_uuid # 忽略,用户不需要传递runonce_uuid入参。
|
|
113
127
|
return self.add_job(push_fun_params_to_broker, trigger, args, kwargs, id, name,
|
|
114
128
|
misfire_grace_time, coalesce, max_instances,
|
|
115
129
|
next_run_time, jobstore, executor,
|
|
@@ -150,37 +164,21 @@ class FunboostBackgroundScheduler(BackgroundScheduler):
|
|
|
150
164
|
或者下一个需要运行的任务的wait_seconds是3600秒后,此时新加了一个动态任务需要3600秒后,
|
|
151
165
|
现在最多只需要1秒就能扫描到动态新增的定时任务了。
|
|
152
166
|
"""
|
|
153
|
-
MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS =
|
|
167
|
+
MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS = 0.5
|
|
154
168
|
wait_seconds = None
|
|
155
169
|
while self.state == STATE_RUNNING:
|
|
156
170
|
if wait_seconds is None:
|
|
157
171
|
wait_seconds = MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS
|
|
158
172
|
self._last_wait_seconds = min(wait_seconds, MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS)
|
|
159
|
-
if wait_seconds in (None,TIMEOUT_MAX):
|
|
173
|
+
if wait_seconds in (None, TIMEOUT_MAX):
|
|
160
174
|
self._last_has_task = False
|
|
161
175
|
else:
|
|
162
176
|
self._last_has_task = True
|
|
163
177
|
time.sleep(self._last_wait_seconds) # 这个要取最小值,不然例如定时间隔0.1秒运行,不取最小值,不会每隔0.1秒运行。
|
|
164
178
|
wait_seconds = self._process_jobs()
|
|
165
179
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
process_jobs_redis_lock_key = f'funboost.BackgroundSchedulerProcessJobsWithinRedisLock'
|
|
169
|
-
|
|
170
|
-
def set_process_jobs_redis_lock_key(self,lock_key):
|
|
171
|
-
self.process_jobs_redis_lock_key = lock_key
|
|
172
|
-
|
|
173
|
-
def _process_jobs(self):
|
|
174
|
-
for i in range(10) :
|
|
175
|
-
with RedisDistributedLockContextManager(RedisMixin().redis_db_frame, self.process_jobs_redis_lock_key, ) as lock:
|
|
176
|
-
if lock.has_aquire_lock:
|
|
177
|
-
wait_seconds = super()._process_jobs()
|
|
178
|
-
return wait_seconds
|
|
179
|
-
else:
|
|
180
|
-
time.sleep(0.1)
|
|
181
|
-
return 0.1
|
|
182
|
-
|
|
183
|
-
|
|
180
|
+
def _create_default_executor(self):
|
|
181
|
+
return ThreadPoolExecutorForAps() # 必须是apscheduler pool的子类
|
|
184
182
|
|
|
185
183
|
|
|
186
184
|
FsdfBackgroundScheduler = FunboostBackgroundScheduler # 兼容一下名字,fsdf是 function-scheduling-distributed-framework 老框架名字的缩写
|
|
@@ -1,25 +1,56 @@
|
|
|
1
1
|
from apscheduler.jobstores.redis import RedisJobStore
|
|
2
|
+
from funboost.utils.redis_manager import RedisMixin
|
|
2
3
|
|
|
3
|
-
from funboost.
|
|
4
|
-
|
|
5
|
-
from funboost.
|
|
4
|
+
from funboost.timing_job import FunboostBackgroundScheduler
|
|
5
|
+
from funboost.funboost_config_deafult import BrokerConnConfig, FunboostCommonConfig
|
|
6
|
+
from funboost.utils.decorators import RedisDistributedBlockLockContextManager
|
|
6
7
|
|
|
7
8
|
"""
|
|
8
|
-
这个是使用redis
|
|
9
|
+
这个是使用redis作为定时任务持久化,支持跨机器好跨进程,外部远程 动态修改/添加/删除定时任务
|
|
9
10
|
"""
|
|
10
11
|
|
|
12
|
+
|
|
13
|
+
class FunboostBackgroundSchedulerProcessJobsWithinRedisLock(FunboostBackgroundScheduler):
|
|
14
|
+
"""
|
|
15
|
+
分布式或多进程都启动某个apscheduler实例,如果都使用的同一个数据库类型的jobstores ,_process_jobs有很大概率会造成报错, 因为_process_jobs使用的是线程锁,管不了其他进程和分布式机器。
|
|
16
|
+
|
|
17
|
+
https://groups.google.com/g/apscheduler/c/Gjc_JQMPePc 问题也提到了这个bug
|
|
18
|
+
|
|
19
|
+
继承 Custom schedulers https://apscheduler.readthedocs.io/en/3.x/extending.html 可以重写 _create_lock
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
process_jobs_redis_lock_key = f'funboost.BackgroundSchedulerProcessJobsWithinRedisLock'
|
|
23
|
+
|
|
24
|
+
def set_process_jobs_redis_lock_key(self, lock_key):
|
|
25
|
+
self.process_jobs_redis_lock_key = lock_key
|
|
26
|
+
|
|
27
|
+
# def _create_lock(self):
|
|
28
|
+
# return RedisDistributedBlockLockContextManager(RedisMixin().redis_db_frame,self.process_jobs_redis_lock_key,) 这个类的写法不适合固定的单例,
|
|
29
|
+
# RedisDistributedBlockLockContextManager的写法不适合 永远用一个 对象,所以还是放到 def _process_jobs 里面运行
|
|
30
|
+
|
|
31
|
+
# def _process_jobs(self):
|
|
32
|
+
# for i in range(10) :
|
|
33
|
+
# with RedisDistributedLockContextManager(RedisMixin().redis_db_frame, self.process_jobs_redis_lock_key, ) as lock:
|
|
34
|
+
# if lock.has_aquire_lock:
|
|
35
|
+
# wait_seconds = super()._process_jobs()
|
|
36
|
+
# return wait_seconds
|
|
37
|
+
# else:
|
|
38
|
+
# time.sleep(0.1)
|
|
39
|
+
# return 0.1
|
|
40
|
+
|
|
41
|
+
def _process_jobs(self):
|
|
42
|
+
with RedisDistributedBlockLockContextManager(RedisMixin().redis_db_frame, self.process_jobs_redis_lock_key, ):
|
|
43
|
+
return super()._process_jobs()
|
|
44
|
+
|
|
45
|
+
|
|
11
46
|
jobstores = {
|
|
12
47
|
"default": RedisJobStore(db=BrokerConnConfig.REDIS_DB, host=BrokerConnConfig.REDIS_HOST,
|
|
13
48
|
port=BrokerConnConfig.REDIS_PORT, password=BrokerConnConfig.REDIS_PASSWORD,
|
|
14
|
-
username=BrokerConnConfig.REDIS_USERNAME,jobs_key='funboost.apscheduler.jobs')
|
|
49
|
+
username=BrokerConnConfig.REDIS_USERNAME, jobs_key='funboost.apscheduler.jobs')
|
|
15
50
|
}
|
|
16
51
|
|
|
17
52
|
funboost_background_scheduler_redis_store = FunboostBackgroundSchedulerProcessJobsWithinRedisLock(timezone=FunboostCommonConfig.TIMEZONE, daemon=False, jobstores=jobstores)
|
|
18
53
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
54
|
"""
|
|
24
55
|
|
|
25
56
|
跨python解释器 跨机器动态修改定时任务配置的例子在
|
|
@@ -27,4 +58,4 @@ funboost_background_scheduler_redis_store = FunboostBackgroundSchedulerProcessJo
|
|
|
27
58
|
test_frame/test_apschedual/test_aps_redis_store.py
|
|
28
59
|
test_frame/test_apschedual/test_change_aps_conf.py
|
|
29
60
|
|
|
30
|
-
"""
|
|
61
|
+
"""
|
funboost/utils/decorators.py
CHANGED
|
@@ -361,6 +361,30 @@ class RedisDistributedLockContextManager(LoggerMixin, LoggerLevelSetterMixin):
|
|
|
361
361
|
return False
|
|
362
362
|
|
|
363
363
|
|
|
364
|
+
class RedisDistributedBlockLockContextManager(RedisDistributedLockContextManager):
|
|
365
|
+
def __init__(self, redis_client, redis_lock_key, expire_seconds=30, check_interval=0.1):
|
|
366
|
+
super().__init__(redis_client,redis_lock_key,expire_seconds)
|
|
367
|
+
self.check_interval = check_interval
|
|
368
|
+
# self.logger.setLevel(logging.DEBUG)
|
|
369
|
+
|
|
370
|
+
def __enter__(self):
|
|
371
|
+
while True:
|
|
372
|
+
self._line = sys._getframe().f_back.f_lineno # 调用此方法的代码的函数
|
|
373
|
+
self._file_name = sys._getframe(1).f_code.co_filename # 哪个文件调了用此方法
|
|
374
|
+
ret = self.redis_client.set(self.redis_lock_key, value=self.identifier, ex=self._expire_seconds, nx=True)
|
|
375
|
+
has_aquire_lock = False if ret is None else True
|
|
376
|
+
if has_aquire_lock:
|
|
377
|
+
log_msg = f'\n"{self._file_name}:{self._line}" 这行代码获得了redis锁 {self.redis_lock_key}'
|
|
378
|
+
else:
|
|
379
|
+
log_msg = f'\n"{self._file_name}:{self._line}" 这行代码此次没有获得redis锁 {self.redis_lock_key}'
|
|
380
|
+
# print(self.logger.level,log_msg)
|
|
381
|
+
self.logger.debug(log_msg)
|
|
382
|
+
if has_aquire_lock:
|
|
383
|
+
break
|
|
384
|
+
else:
|
|
385
|
+
time.sleep(self.check_interval)
|
|
386
|
+
|
|
387
|
+
|
|
364
388
|
"""
|
|
365
389
|
@contextmanager
|
|
366
390
|
def some_generator(<arguments>):
|
|
@@ -372,6 +396,9 @@ class RedisDistributedLockContextManager(LoggerMixin, LoggerLevelSetterMixin):
|
|
|
372
396
|
"""
|
|
373
397
|
|
|
374
398
|
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
|
|
375
402
|
class ExceptionContextManager:
|
|
376
403
|
"""
|
|
377
404
|
用上下文管理器捕获异常,可对代码片段进行错误捕捉,比装饰器更细腻
|
funboost/utils/time_util.py
CHANGED
|
@@ -5,7 +5,7 @@ import datetime
|
|
|
5
5
|
import time
|
|
6
6
|
import re
|
|
7
7
|
import pytz
|
|
8
|
-
|
|
8
|
+
from funboost.core.funboost_time import FunboostTime
|
|
9
9
|
|
|
10
10
|
from funboost.utils import nb_print
|
|
11
11
|
|
|
@@ -75,21 +75,22 @@ class DatetimeConverter:
|
|
|
75
75
|
"""
|
|
76
76
|
:param datetimex: 接受时间戳 datatime类型 和 时间字符串三种类型
|
|
77
77
|
"""
|
|
78
|
-
if isinstance(datetimex, str):
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
elif isinstance(datetimex, (int, float)):
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
elif isinstance(datetimex, datetime.datetime):
|
|
88
|
-
|
|
89
|
-
elif datetimex is None:
|
|
90
|
-
|
|
91
|
-
else:
|
|
92
|
-
|
|
78
|
+
# if isinstance(datetimex, str):
|
|
79
|
+
# if not re.match(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', datetimex):
|
|
80
|
+
# raise ValueError('时间字符串的格式不符合此传参的规定')
|
|
81
|
+
# else:
|
|
82
|
+
# self.datetime_obj = datetime.datetime.strptime(datetimex, self.DATETIME_FORMATTER)
|
|
83
|
+
# elif isinstance(datetimex, (int, float)):
|
|
84
|
+
# if datetimex < 1:
|
|
85
|
+
# datetimex += 86400
|
|
86
|
+
# self.datetime_obj = datetime.datetime.fromtimestamp(datetimex, tz=pytz.timezone(_get_funboost_timezone())) # 时间戳0在windows会出错。
|
|
87
|
+
# elif isinstance(datetimex, datetime.datetime):
|
|
88
|
+
# self.datetime_obj = datetimex
|
|
89
|
+
# elif datetimex is None:
|
|
90
|
+
# self.datetime_obj = datetime.datetime.now(tz=pytz.timezone(_get_funboost_timezone()))
|
|
91
|
+
# else:
|
|
92
|
+
# raise ValueError('实例化时候的传参不符合规定')
|
|
93
|
+
self.datetime_obj = FunboostTime(datetimex).datetime_obj
|
|
93
94
|
|
|
94
95
|
@property
|
|
95
96
|
def datetime_str(self):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: funboost
|
|
3
|
-
Version: 46.
|
|
3
|
+
Version: 46.9
|
|
4
4
|
Summary: pip install funboost,python全功能分布式函数调度框架,funboost的功能是全面性重量级,用户能想得到的功能99%全都有;funboost的使用方式是轻量级,只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件,支持如 celery dramatiq等框架整体作为funboost中间件,python函数加速器,框架包罗万象,用户能想到的控制功能全都有。一统编程思维,兼容50% python业务场景,适用范围广。只需要一行代码即可分布式执行python一切函数,99%用过funboost的pythoner 感受是 简易 方便 强劲 强大,相见恨晚
|
|
5
5
|
Home-page: https://github.com/ydf0509/funboost
|
|
6
6
|
Author: bfzs
|
|
@@ -26,9 +26,9 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
26
26
|
Classifier: Topic :: Software Development :: Libraries
|
|
27
27
|
Description-Content-Type: text/markdown
|
|
28
28
|
License-File: LICENSE
|
|
29
|
-
Requires-Dist: nb-log (>=
|
|
29
|
+
Requires-Dist: nb-log (>=13.2)
|
|
30
30
|
Requires-Dist: nb-libs (>=1.8)
|
|
31
|
-
Requires-Dist: nb-time (>=
|
|
31
|
+
Requires-Dist: nb-time (>=2.0)
|
|
32
32
|
Requires-Dist: pymongo (==4.3.3)
|
|
33
33
|
Requires-Dist: AMQPStorm (==2.10.6)
|
|
34
34
|
Requires-Dist: rabbitpy (==2.0.1)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
funboost/__init__.py,sha256=
|
|
1
|
+
funboost/__init__.py,sha256=qhCEk3zmW8Sta-65sGqfmaYlfK6J36fq4fS1qNWDS2c,3956
|
|
2
2
|
funboost/__init__old.py,sha256=9Kv3cPLnPkbzMRnuJFVkPsuDdx1CdcSIuITkpdncZSc,20382
|
|
3
3
|
funboost/__main__.py,sha256=-6Nogi666Y0LN8fVm3JmHGTOk8xEGWvotW_GDbSaZME,1065
|
|
4
4
|
funboost/constant.py,sha256=STzRDZbuCC5FFV-uD_0r2beGsD1Zni4kregzR11z3Ok,8148
|
|
@@ -24,7 +24,7 @@ funboost/concurrent_pool/bounded_threadpoolexcutor.py,sha256=AWFsTwK7hIiln0XVaiY
|
|
|
24
24
|
funboost/concurrent_pool/concurrent_pool_with_multi_process.py,sha256=qChdRh11Epk7-kmLEdoB1TtM2plx_FZ1u_-g0KNc-MU,1744
|
|
25
25
|
funboost/concurrent_pool/custom_evenlet_pool_executor.py,sha256=tu8Og-uIt5VWqGERnIihBaF7ZFw5D0xww1_ur39OZXY,3252
|
|
26
26
|
funboost/concurrent_pool/custom_gevent_pool_executor.py,sha256=FLcEwEJgFtaEPzH-WCFHtCdEUOTeK0dLSL_oNBcXQW8,5429
|
|
27
|
-
funboost/concurrent_pool/custom_threadpool_executor.py,sha256=
|
|
27
|
+
funboost/concurrent_pool/custom_threadpool_executor.py,sha256=hYgRYqFoCvplyQFTzw57P-oRcmO6ZuxW5qB3U1SPZks,12084
|
|
28
28
|
funboost/concurrent_pool/custom_threadpool_executor000.py,sha256=jJLXy3h-bELap6nZA6yLtdozzTWcvCtZ7IY6MTqLEAM,9317
|
|
29
29
|
funboost/concurrent_pool/fixed_thread_pool.py,sha256=JzaqjuHn6ffo1gZRVJEy5Te5NdCJt7heTmsVZT_AiBg,1609
|
|
30
30
|
funboost/concurrent_pool/flexible_thread_pool.py,sha256=-xvRYcSzh-oNVtDkEsMreRPW6sadQ6PFsN35XzLOdAU,6040
|
|
@@ -35,7 +35,7 @@ funboost/concurrent_pool/backup/async_pool_executor0223.py,sha256=RVUZiylUvpTm6U
|
|
|
35
35
|
funboost/concurrent_pool/backup/async_pool_executor_back.py,sha256=KL6zEQaa1KkZOlAO85mCC1gwLm-YC5Ghn21IUom0UKM,9598
|
|
36
36
|
funboost/concurrent_pool/backup/async_pool_executor_janus.py,sha256=OHMWJ9l3EYTpPpcrPrGGKd4K0tmQ2PN8HiX0Dta0EOo,5728
|
|
37
37
|
funboost/consumers/__init__.py,sha256=ZXY_6Kut1VYNQiF5aWEgIWobsW1ht9YUP0TdRZRWFqI,126
|
|
38
|
-
funboost/consumers/base_consumer.py,sha256=
|
|
38
|
+
funboost/consumers/base_consumer.py,sha256=QznnVHhGFf_4tQFk0hXes0mo4oeIeKp-EjPiuryurYo,82376
|
|
39
39
|
funboost/consumers/celery_consumer.py,sha256=nQpSkzPBJ4bRpxn4i9ms0axrJiq9RWhb4lG2nAdCIig,9254
|
|
40
40
|
funboost/consumers/confirm_mixin.py,sha256=5xC9AAQr_MY4tbSed8U-M6tOVmh69Qv9X0ld0JLT9Tk,6185
|
|
41
41
|
funboost/consumers/dramatiq_consumer.py,sha256=ozmeAfeF0U-YNYHK4suQB0N264h5AZdfMH0O45Mh-8A,2229
|
|
@@ -92,7 +92,7 @@ funboost/core/exceptions.py,sha256=pLF7BkRJAfDiWp2_xGZqencmwdPiSQI1NENbImExknY,1
|
|
|
92
92
|
funboost/core/fabric_deploy_helper.py,sha256=foieeqlNySuU9axJzNF6TavPjIUSYBx9UO3syVKUiyY,9999
|
|
93
93
|
funboost/core/funboost_config_getter.py,sha256=b5nAdAmUxahskY-ohB7ptf2gKywFlDA0Fq1cWroxxbs,384
|
|
94
94
|
funboost/core/funboost_time.py,sha256=IbB4dFCpg3oGUe90ssAJ_x0eDPtAVfvsUr4esdoKaOk,1777
|
|
95
|
-
funboost/core/func_params_model.py,sha256=
|
|
95
|
+
funboost/core/func_params_model.py,sha256=D-Gf3PP-QX-c2yQxDg19-28JkIlxUvG_mt1BewcgJCs,21691
|
|
96
96
|
funboost/core/function_result_status_config.py,sha256=PyjqAQOiwsLt28sRtH-eYRjiI3edPFO4Nde0ILFRReE,1764
|
|
97
97
|
funboost/core/function_result_status_saver.py,sha256=yHKZF9MjmhI-Q4Mkrka7DdweJ0wpgfLmgfAlsfkCeCk,9274
|
|
98
98
|
funboost/core/helper_funs.py,sha256=SsMa7A3iJyLek6v1qRK02kINebDp6kuAmlYkrYLXwQ0,2369
|
|
@@ -172,9 +172,9 @@ funboost/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
172
172
|
funboost/queues/memory_queues_map.py,sha256=e1S_cnjnCVI4DBsA_iupF51S_eX4OvCtlefQCqS1TYA,424
|
|
173
173
|
funboost/queues/peewee_queue.py,sha256=FrwegrilkHZG6Y1cGdl5Bt_UtA25f7m5TJQJMZ9YnJI,5280
|
|
174
174
|
funboost/queues/sqla_queue.py,sha256=b-2QPvvuMxklm41ibZhGKehaAV9trXBQFCOHzgThx_s,11080
|
|
175
|
-
funboost/timing_job/__init__.py,sha256=
|
|
175
|
+
funboost/timing_job/__init__.py,sha256=jaXuyA1O5JnpIJfxzQijOPHP9sI92n8TSVZEr8iBTFM,10340
|
|
176
176
|
funboost/timing_job/apscheduler_use_mysql_store.py,sha256=ss92DiSLzbWuVIo19sTLgC99GessltWLOlqqOd4AIL4,471
|
|
177
|
-
funboost/timing_job/apscheduler_use_redis_store.py,sha256=
|
|
177
|
+
funboost/timing_job/apscheduler_use_redis_store.py,sha256=RNZVerUTTzpSFMFEHWkDrUOSz71B4Ml_hlcavkL1tAA,2933
|
|
178
178
|
funboost/utils/__init__.py,sha256=rAyXE7lgCo_3VdMvGrIJiqsTHv2nZPTJDTj1f6s_KgE,586
|
|
179
179
|
funboost/utils/apscheduler_monkey.py,sha256=CcUISbqX6nMWSxr_QjZ26IvvhUk_ojYZWRaKenpsKfE,3124
|
|
180
180
|
funboost/utils/block_exit.py,sha256=BnfxNYo3lnmhk686RAEoc4u3D4RU_iEMMMgu5L8gIuI,96
|
|
@@ -182,7 +182,7 @@ funboost/utils/bulk_operation.py,sha256=B4FBxlz5f4oqlKDWqer7axn4gnDSfsYoMW2zSUCn
|
|
|
182
182
|
funboost/utils/class_utils.py,sha256=yDsGF4-SaO7k4gtkpYLlACSeejS5FfdIHn087BE_4Cw,3579
|
|
183
183
|
funboost/utils/ctrl_c_end.py,sha256=FgT9An-qsUA5gW-V-UKWqOh5shC7C_uvTFn0fS7i8GI,439
|
|
184
184
|
funboost/utils/custom_pysnooper.py,sha256=7yXLKEMY_JjPRRt0Y0N-wV2CFhILlYNh40Y6uRBUaj8,5923
|
|
185
|
-
funboost/utils/decorators.py,sha256=
|
|
185
|
+
funboost/utils/decorators.py,sha256=gpwof-Nw__iFjeJjVQWx1l-scnxTivxcCI_0XqhMu6c,27885
|
|
186
186
|
funboost/utils/develop_log.py,sha256=Wsx0ongGjTit5xqgk1BztYlVEkC6d0-Y7GENXLedVqY,271
|
|
187
187
|
funboost/utils/expire_lock.py,sha256=AOkd1KlvZeIwQaz8ZoKxLpGxWgqQ4mfNHcFphh04o8Q,4732
|
|
188
188
|
funboost/utils/json_helper.py,sha256=kwN5O-0Z3_9KS3JdBx-BltS9LI3Rfoq329vgkxMPVw4,2286
|
|
@@ -198,7 +198,7 @@ funboost/utils/resource_monitoring.py,sha256=EWq7hqQLM2hYpbkv4sVw9YcpHLxfg8arcGz
|
|
|
198
198
|
funboost/utils/restart_python.py,sha256=bFbV0_24ajL8hBwVRLxWe9v9kTwiX1fGLhXRroNnmgQ,1418
|
|
199
199
|
funboost/utils/simple_data_class.py,sha256=AVaMOyM2o7fszFoG6U8qGly7fIUZt10o5x5mIFYzI9k,2277
|
|
200
200
|
funboost/utils/str_utils.py,sha256=Mxi8BW3-QTBozxioTKpmEh00hOibT-_5dctMjz4miUs,1608
|
|
201
|
-
funboost/utils/time_util.py,sha256=
|
|
201
|
+
funboost/utils/time_util.py,sha256=5Aw0mLTjXfWFmijCcS8XbBEDjzpLaMcl2UBqMS9LJIw,5681
|
|
202
202
|
funboost/utils/un_strict_json_dumps.py,sha256=uh2mXNRCq5dJcqMhb9CkfvehfEGYZAgI6RY1oLcYX_M,408
|
|
203
203
|
funboost/utils/dependency_packages/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
204
204
|
funboost/utils/dependency_packages/mongomq/__init__.py,sha256=yP7LHPsZ5ResiexksmtyJc9HGbMJWwZ0gOvHk2vNcz0,131
|
|
@@ -277,9 +277,9 @@ funboost/utils/pysnooper_ydf/utils.py,sha256=evSmGi_Oul7vSP47AJ0DLjFwoCYCfunJZ1m
|
|
|
277
277
|
funboost/utils/pysnooper_ydf/variables.py,sha256=QejRDESBA06KG9OH4sBT4J1M55eaU29EIHg8K_igaXo,3693
|
|
278
278
|
funboost/utils/times/__init__.py,sha256=Y4bQD3SIA_E7W2YvHq2Qdi0dGM4H2DxyFNdDOuFOq1w,2417
|
|
279
279
|
funboost/utils/times/version.py,sha256=11XfnZVVzOgIhXXdeN_mYfdXThfrsbQHpA0wCjz-hpg,17
|
|
280
|
-
funboost-46.
|
|
281
|
-
funboost-46.
|
|
282
|
-
funboost-46.
|
|
283
|
-
funboost-46.
|
|
284
|
-
funboost-46.
|
|
285
|
-
funboost-46.
|
|
280
|
+
funboost-46.9.dist-info/LICENSE,sha256=9EPP2ktG_lAPB8PjmWV-c9BiaJHc_FP6pPLcUrUwx0E,11562
|
|
281
|
+
funboost-46.9.dist-info/METADATA,sha256=NBn8VHcyN2PhgETIgS6ENUnZjvdEAguk2E7zJ3dbV50,32849
|
|
282
|
+
funboost-46.9.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
|
|
283
|
+
funboost-46.9.dist-info/entry_points.txt,sha256=BQMqRALuw-QT9x2d7puWaUHriXfy3wIzvfzF61AnSSI,97
|
|
284
|
+
funboost-46.9.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
|
|
285
|
+
funboost-46.9.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|