funboost 43.2__py3-none-any.whl → 43.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of funboost might be problematic. Click here for more details.
- funboost/__init__.py +1 -1
- funboost/assist/celery_helper.py +1 -0
- funboost/concurrent_pool/__init__.py +0 -2
- funboost/concurrent_pool/custom_evenlet_pool_executor.py +29 -19
- funboost/concurrent_pool/custom_gevent_pool_executor.py +34 -27
- funboost/consumers/base_consumer.py +4 -4
- funboost/consumers/celery_consumer.py +2 -2
- funboost/contrib/api_publish_msg.py +2 -1
- funboost/core/fabric_deploy_helper.py +9 -3
- funboost/core/lazy_impoter.py +56 -4
- funboost/funboost_config_deafult.py +1 -1
- funboost/utils/decorators.py +56 -4
- {funboost-43.2.dist-info → funboost-43.4.dist-info}/METADATA +2 -4
- {funboost-43.2.dist-info → funboost-43.4.dist-info}/RECORD +18 -18
- {funboost-43.2.dist-info → funboost-43.4.dist-info}/LICENSE +0 -0
- {funboost-43.2.dist-info → funboost-43.4.dist-info}/WHEEL +0 -0
- {funboost-43.2.dist-info → funboost-43.4.dist-info}/entry_points.txt +0 -0
- {funboost-43.2.dist-info → funboost-43.4.dist-info}/top_level.txt +0 -0
funboost/__init__.py
CHANGED
funboost/assist/celery_helper.py
CHANGED
|
@@ -12,8 +12,6 @@ gevent协程
|
|
|
12
12
|
"""
|
|
13
13
|
from .base_pool_type import FunboostBaseConcurrentPool
|
|
14
14
|
from .async_pool_executor import AsyncPoolExecutor
|
|
15
|
-
# from .custom_evenlet_pool_executor import CustomEventletPoolExecutor
|
|
16
|
-
# from .custom_gevent_pool_executor import GeventPoolExecutor
|
|
17
15
|
from .bounded_threadpoolexcutor import BoundedThreadPoolExecutor
|
|
18
16
|
from .custom_threadpool_executor import CustomThreadPoolExecutor
|
|
19
17
|
from .flexible_thread_pool import FlexibleThreadPool
|
|
@@ -4,19 +4,26 @@
|
|
|
4
4
|
import atexit
|
|
5
5
|
import time
|
|
6
6
|
import warnings
|
|
7
|
-
from eventlet import greenpool, monkey_patch, patcher, Timeout
|
|
7
|
+
# from eventlet import greenpool, monkey_patch, patcher, Timeout
|
|
8
8
|
|
|
9
9
|
from funboost.concurrent_pool import FunboostBaseConcurrentPool
|
|
10
10
|
from funboost.core.loggers import get_funboost_file_logger
|
|
11
11
|
# print('eventlet 导入')
|
|
12
|
+
from funboost.core.lazy_impoter import EventletImporter
|
|
13
|
+
|
|
12
14
|
|
|
13
15
|
def check_evenlet_monkey_patch(raise_exc=True):
|
|
14
|
-
|
|
16
|
+
try:
|
|
17
|
+
if not EventletImporter().patcher.is_monkey_patched('socket'): # 随便选一个检测标志
|
|
18
|
+
if raise_exc:
|
|
19
|
+
warnings.warn(f'检测到没有打 evenlet 包的猴子补丁 ,请在起始脚本文件首行加上 import eventlet;eventlet.monkey_patch(all=True) ')
|
|
20
|
+
raise Exception('检测到没有打 evenlet 包的猴子补丁 ,请在起始脚本文件首行加上 import eventlet;eventlet.monkey_patch(all=True)')
|
|
21
|
+
else:
|
|
22
|
+
return 1
|
|
23
|
+
except ModuleNotFoundError:
|
|
15
24
|
if raise_exc:
|
|
16
25
|
warnings.warn(f'检测到没有打 evenlet 包的猴子补丁 ,请在起始脚本文件首行加上 import eventlet;eventlet.monkey_patch(all=True) ')
|
|
17
26
|
raise Exception('检测到没有打 evenlet 包的猴子补丁 ,请在起始脚本文件首行加上 import eventlet;eventlet.monkey_patch(all=True)')
|
|
18
|
-
else:
|
|
19
|
-
return 1
|
|
20
27
|
|
|
21
28
|
|
|
22
29
|
logger_evenlet_timeout_deco = get_funboost_file_logger('evenlet_timeout_deco')
|
|
@@ -25,12 +32,12 @@ logger_evenlet_timeout_deco = get_funboost_file_logger('evenlet_timeout_deco')
|
|
|
25
32
|
def evenlet_timeout_deco(timeout_t):
|
|
26
33
|
def _evenlet_timeout_deco(f):
|
|
27
34
|
def __evenlet_timeout_deco(*args, **kwargs):
|
|
28
|
-
timeout = Timeout(timeout_t, )
|
|
35
|
+
timeout = EventletImporter().Timeout(timeout_t, )
|
|
29
36
|
# timeout.start() # 与gevent不一样,直接start了。
|
|
30
37
|
result = None
|
|
31
38
|
try:
|
|
32
39
|
result = f(*args, **kwargs)
|
|
33
|
-
except Timeout as t:
|
|
40
|
+
except EventletImporter().Timeout as t:
|
|
34
41
|
logger_evenlet_timeout_deco.error(f'函数 {f} 运行超过了 {timeout_t} 秒')
|
|
35
42
|
if t is not timeout:
|
|
36
43
|
print(t)
|
|
@@ -44,24 +51,27 @@ def evenlet_timeout_deco(timeout_t):
|
|
|
44
51
|
return _evenlet_timeout_deco
|
|
45
52
|
|
|
46
53
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
54
|
+
def get_eventlet_pool_executor(*args2, **kwargs2):
|
|
55
|
+
class CustomEventletPoolExecutor(EventletImporter().greenpool.GreenPool, FunboostBaseConcurrentPool):
|
|
56
|
+
def __init__(self, *args, **kwargs):
|
|
57
|
+
super().__init__(*args, **kwargs)
|
|
58
|
+
check_evenlet_monkey_patch() # basecomer.py中检查。
|
|
59
|
+
atexit.register(self.shutdown)
|
|
60
|
+
|
|
61
|
+
def submit(self, *args, **kwargs): # 保持为一直的公有用法。
|
|
62
|
+
# nb_print(args)
|
|
63
|
+
self.spawn_n(*args, **kwargs)
|
|
64
|
+
# self.spawn_n(*args, **kwargs)
|
|
52
65
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
self.spawn_n(*args, **kwargs)
|
|
56
|
-
# self.spawn_n(*args, **kwargs)
|
|
66
|
+
def shutdown(self):
|
|
67
|
+
self.waitall()
|
|
57
68
|
|
|
58
|
-
|
|
59
|
-
self.waitall()
|
|
69
|
+
return CustomEventletPoolExecutor(*args2, **kwargs2)
|
|
60
70
|
|
|
61
71
|
|
|
62
72
|
if __name__ == '__main__':
|
|
63
73
|
# greenpool.GreenPool.waitall()
|
|
64
|
-
monkey_patch(all=True)
|
|
74
|
+
EventletImporter().monkey_patch(all=True)
|
|
65
75
|
|
|
66
76
|
|
|
67
77
|
def f2(x):
|
|
@@ -70,7 +80,7 @@ if __name__ == '__main__':
|
|
|
70
80
|
print(x)
|
|
71
81
|
|
|
72
82
|
|
|
73
|
-
pool =
|
|
83
|
+
pool = get_eventlet_pool_executor(4)
|
|
74
84
|
|
|
75
85
|
for i in range(15):
|
|
76
86
|
print(f'放入{i}')
|
|
@@ -7,23 +7,27 @@ import warnings
|
|
|
7
7
|
# from collections import Callable
|
|
8
8
|
from typing import Callable
|
|
9
9
|
import threading
|
|
10
|
-
import
|
|
11
|
-
from gevent import pool as gevent_pool
|
|
12
|
-
from gevent import monkey
|
|
13
|
-
from gevent.queue import JoinableQueue
|
|
10
|
+
from funboost.core.lazy_impoter import GeventImporter
|
|
14
11
|
|
|
15
12
|
# from nb_log import LoggerMixin, nb_print, LogManager
|
|
16
13
|
from funboost.concurrent_pool import FunboostBaseConcurrentPool
|
|
17
|
-
from funboost.core.loggers import get_funboost_file_logger,FunboostFileLoggerMixin
|
|
14
|
+
from funboost.core.loggers import get_funboost_file_logger, FunboostFileLoggerMixin
|
|
15
|
+
|
|
16
|
+
|
|
18
17
|
# print('gevent 导入')
|
|
19
18
|
|
|
20
19
|
def check_gevent_monkey_patch(raise_exc=True):
|
|
21
|
-
|
|
20
|
+
try:
|
|
21
|
+
if not GeventImporter().monkey.is_module_patched('socket'): # 随便选一个检测标志
|
|
22
|
+
if raise_exc:
|
|
23
|
+
warnings.warn(f'检测到 你还没有打gevent包的猴子补丁,请在所运行的起始脚本第一行写上 【import gevent.monkey;gevent.monkey.patch_all()】 这句话。')
|
|
24
|
+
raise Exception(f'检测到 你还没有打gevent包的猴子补丁,请在所运行的起始脚本第一行写上 【import gevent.monkey;gevent.monkey.patch_all()】 这句话。')
|
|
25
|
+
else:
|
|
26
|
+
return 1
|
|
27
|
+
except ModuleNotFoundError:
|
|
22
28
|
if raise_exc:
|
|
23
29
|
warnings.warn(f'检测到 你还没有打gevent包的猴子补丁,请在所运行的起始脚本第一行写上 【import gevent.monkey;gevent.monkey.patch_all()】 这句话。')
|
|
24
30
|
raise Exception(f'检测到 你还没有打gevent包的猴子补丁,请在所运行的起始脚本第一行写上 【import gevent.monkey;gevent.monkey.patch_all()】 这句话。')
|
|
25
|
-
else:
|
|
26
|
-
return 1
|
|
27
31
|
|
|
28
32
|
|
|
29
33
|
logger_gevent_timeout_deco = get_funboost_file_logger('gevent_timeout_deco')
|
|
@@ -32,12 +36,12 @@ logger_gevent_timeout_deco = get_funboost_file_logger('gevent_timeout_deco')
|
|
|
32
36
|
def gevent_timeout_deco(timeout_t):
|
|
33
37
|
def _gevent_timeout_deco(f):
|
|
34
38
|
def __gevent_timeout_deceo(*args, **kwargs):
|
|
35
|
-
timeout = gevent.Timeout(timeout_t, )
|
|
39
|
+
timeout = GeventImporter().gevent.Timeout(timeout_t, )
|
|
36
40
|
timeout.start()
|
|
37
41
|
result = None
|
|
38
42
|
try:
|
|
39
43
|
result = f(*args, **kwargs)
|
|
40
|
-
except gevent.Timeout as t:
|
|
44
|
+
except GeventImporter().gevent.Timeout as t:
|
|
41
45
|
logger_gevent_timeout_deco.error(f'函数 {f} 运行超过了 {timeout_t} 秒')
|
|
42
46
|
if t is not timeout:
|
|
43
47
|
print(t)
|
|
@@ -51,25 +55,28 @@ def gevent_timeout_deco(timeout_t):
|
|
|
51
55
|
return _gevent_timeout_deco
|
|
52
56
|
|
|
53
57
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
58
|
+
def get_gevent_pool_executor(size=None, greenlet_class=None):
|
|
59
|
+
class GeventPoolExecutor(GeventImporter().gevent_pool.Pool, FunboostBaseConcurrentPool):
|
|
60
|
+
def __init__(self, size2=None, greenlet_class2=None):
|
|
61
|
+
check_gevent_monkey_patch() # basecomer.py中检查。
|
|
62
|
+
super().__init__(size2, greenlet_class2)
|
|
63
|
+
atexit.register(self.shutdown)
|
|
64
|
+
|
|
65
|
+
def submit(self, *args, **kwargs):
|
|
66
|
+
self.spawn(*args, **kwargs)
|
|
59
67
|
|
|
60
|
-
|
|
61
|
-
|
|
68
|
+
def shutdown(self):
|
|
69
|
+
self.join()
|
|
62
70
|
|
|
63
|
-
|
|
64
|
-
self.join()
|
|
71
|
+
return GeventPoolExecutor(size, greenlet_class)
|
|
65
72
|
|
|
66
73
|
|
|
67
|
-
class GeventPoolExecutor2(FunboostFileLoggerMixin,FunboostBaseConcurrentPool):
|
|
74
|
+
class GeventPoolExecutor2(FunboostFileLoggerMixin, FunboostBaseConcurrentPool):
|
|
68
75
|
def __init__(self, max_works, ):
|
|
69
|
-
self._q = JoinableQueue(maxsize=max_works)
|
|
76
|
+
self._q = GeventImporter().JoinableQueue(maxsize=max_works)
|
|
70
77
|
# self._q = Queue(maxsize=max_works)
|
|
71
78
|
for _ in range(max_works):
|
|
72
|
-
gevent.spawn(self.__worker)
|
|
79
|
+
GeventImporter().gevent.spawn(self.__worker)
|
|
73
80
|
# atexit.register(self.__atexit)
|
|
74
81
|
self._q.join(timeout=100)
|
|
75
82
|
|
|
@@ -92,12 +99,12 @@ class GeventPoolExecutor2(FunboostFileLoggerMixin,FunboostBaseConcurrentPool):
|
|
|
92
99
|
self._q.join()
|
|
93
100
|
|
|
94
101
|
|
|
95
|
-
class GeventPoolExecutor3(FunboostFileLoggerMixin,FunboostBaseConcurrentPool):
|
|
102
|
+
class GeventPoolExecutor3(FunboostFileLoggerMixin, FunboostBaseConcurrentPool):
|
|
96
103
|
def __init__(self, max_works, ):
|
|
97
|
-
self._q = gevent.queue.Queue(max_works)
|
|
104
|
+
self._q = GeventImporter().gevent.queue.Queue(max_works)
|
|
98
105
|
self.g_list = []
|
|
99
106
|
for _ in range(max_works):
|
|
100
|
-
self.g_list.append(gevent.spawn(self.__worker))
|
|
107
|
+
self.g_list.append(GeventImporter().gevent.spawn(self.__worker))
|
|
101
108
|
atexit.register(self.__atexit)
|
|
102
109
|
|
|
103
110
|
def __worker(self):
|
|
@@ -112,7 +119,7 @@ class GeventPoolExecutor3(FunboostFileLoggerMixin,FunboostBaseConcurrentPool):
|
|
|
112
119
|
self._q.put((fn, args, kwargs))
|
|
113
120
|
|
|
114
121
|
def joinall(self):
|
|
115
|
-
gevent.joinall(self.g_list)
|
|
122
|
+
GeventImporter().gevent.joinall(self.g_list)
|
|
116
123
|
|
|
117
124
|
def joinall_in_new_thread(self):
|
|
118
125
|
threading.Thread(target=self.joinall)
|
|
@@ -123,7 +130,7 @@ class GeventPoolExecutor3(FunboostFileLoggerMixin,FunboostBaseConcurrentPool):
|
|
|
123
130
|
|
|
124
131
|
|
|
125
132
|
if __name__ == '__main__':
|
|
126
|
-
monkey.patch_all(thread=False)
|
|
133
|
+
GeventImporter().monkey.patch_all(thread=False)
|
|
127
134
|
|
|
128
135
|
|
|
129
136
|
def f2(x):
|
|
@@ -972,11 +972,11 @@ class ConcurrentModeDispatcher(FunboostFileLoggerMixin):
|
|
|
972
972
|
# pool_type = BoundedThreadPoolExecutor
|
|
973
973
|
pool_type = FlexibleThreadPool
|
|
974
974
|
elif self._concurrent_mode == ConcurrentModeEnum.GEVENT:
|
|
975
|
-
from funboost.concurrent_pool.custom_gevent_pool_executor import
|
|
976
|
-
pool_type =
|
|
975
|
+
from funboost.concurrent_pool.custom_gevent_pool_executor import get_gevent_pool_executor
|
|
976
|
+
pool_type = get_gevent_pool_executor
|
|
977
977
|
elif self._concurrent_mode == ConcurrentModeEnum.EVENTLET:
|
|
978
|
-
from funboost.concurrent_pool.custom_evenlet_pool_executor import
|
|
979
|
-
pool_type =
|
|
978
|
+
from funboost.concurrent_pool.custom_evenlet_pool_executor import get_eventlet_pool_executor
|
|
979
|
+
pool_type = get_eventlet_pool_executor
|
|
980
980
|
elif self._concurrent_mode == ConcurrentModeEnum.ASYNC:
|
|
981
981
|
pool_type = AsyncPoolExecutor
|
|
982
982
|
elif self._concurrent_mode == ConcurrentModeEnum.SINGLE_THREAD:
|
|
@@ -175,10 +175,10 @@ class CeleryConsumer(AbstractConsumer):
|
|
|
175
175
|
# print(this.request.__dict__,dir(this))
|
|
176
176
|
if this.request.retries != self.consumer_params.max_retry_times:
|
|
177
177
|
log_msg = f'fun: {self.consuming_function} args: {args} , kwargs: {kwargs} 消息第{this.request.retries}次运行出错, {exc} \n'
|
|
178
|
-
self.
|
|
178
|
+
self.logger.error(log_msg, exc_info=self.consumer_params.is_print_detail_exception)
|
|
179
179
|
else:
|
|
180
180
|
log_msg = f'fun: {self.consuming_function} args: {args} , kwargs: {kwargs} 消息达到最大重试次数{this.request.retries}次仍然出错, {exc} \n'
|
|
181
|
-
self.
|
|
181
|
+
self.logger.critical(log_msg, exc_info=self.consumer_params.is_print_detail_exception)
|
|
182
182
|
# 发生异常,尝试重试任务,countdown 是多少秒后重试
|
|
183
183
|
raise this.retry(exc=exc, countdown=5)
|
|
184
184
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import traceback
|
|
2
|
+
import typing
|
|
2
3
|
|
|
3
4
|
from funboost import AioAsyncResult, AsyncResult
|
|
4
5
|
|
|
@@ -18,7 +19,7 @@ class MsgItem(BaseModel):
|
|
|
18
19
|
class PublishResponse(BaseModel):
|
|
19
20
|
succ: bool
|
|
20
21
|
msg: str
|
|
21
|
-
status_and_result: dict = None # 消费函数的消费状态和结果.
|
|
22
|
+
status_and_result: typing.Optional[dict] = None # 消费函数的消费状态和结果.
|
|
22
23
|
|
|
23
24
|
|
|
24
25
|
# 创建 FastAPI 应用实例
|
|
@@ -3,6 +3,7 @@ import re
|
|
|
3
3
|
import sys
|
|
4
4
|
import threading
|
|
5
5
|
import time
|
|
6
|
+
import os
|
|
6
7
|
|
|
7
8
|
from fabric2 import Connection
|
|
8
9
|
|
|
@@ -14,6 +15,11 @@ from funboost.core.loggers import get_funboost_file_logger
|
|
|
14
15
|
from funboost.core.booster import Booster
|
|
15
16
|
logger = get_funboost_file_logger(__name__)
|
|
16
17
|
|
|
18
|
+
def _remove_drive_letter(path):
|
|
19
|
+
if os.name == 'nt' and len(path) >= 3 and path[1] == ':':
|
|
20
|
+
return path[2:]
|
|
21
|
+
else:
|
|
22
|
+
return path
|
|
17
23
|
|
|
18
24
|
# noinspection PyDefaultArgument
|
|
19
25
|
def fabric_deploy(booster:Booster, host, port, user, password,
|
|
@@ -65,11 +71,11 @@ def fabric_deploy(booster:Booster, host, port, user, password,
|
|
|
65
71
|
task_fun.fabric_deploy('192.168.6.133', 22, 'ydf', '123456', process_num=2) 只需要这样就可以自动部署在远程机器运行,无需任何额外操作。
|
|
66
72
|
"""
|
|
67
73
|
# print(locals())
|
|
68
|
-
python_proj_dir = sys.path[1].replace('\\', '/') + '/'
|
|
74
|
+
python_proj_dir = _remove_drive_letter(sys.path[1].replace('\\', '/') + '/')
|
|
69
75
|
python_proj_dir_short = python_proj_dir.split('/')[-2]
|
|
70
76
|
# 获取被调用函数所在模块文件名
|
|
71
|
-
file_name = sys._getframe(2).f_code.co_filename.replace('\\', '/') # noqa\
|
|
72
|
-
|
|
77
|
+
file_name = _remove_drive_letter(sys._getframe(2).f_code.co_filename.replace('\\', '/')) # noqa\
|
|
78
|
+
print(file_name,python_proj_dir)
|
|
73
79
|
relative_file_name = re.sub(f'^{python_proj_dir}', '', file_name)
|
|
74
80
|
relative_module = relative_file_name.replace('/', '.')[:-3] # -3是去掉.py
|
|
75
81
|
if user == 'root': # 文件夹会被自动创建,无需用户创建。
|
funboost/core/lazy_impoter.py
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
import abc
|
|
2
2
|
|
|
3
|
+
from funboost.utils.decorators import cached_method_result, singleton, SingletonBaseNew, SingletonBaseCustomInit
|
|
3
4
|
|
|
4
|
-
|
|
5
|
+
|
|
6
|
+
# @singleton # 不方便代码补全
|
|
7
|
+
|
|
8
|
+
class LazyImpoter(SingletonBaseNew):
|
|
5
9
|
"""
|
|
6
10
|
延迟导入,避免需要互相导入.
|
|
7
11
|
"""
|
|
@@ -18,9 +22,57 @@ class LazyImpoter:
|
|
|
18
22
|
# from funboost.core.current_task import get_current_taskid
|
|
19
23
|
# return get_current_taskid
|
|
20
24
|
|
|
25
|
+
|
|
21
26
|
lazy_impoter = LazyImpoter()
|
|
22
27
|
|
|
28
|
+
|
|
29
|
+
class GeventImporter(SingletonBaseNew):
|
|
30
|
+
"""
|
|
31
|
+
import gevent
|
|
32
|
+
from gevent import pool as gevent_pool
|
|
33
|
+
from gevent import monkey
|
|
34
|
+
from gevent.queue import JoinableQueue
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
@cached_method_result
|
|
39
|
+
def gevent(self):
|
|
40
|
+
import gevent
|
|
41
|
+
return gevent
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
@cached_method_result
|
|
45
|
+
def gevent_pool(self):
|
|
46
|
+
from gevent import pool as gevent_pool
|
|
47
|
+
return gevent_pool
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
@cached_method_result
|
|
51
|
+
def monkey(self):
|
|
52
|
+
from gevent import monkey
|
|
53
|
+
return monkey
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
@cached_method_result
|
|
57
|
+
def JoinableQueue(self):
|
|
58
|
+
from gevent.queue import JoinableQueue
|
|
59
|
+
return JoinableQueue
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class EventletImporter(SingletonBaseCustomInit):
|
|
63
|
+
"""
|
|
64
|
+
from eventlet import greenpool, monkey_patch, patcher, Timeout
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
def _custom_init(self, ):
|
|
68
|
+
from eventlet import greenpool, monkey_patch, patcher, Timeout
|
|
69
|
+
self.greenpool = greenpool
|
|
70
|
+
self.monkey_patch = monkey_patch
|
|
71
|
+
self.patcher = patcher
|
|
72
|
+
self.Timeout = Timeout
|
|
73
|
+
|
|
74
|
+
|
|
23
75
|
if __name__ == '__main__':
|
|
24
76
|
for i in range(10000):
|
|
25
|
-
lazy_impoter.BoostersManager
|
|
26
|
-
|
|
77
|
+
# lazy_impoter.BoostersManager
|
|
78
|
+
EventletImporter().greenpool
|
|
@@ -100,7 +100,7 @@ class FunboostCommonConfig(DataClassBase):
|
|
|
100
100
|
# NB_LOG_FORMATER_INDEX_FOR_CONSUMER_AND_PUBLISHER = 11 # 7是简短的不可跳转,5是可点击跳转的,11是可显示ip 进程 线程的模板,也可以亲自设置日志模板不传递数字。
|
|
101
101
|
NB_LOG_FORMATER_INDEX_FOR_CONSUMER_AND_PUBLISHER = logging.Formatter(
|
|
102
102
|
f'%(asctime)s-({nb_log_config_default.computer_ip},{nb_log_config_default.computer_name})-[p%(process)d_t%(thread)d] - %(name)s - "%(filename)s:%(lineno)d" - %(funcName)s - %(levelname)s - %(task_id)s - %(message)s',
|
|
103
|
-
"%Y-%m-%d %H:%M:%S",) # 这个是带task_id的日志模板,日志可以显示task_id
|
|
103
|
+
"%Y-%m-%d %H:%M:%S",) # 这个是带task_id的日志模板,日志可以显示task_id,方便用户串联起来排查某一个任务消息的所有日志.
|
|
104
104
|
|
|
105
105
|
TIMEZONE = 'Asia/Shanghai' # 时区
|
|
106
106
|
|
funboost/utils/decorators.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# coding=utf-8
|
|
2
2
|
import base64
|
|
3
3
|
import copy
|
|
4
|
+
import abc
|
|
4
5
|
import logging
|
|
5
6
|
import random
|
|
6
7
|
import uuid
|
|
@@ -165,6 +166,60 @@ def singleton(cls):
|
|
|
165
166
|
return _singleton
|
|
166
167
|
|
|
167
168
|
|
|
169
|
+
class SingletonMeta(type):
|
|
170
|
+
_instances = {}
|
|
171
|
+
|
|
172
|
+
def __call__(cls, *args, **kwargs):
|
|
173
|
+
if cls not in cls._instances:
|
|
174
|
+
cls._instances[cls] = super().__call__(*args, **kwargs)
|
|
175
|
+
return cls._instances[cls]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class SingletonBaseCall(metaclass=SingletonMeta):
|
|
179
|
+
"""
|
|
180
|
+
单例基类。任何继承自这个基类的子类都会自动成为单例。
|
|
181
|
+
|
|
182
|
+
示例:
|
|
183
|
+
class MyClass(SingletonBase):
|
|
184
|
+
pass
|
|
185
|
+
|
|
186
|
+
instance1 = MyClass()
|
|
187
|
+
instance2 = MyClass()
|
|
188
|
+
|
|
189
|
+
assert instance1 is instance2 # 实例1和实例2实际上是同一个对象
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
def __init_subclass__(cls, **kwargs):
|
|
193
|
+
super().__init_subclass__(**kwargs)
|
|
194
|
+
# 可以在此处添加对子类的额外处理,比如检查其是否符合单例要求等
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class SingletonBaseNew:
|
|
198
|
+
_instance = None
|
|
199
|
+
|
|
200
|
+
def __new__(cls, *args, **kwargs):
|
|
201
|
+
if not cls._instance:
|
|
202
|
+
cls._instance = super().__new__(cls)
|
|
203
|
+
return cls._instance
|
|
204
|
+
|
|
205
|
+
def __init_subclass__(cls, **kwargs):
|
|
206
|
+
super().__init_subclass__(**kwargs)
|
|
207
|
+
# 可以在此处添加对子类的额外处理,比如检查其是否符合单例要求等
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
class SingletonBaseCustomInit(metaclass=abc.ABCMeta):
|
|
211
|
+
_instance = None
|
|
212
|
+
|
|
213
|
+
def __new__(cls, *args, **kwargs):
|
|
214
|
+
if not cls._instance:
|
|
215
|
+
cls._instance = super().__new__(cls)
|
|
216
|
+
cls._instance._custom_init(*args, **kwargs)
|
|
217
|
+
return cls._instance
|
|
218
|
+
|
|
219
|
+
def _custom_init(self, *args, **kwargs):
|
|
220
|
+
raise NotImplemented
|
|
221
|
+
|
|
222
|
+
|
|
168
223
|
def flyweight(cls):
|
|
169
224
|
_instance = {}
|
|
170
225
|
|
|
@@ -291,6 +346,7 @@ class RedisDistributedLockContextManager(LoggerMixin, LoggerLevelSetterMixin):
|
|
|
291
346
|
else:
|
|
292
347
|
return False
|
|
293
348
|
|
|
349
|
+
|
|
294
350
|
"""
|
|
295
351
|
@contextmanager
|
|
296
352
|
def some_generator(<arguments>):
|
|
@@ -536,8 +592,6 @@ def timeout(seconds):
|
|
|
536
592
|
|
|
537
593
|
def timeout_decorator(func):
|
|
538
594
|
|
|
539
|
-
|
|
540
|
-
|
|
541
595
|
def _(*args, **kwargs):
|
|
542
596
|
def _new_func(oldfunc, result, oldfunc_args, oldfunc_kwargs):
|
|
543
597
|
result.append(oldfunc(*oldfunc_args, **oldfunc_kwargs))
|
|
@@ -738,8 +792,6 @@ class _Test(unittest.TestCase):
|
|
|
738
792
|
f(5)
|
|
739
793
|
|
|
740
794
|
|
|
741
|
-
|
|
742
|
-
|
|
743
795
|
if __name__ == '__main__':
|
|
744
796
|
pass
|
|
745
797
|
unittest.main()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: funboost
|
|
3
|
-
Version: 43.
|
|
3
|
+
Version: 43.4
|
|
4
4
|
Summary: pip install funboost,python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件,支持如 celery dramatiq等框架整体作为funboost中间件,python函数加速器,框架包罗万象,用户能想到的控制功能全都有。一统编程思维,兼容50% python业务场景,适用范围广。只需要一行代码即可分布式执行python一切函数,99%用过funboost的pythoner 感受是 简易 方便 强劲 强大,相见恨晚
|
|
5
5
|
Home-page: https://github.com/ydf0509/funboost
|
|
6
6
|
Author: bfzs
|
|
@@ -26,9 +26,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
26
26
|
Classifier: Topic :: Software Development :: Libraries
|
|
27
27
|
Description-Content-Type: text/markdown
|
|
28
28
|
License-File: LICENSE
|
|
29
|
-
Requires-Dist: nb-log >=12.
|
|
30
|
-
Requires-Dist: eventlet ==0.33.3
|
|
31
|
-
Requires-Dist: gevent ==22.10.2
|
|
29
|
+
Requires-Dist: nb-log >=12.6
|
|
32
30
|
Requires-Dist: pymongo ==4.3.3
|
|
33
31
|
Requires-Dist: AMQPStorm ==2.10.6
|
|
34
32
|
Requires-Dist: rabbitpy ==2.0.1
|
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
funboost/__init__.py,sha256=
|
|
1
|
+
funboost/__init__.py,sha256=Asj6x0xGXr9H1kCXh2hFMM15NJhNzrf24jbHxUQiCHA,3832
|
|
2
2
|
funboost/__init__old.py,sha256=07A1MLsxLtuRQQOIfDyphddOwNBrGe34enoHWAnjV14,20379
|
|
3
3
|
funboost/__main__.py,sha256=-6Nogi666Y0LN8fVm3JmHGTOk8xEGWvotW_GDbSaZME,1065
|
|
4
4
|
funboost/constant.py,sha256=Yxt3WJt9D8ybcrgiojOy0qjnq5mffwTnTJplerGL0Oo,7188
|
|
5
|
-
funboost/funboost_config_deafult.py,sha256=
|
|
5
|
+
funboost/funboost_config_deafult.py,sha256=rwfHVttpqWX5tGoO7MdiCquHSrK-z56U81df-jnavlc,6613
|
|
6
6
|
funboost/set_frame_config.py,sha256=bEsW4a4EuE3PpyypNuWSfsB-_bvT94pktJGKk4r-rlo,14328
|
|
7
7
|
funboost/assist/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
-
funboost/assist/celery_helper.py,sha256=
|
|
8
|
+
funboost/assist/celery_helper.py,sha256=EOxsl-y65JPwHrmL0LTGAj1nKLbS0qyRbuXcJ0xo9wc,5721
|
|
9
9
|
funboost/assist/dramatiq_helper.py,sha256=9mUyfBMAJXzwvB8LwOmapn3rY30a6UXt3tNOfL6OXoM,2106
|
|
10
10
|
funboost/assist/huey_helper.py,sha256=PuJHIzK5oRd5RzbuxLMHhWlkKO3J-ObRK0mrMs_iQWs,1770
|
|
11
11
|
funboost/assist/rocketry_helper.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
12
|
funboost/assist/rq_helper.py,sha256=HYvQ7VqIMx7dAVZZhtqQmGReCJavEuc1BQPYvfm1EvY,1549
|
|
13
13
|
funboost/assist/rq_windows_worker.py,sha256=jQlGmU0FWPVoKVP8cyuwTuAca9VfSd75F5Qw_hR04y0,4831
|
|
14
14
|
funboost/beggar_version_implementation/beggar_redis_consumer.py,sha256=x5cH6Vc3_UooY2oPeC9MlpMewrGd9qXCx6gEWi1fGa0,3941
|
|
15
|
-
funboost/concurrent_pool/__init__.py,sha256=
|
|
15
|
+
funboost/concurrent_pool/__init__.py,sha256=C27xYXj7c1NGuVeG7K2LxKH7JKnGwfPfITyon6xFaoE,803
|
|
16
16
|
funboost/concurrent_pool/async_helper.py,sha256=iyb0Jcjyx-vkUGC_saSUWqN657kcR5K7B-L_SB6cDCE,3256
|
|
17
17
|
funboost/concurrent_pool/async_pool_executor.py,sha256=n2T4WgUitloFgHfRXTaD5Ro29WkR8qEw1ftEIub03Ug,7515
|
|
18
18
|
funboost/concurrent_pool/base_pool_type.py,sha256=h3xcadufMAf49CoNe5VkUyIxlniMeNtDjadqB5IsiKA,194
|
|
@@ -20,8 +20,8 @@ funboost/concurrent_pool/bounded_processpoolexcutor_gt_py37.py,sha256=pWqpKAP0Z0
|
|
|
20
20
|
funboost/concurrent_pool/bounded_processpoolexcutor_py36.py,sha256=CMRSjwnUAhNT2fj44SVNEYRehprlxjQADYkgw9hTVlw,3063
|
|
21
21
|
funboost/concurrent_pool/bounded_threadpoolexcutor.py,sha256=AWFsTwK7hIiln0XVaiY8BXFO8MffIPWgYBRqLpvwEHQ,1675
|
|
22
22
|
funboost/concurrent_pool/concurrent_pool_with_multi_process.py,sha256=qChdRh11Epk7-kmLEdoB1TtM2plx_FZ1u_-g0KNc-MU,1744
|
|
23
|
-
funboost/concurrent_pool/custom_evenlet_pool_executor.py,sha256=
|
|
24
|
-
funboost/concurrent_pool/custom_gevent_pool_executor.py,sha256=
|
|
23
|
+
funboost/concurrent_pool/custom_evenlet_pool_executor.py,sha256=tu8Og-uIt5VWqGERnIihBaF7ZFw5D0xww1_ur39OZXY,3252
|
|
24
|
+
funboost/concurrent_pool/custom_gevent_pool_executor.py,sha256=FLcEwEJgFtaEPzH-WCFHtCdEUOTeK0dLSL_oNBcXQW8,5429
|
|
25
25
|
funboost/concurrent_pool/custom_threadpool_executor.py,sha256=i3OGtOjk5GOGvGjEMdYVzFVscSVPRFWmEe1yZeZLsCQ,11872
|
|
26
26
|
funboost/concurrent_pool/custom_threadpool_executor000.py,sha256=jJLXy3h-bELap6nZA6yLtdozzTWcvCtZ7IY6MTqLEAM,9317
|
|
27
27
|
funboost/concurrent_pool/fixed_thread_pool.py,sha256=JzaqjuHn6ffo1gZRVJEy5Te5NdCJt7heTmsVZT_AiBg,1609
|
|
@@ -33,8 +33,8 @@ funboost/concurrent_pool/backup/async_pool_executor0223.py,sha256=RVUZiylUvpTm6U
|
|
|
33
33
|
funboost/concurrent_pool/backup/async_pool_executor_back.py,sha256=KL6zEQaa1KkZOlAO85mCC1gwLm-YC5Ghn21IUom0UKM,9598
|
|
34
34
|
funboost/concurrent_pool/backup/async_pool_executor_janus.py,sha256=OHMWJ9l3EYTpPpcrPrGGKd4K0tmQ2PN8HiX0Dta0EOo,5728
|
|
35
35
|
funboost/consumers/__init__.py,sha256=ZXY_6Kut1VYNQiF5aWEgIWobsW1ht9YUP0TdRZRWFqI,126
|
|
36
|
-
funboost/consumers/base_consumer.py,sha256=
|
|
37
|
-
funboost/consumers/celery_consumer.py,sha256=
|
|
36
|
+
funboost/consumers/base_consumer.py,sha256=5e-nHq6lGAnYzP0zLkbvtRFJPhHhZawTiiv1sJfYisg,74479
|
|
37
|
+
funboost/consumers/celery_consumer.py,sha256=9gtz7nlZkmv3ErmaseT0_Q__ltSPx-fOcwi-TMPoaLA,9220
|
|
38
38
|
funboost/consumers/confirm_mixin.py,sha256=eY6fNwx51Hn4bQSYRjyTRwOqfCGsikVnd2Ga_Ep31N4,6062
|
|
39
39
|
funboost/consumers/dramatiq_consumer.py,sha256=ozmeAfeF0U-YNYHK4suQB0N264h5AZdfMH0O45Mh-8A,2229
|
|
40
40
|
funboost/consumers/http_consumer.py,sha256=3HF8tsH90fUPX3iOmVid_nqW_7hZCFaL7feOkuAM36U,2025
|
|
@@ -74,7 +74,7 @@ funboost/consumers/txt_file_consumer.py,sha256=MlCv9INF6DkLgdU4NsNWynHq1Tgs10a2H
|
|
|
74
74
|
funboost/consumers/udp_consumer.py,sha256=J-G1ZYktXZ_er_1fg3FdSPVl4V_eEIHZXlBadCNpJmE,1643
|
|
75
75
|
funboost/consumers/zeromq_consumer.py,sha256=mRU1eC1OhhPnW6L9cQIztKcDg6bfVttsGRE-plC0AqE,4232
|
|
76
76
|
funboost/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
77
|
-
funboost/contrib/api_publish_msg.py,sha256=
|
|
77
|
+
funboost/contrib/api_publish_msg.py,sha256=8khUiOjbiaTuNlOU4_FMBYYNLkwHssPw-vnlE0iixVk,2480
|
|
78
78
|
funboost/contrib/django_db_deco.py,sha256=RJaRUYdVqS10gWqM4Ncs0Lngox52SUaqIIn5GK5a8Uo,865
|
|
79
79
|
funboost/contrib/queue2queue.py,sha256=4-28ULM7PTbmbOw8DG9rjlMUQCDkJNcUqkmdHAkGg2c,4898
|
|
80
80
|
funboost/contrib/redis_consume_latest_msg_broker.py,sha256=ESortBZ2qu_4PBCa3e3FeL2k_PClZNb74_v55HV-BOg,1902
|
|
@@ -84,13 +84,13 @@ funboost/core/active_cousumer_info_getter.py,sha256=09fEc-BTEIRfDDfHmOvKnMjLjtOy
|
|
|
84
84
|
funboost/core/booster.py,sha256=Y_HqdK1mOFUyTzuEU070kTWQfXR7j0WVHIOfrAdPhAQ,15831
|
|
85
85
|
funboost/core/current_task.py,sha256=rAIQVLqYqtlHVRkjl17yki-mqvuMb640ssGmto4RSdA,4864
|
|
86
86
|
funboost/core/exceptions.py,sha256=pLF7BkRJAfDiWp2_xGZqencmwdPiSQI1NENbImExknY,1311
|
|
87
|
-
funboost/core/fabric_deploy_helper.py,sha256=
|
|
87
|
+
funboost/core/fabric_deploy_helper.py,sha256=vI68U_3HsL5LtsP-CvnTdKCVC7qhdVI0QdW6l-XW9rs,9163
|
|
88
88
|
funboost/core/func_params_model.py,sha256=ChxvSq_iVuYnGR_CdN9h9qNtvBq8sZ4BocdzCjgsQHc,18167
|
|
89
89
|
funboost/core/function_result_status_config.py,sha256=PyjqAQOiwsLt28sRtH-eYRjiI3edPFO4Nde0ILFRReE,1764
|
|
90
90
|
funboost/core/function_result_status_saver.py,sha256=UdokGSwU630t70AZnT9Ecj7GpYXORBDivlc9kadoI2E,9172
|
|
91
91
|
funboost/core/helper_funs.py,sha256=M9Ad9EzgHdP581X-vuFgCavJRoezLGXlXSFg7zyMWD0,1578
|
|
92
92
|
funboost/core/kill_remote_task.py,sha256=MZ5vWLGt6SxyN76h5Lf_id9tyVUzjR-qXNyJwXaGlZY,8129
|
|
93
|
-
funboost/core/lazy_impoter.py,sha256=
|
|
93
|
+
funboost/core/lazy_impoter.py,sha256=3t2t0h9rUHd59wMh6xQQ-xYgxs1BP1Ntcjv6KuitSLM,1930
|
|
94
94
|
funboost/core/loggers.py,sha256=173aXdqE8nAe8t6OcVMNAFsCUClVrWQovdQhTAg9IyM,2407
|
|
95
95
|
funboost/core/msg_result_getter.py,sha256=oZDuLDR5XQNzzvgDTsA7wroICToPwrkU9-OAaXXUQSk,8031
|
|
96
96
|
funboost/core/muliti_process_enhance.py,sha256=64rkVa5Eel-0EY2B7lc1dQTRwX4ehARVvcxQVDa6jr0,3568
|
|
@@ -171,7 +171,7 @@ funboost/utils/block_exit.py,sha256=BnfxNYo3lnmhk686RAEoc4u3D4RU_iEMMMgu5L8gIuI,
|
|
|
171
171
|
funboost/utils/bulk_operation.py,sha256=u1cBL9qo3G-Pvr75coMg5p6EQxDkc8Iuf4C_PJI7BJQ,10013
|
|
172
172
|
funboost/utils/ctrl_c_end.py,sha256=FgT9An-qsUA5gW-V-UKWqOh5shC7C_uvTFn0fS7i8GI,439
|
|
173
173
|
funboost/utils/custom_pysnooper.py,sha256=7yXLKEMY_JjPRRt0Y0N-wV2CFhILlYNh40Y6uRBUaj8,5923
|
|
174
|
-
funboost/utils/decorators.py,sha256=
|
|
174
|
+
funboost/utils/decorators.py,sha256=TQ_LR5eligJNnVrZETiJvaq1LZ3OqVd8aRBJY_9TOTY,26063
|
|
175
175
|
funboost/utils/develop_log.py,sha256=Wsx0ongGjTit5xqgk1BztYlVEkC6d0-Y7GENXLedVqY,271
|
|
176
176
|
funboost/utils/expire_lock.py,sha256=AOkd1KlvZeIwQaz8ZoKxLpGxWgqQ4mfNHcFphh04o8Q,4732
|
|
177
177
|
funboost/utils/json_helper.py,sha256=HDdtLyZBGpWbUm7vmTypKXd8K-Hb-9BaxpdmRlKMYUI,1849
|
|
@@ -264,9 +264,9 @@ funboost/utils/pysnooper_ydf/utils.py,sha256=evSmGi_Oul7vSP47AJ0DLjFwoCYCfunJZ1m
|
|
|
264
264
|
funboost/utils/pysnooper_ydf/variables.py,sha256=QejRDESBA06KG9OH4sBT4J1M55eaU29EIHg8K_igaXo,3693
|
|
265
265
|
funboost/utils/times/__init__.py,sha256=Y4bQD3SIA_E7W2YvHq2Qdi0dGM4H2DxyFNdDOuFOq1w,2417
|
|
266
266
|
funboost/utils/times/version.py,sha256=11XfnZVVzOgIhXXdeN_mYfdXThfrsbQHpA0wCjz-hpg,17
|
|
267
|
-
funboost-43.
|
|
268
|
-
funboost-43.
|
|
269
|
-
funboost-43.
|
|
270
|
-
funboost-43.
|
|
271
|
-
funboost-43.
|
|
272
|
-
funboost-43.
|
|
267
|
+
funboost-43.4.dist-info/LICENSE,sha256=9EPP2ktG_lAPB8PjmWV-c9BiaJHc_FP6pPLcUrUwx0E,11562
|
|
268
|
+
funboost-43.4.dist-info/METADATA,sha256=3mgY8IrUDSZ0pWYP4dnHF4WfcfWzY1UHLJCH1_VhejA,30410
|
|
269
|
+
funboost-43.4.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
|
270
|
+
funboost-43.4.dist-info/entry_points.txt,sha256=yMSSAGRzRAAhGyNNQHw24MooKlDZsaJ499_D6fPl58A,96
|
|
271
|
+
funboost-43.4.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
|
|
272
|
+
funboost-43.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|