funboost 50.2__py3-none-any.whl → 50.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of funboost might be problematic. Click here for more details.
- funboost/__init__.py +1 -1
- funboost/consumers/base_consumer.py +4 -0
- funboost/consumers/grpc_consumer.py +1 -18
- funboost/consumers/http_consumer.py +102 -35
- funboost/consumers/http_consumer_aiohttp_old.py +113 -0
- funboost/consumers/kafka_consumer.py +1 -4
- funboost/core/booster.py +31 -2
- funboost/core/funboost_time.py +65 -21
- funboost/core/func_params_model.py +21 -3
- funboost/core/helper_funs.py +9 -8
- funboost/core/msg_result_getter.py +27 -0
- funboost/factories/broker_kind__publsiher_consumer_type_map.py +8 -3
- funboost/publishers/base_publisher.py +5 -0
- funboost/publishers/http_publisher.py +20 -2
- {funboost-50.2.dist-info → funboost-50.3.dist-info}/METADATA +7 -5
- {funboost-50.2.dist-info → funboost-50.3.dist-info}/RECORD +20 -19
- {funboost-50.2.dist-info → funboost-50.3.dist-info}/LICENSE +0 -0
- {funboost-50.2.dist-info → funboost-50.3.dist-info}/WHEEL +0 -0
- {funboost-50.2.dist-info → funboost-50.3.dist-info}/entry_points.txt +0 -0
- {funboost-50.2.dist-info → funboost-50.3.dist-info}/top_level.txt +0 -0
funboost/__init__.py
CHANGED
|
@@ -639,6 +639,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
639
639
|
def _frame_custom_record_process_info_func(self,current_function_result_status: FunctionResultStatus,kw:dict):
|
|
640
640
|
pass
|
|
641
641
|
|
|
642
|
+
async def _aio_frame_custom_record_process_info_func(self,current_function_result_status: FunctionResultStatus,kw:dict):
|
|
643
|
+
pass
|
|
644
|
+
|
|
642
645
|
def user_custom_record_process_info_func(self, current_function_result_status: FunctionResultStatus,): # 这个可以继承
|
|
643
646
|
pass
|
|
644
647
|
|
|
@@ -921,6 +924,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
921
924
|
self.metric_calculation.cal(t_start_run_fun, current_function_result_status)
|
|
922
925
|
|
|
923
926
|
self._frame_custom_record_process_info_func(current_function_result_status,kw)
|
|
927
|
+
await self._aio_frame_custom_record_process_info_func(current_function_result_status,kw)
|
|
924
928
|
self.user_custom_record_process_info_func(current_function_result_status,) # 两种方式都可以自定义,记录结果.建议使用文档4.21.b的方式继承来重写
|
|
925
929
|
await self.aio_user_custom_record_process_info_func(current_function_result_status,)
|
|
926
930
|
if self.consumer_params.user_custom_record_process_info_func:
|
|
@@ -10,29 +10,12 @@ import time
|
|
|
10
10
|
from funboost import FunctionResultStatus
|
|
11
11
|
from funboost.assist.grpc_helper import funboost_grpc_pb2_grpc, funboost_grpc_pb2
|
|
12
12
|
from funboost.consumers.base_consumer import AbstractConsumer
|
|
13
|
+
from funboost.core.msg_result_getter import FutureStatusResult
|
|
13
14
|
from funboost.core.serialization import Serialization
|
|
14
15
|
from funboost.core.exceptions import FunboostWaitRpcResultTimeout
|
|
15
16
|
from funboost.concurrent_pool.flexible_thread_pool import FlexibleThreadPool
|
|
16
17
|
|
|
17
18
|
|
|
18
|
-
class FutureStatusResult:
|
|
19
|
-
def __init__(self,call_type:str):
|
|
20
|
-
self.execute_finish_event = threading.Event()
|
|
21
|
-
self.staus_result_obj: FunctionResultStatus = None
|
|
22
|
-
self.call_type = call_type # sync_call or publish
|
|
23
|
-
|
|
24
|
-
def set_finish(self):
|
|
25
|
-
self.execute_finish_event.set()
|
|
26
|
-
|
|
27
|
-
def wait_finish(self,rpc_timeout):
|
|
28
|
-
return self.execute_finish_event.wait(rpc_timeout)
|
|
29
|
-
|
|
30
|
-
def set_staus_result_obj(self, staus_result_obj:FunctionResultStatus):
|
|
31
|
-
self.staus_result_obj = staus_result_obj
|
|
32
|
-
|
|
33
|
-
def get_staus_result_obj(self):
|
|
34
|
-
return self.staus_result_obj
|
|
35
|
-
|
|
36
19
|
|
|
37
20
|
|
|
38
21
|
|
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
# -*- coding: utf-8 -*-
|
|
2
2
|
# @Author : ydf
|
|
3
3
|
# @Time : 2022/8/8 0008 13:32
|
|
4
|
-
import
|
|
5
|
-
import
|
|
4
|
+
import logging
|
|
5
|
+
import threading
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
|
|
8
|
+
from flask import Flask, request
|
|
9
9
|
|
|
10
10
|
from funboost.consumers.base_consumer import AbstractConsumer
|
|
11
|
-
from funboost.core.
|
|
11
|
+
from funboost.core.function_result_status_saver import FunctionResultStatus
|
|
12
|
+
from funboost.core.msg_result_getter import FutureStatusResult
|
|
13
|
+
from funboost.core.serialization import Serialization
|
|
14
|
+
|
|
15
|
+
|
|
12
16
|
|
|
13
17
|
|
|
14
18
|
class HTTPConsumer(AbstractConsumer, ):
|
|
15
19
|
"""
|
|
16
|
-
|
|
20
|
+
flask 作为消息队列实现 consumer
|
|
17
21
|
"""
|
|
18
22
|
BROKER_EXCLUSIVE_CONFIG_DEFAULT = {'host': '127.0.0.1', 'port': None}
|
|
19
23
|
|
|
@@ -30,42 +34,105 @@ class HTTPConsumer(AbstractConsumer, ):
|
|
|
30
34
|
if self._port is None:
|
|
31
35
|
raise ValueError('please specify port')
|
|
32
36
|
|
|
33
|
-
# noinspection DuplicatedCode
|
|
34
37
|
def _shedual_task(self):
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
# kw = {'body': json.loads(msg)}
|
|
41
|
-
# self._submit_task(kw)
|
|
42
|
-
# return 'finish'
|
|
43
|
-
#
|
|
44
|
-
# flask_app.run('0.0.0.0', port=self._port,debug=False)
|
|
38
|
+
"""
|
|
39
|
+
使用Flask实现HTTP服务器
|
|
40
|
+
相比aiohttp,Flask是同步框架,避免了异步阻塞问题
|
|
41
|
+
"""
|
|
42
|
+
|
|
45
43
|
|
|
46
|
-
|
|
44
|
+
# 创建Flask应用
|
|
45
|
+
flask_app = Flask(__name__)
|
|
46
|
+
# 关闭Flask的日志,避免干扰funboost的日志
|
|
47
|
+
flask_app.logger.disabled = True
|
|
48
|
+
logging.getLogger('werkzeug').disabled = True
|
|
49
|
+
|
|
50
|
+
@flask_app.route('/', methods=['GET'])
|
|
51
|
+
def hello():
|
|
52
|
+
"""健康检查接口"""
|
|
53
|
+
return "Hello, from funboost (Flask version)"
|
|
54
|
+
|
|
55
|
+
@flask_app.route('/queue', methods=['POST'])
|
|
56
|
+
def recv_msg():
|
|
57
|
+
"""
|
|
58
|
+
接收消息的核心接口
|
|
59
|
+
支持两种调用类型:
|
|
60
|
+
1. publish: 异步发布,立即返回
|
|
61
|
+
2. sync_call: 同步调用,等待结果返回
|
|
62
|
+
"""
|
|
63
|
+
try:
|
|
64
|
+
# 获取请求数据
|
|
65
|
+
msg = request.form.get('msg')
|
|
66
|
+
call_type = request.form.get('call_type', 'publish')
|
|
67
|
+
|
|
68
|
+
if not msg:
|
|
69
|
+
return {"error": "msg parameter is required"}, 400
|
|
70
|
+
|
|
71
|
+
# 构造消息数据
|
|
72
|
+
kw = {
|
|
73
|
+
'body': msg,
|
|
74
|
+
'call_type': call_type,
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if call_type == 'sync_call':
|
|
78
|
+
# 同步调用:需要等待执行结果
|
|
79
|
+
future_status_result = FutureStatusResult(call_type=call_type)
|
|
80
|
+
kw['future_status_result'] = future_status_result
|
|
81
|
+
|
|
82
|
+
# 提交任务到线程池执行
|
|
83
|
+
self._submit_task(kw)
|
|
84
|
+
|
|
85
|
+
# 等待任务完成(带超时)
|
|
86
|
+
if future_status_result.wait_finish(self.consumer_params.rpc_timeout):
|
|
87
|
+
# 返回执行结果
|
|
88
|
+
result = future_status_result.get_staus_result_obj()
|
|
89
|
+
return Serialization.to_json_str(
|
|
90
|
+
result.get_status_dict(without_datetime_obj=True)
|
|
91
|
+
)
|
|
92
|
+
else:
|
|
93
|
+
# 超时处理
|
|
94
|
+
self.logger.error(f'sync_call wait timeout after {self.consumer_params.rpc_timeout}s')
|
|
95
|
+
return {"error": "execution timeout"}, 408
|
|
96
|
+
|
|
97
|
+
else:
|
|
98
|
+
# 异步发布:直接提交任务,立即返回
|
|
99
|
+
self._submit_task(kw)
|
|
100
|
+
return "finish"
|
|
101
|
+
|
|
102
|
+
except Exception as e:
|
|
103
|
+
self.logger.error(f'处理HTTP请求时出错: {e}', exc_info=True)
|
|
104
|
+
return {"error": str(e)}, 500
|
|
105
|
+
|
|
106
|
+
# 启动Flask服务器
|
|
107
|
+
# 注意:Flask默认是单线程的,但funboost使用线程池处理任务,所以这里threaded=True
|
|
108
|
+
self.logger.info(f'启动Flask HTTP服务器,监听 {self._ip}:{self._port}')
|
|
47
109
|
|
|
48
|
-
#
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
110
|
+
# flask_app.run(
|
|
111
|
+
# host='0.0.0.0', # 监听所有接口
|
|
112
|
+
# port=self._port,
|
|
113
|
+
# debug=False, # 生产环境关闭debug
|
|
114
|
+
# threaded=True, # 开启多线程支持
|
|
115
|
+
# use_reloader=False, # 关闭自动重载
|
|
116
|
+
# )
|
|
52
117
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
data = await request.post()
|
|
56
|
-
msg = data['msg']
|
|
57
|
-
kw = {'body': msg}
|
|
58
|
-
self._submit_task(kw)
|
|
59
|
-
return AioHttpImporter().web.Response(text="finish")
|
|
118
|
+
import waitress
|
|
119
|
+
waitress.serve(flask_app, host='0.0.0.0', port=self._port,threads=self.consumer_params.concurrent_num)
|
|
60
120
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
121
|
+
def _frame_custom_record_process_info_func(self, current_function_result_status: FunctionResultStatus, kw: dict):
|
|
122
|
+
"""
|
|
123
|
+
任务执行完成后的回调函数
|
|
124
|
+
对于sync_call模式,需要通知等待的HTTP请求
|
|
125
|
+
"""
|
|
126
|
+
if kw['call_type'] == "sync_call":
|
|
127
|
+
future_status_result: FutureStatusResult = kw['future_status_result']
|
|
128
|
+
future_status_result.set_staus_result_obj(current_function_result_status)
|
|
129
|
+
future_status_result.set_finish()
|
|
130
|
+
# self.logger.info('sync_call任务执行完成,通知HTTP请求返回结果')
|
|
66
131
|
|
|
67
132
|
def _confirm_consume(self, kw):
|
|
68
|
-
|
|
133
|
+
"""HTTP模式没有确认消费的功能"""
|
|
134
|
+
pass
|
|
69
135
|
|
|
70
136
|
def _requeue(self, kw):
|
|
137
|
+
"""HTTP模式没有重新入队的功能"""
|
|
71
138
|
pass
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# @Author : ydf
|
|
3
|
+
# @Time : 2022/8/8 0008 13:32
|
|
4
|
+
import asyncio
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
# from aiohttp import web
|
|
8
|
+
# from aiohttp.web_request import Request
|
|
9
|
+
|
|
10
|
+
from funboost.consumers.base_consumer import AbstractConsumer
|
|
11
|
+
from funboost.core.function_result_status_saver import FunctionResultStatus
|
|
12
|
+
from funboost.core.lazy_impoter import AioHttpImporter
|
|
13
|
+
from funboost.core.serialization import Serialization
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class AioFutureStatusResult:
|
|
17
|
+
def __init__(self,call_type:str):
|
|
18
|
+
self.execute_finish_event = asyncio.Event()
|
|
19
|
+
self.staus_result_obj: FunctionResultStatus = None
|
|
20
|
+
self.call_type = call_type # sync_call or publish
|
|
21
|
+
|
|
22
|
+
def set_finish(self):
|
|
23
|
+
self.execute_finish_event.set()
|
|
24
|
+
|
|
25
|
+
async def wait_finish(self,rpc_timeout):
|
|
26
|
+
return await self.execute_finish_event.wait()
|
|
27
|
+
|
|
28
|
+
def set_staus_result_obj(self, staus_result_obj:FunctionResultStatus):
|
|
29
|
+
self.staus_result_obj = staus_result_obj
|
|
30
|
+
|
|
31
|
+
def get_staus_result_obj(self):
|
|
32
|
+
return self.staus_result_obj
|
|
33
|
+
|
|
34
|
+
class HTTPConsumer(AbstractConsumer, ):
|
|
35
|
+
"""
|
|
36
|
+
aiohttp 实现消息队列,不支持持久化,但不需要安装软件。
|
|
37
|
+
"""
|
|
38
|
+
BROKER_EXCLUSIVE_CONFIG_DEFAULT = {'host': '127.0.0.1', 'port': None}
|
|
39
|
+
|
|
40
|
+
# noinspection PyAttributeOutsideInit
|
|
41
|
+
def custom_init(self):
|
|
42
|
+
# try:
|
|
43
|
+
# self._ip, self._port = self.queue_name.split(':')
|
|
44
|
+
# self._port = int(self._port)
|
|
45
|
+
# except BaseException as e:
|
|
46
|
+
# self.logger.critical(f'http作为消息队列时候,队列名字必须设置为 例如 192.168.1.101:8200 这种, ip:port')
|
|
47
|
+
# raise e
|
|
48
|
+
self._ip = self.consumer_params.broker_exclusive_config['host']
|
|
49
|
+
self._port = self.consumer_params.broker_exclusive_config['port']
|
|
50
|
+
if self._port is None:
|
|
51
|
+
raise ValueError('please specify port')
|
|
52
|
+
|
|
53
|
+
# noinspection DuplicatedCode
|
|
54
|
+
def _shedual_task(self):
|
|
55
|
+
# flask_app = Flask(__name__)
|
|
56
|
+
#
|
|
57
|
+
# @flask_app.route('/queue', methods=['post'])
|
|
58
|
+
# def recv_msg():
|
|
59
|
+
# msg = request.form['msg']
|
|
60
|
+
# kw = {'body': json.loads(msg)}
|
|
61
|
+
# self._submit_task(kw)
|
|
62
|
+
# return 'finish'
|
|
63
|
+
#
|
|
64
|
+
# flask_app.run('0.0.0.0', port=self._port,debug=False)
|
|
65
|
+
|
|
66
|
+
routes = AioHttpImporter().web.RouteTableDef()
|
|
67
|
+
|
|
68
|
+
# noinspection PyUnusedLocal
|
|
69
|
+
@routes.get('/')
|
|
70
|
+
async def hello(request):
|
|
71
|
+
return AioHttpImporter().web.Response(text="Hello, from funboost")
|
|
72
|
+
|
|
73
|
+
@routes.post('/queue')
|
|
74
|
+
async def recv_msg(request: AioHttpImporter().Request):
|
|
75
|
+
data = await request.post()
|
|
76
|
+
msg = data['msg']
|
|
77
|
+
call_type = data['call_type']
|
|
78
|
+
kw = {'body': msg,'call_type': call_type,}
|
|
79
|
+
if call_type == 'sync_call':
|
|
80
|
+
aio_future_status_result = AioFutureStatusResult(call_type=call_type)
|
|
81
|
+
kw['aio_future_status_result'] = aio_future_status_result
|
|
82
|
+
self._submit_task(kw)
|
|
83
|
+
if data['call_type'] == 'sync_call':
|
|
84
|
+
await aio_future_status_result.wait_finish(self.consumer_params.rpc_timeout)
|
|
85
|
+
return AioHttpImporter().web.Response(text=Serialization.to_json_str(
|
|
86
|
+
aio_future_status_result.get_staus_result_obj().get_status_dict(without_datetime_obj=True)))
|
|
87
|
+
return AioHttpImporter().web.Response(text="finish")
|
|
88
|
+
|
|
89
|
+
app = AioHttpImporter().web.Application()
|
|
90
|
+
app.add_routes(routes)
|
|
91
|
+
loop = asyncio.new_event_loop()
|
|
92
|
+
asyncio.set_event_loop(loop)
|
|
93
|
+
AioHttpImporter().web.run_app(app, host='0.0.0.0', port=self._port, )
|
|
94
|
+
|
|
95
|
+
def _frame_custom_record_process_info_func(self,current_function_result_status: FunctionResultStatus,kw:dict):
|
|
96
|
+
if kw['call_type'] == "sync_call":
|
|
97
|
+
aio_future_status_result: AioFutureStatusResult = kw['aio_future_status_result']
|
|
98
|
+
aio_future_status_result.set_staus_result_obj(current_function_result_status)
|
|
99
|
+
aio_future_status_result.set_finish()
|
|
100
|
+
self.logger.info(f'aio_future_status_result.set_finish()')
|
|
101
|
+
|
|
102
|
+
# async def _aio_frame_custom_record_process_info_func(self,current_function_result_status: FunctionResultStatus,kw:dict):
|
|
103
|
+
# self.logger.info(666666)
|
|
104
|
+
# if kw['call_type'] == "sync_call":
|
|
105
|
+
# aio_future_status_result: AioFutureStatusResult = kw['aio_future_status_result']
|
|
106
|
+
# aio_future_status_result.set_staus_result_obj(current_function_result_status)
|
|
107
|
+
# aio_future_status_result.set_finish()
|
|
108
|
+
# self.logger.info(f'aio_future_status_result.set_finish()')
|
|
109
|
+
def _confirm_consume(self, kw):
|
|
110
|
+
pass # 没有确认消费的功能。
|
|
111
|
+
|
|
112
|
+
def _requeue(self, kw):
|
|
113
|
+
pass
|
|
@@ -60,10 +60,7 @@ class KafkaConsumer(AbstractConsumer):
|
|
|
60
60
|
# REMIND 好处是并发高。topic像翻书一样,随时可以设置偏移量重新消费。多个分组消费同一个主题,每个分组对相同主题的偏移量互不干扰 。
|
|
61
61
|
for message in consumer:
|
|
62
62
|
# 注意: message ,value都是原始的字节数据,需要decode
|
|
63
|
-
|
|
64
|
-
self.logger.debug(
|
|
65
|
-
f'从kafka的 [{message.topic}] 主题,分区 {message.partition} 中 取出的消息是: {message.value.decode()}')
|
|
66
|
-
kw = {'consumer': consumer, 'message': message, 'body': message.value}
|
|
63
|
+
kw = {'consumer': consumer, 'message': message, 'body': message.value.decode('utf-8')}
|
|
67
64
|
self._submit_task(kw)
|
|
68
65
|
|
|
69
66
|
def _confirm_consume(self, kw):
|
funboost/core/booster.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
import copy
|
|
3
3
|
import inspect
|
|
4
|
+
from multiprocessing import Process
|
|
4
5
|
import os
|
|
5
6
|
import sys
|
|
6
7
|
import types
|
|
@@ -242,10 +243,10 @@ class BoostersManager:
|
|
|
242
243
|
"""
|
|
243
244
|
|
|
244
245
|
# pid_queue_name__booster_map字典存放 {(进程id,queue_name):Booster对象}
|
|
245
|
-
pid_queue_name__booster_map
|
|
246
|
+
pid_queue_name__booster_map :typing.Dict[typing.Tuple[int,str],Booster]= {}
|
|
246
247
|
|
|
247
248
|
# queue_name__boost_params_consuming_function_map 字典存放 {queue_name,(@boost的入参字典,@boost装饰的消费函数)}
|
|
248
|
-
queue_name__boost_params_map
|
|
249
|
+
queue_name__boost_params_map :typing.Dict[str,BoosterParams]= {}
|
|
249
250
|
|
|
250
251
|
pid_queue_name__has_start_consume_set = set()
|
|
251
252
|
|
|
@@ -392,6 +393,34 @@ class BoostersManager:
|
|
|
392
393
|
|
|
393
394
|
m_consume = multi_process_consume_queues
|
|
394
395
|
|
|
396
|
+
@classmethod
|
|
397
|
+
def consume_group(cls, booster_group:str,block=False):
|
|
398
|
+
"""
|
|
399
|
+
根据@boost装饰器的 booster_group消费分组名字,启动多个消费函数;
|
|
400
|
+
"""
|
|
401
|
+
if booster_group is None:
|
|
402
|
+
raise ValueError('booster_group 不能为None')
|
|
403
|
+
need_consume_queue_names = []
|
|
404
|
+
for queue_name in cls.get_all_queues():
|
|
405
|
+
booster= cls.get_or_create_booster_by_queue_name(queue_name)
|
|
406
|
+
if booster.boost_params.booster_group == booster_group:
|
|
407
|
+
need_consume_queue_names.append(queue_name)
|
|
408
|
+
flogger.info(f'according to booster_group:{booster_group} ,start consume queues: {need_consume_queue_names}')
|
|
409
|
+
for queue_name in need_consume_queue_names:
|
|
410
|
+
cls.get_or_create_booster_by_queue_name(queue_name).consume()
|
|
411
|
+
if block:
|
|
412
|
+
ctrl_c_recv()
|
|
413
|
+
|
|
414
|
+
@classmethod
|
|
415
|
+
def multi_process_consume_group(cls, booster_group:str, process_num=1):
|
|
416
|
+
"""
|
|
417
|
+
根据@boost装饰器的 booster_group消费分组名字,启动多个消费函数;
|
|
418
|
+
"""
|
|
419
|
+
for _ in range(process_num):
|
|
420
|
+
Process(target=cls.consume_group,args=(booster_group,True)).start()
|
|
421
|
+
|
|
422
|
+
m_consume_group = multi_process_consume_group
|
|
423
|
+
|
|
395
424
|
@classmethod
|
|
396
425
|
def multi_process_consume_all_queues(cls, process_num=1):
|
|
397
426
|
"""
|
funboost/core/funboost_time.py
CHANGED
|
@@ -4,19 +4,20 @@ import sys
|
|
|
4
4
|
import datetime
|
|
5
5
|
|
|
6
6
|
import typing
|
|
7
|
-
|
|
7
|
+
import threading
|
|
8
8
|
from nb_time import NbTime
|
|
9
9
|
from funboost.funboost_config_deafult import FunboostCommonConfig
|
|
10
10
|
|
|
11
|
+
|
|
11
12
|
class FunboostTime(NbTime):
|
|
12
13
|
default_formatter = NbTime.FORMATTER_DATETIME_NO_ZONE
|
|
13
14
|
|
|
14
|
-
def get_time_zone_str(self,time_zone: typing.Union[str, datetime.tzinfo,None] = None):
|
|
15
|
-
return time_zone or self.default_time_zone or
|
|
15
|
+
def get_time_zone_str(self, time_zone: typing.Union[str, datetime.tzinfo, None] = None):
|
|
16
|
+
return time_zone or self.default_time_zone or FunboostCommonConfig.TIMEZONE or self.get_localzone_name()
|
|
16
17
|
|
|
17
18
|
@staticmethod
|
|
18
|
-
def _get_tow_digist(num:int)->str:
|
|
19
|
-
if len(str(num)) ==1:
|
|
19
|
+
def _get_tow_digist(num: int) -> str:
|
|
20
|
+
if len(str(num)) == 1:
|
|
20
21
|
return f'0{num}'
|
|
21
22
|
return str(num)
|
|
22
23
|
|
|
@@ -28,12 +29,18 @@ class FunboostTime(NbTime):
|
|
|
28
29
|
return t_str
|
|
29
30
|
|
|
30
31
|
|
|
31
|
-
|
|
32
|
-
|
|
33
32
|
# 缓存时区对象,提升性能(避免重复解析)
|
|
34
33
|
_tz_cache = {}
|
|
35
34
|
|
|
36
|
-
|
|
35
|
+
_DIGIT_MAP = {i: f'{i:02d}' for i in range(100)}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _gen_2_dig_number(n):
|
|
39
|
+
return _DIGIT_MAP[n]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def get_now_time_str_by_tz(tz_name: str = None) -> str:
|
|
43
|
+
# 生成100万次当前时间字符串%Y-%m-%d %H:%M:%S仅需1.9秒.
|
|
37
44
|
"""
|
|
38
45
|
根据时区名(如 'Asia/Shanghai')返回当前时间字符串,格式:'%Y-%m-%d %H:%M:%S'
|
|
39
46
|
|
|
@@ -50,27 +57,64 @@ def get_now_time_str_by_tz(tz_name: str=None) -> str:
|
|
|
50
57
|
_tz_cache[tz_name] = ZoneInfo(tz_name)
|
|
51
58
|
else:
|
|
52
59
|
# Python < 3.9,使用 pytz
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
_tz_cache[tz_name] = pytz.timezone(tz_name)
|
|
56
|
-
except ImportError:
|
|
57
|
-
raise RuntimeError(
|
|
58
|
-
f"Python < 3.9 requires 'pytz' to handle timezones. "
|
|
59
|
-
f"Install it with: pip install pytz"
|
|
60
|
-
) from None
|
|
61
|
-
except pytz.UnknownTimeZoneError:
|
|
62
|
-
raise pytz.UnknownTimeZoneError(tz_name)
|
|
60
|
+
_tz_cache[tz_name] = pytz.timezone(tz_name)
|
|
61
|
+
|
|
63
62
|
|
|
64
63
|
tz = _tz_cache[tz_name]
|
|
65
|
-
|
|
64
|
+
|
|
66
65
|
# 获取当前时间并格式化(注意:datetime.now(tz) 是最高效的方式)
|
|
67
66
|
now = datetime.datetime.now(tz)
|
|
68
|
-
return f'{now.year:04d}-{now.month:02d}-{now.day:02d} {now.hour:02d}:{now.minute:02d}:{now.second:02d}'
|
|
67
|
+
# return f'{now.year:04d}-{now.month:02d}-{now.day:02d} {now.hour:02d}:{now.minute:02d}:{now.second:02d}'
|
|
69
68
|
# return now.strftime("%Y-%m-%d %H:%M:%S")
|
|
69
|
+
return f'{now.year}-{_gen_2_dig_number(now.month)}-{_gen_2_dig_number(now.day)} {_gen_2_dig_number(now.hour)}:{_gen_2_dig_number(now.minute)}:{_gen_2_dig_number(now.second)}'
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class NowTimeStrCache:
|
|
73
|
+
# 生成100万次当前时间字符串%Y-%m-%d %H:%M:%S仅需0.4秒.
|
|
74
|
+
# 全局变量,用于存储缓存的时间字符串和对应的整秒时间戳
|
|
75
|
+
_cached_time_str: typing.Optional[str] = None
|
|
76
|
+
_cached_time_second: int = 0
|
|
77
|
+
|
|
78
|
+
# 为了线程安全,使用锁。在极高并发下,锁的开销远小于每毫秒都进行时间格式化。
|
|
79
|
+
_time_cache_lock = threading.Lock()
|
|
80
|
+
|
|
81
|
+
@classmethod
|
|
82
|
+
def fast_get_now_time_str(cls,timezone_str:str=None) -> str:
|
|
83
|
+
"""
|
|
84
|
+
获取当前时间字符串,格式为 '%Y-%m-%d %H:%M:%S'。
|
|
85
|
+
通过缓存机制,同一秒内的多次调用直接返回缓存结果,极大提升性能。
|
|
86
|
+
适用于对时间精度要求不高(秒级即可)的高并发场景。
|
|
87
|
+
:return: 格式化后的时间字符串,例如 '2024-06-12 15:30:45'
|
|
88
|
+
"""
|
|
89
|
+
timezone_str = timezone_str or FunboostCommonConfig.TIMEZONE
|
|
90
|
+
|
|
91
|
+
# 获取当前的整秒时间戳(去掉小数部分)
|
|
92
|
+
current_second = int(time.time())
|
|
93
|
+
|
|
94
|
+
# 如果缓存的时间戳与当前秒数一致,直接返回缓存的字符串。
|
|
95
|
+
if current_second == cls._cached_time_second:
|
|
96
|
+
return cls._cached_time_str
|
|
97
|
+
|
|
98
|
+
# 如果不一致,说明进入新的一秒,需要重新计算并更新缓存。
|
|
99
|
+
# 使用锁确保在多线程环境下,只有一个线程会执行更新操作。
|
|
100
|
+
|
|
101
|
+
with cls._time_cache_lock:
|
|
102
|
+
# 双重检查锁定 (Double-Checked Locking),防止在等待锁的过程中,其他线程已经更新了缓存。
|
|
103
|
+
if current_second == cls._cached_time_second:
|
|
104
|
+
return cls._cached_time_str
|
|
105
|
+
|
|
106
|
+
# 重新计算时间字符串。这里直接使用 time.strftime,因为它在秒级更新的场景下性能足够。
|
|
107
|
+
# 我们不需要像 Funboost 那样为每一毫秒的调用都去做查表优化。
|
|
108
|
+
now = datetime.datetime.now(tz=pytz.timezone(timezone_str))
|
|
109
|
+
cls._cached_time_str = now.strftime('%Y-%m-%d %H:%M:%S', )
|
|
110
|
+
cls._cached_time_second = current_second
|
|
111
|
+
|
|
112
|
+
return cls._cached_time_str
|
|
113
|
+
|
|
70
114
|
|
|
71
115
|
if __name__ == '__main__':
|
|
72
116
|
print(FunboostTime().get_str())
|
|
73
|
-
tz=pytz.timezone(FunboostCommonConfig.TIMEZONE)
|
|
117
|
+
tz = pytz.timezone(FunboostCommonConfig.TIMEZONE)
|
|
74
118
|
for i in range(1000000):
|
|
75
119
|
pass
|
|
76
120
|
# FunboostTime()#.get_str_fast()
|
|
@@ -201,12 +201,17 @@ class BoosterParams(BaseJsonAbleModel):
|
|
|
201
201
|
schedule_tasks_on_main_thread: bool = False # 直接在主线程调度任务,意味着不能直接在当前主线程同时开启两个消费者。
|
|
202
202
|
|
|
203
203
|
is_auto_start_consuming_message: bool = False # 是否在定义后就自动启动消费,无需用户手动写 .consume() 来启动消息消费。
|
|
204
|
+
|
|
205
|
+
# booster_group :消费分组名字, BoostersManager.consume_group 时候根据 booster_group 启动多个消费函数,减少需要写 f1.consume() f2.consume() ...这种。
|
|
206
|
+
# 不像BoostersManager.consume_all() 会启动所有不相关消费函数,也不像 f1.consume() f2.consume() 这样需要逐个启动消费函数。
|
|
207
|
+
# 可以根据业务逻辑创建不同的分组,实现灵活的消费启动策略。
|
|
208
|
+
# 用法见文档 4.2d.3 章节. 使用 BoostersManager ,通过 consume_group 启动一组消费函数
|
|
209
|
+
booster_group:typing.Union[str, None] = None
|
|
204
210
|
|
|
205
211
|
consuming_function: typing.Optional[typing.Callable] = None # 消费函数,在@boost时候不用指定,因为装饰器知道下面的函数.
|
|
206
212
|
consuming_function_raw: typing.Optional[typing.Callable] = None # 不需要传递,自动生成
|
|
207
213
|
consuming_function_name: str = '' # 不需要传递,自动生成
|
|
208
214
|
|
|
209
|
-
|
|
210
215
|
|
|
211
216
|
broker_exclusive_config: dict = {} # 加上一个不同种类中间件非通用的配置,不同中间件自身独有的配置,不是所有中间件都兼容的配置,因为框架支持30种消息队列,消息队列不仅仅是一般的先进先出queue这么简单的概念,
|
|
212
217
|
# 例如kafka支持消费者组,rabbitmq也支持各种独特概念例如各种ack机制 复杂路由机制,有的中间件原生能支持消息优先级有的中间件不支持,每一种消息队列都有独特的配置参数意义,可以通过这里传递。每种中间件能传递的键值对可以看consumer类的 BROKER_EXCLUSIVE_CONFIG_DEFAULT
|
|
@@ -228,9 +233,20 @@ class BoosterParams(BaseJsonAbleModel):
|
|
|
228
233
|
COMMON_FUNCTION = 'COMMON_FUNCTION'
|
|
229
234
|
"""
|
|
230
235
|
|
|
236
|
+
"""
|
|
237
|
+
user_options:
|
|
238
|
+
用户额外自定义的配置,高级用户或者奇葩需求可以用得到,用户可以自由发挥,存放任何设置.
|
|
239
|
+
user_options 提供了一个统一的、用户自定义的命名空间,让用户可以为自己的“奇葩需求”或“高级定制”传递配置,而无需等待框架开发者添加官方支持。
|
|
240
|
+
funboost 是自由框架不是奴役框架,不仅消费函数逻辑自由,目录层级结构自由,自定义奇葩扩展也要追求自由,用户不用改funboost BoosterParams 源码来加装饰器参数
|
|
241
|
+
|
|
242
|
+
使用场景见文档 4b.6 章节.
|
|
243
|
+
"""
|
|
244
|
+
user_options: dict = {} # 用户自定义的配置,高级用户或者奇葩需求可以用得到,用户可以自由发挥,存放任何设置.
|
|
245
|
+
|
|
246
|
+
|
|
231
247
|
auto_generate_info: dict = {} # 自动生成的信息,不需要用户主动传参.
|
|
232
|
-
|
|
233
|
-
|
|
248
|
+
|
|
249
|
+
|
|
234
250
|
|
|
235
251
|
@root_validator(skip_on_failure=True, )
|
|
236
252
|
def check_values(cls, values: dict):
|
|
@@ -358,6 +374,8 @@ class PublisherParams(BaseJsonAbleModel):
|
|
|
358
374
|
publish_msg_log_use_full_msg: bool = False # 发布到消息队列的消息内容的日志,是否显示消息的完整体,还是只显示函数入参。
|
|
359
375
|
consuming_function_kind: typing.Optional[str] = None # 自动生成的信息,不需要用户主动传参.
|
|
360
376
|
rpc_timeout: int = 1800 # rpc模式下,等待rpc结果返回的超时时间
|
|
377
|
+
user_options: dict = {} # 用户自定义的配置,高级用户或者奇葩需求可以用得到,用户可以自由发挥,存放任何设置.
|
|
378
|
+
|
|
361
379
|
|
|
362
380
|
if __name__ == '__main__':
|
|
363
381
|
from funboost.concurrent_pool import FlexibleThreadPool
|
funboost/core/helper_funs.py
CHANGED
|
@@ -3,7 +3,7 @@ import pytz
|
|
|
3
3
|
import time
|
|
4
4
|
import uuid
|
|
5
5
|
import datetime
|
|
6
|
-
from funboost.core.funboost_time import FunboostTime, get_now_time_str_by_tz
|
|
6
|
+
from funboost.core.funboost_time import FunboostTime, get_now_time_str_by_tz, NowTimeStrCache
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
def get_publish_time(paramsx: dict):
|
|
@@ -57,13 +57,13 @@ class MsgGenerater:
|
|
|
57
57
|
def generate_publish_time() -> float:
|
|
58
58
|
return round(time.time(),4)
|
|
59
59
|
|
|
60
|
-
|
|
61
|
-
# def generate_publish_time_format() -> str:
|
|
62
|
-
# return FunboostTime().get_str()
|
|
63
|
-
|
|
60
|
+
|
|
64
61
|
@staticmethod
|
|
65
62
|
def generate_publish_time_format() -> str:
|
|
66
|
-
return
|
|
63
|
+
# return FunboostTime().get_str() # 性能不好
|
|
64
|
+
# return get_now_time_str_by_tz() # 2秒100万次
|
|
65
|
+
return NowTimeStrCache.fast_get_now_time_str() # 0.4秒100万次
|
|
66
|
+
|
|
67
67
|
|
|
68
68
|
@classmethod
|
|
69
69
|
def generate_pulish_time_and_task_id(cls,queue_name:str,task_id=None):
|
|
@@ -81,8 +81,9 @@ if __name__ == '__main__':
|
|
|
81
81
|
print(FunboostTime())
|
|
82
82
|
for i in range(1000000):
|
|
83
83
|
# time.time()
|
|
84
|
-
|
|
84
|
+
MsgGenerater.generate_publish_time_format()
|
|
85
|
+
# FunboostTime().get_str()
|
|
85
86
|
|
|
86
|
-
datetime.datetime.now(tz=pytz.timezone(FunboostCommonConfig.TIMEZONE)).strftime(FunboostTime.FORMATTER_DATETIME_NO_ZONE)
|
|
87
|
+
# datetime.datetime.now(tz=pytz.timezone(FunboostCommonConfig.TIMEZONE)).strftime(FunboostTime.FORMATTER_DATETIME_NO_ZONE)
|
|
87
88
|
|
|
88
89
|
print(FunboostTime())
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
+
import threading
|
|
2
3
|
import time
|
|
3
4
|
|
|
4
5
|
import typing
|
|
@@ -278,6 +279,32 @@ class ResultFromMongo(MongoMixin):
|
|
|
278
279
|
return (self.mongo_row or {}).get('result', NO_RESULT)
|
|
279
280
|
|
|
280
281
|
|
|
282
|
+
class FutureStatusResult:
|
|
283
|
+
"""
|
|
284
|
+
用于sync_call模式的结果等待和通知
|
|
285
|
+
使用threading.Event实现同步等待
|
|
286
|
+
"""
|
|
287
|
+
def __init__(self, call_type: str):
|
|
288
|
+
self.execute_finish_event = threading.Event()
|
|
289
|
+
self.staus_result_obj: FunctionResultStatus = None
|
|
290
|
+
self.call_type = call_type # sync_call or publish
|
|
291
|
+
|
|
292
|
+
def set_finish(self):
|
|
293
|
+
"""标记任务完成"""
|
|
294
|
+
self.execute_finish_event.set()
|
|
295
|
+
|
|
296
|
+
def wait_finish(self, rpc_timeout):
|
|
297
|
+
"""等待任务完成,带超时"""
|
|
298
|
+
return self.execute_finish_event.wait(rpc_timeout)
|
|
299
|
+
|
|
300
|
+
def set_staus_result_obj(self, staus_result_obj: FunctionResultStatus):
|
|
301
|
+
"""设置任务执行结果"""
|
|
302
|
+
self.staus_result_obj = staus_result_obj
|
|
303
|
+
|
|
304
|
+
def get_staus_result_obj(self):
|
|
305
|
+
"""获取任务执行结果"""
|
|
306
|
+
return self.staus_result_obj
|
|
307
|
+
|
|
281
308
|
if __name__ == '__main__':
|
|
282
309
|
print(ResultFromMongo('test_queue77h6_result:764a1ba2-14eb-49e2-9209-ac83fc5db1e8').get_status_and_result())
|
|
283
310
|
print(ResultFromMongo('test_queue77h6_result:5cdb4386-44cc-452f-97f4-9e5d2882a7c1').get_result())
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import typing
|
|
2
2
|
|
|
3
3
|
from funboost.publishers.empty_publisher import EmptyPublisher
|
|
4
|
-
|
|
4
|
+
|
|
5
5
|
from funboost.publishers.nats_publisher import NatsPublisher
|
|
6
6
|
from funboost.publishers.peewee_publisher import PeeweePublisher
|
|
7
7
|
from funboost.publishers.redis_publisher_lpush import RedisPublisherLpush
|
|
@@ -28,7 +28,7 @@ from funboost.publishers.httpsqs_publisher import HttpsqsPublisher
|
|
|
28
28
|
from funboost.consumers.empty_consumer import EmptyConsumer
|
|
29
29
|
from funboost.consumers.redis_consumer_priority import RedisPriorityConsumer
|
|
30
30
|
from funboost.consumers.redis_pubsub_consumer import RedisPbSubConsumer
|
|
31
|
-
|
|
31
|
+
|
|
32
32
|
from funboost.consumers.kafka_consumer import KafkaConsumer
|
|
33
33
|
from funboost.consumers.local_python_queue_consumer import LocalPythonQueueConsumer
|
|
34
34
|
from funboost.consumers.mongomq_consumer import MongoMqConsumer
|
|
@@ -74,7 +74,7 @@ broker_kind__publsiher_consumer_type_map = {
|
|
|
74
74
|
BrokerEnum.HTTPSQS: (HttpsqsPublisher, HttpsqsConsumer),
|
|
75
75
|
BrokerEnum.UDP: (UDPPublisher, UDPConsumer),
|
|
76
76
|
BrokerEnum.TCP: (TCPPublisher, TCPConsumer),
|
|
77
|
-
|
|
77
|
+
|
|
78
78
|
BrokerEnum.NATS: (NatsPublisher, NatsConsumer),
|
|
79
79
|
BrokerEnum.TXT_FILE: (TxtFilePublisher, TxtFileConsumer),
|
|
80
80
|
BrokerEnum.PEEWEE: (PeeweePublisher, PeeweeConsumer),
|
|
@@ -184,6 +184,11 @@ def regist_to_funboost(broker_kind: str):
|
|
|
184
184
|
from funboost.consumers.mysql_cdc_consumer import MysqlCdcConsumer
|
|
185
185
|
from funboost.publishers.mysql_cdc_publisher import MysqlCdcPublisher
|
|
186
186
|
register_custom_broker(broker_kind, MysqlCdcPublisher, MysqlCdcConsumer)
|
|
187
|
+
|
|
188
|
+
if broker_kind == BrokerEnum.HTTP:
|
|
189
|
+
from funboost.consumers.http_consumer import HTTPConsumer
|
|
190
|
+
from funboost.publishers.http_publisher import HTTPPublisher
|
|
191
|
+
register_custom_broker(broker_kind, HTTPPublisher, HTTPConsumer)
|
|
187
192
|
|
|
188
193
|
if __name__ == '__main__':
|
|
189
194
|
import sys
|
|
@@ -21,6 +21,7 @@ from threading import Lock
|
|
|
21
21
|
import nb_log
|
|
22
22
|
from funboost.constant import ConstStrForClassMethod, FunctionKind
|
|
23
23
|
from funboost.core.func_params_model import PublisherParams, PriorityConsumingControlConfig
|
|
24
|
+
from funboost.core.function_result_status_saver import FunctionResultStatus
|
|
24
25
|
from funboost.core.helper_funs import MsgGenerater
|
|
25
26
|
from funboost.core.loggers import develop_logger
|
|
26
27
|
|
|
@@ -315,6 +316,10 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
315
316
|
def concrete_realization_of_publish(self, msg: str):
|
|
316
317
|
raise NotImplementedError
|
|
317
318
|
|
|
319
|
+
def sync_call(self, msg_dict: dict, is_return_rpc_data_obj=True) -> typing.Union[dict, FunctionResultStatus]:
|
|
320
|
+
"""仅有部分中间件支持同步调用并阻塞等待返回结果,不依赖AsyncResult + redis作为rpc,例如 http grpc 等"""
|
|
321
|
+
raise NotImplementedError(f'broker {self.publisher_params.broker_kind} not support sync_call method')
|
|
322
|
+
|
|
318
323
|
@abc.abstractmethod
|
|
319
324
|
def clear(self):
|
|
320
325
|
raise NotImplementedError
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
# -*- coding: utf-8 -*-
|
|
2
2
|
# @Author : ydf
|
|
3
3
|
# @Time : 2022/8/8 0008 12:12
|
|
4
|
+
import threading
|
|
4
5
|
|
|
6
|
+
from funboost.core.function_result_status_saver import FunctionResultStatus
|
|
7
|
+
from funboost.core.serialization import Serialization
|
|
5
8
|
from funboost.publishers.base_publisher import AbstractPublisher
|
|
6
9
|
from urllib3 import PoolManager
|
|
7
10
|
|
|
@@ -13,7 +16,7 @@ class HTTPPublisher(AbstractPublisher, ):
|
|
|
13
16
|
|
|
14
17
|
# noinspection PyAttributeOutsideInit
|
|
15
18
|
def custom_init(self):
|
|
16
|
-
self._http = PoolManager(
|
|
19
|
+
self._http = PoolManager(maxsize=100)
|
|
17
20
|
self._ip = self.publisher_params.broker_exclusive_config['host']
|
|
18
21
|
self._port = self.publisher_params.broker_exclusive_config['port']
|
|
19
22
|
self._ip_port_str = f'{self._ip}:{self._port}'
|
|
@@ -23,7 +26,22 @@ class HTTPPublisher(AbstractPublisher, ):
|
|
|
23
26
|
|
|
24
27
|
def concrete_realization_of_publish(self, msg):
|
|
25
28
|
url = self._ip_port_str + '/queue'
|
|
26
|
-
self._http.request('post', url, fields={'msg': msg})
|
|
29
|
+
self._http.request('post', url, fields={'msg': msg,'call_type':'publish'})
|
|
30
|
+
|
|
31
|
+
def sync_call(self, msg_dict: dict, is_return_rpc_data_obj=True):
|
|
32
|
+
url = self._ip_port_str + '/queue'
|
|
33
|
+
response = self._http.request('post', url,
|
|
34
|
+
fields={'msg': Serialization.to_json_str(msg_dict),'call_type':'sync_call'})
|
|
35
|
+
json_resp = response.data.decode('utf-8')
|
|
36
|
+
# import requests
|
|
37
|
+
# response = requests.request('post', url,
|
|
38
|
+
# data={'msg': Serialization.to_json_str(msg_dict),'call_type':'sync_call'})
|
|
39
|
+
# json_resp = response.text()
|
|
40
|
+
if is_return_rpc_data_obj:
|
|
41
|
+
return FunctionResultStatus.parse_status_and_result_to_obj(Serialization.to_dict(json_resp))
|
|
42
|
+
else:
|
|
43
|
+
return Serialization.to_dict(json_resp)
|
|
44
|
+
|
|
27
45
|
|
|
28
46
|
def clear(self):
|
|
29
47
|
pass # udp没有保存消息
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: funboost
|
|
3
|
-
Version: 50.
|
|
3
|
+
Version: 50.3
|
|
4
4
|
Summary: pip install funboost,python全功能分布式函数调度框架,funboost的功能是全面性重量级,用户能想得到的功能99%全都有;funboost的使用方式是轻量级,只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件,支持如 celery dramatiq等框架整体作为funboost中间件,python函数加速器,框架包罗万象,用户能想到的控制功能全都有。一统编程思维,兼容50% python业务场景,适用范围广。只需要一行代码即可分布式执行python一切函数,funboost web manager 方便查看和管理消费函数;99%用过funboost的pythoner 感受是 简易 方便 强劲 强大,相见恨晚
|
|
5
5
|
Home-page: https://github.com/ydf0509/funboost
|
|
6
6
|
Author: bfzs
|
|
@@ -86,6 +86,7 @@ Requires-Dist: mysql-replication ==1.0.9 ; extra == 'all'
|
|
|
86
86
|
Requires-Dist: grpcio ==1.60.0 ; extra == 'all'
|
|
87
87
|
Requires-Dist: grpcio-tools ==1.60.0 ; extra == 'all'
|
|
88
88
|
Requires-Dist: protobuf ==4.25.1 ; extra == 'all'
|
|
89
|
+
Requires-Dist: waitress ; extra == 'all'
|
|
89
90
|
Requires-Dist: flask ; extra == 'all'
|
|
90
91
|
Requires-Dist: flask-bootstrap ; extra == 'all'
|
|
91
92
|
Requires-Dist: flask-wtf ; extra == 'all'
|
|
@@ -120,6 +121,7 @@ Requires-Dist: mysql-replication ==1.0.9 ; extra == 'extra_brokers'
|
|
|
120
121
|
Requires-Dist: grpcio ==1.60.0 ; extra == 'extra_brokers'
|
|
121
122
|
Requires-Dist: grpcio-tools ==1.60.0 ; extra == 'extra_brokers'
|
|
122
123
|
Requires-Dist: protobuf ==4.25.1 ; extra == 'extra_brokers'
|
|
124
|
+
Requires-Dist: waitress ; extra == 'extra_brokers'
|
|
123
125
|
Requires-Dist: pulsar-client ==3.1.0 ; (python_version >= "3.7") and extra == 'extra_brokers'
|
|
124
126
|
Provides-Extra: flask
|
|
125
127
|
Requires-Dist: flask ; extra == 'flask'
|
|
@@ -168,7 +170,7 @@ pip install funboost --upgrade
|
|
|
168
170
|
`funboost`是`函数增强器,属于轻型自由框架`,你可以对任意项目任意位置的新旧函数加上`@boost`装饰器,是给你函数赋能插上强大翅膀,用户不需要围绕`funboost`或某个中央app实例来组织代码结构,用户函数自身就是一等公民
|
|
169
171
|
|
|
170
172
|
2个框架最显而易见明显差别就是 `funboost` 无需 `@app.boost` 而是直接`@boost`,这个小区别,造成影响深远的框架用法和理念区别.
|
|
171
|
-
`funboost`任务控制功能更多,支持broker中间件种类更多,并发方式更多,发布性能超越celery 22倍,消费性能超越 celery
|
|
173
|
+
`funboost`任务控制功能更多,支持broker中间件种类更多,并发方式更多,发布性能超越celery 22倍,消费性能超越 celery 46倍,性能是高几个数量级的断崖式遥遥领先,但反而使用比celery简单得多.
|
|
172
174
|
```
|
|
173
175
|
|
|
174
176
|
#### **funboost 支持的并发模式:**
|
|
@@ -206,7 +208,7 @@ pip install funboost --upgrade
|
|
|
206
208
|
通过`funboost web manager` 管理系统,支持全面 查看 监控 管理 `funboost`的任务消费。
|
|
207
209
|
|
|
208
210
|
#### **funboost的性能超过`celery`一个数量级,不是一个档次上:**
|
|
209
|
-
`funboost`发布性能是`celery`的22倍,`funboost`消费性能是`celery`的
|
|
211
|
+
`funboost`发布性能是`celery`的22倍,`funboost`消费性能是`celery`的46倍! 控制变量法对比方式,见文档2.6章节
|
|
210
212
|
|
|
211
213
|
|
|
212
214
|
#### **funboost框架评价:**
|
|
@@ -235,7 +237,7 @@ funboost 框架和一般的框架不一样,因为只有一行代码需要掌
|
|
|
235
237
|
|
|
236
238
|
只要用过 `funboost` 的用户,都评价比 `celery` 的用法简单几百倍.
|
|
237
239
|
|
|
238
|
-
用户可以看文档`
|
|
240
|
+
用户可以看文档`14`章节,怎么正确的用`ai`大模型掌握`funboost`的用法
|
|
239
241
|
```
|
|
240
242
|
|
|
241
243
|
[**1.python万能分布式函数调度框架简funboost简介**](https://funboost.readthedocs.io/zh-cn/latest/articles/c1.html)
|
|
@@ -739,7 +741,7 @@ python比其他语言更需要分布式函数调度框架来执行函数,有
|
|
|
739
741
|
大部分框架,都要深入使用里面的很多个类,还需要继承组合一顿。
|
|
740
742
|
```
|
|
741
743
|
|
|
742
|
-
用户也可以按照 文档
|
|
744
|
+
用户也可以按照 文档14章节的方式,使用ai来掌握`funboost`
|
|
743
745
|
|
|
744
746
|
## 1.6 funboost支持支持celery框架整体作为funboost的broker (2023.4新增)
|
|
745
747
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
funboost/__init__.py,sha256=
|
|
1
|
+
funboost/__init__.py,sha256=8kd6cYLc0abvVhKKfGN2yyx9auqp_CwSInI7k-cy9ZI,4093
|
|
2
2
|
funboost/__init__old.py,sha256=9Kv3cPLnPkbzMRnuJFVkPsuDdx1CdcSIuITkpdncZSc,20382
|
|
3
3
|
funboost/__main__.py,sha256=BetXBv7PkVeeK-UENiFq_KEEIzvObMI0rd8S1DHRdLU,1139
|
|
4
4
|
funboost/constant.py,sha256=tgoui4EVwnAMMzA9qshqyRKS1oi90bYcFVNu0Ss_Jc4,14895
|
|
@@ -36,18 +36,19 @@ funboost/concurrent_pool/backup/async_pool_executor_back.py,sha256=x0pyPgxzTENOs
|
|
|
36
36
|
funboost/concurrent_pool/backup/async_pool_executor_janus.py,sha256=OHMWJ9l3EYTpPpcrPrGGKd4K0tmQ2PN8HiX0Dta0EOo,5728
|
|
37
37
|
funboost/concurrent_pool/backup/grok_async_pool.py,sha256=3DgyB2aT0iHakb-pxd51WRKGIF7EKNNcX_ogDTF2hik,5825
|
|
38
38
|
funboost/consumers/__init__.py,sha256=ZXY_6Kut1VYNQiF5aWEgIWobsW1ht9YUP0TdRZRWFqI,126
|
|
39
|
-
funboost/consumers/base_consumer.py,sha256=
|
|
39
|
+
funboost/consumers/base_consumer.py,sha256=YSYhVMUuv5V2nYZQBd-wDVdfDBTXTtqUfv25VasixKQ,90359
|
|
40
40
|
funboost/consumers/celery_consumer.py,sha256=6BPZa2O36BQFVu7uWuuFCfJTTaw-36DJJPSpvEbamU8,9371
|
|
41
41
|
funboost/consumers/confirm_mixin.py,sha256=5xC9AAQr_MY4tbSed8U-M6tOVmh69Qv9X0ld0JLT9Tk,6185
|
|
42
42
|
funboost/consumers/dramatiq_consumer.py,sha256=ozmeAfeF0U-YNYHK4suQB0N264h5AZdfMH0O45Mh-8A,2229
|
|
43
43
|
funboost/consumers/empty_consumer.py,sha256=qbkQX2Qlw2Wm8dFEJqjSEEvTA-uh184sqQbFo_5EOiI,1501
|
|
44
44
|
funboost/consumers/faststream_consumer.py,sha256=4UEYGSjUwJElhy4ZGzgL7QhYVCbdVGjyZ6QuM-F0oVA,2361
|
|
45
|
-
funboost/consumers/grpc_consumer.py,sha256=
|
|
46
|
-
funboost/consumers/http_consumer.py,sha256=
|
|
45
|
+
funboost/consumers/grpc_consumer.py,sha256=rytrYXslxNQdWrmct1pdV4E3Rxkcju3hJc6UVh3gLYY,3498
|
|
46
|
+
funboost/consumers/http_consumer.py,sha256=EZiZYo14BoevFK0g1EzebTBafLSCx-CZf5c5vtWmhW4,5605
|
|
47
47
|
funboost/consumers/http_consumer000.py,sha256=PiiSLSQB-hHgS1vAn8DoHD2siC3zO6_kKjVW0g6AFIc,4379
|
|
48
|
+
funboost/consumers/http_consumer_aiohttp_old.py,sha256=kyqDYQT3kkDBSkQN5LhoTHHUJBYd2mPQcz3D7QMIGeU,4873
|
|
48
49
|
funboost/consumers/httpsqs_consumer.py,sha256=kaqOcrKMLrSR27XqeiheRDmpF1KDVDghgbHcefTjqt8,1171
|
|
49
50
|
funboost/consumers/huey_consumer.py,sha256=cW10ZPxdZQzUuJwdqQpJIRPTj2vCbZS0uXFJ7L8bpa4,1843
|
|
50
|
-
funboost/consumers/kafka_consumer.py,sha256=
|
|
51
|
+
funboost/consumers/kafka_consumer.py,sha256=fCcClME4q_MN5Yoz_JycZhmm9fvPbh96DTG_vX7KtPU,4406
|
|
51
52
|
funboost/consumers/kafka_consumer_manually_commit.py,sha256=XbExZKewuxt-z8JFo_yV9yjjw7rDbTK0F20V84Y6E_A,9929
|
|
52
53
|
funboost/consumers/kombu_consumer.py,sha256=ZbzCiKQDZtbD_aK7aWxSvzARYjNuHVlyef4Feb4R7q0,7938
|
|
53
54
|
funboost/consumers/local_python_queue_consumer.py,sha256=4Cel1WaNwbRpDux22USP8is5R9__A_-LlqyHjcw02Z4,1160
|
|
@@ -91,20 +92,20 @@ funboost/contrib/cdc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
|
|
|
91
92
|
funboost/contrib/cdc/mysql2mysql.py,sha256=J7jLHTB27WO3iMMNU4s556b_p0x51dgnCz5c7eVWtKk,2317
|
|
92
93
|
funboost/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
93
94
|
funboost/core/active_cousumer_info_getter.py,sha256=RF_xJMCskQImWwi7aLcIVipsp5KgDZZ4vtxBEhoPeTI,16086
|
|
94
|
-
funboost/core/booster.py,sha256=
|
|
95
|
+
funboost/core/booster.py,sha256=piykdtMO2jwnnEzuBcyPuQ5MZNZls1mVJD4SjXRkAqA,22811
|
|
95
96
|
funboost/core/current_task.py,sha256=6f7IbGZaSnojVxIeyXPkk5LE2yUiRRTJBjTNqwLL2WU,7270
|
|
96
97
|
funboost/core/exceptions.py,sha256=twp5eAUCds3sWh3Ar7WVNhCn3y_TFqb-Ajwbjb9scGU,1564
|
|
97
98
|
funboost/core/fabric_deploy_helper.py,sha256=foieeqlNySuU9axJzNF6TavPjIUSYBx9UO3syVKUiyY,9999
|
|
98
99
|
funboost/core/funboost_config_getter.py,sha256=b5nAdAmUxahskY-ohB7ptf2gKywFlDA0Fq1cWroxxbs,384
|
|
99
|
-
funboost/core/funboost_time.py,sha256=
|
|
100
|
-
funboost/core/func_params_model.py,sha256=
|
|
100
|
+
funboost/core/funboost_time.py,sha256=Wz7Zto7RrDoJk70-qTPncTJsIwA7HmvJBmgQLqXWzhU,5538
|
|
101
|
+
funboost/core/func_params_model.py,sha256=n7fDmmdMeGcFqeqq26QmP9wU36_BqX-5ogIVSV-U9EY,26380
|
|
101
102
|
funboost/core/function_result_status_config.py,sha256=PyjqAQOiwsLt28sRtH-eYRjiI3edPFO4Nde0ILFRReE,1764
|
|
102
103
|
funboost/core/function_result_status_saver.py,sha256=LMLF6AROCaaYB1HIXT13L5NB6r-yezaCGeB9VSSxnQ0,9871
|
|
103
|
-
funboost/core/helper_funs.py,sha256=
|
|
104
|
+
funboost/core/helper_funs.py,sha256=7Pe9ax0GmS5cf3jPGPJPEjL0lrF5JB3g00DYsxX2agQ,2800
|
|
104
105
|
funboost/core/kill_remote_task.py,sha256=lfclwtNhMDGLKX2UCpK_wyhnKPKkoxCZxesRA6KHOrc,8186
|
|
105
106
|
funboost/core/lazy_impoter.py,sha256=yyJqwmbJziMfRTESn9magqso-_8ppl8yzHFCS5qzxkI,5104
|
|
106
107
|
funboost/core/loggers.py,sha256=YY69MAP_o0Eq-CHp5UNWrKDYpoJsiHZ92E2i_fxcxRI,2358
|
|
107
|
-
funboost/core/msg_result_getter.py,sha256=
|
|
108
|
+
funboost/core/msg_result_getter.py,sha256=q_4jzpSGalNkIv_prRULmkreF6-yseMgUne2cna4GMo,12637
|
|
108
109
|
funboost/core/muliti_process_enhance.py,sha256=tI3178inc5sqPh-jQc0XaTuUD1diIZyHuukBRk1Gp6Y,3595
|
|
109
110
|
funboost/core/serialization.py,sha256=-5YdIppOSx81xaOXCryZa2nsmJ-Ro3pfV6yePcbkoEA,1240
|
|
110
111
|
funboost/core/task_id_logger.py,sha256=lR19HQcX6Pp8laURCD656xNpF_JP6nLB3zUKI69EWzE,864
|
|
@@ -113,7 +114,7 @@ funboost/core/cli/discovery_boosters.py,sha256=mbEyv0bUIGcmgkfXLI_Q1IK1QvVwKyro8
|
|
|
113
114
|
funboost/core/cli/funboost_cli_user_templ.py,sha256=XUpKLxRKtYfebPUM8wii64kB0HW8L7j9LnRpT0xCfQI,2243
|
|
114
115
|
funboost/core/cli/funboost_fire.py,sha256=n2Zny_UJ7zx4kRXD_YzNBHqdHMc7n0SE7gL28tuwiPU,5387
|
|
115
116
|
funboost/factories/__init__.py,sha256=s7kKKjR1HU5eMjPD6r5b-SXTVMo1zBp2JjOAtkyt5Yo,178
|
|
116
|
-
funboost/factories/broker_kind__publsiher_consumer_type_map.py,sha256=
|
|
117
|
+
funboost/factories/broker_kind__publsiher_consumer_type_map.py,sha256=8mkp4hVYKGaHsrJPyqKykWbU-OuiEYdMdT4Eb0mS7sM,10873
|
|
117
118
|
funboost/factories/consumer_factory.py,sha256=3RbdcH5fNAnumDkZhtRpDP7RsfJw50NLCgl_-YgJclE,1494
|
|
118
119
|
funboost/factories/publisher_factotry.py,sha256=W5giVsQnT6pHRIC6nRANj0-XBovkuzkl0Zqcb4u5o_U,2529
|
|
119
120
|
funboost/function_result_web/app.py,sha256=bEr9vdceU2cX4ZKqHDXUOQVKlobSLI_qzzzE-A3DABo,14895
|
|
@@ -174,7 +175,7 @@ funboost/function_result_web/templates/rpc_call.html,sha256=qFznWysEFTvJKUQYqnJa
|
|
|
174
175
|
funboost/function_result_web/templates/running_consumer_by_ip.html,sha256=2Rcxbi80c1JEIRCnNe1MG55axdb0vBlkB6yL9rxw53c,10248
|
|
175
176
|
funboost/function_result_web/templates/running_consumer_by_queue_name.html,sha256=r5EYlfp0fE8RFWzI0k3K571EUmirwcPX9NdnSEfAWiQ,10301
|
|
176
177
|
funboost/publishers/__init__.py,sha256=xqBHlvsJQVPfbdvP84G0LHmVB7-pFBS7vDnX1Uo9pVY,131
|
|
177
|
-
funboost/publishers/base_publisher.py,sha256=
|
|
178
|
+
funboost/publishers/base_publisher.py,sha256=THtqnyemMKaS13KC1B66MeqzzFn0JNOQRFlUwWzSYkQ,19255
|
|
178
179
|
funboost/publishers/celery_publisher.py,sha256=uc9N1uLW74skUCw8dsnvxORM2O3cy4SiI7tUZRmvkHA,2336
|
|
179
180
|
funboost/publishers/celery_publisher000.py,sha256=2XLOyU2__vlIUTi5L15uf0BJqAIjxbc3kCLIRDSOY9w,3966
|
|
180
181
|
funboost/publishers/confluent_kafka_publisher.py,sha256=B4rF6gljixOMyN6L2eL1gzqTv97uoy7TTzgKUhHljEQ,4749
|
|
@@ -182,7 +183,7 @@ funboost/publishers/dramatiq_publisher.py,sha256=IH9F-Ps1r94WDu2a7cZbJqWlBgblDbE
|
|
|
182
183
|
funboost/publishers/empty_publisher.py,sha256=Com5m-mkjXpcWxKZZneymhLNaRJNaGAtvwjhwHmECgg,870
|
|
183
184
|
funboost/publishers/faststream_publisher.py,sha256=TS_hFDZ4Q1HGyGJ_3D9m9SroUOmMo-f2hCMTyuEoEIU,2303
|
|
184
185
|
funboost/publishers/grpc_publisher.py,sha256=kQ8c9VMmFXXwMz0-KyDV8VWKeAzZ20pp6KrI3rZmbRQ,1967
|
|
185
|
-
funboost/publishers/http_publisher.py,sha256=
|
|
186
|
+
funboost/publishers/http_publisher.py,sha256=CXjrcuVXC2gsJOw-KomWX4k6jzlFyTehMN89DQUNpZE,1975
|
|
186
187
|
funboost/publishers/httpsqs_publisher.py,sha256=PS6h8-mn3wYFfMOsPt4tal8p0yZgYgrYYO9ZnIl9CwU,2737
|
|
187
188
|
funboost/publishers/huey_publisher.py,sha256=9HBrsqTO61iPB1nI5fYOQNPuOaX4I4Wmb1BRNODAE_0,1118
|
|
188
189
|
funboost/publishers/kafka_publisher.py,sha256=psjQbOmjpNHZNxyo14a09XsO0JLSSPPaBLc5mLUJTRI,2343
|
|
@@ -326,9 +327,9 @@ funboost/utils/func_timeout/dafunc.py,sha256=Yy98BzsmgWi07ja5zM4ElLwb1h8NXahxtRG
|
|
|
326
327
|
funboost/utils/func_timeout/exceptions.py,sha256=tUEaspemq_dS460EQvUMMSxeeyjIbgfEHIdxIC6ZhaU,3974
|
|
327
328
|
funboost/utils/func_timeout/py2_raise.py,sha256=9tpZLQ3-zIgU_ixazydwZmw8rFg7ybjI9alNYfSvwRk,169
|
|
328
329
|
funboost/utils/func_timeout/py3_raise.py,sha256=Odvg1FtXTEC--Ru1EIfsHASamBpOm9hdXY7OnlEUObA,280
|
|
329
|
-
funboost-50.
|
|
330
|
-
funboost-50.
|
|
331
|
-
funboost-50.
|
|
332
|
-
funboost-50.
|
|
333
|
-
funboost-50.
|
|
334
|
-
funboost-50.
|
|
330
|
+
funboost-50.3.dist-info/LICENSE,sha256=9EPP2ktG_lAPB8PjmWV-c9BiaJHc_FP6pPLcUrUwx0E,11562
|
|
331
|
+
funboost-50.3.dist-info/METADATA,sha256=Z2Zsq9Y2qleAbv1r1R22msVkRqIUQmmsogNzv0p_j9k,42216
|
|
332
|
+
funboost-50.3.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
|
333
|
+
funboost-50.3.dist-info/entry_points.txt,sha256=yMSSAGRzRAAhGyNNQHw24MooKlDZsaJ499_D6fPl58A,96
|
|
334
|
+
funboost-50.3.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
|
|
335
|
+
funboost-50.3.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|