funboost 50.2__py3-none-any.whl → 50.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of funboost might be problematic. Click here for more details.
- funboost/__init__.py +1 -1
- funboost/constant.py +4 -0
- funboost/consumers/base_consumer.py +95 -96
- funboost/consumers/celery_consumer.py +1 -1
- funboost/consumers/dramatiq_consumer.py +0 -5
- funboost/consumers/grpc_consumer.py +2 -19
- funboost/consumers/http_consumer.py +107 -40
- funboost/consumers/http_consumer_aiohttp_old.py +113 -0
- funboost/consumers/huey_consumer.py +2 -5
- funboost/consumers/kafka_consumer.py +1 -6
- funboost/consumers/kafka_consumer_manually_commit.py +0 -1
- funboost/consumers/kombu_consumer.py +0 -39
- funboost/consumers/mysql_cdc_consumer.py +1 -3
- funboost/consumers/pulsar_consumer.py +10 -5
- funboost/consumers/rabbitmq_amqpstorm_consumer.py +7 -8
- funboost/consumers/rabbitmq_complex_routing_consumer.py +54 -0
- funboost/consumers/redis_consumer.py +1 -1
- funboost/consumers/redis_consumer_ack_able.py +1 -1
- funboost/consumers/redis_consumer_ack_using_timeout.py +2 -6
- funboost/consumers/redis_consumer_priority.py +1 -1
- funboost/consumers/redis_stream_consumer.py +1 -3
- funboost/consumers/tcp_consumer.py +1 -1
- funboost/consumers/udp_consumer.py +1 -1
- funboost/consumers/zeromq_consumer.py +1 -1
- funboost/contrib/save_function_result_status/__init__.py +0 -0
- funboost/contrib/{save_result_status_to_sqldb.py → save_function_result_status/save_result_status_to_sqldb.py} +8 -41
- funboost/contrib/save_function_result_status/save_result_status_use_dataset.py +47 -0
- funboost/core/booster.py +38 -3
- funboost/core/broker_kind__exclusive_config_default_define.py +229 -0
- funboost/core/funboost_time.py +10 -45
- funboost/core/func_params_model.py +28 -4
- funboost/core/helper_funs.py +9 -8
- funboost/core/msg_result_getter.py +27 -0
- funboost/factories/broker_kind__publsiher_consumer_type_map.py +13 -3
- funboost/funboost_config_deafult.py +0 -3
- funboost/function_result_web/templates/fun_result_table.html +1 -1
- funboost/publishers/base_publisher.py +8 -2
- funboost/publishers/http_publisher.py +20 -2
- funboost/publishers/rabbitmq_amqpstorm_publisher.py +8 -7
- funboost/publishers/rabbitmq_complex_routing_publisher.py +84 -0
- funboost/utils/redis_manager.py +11 -5
- {funboost-50.2.dist-info → funboost-50.4.dist-info}/METADATA +159 -98
- {funboost-50.2.dist-info → funboost-50.4.dist-info}/RECORD +46 -41
- {funboost-50.2.dist-info → funboost-50.4.dist-info}/WHEEL +1 -1
- funboost-50.2.dist-info/LICENSE +0 -203
- {funboost-50.2.dist-info → funboost-50.4.dist-info}/entry_points.txt +0 -0
- {funboost-50.2.dist-info → funboost-50.4.dist-info}/top_level.txt +0 -0
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
# -*- coding: utf-8 -*-
|
|
2
2
|
import logging
|
|
3
3
|
from pathlib import Path
|
|
4
|
-
import pytz
|
|
5
|
-
from funboost.constant import BrokerEnum, ConcurrentModeEnum
|
|
6
|
-
from funboost.core.func_params_model import FunctionResultStatusPersistanceConfig
|
|
7
4
|
from funboost.utils.simple_data_class import DataClassBase
|
|
8
5
|
from nb_log import nb_log_config_default
|
|
9
6
|
|
|
@@ -325,7 +325,7 @@
|
|
|
325
325
|
var time_start_obj = new Date(item.time_start * 1000);
|
|
326
326
|
var time_start_str = dateToString(time_start_obj);
|
|
327
327
|
|
|
328
|
-
tr = tr.format(displayLevel, item.host_process + ' - ' + item.script_name, item.function, item.params_str, item.result, item.
|
|
328
|
+
tr = tr.format(displayLevel, item.host_process + ' - ' + item.script_name, item.function, item.params_str, item.result, item.publish_time_format,
|
|
329
329
|
time_start_str, item.time_cost, item.run_times, run_status_text, successText, item.exception, item.total_thread);
|
|
330
330
|
html += tr;
|
|
331
331
|
}
|
|
@@ -10,7 +10,6 @@ import atexit
|
|
|
10
10
|
import json
|
|
11
11
|
import logging
|
|
12
12
|
import multiprocessing
|
|
13
|
-
from re import S
|
|
14
13
|
import sys
|
|
15
14
|
import threading
|
|
16
15
|
import time
|
|
@@ -20,7 +19,9 @@ from threading import Lock
|
|
|
20
19
|
|
|
21
20
|
import nb_log
|
|
22
21
|
from funboost.constant import ConstStrForClassMethod, FunctionKind
|
|
22
|
+
from funboost.core.broker_kind__exclusive_config_default_define import generate_broker_exclusive_config
|
|
23
23
|
from funboost.core.func_params_model import PublisherParams, PriorityConsumingControlConfig
|
|
24
|
+
from funboost.core.function_result_status_saver import FunctionResultStatus
|
|
24
25
|
from funboost.core.helper_funs import MsgGenerater
|
|
25
26
|
from funboost.core.loggers import develop_logger
|
|
26
27
|
|
|
@@ -139,11 +140,12 @@ class PublishParamsChecker(FunboostFileLoggerMixin):
|
|
|
139
140
|
class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
140
141
|
def __init__(self, publisher_params: PublisherParams, ):
|
|
141
142
|
self.publisher_params = publisher_params
|
|
143
|
+
|
|
142
144
|
self.queue_name = self._queue_name = publisher_params.queue_name
|
|
143
145
|
self.logger: logging.Logger
|
|
144
146
|
self._build_logger()
|
|
145
147
|
self.publish_params_checker = PublishParamsChecker(publisher_params.consuming_function) if publisher_params.consuming_function else None
|
|
146
|
-
|
|
148
|
+
self.publisher_params.broker_exclusive_config = generate_broker_exclusive_config(self.publisher_params.broker_kind,self.publisher_params.broker_exclusive_config,self.logger)
|
|
147
149
|
self.has_init_broker = 0
|
|
148
150
|
self._lock_for_count = Lock()
|
|
149
151
|
self._current_time = None
|
|
@@ -315,6 +317,10 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
315
317
|
def concrete_realization_of_publish(self, msg: str):
|
|
316
318
|
raise NotImplementedError
|
|
317
319
|
|
|
320
|
+
def sync_call(self, msg_dict: dict, is_return_rpc_data_obj=True) -> typing.Union[dict, FunctionResultStatus]:
|
|
321
|
+
"""仅有部分中间件支持同步调用并阻塞等待返回结果,不依赖AsyncResult + redis作为rpc,例如 http grpc 等"""
|
|
322
|
+
raise NotImplementedError(f'broker {self.publisher_params.broker_kind} not support sync_call method')
|
|
323
|
+
|
|
318
324
|
@abc.abstractmethod
|
|
319
325
|
def clear(self):
|
|
320
326
|
raise NotImplementedError
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
# -*- coding: utf-8 -*-
|
|
2
2
|
# @Author : ydf
|
|
3
3
|
# @Time : 2022/8/8 0008 12:12
|
|
4
|
+
import threading
|
|
4
5
|
|
|
6
|
+
from funboost.core.function_result_status_saver import FunctionResultStatus
|
|
7
|
+
from funboost.core.serialization import Serialization
|
|
5
8
|
from funboost.publishers.base_publisher import AbstractPublisher
|
|
6
9
|
from urllib3 import PoolManager
|
|
7
10
|
|
|
@@ -13,7 +16,7 @@ class HTTPPublisher(AbstractPublisher, ):
|
|
|
13
16
|
|
|
14
17
|
# noinspection PyAttributeOutsideInit
|
|
15
18
|
def custom_init(self):
|
|
16
|
-
self._http = PoolManager(
|
|
19
|
+
self._http = PoolManager(maxsize=100)
|
|
17
20
|
self._ip = self.publisher_params.broker_exclusive_config['host']
|
|
18
21
|
self._port = self.publisher_params.broker_exclusive_config['port']
|
|
19
22
|
self._ip_port_str = f'{self._ip}:{self._port}'
|
|
@@ -23,7 +26,22 @@ class HTTPPublisher(AbstractPublisher, ):
|
|
|
23
26
|
|
|
24
27
|
def concrete_realization_of_publish(self, msg):
|
|
25
28
|
url = self._ip_port_str + '/queue'
|
|
26
|
-
self._http.request('post', url, fields={'msg': msg})
|
|
29
|
+
self._http.request('post', url, fields={'msg': msg,'call_type':'publish'})
|
|
30
|
+
|
|
31
|
+
def sync_call(self, msg_dict: dict, is_return_rpc_data_obj=True):
|
|
32
|
+
url = self._ip_port_str + '/queue'
|
|
33
|
+
response = self._http.request('post', url,
|
|
34
|
+
fields={'msg': Serialization.to_json_str(msg_dict),'call_type':'sync_call'})
|
|
35
|
+
json_resp = response.data.decode('utf-8')
|
|
36
|
+
# import requests
|
|
37
|
+
# response = requests.request('post', url,
|
|
38
|
+
# data={'msg': Serialization.to_json_str(msg_dict),'call_type':'sync_call'})
|
|
39
|
+
# json_resp = response.text()
|
|
40
|
+
if is_return_rpc_data_obj:
|
|
41
|
+
return FunctionResultStatus.parse_status_and_result_to_obj(Serialization.to_dict(json_resp))
|
|
42
|
+
else:
|
|
43
|
+
return Serialization.to_dict(json_resp)
|
|
44
|
+
|
|
27
45
|
|
|
28
46
|
def clear(self):
|
|
29
47
|
pass # udp没有保存消息
|
|
@@ -12,19 +12,20 @@ from funboost.utils import decorators
|
|
|
12
12
|
class RabbitmqPublisherUsingAmqpStorm(AbstractPublisher):
|
|
13
13
|
# 使用amqpstorm包实现的mq操作。
|
|
14
14
|
# 实例属性没在__init__里面写,造成代码补全很麻烦,写在这里做类属性,方便pycharm补全
|
|
15
|
-
connection
|
|
16
|
-
channel
|
|
17
|
-
channel_wrapper_by_ampqstormbaic
|
|
18
|
-
queue
|
|
19
|
-
|
|
15
|
+
connection : amqpstorm.UriConnection
|
|
16
|
+
channel : amqpstorm.Channel
|
|
17
|
+
channel_wrapper_by_ampqstormbaic : AmqpStormBasic
|
|
18
|
+
queue : AmqpStormQueue
|
|
19
|
+
|
|
20
20
|
|
|
21
21
|
def custom_init(self):
|
|
22
22
|
arguments = {} # {'x-queue-type':'classic'} classic stream lazy quorum
|
|
23
23
|
if self.publisher_params.broker_exclusive_config.get('x-max-priority'):
|
|
24
24
|
arguments['x-max-priority'] = self.publisher_params.broker_exclusive_config['x-max-priority']
|
|
25
|
-
self.
|
|
25
|
+
self._queue_durable = self.publisher_params.broker_exclusive_config['queue_durable']
|
|
26
|
+
self.queue_declare_params = dict(queue=self._queue_name, durable=self._queue_durable, arguments=arguments,auto_delete=False)
|
|
26
27
|
|
|
27
|
-
# noinspection
|
|
28
|
+
# noinspection PyAttrib_durableuteOutsideInit
|
|
28
29
|
# @decorators.synchronized
|
|
29
30
|
def init_broker(self):
|
|
30
31
|
# username=app_config.RABBITMQ_USER, password=app_config.RABBITMQ_PASS, host=app_config.RABBITMQ_HOST, port=app_config.RABBITMQ_PORT, virtual_host=app_config.RABBITMQ_VIRTUAL_HOST, heartbeat=60 * 10
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# @Author : ydf
|
|
3
|
+
# @Time : 2022/8/8 0008 12:06
|
|
4
|
+
import amqpstorm
|
|
5
|
+
from amqpstorm.basic import Basic as AmqpStormBasic
|
|
6
|
+
from amqpstorm.queue import Queue as AmqpStormQueue
|
|
7
|
+
from funboost.funboost_config_deafult import BrokerConnConfig
|
|
8
|
+
from funboost.publishers.base_publisher import AbstractPublisher, deco_mq_conn_error
|
|
9
|
+
from funboost.utils import decorators
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class RabbitmqComplexRoutingPublisher(AbstractPublisher):
|
|
13
|
+
# 使用amqpstorm包实现的mq操作。
|
|
14
|
+
# 实例属性没在__init__里面写,造成代码补全很麻烦,写在这里做类属性,方便pycharm补全
|
|
15
|
+
connection : amqpstorm.UriConnection
|
|
16
|
+
channel : amqpstorm.Channel
|
|
17
|
+
channel_wrapper_by_ampqstormbaic : AmqpStormBasic
|
|
18
|
+
queue : AmqpStormQueue
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def custom_init(self):
|
|
22
|
+
arguments = {} # {'x-queue-type':'classic'} classic stream lazy quorum
|
|
23
|
+
if self.publisher_params.broker_exclusive_config.get('x-max-priority'):
|
|
24
|
+
arguments['x-max-priority'] = self.publisher_params.broker_exclusive_config['x-max-priority']
|
|
25
|
+
self._queue_durable = self.publisher_params.broker_exclusive_config['queue_durable']
|
|
26
|
+
self.queue_declare_params = dict(queue=self._queue_name, durable=self._queue_durable,
|
|
27
|
+
arguments=arguments,auto_delete=False)
|
|
28
|
+
|
|
29
|
+
self._exchange_name = self.publisher_params.broker_exclusive_config['exchange_name']
|
|
30
|
+
self._exchange_type = self.publisher_params.broker_exclusive_config['exchange_type']
|
|
31
|
+
# 如果用户没有在 broker_exclusive_config 中指定 routing_key_for_publish,则默认使用队列名作为 routing_key
|
|
32
|
+
self._routing_key_for_publish = self.publisher_params.broker_exclusive_config.get('routing_key_for_publish') or self._queue_name # 这里你已经用了很好的实践
|
|
33
|
+
|
|
34
|
+
self._exchange_declare_durable = self.publisher_params.broker_exclusive_config['exchange_declare_durable']
|
|
35
|
+
|
|
36
|
+
self.init_broker()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# noinspection PyAttributeOutsideInit
|
|
40
|
+
# @decorators.synchronized
|
|
41
|
+
def init_broker(self):
|
|
42
|
+
# username=app_config.RABBITMQ_USER, password=app_config.RABBITMQ_PASS, host=app_config.RABBITMQ_HOST, port=app_config.RABBITMQ_PORT, virtual_host=app_config.RABBITMQ_VIRTUAL_HOST, heartbeat=60 * 10
|
|
43
|
+
self.logger.warning(f'使用AmqpStorm包 链接mq')
|
|
44
|
+
self.connection = amqpstorm.UriConnection(
|
|
45
|
+
f'amqp://{BrokerConnConfig.RABBITMQ_USER}:{BrokerConnConfig.RABBITMQ_PASS}@{BrokerConnConfig.RABBITMQ_HOST}:{BrokerConnConfig.RABBITMQ_PORT}/{BrokerConnConfig.RABBITMQ_VIRTUAL_HOST}?heartbeat={60 * 10}&timeout=20000'
|
|
46
|
+
)
|
|
47
|
+
self.channel = self.connection.channel() # type:amqpstorm.Channel
|
|
48
|
+
self.channel_wrapper_by_ampqstormbaic = AmqpStormBasic(self.channel)
|
|
49
|
+
# 发布者不声明队列,队列的声明和绑定完全由消费者负责。
|
|
50
|
+
if self._exchange_name:
|
|
51
|
+
self.channel.exchange.declare(exchange=self._exchange_name, exchange_type=self._exchange_type,
|
|
52
|
+
durable=self._exchange_declare_durable)
|
|
53
|
+
|
|
54
|
+
# @decorators.tomorrow_threads(10)
|
|
55
|
+
@deco_mq_conn_error
|
|
56
|
+
def concrete_realization_of_publish(self, msg: str):
|
|
57
|
+
routing_key_publish_dynamic = self._get_from_other_extra_params('routing_key_for_publish', msg)
|
|
58
|
+
routing_key_publish = routing_key_publish_dynamic if routing_key_publish_dynamic is not None else self._routing_key_for_publish
|
|
59
|
+
# 核心修正2:为 headers 交换机添加 headers 支持
|
|
60
|
+
headers = self._get_from_other_extra_params('headers_for_publish', msg)
|
|
61
|
+
|
|
62
|
+
self.channel_wrapper_by_ampqstormbaic.publish(exchange=self._exchange_name,
|
|
63
|
+
routing_key=routing_key_publish,
|
|
64
|
+
body=msg,
|
|
65
|
+
properties={'delivery_mode': 2,
|
|
66
|
+
'priority': self._get_from_other_extra_params('priority', msg),
|
|
67
|
+
'headers': headers}, )
|
|
68
|
+
# nb_print(msg)
|
|
69
|
+
|
|
70
|
+
@deco_mq_conn_error
|
|
71
|
+
def clear(self):
|
|
72
|
+
AmqpStormQueue(self.channel).purge(self._queue_name)
|
|
73
|
+
self.logger.warning(f'清除 {self._queue_name} 队列中的消息成功')
|
|
74
|
+
|
|
75
|
+
@deco_mq_conn_error
|
|
76
|
+
def get_message_count(self):
|
|
77
|
+
# noinspection PyUnresolvedReferences
|
|
78
|
+
return AmqpStormQueue(self.channel).declare(**self.queue_declare_params)['message_count']
|
|
79
|
+
|
|
80
|
+
# @deco_mq_conn_error
|
|
81
|
+
def close(self):
|
|
82
|
+
self.channel.close()
|
|
83
|
+
self.connection.close()
|
|
84
|
+
self.logger.warning('关闭amqpstorm包 链接mq')
|
funboost/utils/redis_manager.py
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# coding=utf8
|
|
2
2
|
|
|
3
3
|
import copy
|
|
4
|
+
import os
|
|
5
|
+
import threading
|
|
4
6
|
# import redis2 as redis
|
|
5
7
|
# import redis3
|
|
6
8
|
import redis5
|
|
@@ -25,14 +27,18 @@ def _get_redis_conn_kwargs_by_db(db):
|
|
|
25
27
|
|
|
26
28
|
class RedisManager(object):
|
|
27
29
|
_redis_db__conn_map = {}
|
|
30
|
+
_lock = threading.Lock()
|
|
28
31
|
|
|
29
32
|
def __init__(self, host='127.0.0.1', port=6379, db=0, username='', password='',ssl=False):
|
|
30
|
-
|
|
33
|
+
pid = os.getpid()
|
|
34
|
+
self._key = (host, port, db, username, password,ssl,pid)
|
|
31
35
|
if self._key not in self.__class__._redis_db__conn_map:
|
|
32
|
-
self.__class__.
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
+
with self.__class__._lock:
|
|
37
|
+
if self._key not in self.__class__._redis_db__conn_map:
|
|
38
|
+
self.__class__._redis_db__conn_map[self._key] = redis5.Redis(host=host, port=port, db=db, username=username,
|
|
39
|
+
password=password, max_connections=100,
|
|
40
|
+
ssl=ssl,
|
|
41
|
+
decode_responses=True)
|
|
36
42
|
self.redis = self.__class__._redis_db__conn_map[self._key]
|
|
37
43
|
|
|
38
44
|
def get_redis(self) -> redis5.Redis:
|
|
@@ -1,19 +1,18 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: funboost
|
|
3
|
-
Version: 50.
|
|
3
|
+
Version: 50.4
|
|
4
4
|
Summary: pip install funboost,python全功能分布式函数调度框架,funboost的功能是全面性重量级,用户能想得到的功能99%全都有;funboost的使用方式是轻量级,只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件,支持如 celery dramatiq等框架整体作为funboost中间件,python函数加速器,框架包罗万象,用户能想到的控制功能全都有。一统编程思维,兼容50% python业务场景,适用范围广。只需要一行代码即可分布式执行python一切函数,funboost web manager 方便查看和管理消费函数;99%用过funboost的pythoner 感受是 简易 方便 强劲 强大,相见恨晚
|
|
5
5
|
Home-page: https://github.com/ydf0509/funboost
|
|
6
6
|
Author: bfzs
|
|
7
7
|
Author-email: ydf0509@sohu.com
|
|
8
8
|
Maintainer: ydf
|
|
9
9
|
Maintainer-email: ydf0509@sohu.com
|
|
10
|
-
License: BSD
|
|
10
|
+
License: BSD-3-Clause
|
|
11
11
|
Keywords: funboost,distributed-framework,function-scheduling,rabbitmq,rocketmq,kafka,nsq,redis,disk,sqlachemy,consume-confirm,timing,task-scheduling,apscheduler,pulsar,mqtt,kombu,的,celery,框架,分布式调度
|
|
12
12
|
Platform: all
|
|
13
13
|
Classifier: Development Status :: 4 - Beta
|
|
14
14
|
Classifier: Operating System :: OS Independent
|
|
15
15
|
Classifier: Intended Audience :: Developers
|
|
16
|
-
Classifier: License :: OSI Approved :: BSD License
|
|
17
16
|
Classifier: Programming Language :: Python
|
|
18
17
|
Classifier: Programming Language :: Python :: Implementation
|
|
19
18
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -28,30 +27,29 @@ Classifier: Programming Language :: Python :: 3.14
|
|
|
28
27
|
Classifier: Programming Language :: Python :: 3 :: Only
|
|
29
28
|
Classifier: Topic :: Software Development :: Libraries
|
|
30
29
|
Description-Content-Type: text/markdown
|
|
31
|
-
|
|
32
|
-
Requires-Dist:
|
|
33
|
-
Requires-Dist:
|
|
34
|
-
Requires-Dist:
|
|
35
|
-
Requires-Dist:
|
|
36
|
-
Requires-Dist:
|
|
37
|
-
Requires-Dist:
|
|
38
|
-
Requires-Dist:
|
|
39
|
-
Requires-Dist:
|
|
40
|
-
Requires-Dist:
|
|
41
|
-
Requires-Dist: apscheduler <4.0.0,>=3.10.1
|
|
30
|
+
Requires-Dist: nb_log>=13.9
|
|
31
|
+
Requires-Dist: nb_libs>=1.9
|
|
32
|
+
Requires-Dist: nb_time>=2.7
|
|
33
|
+
Requires-Dist: pymongo>=4.6.3
|
|
34
|
+
Requires-Dist: AMQPStorm==2.10.6
|
|
35
|
+
Requires-Dist: rabbitpy==2.0.1
|
|
36
|
+
Requires-Dist: decorator==5.1.1
|
|
37
|
+
Requires-Dist: tomorrow3==1.1.0
|
|
38
|
+
Requires-Dist: persist-queue>=0.4.2
|
|
39
|
+
Requires-Dist: apscheduler<4.0.0,>=3.10.1
|
|
42
40
|
Requires-Dist: pikav0
|
|
43
41
|
Requires-Dist: pikav1
|
|
44
42
|
Requires-Dist: redis2
|
|
45
43
|
Requires-Dist: redis3
|
|
46
44
|
Requires-Dist: redis5
|
|
47
45
|
Requires-Dist: redis
|
|
48
|
-
Requires-Dist:
|
|
49
|
-
Requires-Dist: fabric2
|
|
50
|
-
Requires-Dist:
|
|
46
|
+
Requires-Dist: setuptools_rust
|
|
47
|
+
Requires-Dist: fabric2>=2.6.0
|
|
48
|
+
Requires-Dist: nb_filelock
|
|
51
49
|
Requires-Dist: pysnooper
|
|
52
50
|
Requires-Dist: deprecated
|
|
53
51
|
Requires-Dist: cryptography
|
|
54
|
-
Requires-Dist:
|
|
52
|
+
Requires-Dist: auto_run_on_remote
|
|
55
53
|
Requires-Dist: frozenlist
|
|
56
54
|
Requires-Dist: fire
|
|
57
55
|
Requires-Dist: pydantic
|
|
@@ -59,80 +57,112 @@ Requires-Dist: orjson
|
|
|
59
57
|
Requires-Dist: async-timeout
|
|
60
58
|
Requires-Dist: typing-extensions
|
|
61
59
|
Provides-Extra: all
|
|
62
|
-
Requires-Dist:
|
|
63
|
-
Requires-Dist:
|
|
64
|
-
Requires-Dist:
|
|
65
|
-
Requires-Dist:
|
|
66
|
-
Requires-Dist:
|
|
67
|
-
Requires-Dist: sqlalchemy
|
|
68
|
-
Requires-Dist:
|
|
69
|
-
Requires-Dist:
|
|
70
|
-
Requires-Dist:
|
|
71
|
-
Requires-Dist:
|
|
72
|
-
Requires-Dist:
|
|
73
|
-
Requires-Dist:
|
|
74
|
-
Requires-Dist:
|
|
75
|
-
Requires-Dist:
|
|
76
|
-
Requires-Dist:
|
|
77
|
-
Requires-Dist:
|
|
78
|
-
Requires-Dist:
|
|
79
|
-
Requires-Dist:
|
|
80
|
-
Requires-Dist:
|
|
81
|
-
Requires-Dist:
|
|
82
|
-
Requires-Dist:
|
|
83
|
-
Requires-Dist:
|
|
84
|
-
Requires-Dist:
|
|
85
|
-
Requires-Dist:
|
|
86
|
-
Requires-Dist:
|
|
87
|
-
Requires-Dist: grpcio
|
|
88
|
-
Requires-Dist:
|
|
89
|
-
Requires-Dist:
|
|
90
|
-
Requires-Dist:
|
|
91
|
-
Requires-Dist: flask
|
|
92
|
-
Requires-Dist:
|
|
93
|
-
Requires-Dist:
|
|
94
|
-
Requires-Dist:
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
Requires-Dist:
|
|
98
|
-
Requires-Dist:
|
|
99
|
-
Requires-Dist:
|
|
100
|
-
Requires-Dist:
|
|
101
|
-
Requires-Dist:
|
|
102
|
-
Requires-Dist:
|
|
103
|
-
Requires-Dist:
|
|
104
|
-
Requires-Dist:
|
|
105
|
-
Requires-Dist:
|
|
106
|
-
Requires-Dist:
|
|
107
|
-
Requires-Dist:
|
|
108
|
-
Requires-Dist:
|
|
109
|
-
Requires-Dist:
|
|
110
|
-
Requires-Dist:
|
|
111
|
-
Requires-Dist:
|
|
112
|
-
Requires-Dist:
|
|
113
|
-
Requires-Dist:
|
|
114
|
-
Requires-Dist:
|
|
115
|
-
Requires-Dist:
|
|
116
|
-
Requires-Dist:
|
|
117
|
-
Requires-Dist:
|
|
118
|
-
Requires-Dist:
|
|
119
|
-
Requires-Dist:
|
|
120
|
-
Requires-Dist:
|
|
121
|
-
Requires-Dist:
|
|
122
|
-
Requires-Dist:
|
|
123
|
-
Requires-Dist:
|
|
60
|
+
Requires-Dist: confluent_kafka==1.7.0; extra == "all"
|
|
61
|
+
Requires-Dist: pulsar-client==3.1.0; python_version >= "3.7" and extra == "all"
|
|
62
|
+
Requires-Dist: celery; extra == "all"
|
|
63
|
+
Requires-Dist: flower; extra == "all"
|
|
64
|
+
Requires-Dist: nameko==2.14.1; extra == "all"
|
|
65
|
+
Requires-Dist: sqlalchemy==1.4.13; extra == "all"
|
|
66
|
+
Requires-Dist: sqlalchemy_utils==0.36.1; extra == "all"
|
|
67
|
+
Requires-Dist: dramatiq==1.14.2; extra == "all"
|
|
68
|
+
Requires-Dist: huey==2.4.5; extra == "all"
|
|
69
|
+
Requires-Dist: rq==1.15.0; extra == "all"
|
|
70
|
+
Requires-Dist: kombu; extra == "all"
|
|
71
|
+
Requires-Dist: elasticsearch; extra == "all"
|
|
72
|
+
Requires-Dist: gnsq==1.0.1; extra == "all"
|
|
73
|
+
Requires-Dist: psutil; extra == "all"
|
|
74
|
+
Requires-Dist: peewee==3.17.3; extra == "all"
|
|
75
|
+
Requires-Dist: nats-python; extra == "all"
|
|
76
|
+
Requires-Dist: aiohttp==3.8.3; extra == "all"
|
|
77
|
+
Requires-Dist: paho-mqtt; extra == "all"
|
|
78
|
+
Requires-Dist: rocketmq; extra == "all"
|
|
79
|
+
Requires-Dist: zmq; extra == "all"
|
|
80
|
+
Requires-Dist: pyzmq; extra == "all"
|
|
81
|
+
Requires-Dist: kafka-python==2.0.2; extra == "all"
|
|
82
|
+
Requires-Dist: eventlet==0.33.3; extra == "all"
|
|
83
|
+
Requires-Dist: gevent==22.10.2; extra == "all"
|
|
84
|
+
Requires-Dist: mysql-replication==1.0.9; extra == "all"
|
|
85
|
+
Requires-Dist: grpcio==1.60.0; extra == "all"
|
|
86
|
+
Requires-Dist: grpcio-tools==1.60.0; extra == "all"
|
|
87
|
+
Requires-Dist: protobuf==4.25.1; extra == "all"
|
|
88
|
+
Requires-Dist: waitress; extra == "all"
|
|
89
|
+
Requires-Dist: flask; extra == "all"
|
|
90
|
+
Requires-Dist: flask_bootstrap; extra == "all"
|
|
91
|
+
Requires-Dist: flask_wtf; extra == "all"
|
|
92
|
+
Requires-Dist: wtforms; extra == "all"
|
|
93
|
+
Requires-Dist: flask_login; extra == "all"
|
|
94
|
+
Provides-Extra: extra-brokers
|
|
95
|
+
Requires-Dist: confluent_kafka==1.7.0; extra == "extra-brokers"
|
|
96
|
+
Requires-Dist: pulsar-client==3.1.0; python_version >= "3.7" and extra == "extra-brokers"
|
|
97
|
+
Requires-Dist: celery; extra == "extra-brokers"
|
|
98
|
+
Requires-Dist: flower; extra == "extra-brokers"
|
|
99
|
+
Requires-Dist: nameko==2.14.1; extra == "extra-brokers"
|
|
100
|
+
Requires-Dist: sqlalchemy==1.4.13; extra == "extra-brokers"
|
|
101
|
+
Requires-Dist: sqlalchemy_utils==0.36.1; extra == "extra-brokers"
|
|
102
|
+
Requires-Dist: dramatiq==1.14.2; extra == "extra-brokers"
|
|
103
|
+
Requires-Dist: huey==2.4.5; extra == "extra-brokers"
|
|
104
|
+
Requires-Dist: rq==1.15.0; extra == "extra-brokers"
|
|
105
|
+
Requires-Dist: kombu; extra == "extra-brokers"
|
|
106
|
+
Requires-Dist: elasticsearch; extra == "extra-brokers"
|
|
107
|
+
Requires-Dist: gnsq==1.0.1; extra == "extra-brokers"
|
|
108
|
+
Requires-Dist: psutil; extra == "extra-brokers"
|
|
109
|
+
Requires-Dist: peewee==3.17.3; extra == "extra-brokers"
|
|
110
|
+
Requires-Dist: nats-python; extra == "extra-brokers"
|
|
111
|
+
Requires-Dist: aiohttp==3.8.3; extra == "extra-brokers"
|
|
112
|
+
Requires-Dist: paho-mqtt; extra == "extra-brokers"
|
|
113
|
+
Requires-Dist: rocketmq; extra == "extra-brokers"
|
|
114
|
+
Requires-Dist: zmq; extra == "extra-brokers"
|
|
115
|
+
Requires-Dist: pyzmq; extra == "extra-brokers"
|
|
116
|
+
Requires-Dist: kafka-python==2.0.2; extra == "extra-brokers"
|
|
117
|
+
Requires-Dist: eventlet==0.33.3; extra == "extra-brokers"
|
|
118
|
+
Requires-Dist: gevent==22.10.2; extra == "extra-brokers"
|
|
119
|
+
Requires-Dist: mysql-replication==1.0.9; extra == "extra-brokers"
|
|
120
|
+
Requires-Dist: grpcio==1.60.0; extra == "extra-brokers"
|
|
121
|
+
Requires-Dist: grpcio-tools==1.60.0; extra == "extra-brokers"
|
|
122
|
+
Requires-Dist: protobuf==4.25.1; extra == "extra-brokers"
|
|
123
|
+
Requires-Dist: waitress; extra == "extra-brokers"
|
|
124
124
|
Provides-Extra: flask
|
|
125
|
-
Requires-Dist: flask
|
|
126
|
-
Requires-Dist:
|
|
127
|
-
Requires-Dist:
|
|
128
|
-
Requires-Dist: wtforms
|
|
129
|
-
Requires-Dist:
|
|
125
|
+
Requires-Dist: flask; extra == "flask"
|
|
126
|
+
Requires-Dist: flask_bootstrap; extra == "flask"
|
|
127
|
+
Requires-Dist: flask_wtf; extra == "flask"
|
|
128
|
+
Requires-Dist: wtforms; extra == "flask"
|
|
129
|
+
Requires-Dist: flask_login; extra == "flask"
|
|
130
|
+
Dynamic: author
|
|
131
|
+
Dynamic: author-email
|
|
132
|
+
Dynamic: classifier
|
|
133
|
+
Dynamic: description
|
|
134
|
+
Dynamic: description-content-type
|
|
135
|
+
Dynamic: home-page
|
|
136
|
+
Dynamic: keywords
|
|
137
|
+
Dynamic: license
|
|
138
|
+
Dynamic: maintainer
|
|
139
|
+
Dynamic: maintainer-email
|
|
140
|
+
Dynamic: platform
|
|
141
|
+
Dynamic: provides-extra
|
|
142
|
+
Dynamic: requires-dist
|
|
143
|
+
Dynamic: summary
|
|
130
144
|
|
|
131
145
|
# 1.python万能分布式函数调度框架简funboost简介
|
|
132
146
|
|
|
147
|
+
funboost教程: [https://funboost.readthedocs.io/zh-cn/latest/index.html](https://funboost.readthedocs.io/zh-cn/latest/index.html)
|
|
148
|
+
|
|
133
149
|
## 1.0 funboost 框架说明介绍
|
|
134
150
|
|
|
135
|
-
`funboost`是一个
|
|
151
|
+
`funboost`是一个 万能 强大 简单 自由 的 `python` 全功能分布式调度框架,它的作用是给用户任意项目的任意函数赋能.
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
<h4>📹 观看 funboost 视频</h4>
|
|
155
|
+
<video controls width="800"
|
|
156
|
+
src="https://ydf0509.github.io/funboost_git_pages/%E8%A7%86%E9%A2%91-Funboost_%E8%A7%86%E9%A2%91.mp4">
|
|
157
|
+
您的浏览器不支持视频播放。
|
|
158
|
+
</video>
|
|
159
|
+
|
|
160
|
+
<h4>🎧 收听 funboost 音频</h4>
|
|
161
|
+
<audio controls
|
|
162
|
+
src="https://ydf0509.github.io/funboost_git_pages/%E9%9F%B3%E9%A2%91-funboost_%E9%9F%B3%E9%A2%91.mp4">
|
|
163
|
+
您的浏览器不支持音频播放。
|
|
164
|
+
</audio>
|
|
165
|
+
|
|
136
166
|
|
|
137
167
|
### 1.0.0 funboost 框架安装方式
|
|
138
168
|
|
|
@@ -154,6 +184,36 @@ pip install funboost --upgrade
|
|
|
154
184
|
它证明了一个框架可以既功能丰富又极其易用,这是对传统Python框架设计的一次巧妙超越。
|
|
155
185
|
只需要一行`@boost`代码即可分布式执行`python`一切任意函数,99%用过`funboost`的`pythoner` 感受是 方便 高速 强大 自由。
|
|
156
186
|
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
#### funboost的适用场景:
|
|
190
|
+
```
|
|
191
|
+
`funboost`是python函数加速器,框架包罗万象,一统编程思维,兼容50% `python`编程业务场景,适用范围广,任何新老项目都能用到。
|
|
192
|
+
`funboost` 用途概念就是常规经典的 生产者 + 消息队列中间件 + 消费者 编程思想。
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
```
|
|
196
|
+
需要分布式? 好,funboost支持40多种消息队列,任何只要是稍微小有名气的消息队列和甚至任务框架,funboost全都支持.
|
|
197
|
+
需要并发? 好,那就把Python所有的并发模式(线程、协程、多进程)都给你,并且让它们可以叠加。
|
|
198
|
+
需要可靠性? 好,那就把消费确认(ACK)、自动重试、死信队列(DLQ)、断点续爬做到极致,让你无惧任何宕机。
|
|
199
|
+
需要控制力? 好,那就给你精准的QPS控频、分布式控频、定时任务、延时任务、超时杀死、任务过滤等三十多种控制武器。
|
|
200
|
+
需要监控? 好,那就给你一个开箱即用的Web UI,让你对任务状态、队列情况、消费者实例了如指掌。
|
|
201
|
+
需要自由? 好,那就让你用最普通的Python函数,不侵入你的代码,不规定你的项目结构,让你随时能用,随时能走。
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
#### 有人问 funboost 是做什么的? 怎么回答最合适?
|
|
205
|
+
|
|
206
|
+
这个问题很难精确的一句话概括回答,因为funboost是万能框架,几乎所有的python编程业务场景都能用到,答案是发散的不是唯一的。
|
|
207
|
+
|
|
208
|
+
发散的答案, 见文档 6.0b 章节 [https://funboost.readthedocs.io/zh-cn/latest/articles/c6.html#b-funboost](https://funboost.readthedocs.io/zh-cn/latest/articles/c6.html#b-funboost)
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
#### 最最最重要的第一个问题: "funboost这个框架怎么样呀,值得我学习使用吗?"
|
|
212
|
+
|
|
213
|
+
如果选择学习使用了一个 用途狭窄 功能不好 性能不好 用法复杂 写法不自由 的框架,简直是浪费时间浪费生命.
|
|
214
|
+
|
|
215
|
+
问题答案, 见文档 6.0 章节 [https://funboost.readthedocs.io/zh-cn/latest/articles/c6.html#funboost](https://funboost.readthedocs.io/zh-cn/latest/articles/c6.html#funboost)
|
|
216
|
+
|
|
157
217
|
#### funboost与celery的理念区别
|
|
158
218
|
**共同点:**
|
|
159
219
|
```
|
|
@@ -164,11 +224,15 @@ pip install funboost --upgrade
|
|
|
164
224
|
|
|
165
225
|
**区别:**
|
|
166
226
|
```
|
|
167
|
-
`celery`是`围绕celery框架组织代码,属于重型奴役框架`,你围绕`celery`项目结构和`celery app`这样中央app实例 转,去新增定义`@app.task`函数,
|
|
168
|
-
|
|
227
|
+
`celery`是`围绕celery框架组织代码,属于重型奴役框架`,你围绕`celery`项目结构和`celery app`这样中央app实例 转,去新增定义`@app.task`函数,
|
|
228
|
+
app才是一等公民,task函数是二等公民
|
|
229
|
+
|
|
230
|
+
`funboost`是`函数增强器,属于轻型自由框架`,你可以对任意项目任意位置的新旧函数加上`@boost`装饰器,是给你函数赋能插上强大翅膀,
|
|
231
|
+
用户不需要围绕`funboost`或某个中央app实例来组织代码结构,用户函数自身就是一等公民
|
|
169
232
|
|
|
170
233
|
2个框架最显而易见明显差别就是 `funboost` 无需 `@app.boost` 而是直接`@boost`,这个小区别,造成影响深远的框架用法和理念区别.
|
|
171
|
-
`funboost`任务控制功能更多,支持broker中间件种类更多,并发方式更多,发布性能超越celery 22倍,消费性能超越 celery
|
|
234
|
+
`funboost`任务控制功能更多,支持broker中间件种类更多,并发方式更多,发布性能超越celery 22倍,消费性能超越 celery 46倍,
|
|
235
|
+
性能是高几个数量级的断崖式遥遥领先,但反而使用比celery简单得多.
|
|
172
236
|
```
|
|
173
237
|
|
|
174
238
|
#### **funboost 支持的并发模式:**
|
|
@@ -183,11 +247,8 @@ pip install funboost --upgrade
|
|
|
183
247
|
并且`funboost` 将 `mysql cdc` 这种`mysql binlog`变化捕获作为`broker`,使得`funboost`可以是事件驱动的,远超`celery`的理念.
|
|
184
248
|
`funboost` 还轻松内置了将各种三方消费框架作为`broker`,例如直接将 `celery` `dramatiq` `huey` `rq` `nameko` 作为`broker`,使用这些框架的核心来执行用户的函数
|
|
185
249
|
```
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
`funboost`是python函数加速器,框架包罗万象,一统编程思维,兼容50% `python`编程业务场景,适用范围广,任何新老项目都能用到。
|
|
189
|
-
`funboost` 用途概念就是常规经典的 生产者 + 消息队列中间件 + 消费者 编程思想。
|
|
190
|
-
```
|
|
250
|
+
|
|
251
|
+
|
|
191
252
|
|
|
192
253
|
|
|
193
254
|
#### **funboost 学习难吗?**
|
|
@@ -206,7 +267,7 @@ pip install funboost --upgrade
|
|
|
206
267
|
通过`funboost web manager` 管理系统,支持全面 查看 监控 管理 `funboost`的任务消费。
|
|
207
268
|
|
|
208
269
|
#### **funboost的性能超过`celery`一个数量级,不是一个档次上:**
|
|
209
|
-
`funboost`发布性能是`celery`的22倍,`funboost`消费性能是`celery`的
|
|
270
|
+
`funboost`发布性能是`celery`的22倍,`funboost`消费性能是`celery`的46倍! 控制变量法对比方式,见文档2.6章节
|
|
210
271
|
|
|
211
272
|
|
|
212
273
|
#### **funboost框架评价:**
|
|
@@ -235,7 +296,7 @@ funboost 框架和一般的框架不一样,因为只有一行代码需要掌
|
|
|
235
296
|
|
|
236
297
|
只要用过 `funboost` 的用户,都评价比 `celery` 的用法简单几百倍.
|
|
237
298
|
|
|
238
|
-
用户可以看文档`
|
|
299
|
+
用户可以看文档`14`章节,怎么正确的用`ai`大模型掌握`funboost`的用法
|
|
239
300
|
```
|
|
240
301
|
|
|
241
302
|
[**1.python万能分布式函数调度框架简funboost简介**](https://funboost.readthedocs.io/zh-cn/latest/articles/c1.html)
|
|
@@ -739,7 +800,7 @@ python比其他语言更需要分布式函数调度框架来执行函数,有
|
|
|
739
800
|
大部分框架,都要深入使用里面的很多个类,还需要继承组合一顿。
|
|
740
801
|
```
|
|
741
802
|
|
|
742
|
-
用户也可以按照 文档
|
|
803
|
+
用户也可以按照 文档14章节的方式,使用ai来掌握`funboost`
|
|
743
804
|
|
|
744
805
|
## 1.6 funboost支持支持celery框架整体作为funboost的broker (2023.4新增)
|
|
745
806
|
|