funboost 43.7__py3-none-any.whl → 43.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of funboost might be problematic. Click here for more details.
- funboost/__init__.py +1 -1
- funboost/consumers/base_consumer.py +39 -10
- funboost/core/booster.py +3 -1
- funboost/core/fabric_deploy_helper.py +6 -11
- funboost/core/func_params_model.py +12 -1
- funboost/core/helper_funs.py +5 -1
- funboost/core/lazy_impoter.py +14 -0
- funboost/publishers/base_publisher.py +3 -2
- funboost/queues/peewee_queue.py +13 -13
- funboost/utils/paramiko_util.py +7 -2
- {funboost-43.7.dist-info → funboost-43.9.dist-info}/METADATA +1 -2
- {funboost-43.7.dist-info → funboost-43.9.dist-info}/RECORD +16 -16
- {funboost-43.7.dist-info → funboost-43.9.dist-info}/LICENSE +0 -0
- {funboost-43.7.dist-info → funboost-43.9.dist-info}/WHEEL +0 -0
- {funboost-43.7.dist-info → funboost-43.9.dist-info}/entry_points.txt +0 -0
- {funboost-43.7.dist-info → funboost-43.9.dist-info}/top_level.txt +0 -0
funboost/__init__.py
CHANGED
|
@@ -33,7 +33,7 @@ import nb_log
|
|
|
33
33
|
from funboost.core.current_task import funboost_current_task, get_current_taskid
|
|
34
34
|
from funboost.core.loggers import develop_logger
|
|
35
35
|
|
|
36
|
-
from funboost.core.func_params_model import BoosterParams, PublisherParams
|
|
36
|
+
from funboost.core.func_params_model import BoosterParams, PublisherParams, BaseJsonAbleModel
|
|
37
37
|
from funboost.core.task_id_logger import TaskIdLogger
|
|
38
38
|
from nb_log import (get_logger, LoggerLevelSetterMixin, LogManager, CompatibleLogger,
|
|
39
39
|
LoggerMixinDefaultWithFileHandler, stdout_write, is_main_process,
|
|
@@ -49,7 +49,7 @@ from funboost.concurrent_pool.single_thread_executor import SoloExecutor
|
|
|
49
49
|
|
|
50
50
|
from funboost.core.function_result_status_saver import ResultPersistenceHelper, FunctionResultStatus, RunStatus
|
|
51
51
|
|
|
52
|
-
from funboost.core.helper_funs import delete_keys_and_return_new_dict, get_publish_time
|
|
52
|
+
from funboost.core.helper_funs import delete_keys_and_return_new_dict, get_publish_time, generate_task_id
|
|
53
53
|
|
|
54
54
|
from funboost.concurrent_pool.async_helper import simple_run_in_executor
|
|
55
55
|
from funboost.concurrent_pool.async_pool_executor import AsyncPoolExecutor
|
|
@@ -215,13 +215,15 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
215
215
|
self._consuming_function_is_asyncio = inspect.iscoroutinefunction(self.consuming_function)
|
|
216
216
|
self.custom_init()
|
|
217
217
|
# develop_logger.warning(consumer_params._log_filename)
|
|
218
|
-
self.publisher_params = PublisherParams(queue_name=consumer_params.queue_name, consuming_function=consumer_params.consuming_function,
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
218
|
+
# self.publisher_params = PublisherParams(queue_name=consumer_params.queue_name, consuming_function=consumer_params.consuming_function,
|
|
219
|
+
# broker_kind=self.BROKER_KIND, log_level=consumer_params.log_level,
|
|
220
|
+
# logger_prefix=consumer_params.logger_prefix,
|
|
221
|
+
# create_logger_file=consumer_params.create_logger_file,
|
|
222
|
+
# log_filename=consumer_params.log_filename,
|
|
223
|
+
# logger_name=consumer_params.logger_name,
|
|
224
|
+
# broker_exclusive_config=self.consumer_params.broker_exclusive_config)
|
|
225
|
+
self.publisher_params = BaseJsonAbleModel.init_by_another_model(PublisherParams, consumer_params)
|
|
226
|
+
# print(self.publisher_params)
|
|
225
227
|
if is_main_process:
|
|
226
228
|
self.logger.info(f'{self.queue_name} consumer 的消费者配置:\n {self.consumer_params.json_str_value()}')
|
|
227
229
|
atexit.register(self.join_shedual_task_thread)
|
|
@@ -381,6 +383,33 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
381
383
|
"""
|
|
382
384
|
raise NotImplementedError
|
|
383
385
|
|
|
386
|
+
def __auto_fill_msg(self, msg: dict):
|
|
387
|
+
"""填充消息,消息没有使用funboost来发送,并且没有extra相关字段时候"""
|
|
388
|
+
""" 一般消息至少包含这样
|
|
389
|
+
{
|
|
390
|
+
"a": 42,
|
|
391
|
+
"b": 84,
|
|
392
|
+
"extra": {
|
|
393
|
+
"task_id": "queue_2_result:9b79a372-f765-4a33-8639-9d15d7a95f61",
|
|
394
|
+
"publish_time": 1701687443.3596,
|
|
395
|
+
"publish_time_format": "2023-12-04 18:57:23"
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
"""
|
|
399
|
+
"""
|
|
400
|
+
extra_params = {'task_id': task_id, 'publish_time': round(time.time(), 4),
|
|
401
|
+
'publish_time_format': time.strftime('%Y-%m-%d %H:%M:%S')}
|
|
402
|
+
"""
|
|
403
|
+
if 'extra' not in msg:
|
|
404
|
+
msg['extra'] = {}
|
|
405
|
+
extra = msg['extra']
|
|
406
|
+
if 'task_id' not in extra:
|
|
407
|
+
extra['task_id'] = generate_task_id(self._queue_name)
|
|
408
|
+
if 'publish_time' not in extra:
|
|
409
|
+
extra['publish_time'] = round(time.time(), 4)
|
|
410
|
+
if 'publish_time_format':
|
|
411
|
+
extra['publish_time_format'] = time.strftime('%Y-%m-%d %H:%M:%S')
|
|
412
|
+
|
|
384
413
|
def _submit_task(self, kw):
|
|
385
414
|
while 1: # 这一块的代码为支持暂停消费。
|
|
386
415
|
# print(self._pause_flag)
|
|
@@ -396,7 +425,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
396
425
|
self._requeue(kw)
|
|
397
426
|
time.sleep(self.time_interval_for_check_do_not_run_time)
|
|
398
427
|
return
|
|
399
|
-
|
|
428
|
+
self.__auto_fill_msg(kw['body'])
|
|
400
429
|
function_only_params = delete_keys_and_return_new_dict(kw['body'], )
|
|
401
430
|
if self._get_priority_conf(kw, 'do_task_filtering') and self._redis_filter.check_value_exists(
|
|
402
431
|
function_only_params): # 对函数的参数进行检查,过滤已经执行过并且成功的任务。
|
funboost/core/booster.py
CHANGED
|
@@ -155,7 +155,9 @@ class Booster:
|
|
|
155
155
|
file_volume_limit=1000 * 1000, sftp_log_level=20, extra_shell_str='',
|
|
156
156
|
invoke_runner_kwargs={'hide': None, 'pty': True, 'warn': False},
|
|
157
157
|
python_interpreter='python3',
|
|
158
|
-
process_num=1
|
|
158
|
+
process_num=1,
|
|
159
|
+
pkey_file_path=None,
|
|
160
|
+
):
|
|
159
161
|
"""
|
|
160
162
|
入参见 fabric_deploy 函数。这里重复入参是为了代码在pycharm补全提示。
|
|
161
163
|
"""
|
|
@@ -1,17 +1,12 @@
|
|
|
1
1
|
# noinspection PyDefaultArgument
|
|
2
|
-
import re
|
|
3
2
|
import sys
|
|
4
3
|
import threading
|
|
5
4
|
import time
|
|
6
|
-
import os
|
|
7
|
-
import typing
|
|
8
5
|
from pathlib import Path
|
|
9
6
|
from fabric2 import Connection
|
|
10
7
|
from nb_libs.path_helper import PathHelper
|
|
11
|
-
# from funboost.core.booster import Booster
|
|
12
8
|
from funboost.utils.paramiko_util import ParamikoFolderUploader
|
|
13
9
|
|
|
14
|
-
# import nb_log
|
|
15
10
|
from funboost.core.loggers import get_funboost_file_logger
|
|
16
11
|
from funboost.core.booster import Booster
|
|
17
12
|
|
|
@@ -19,9 +14,6 @@ logger = get_funboost_file_logger(__name__)
|
|
|
19
14
|
|
|
20
15
|
|
|
21
16
|
# noinspection PyDefaultArgument
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
17
|
def fabric_deploy(booster: Booster, host, port, user, password,
|
|
26
18
|
path_pattern_exluded_tuple=('/.git/', '/.idea/', '/dist/', '/build/'),
|
|
27
19
|
file_suffix_tuple_exluded=('.pyc', '.log', '.gz'),
|
|
@@ -29,7 +21,9 @@ def fabric_deploy(booster: Booster, host, port, user, password,
|
|
|
29
21
|
file_volume_limit=1000 * 1000, sftp_log_level=20, extra_shell_str='',
|
|
30
22
|
invoke_runner_kwargs={'hide': None, 'pty': True, 'warn': False},
|
|
31
23
|
python_interpreter='python3',
|
|
32
|
-
process_num=1
|
|
24
|
+
process_num=1,
|
|
25
|
+
pkey_file_path=None,
|
|
26
|
+
):
|
|
33
27
|
"""
|
|
34
28
|
不依赖阿里云codepipeline 和任何运维发布管理工具,只需要在python代码层面就能实现多机器远程部署。
|
|
35
29
|
这实现了函数级别的精确部署,而非是部署一个 .py的代码,远程部署一个函数实现难度比远程部署一个脚本更高一点,部署更灵活。
|
|
@@ -65,6 +59,7 @@ def fabric_deploy(booster: Booster, host, port, user, password,
|
|
|
65
59
|
pty 的意思是,远程机器的部署的代码进程是否随着当前脚本的结束而结束。如果为True,本机代码结束远程进程就会结束。如果为False,即使本机代码被关闭结束,远程机器还在运行代码。
|
|
66
60
|
warn 的意思是如果远程机器控制台返回了异常码本机代码是否立即退出。warn为True这只是警告一下,warn为False,远程机器返回异常code码则本机代码直接终止退出。
|
|
67
61
|
:param process_num:启动几个进程,要达到最大cpu性能就开启cpu核数个进程就可以了。每个进程内部都有任务函数本身指定的并发方式和并发数量,所以是多进程+线程/协程。
|
|
62
|
+
:param pkey_file_path: 私钥文件路径,如果设置了这个参数,则使用ssh私钥登录远程机器,如果没设置,则使用密码登录。
|
|
68
63
|
:return:
|
|
69
64
|
|
|
70
65
|
|
|
@@ -80,7 +75,7 @@ def fabric_deploy(booster: Booster, host, port, user, password,
|
|
|
80
75
|
func_name = booster.consuming_function.__name__
|
|
81
76
|
|
|
82
77
|
"""以下这种是为了兼容 函数没有@boost,而是使用 boosterxx = BoostersManager.build_booster() 来创建的booster. 下面的 python_exec_str 中需要用到 func_name
|
|
83
|
-
也可以远程时候使用 BoostersManager.get_booster(queue_name),然后启动消费. 因为import
|
|
78
|
+
也可以远程时候使用 BoostersManager.get_booster(queue_name),然后启动消费. 因为import模块后,就注册booster信息到BoostersManager,所以能启动了.
|
|
84
79
|
"""
|
|
85
80
|
module_obj = PathHelper(sys._getframe(2).f_code.co_filename).import_as_module() # noqa
|
|
86
81
|
for var_name,var_value in module_obj.__dict__.items():
|
|
@@ -99,7 +94,7 @@ def fabric_deploy(booster: Booster, host, port, user, password,
|
|
|
99
94
|
t_start = time.perf_counter()
|
|
100
95
|
uploader = ParamikoFolderUploader(host, port, user, password, python_proj_dir, remote_dir,
|
|
101
96
|
path_pattern_exluded_tuple, file_suffix_tuple_exluded,
|
|
102
|
-
only_upload_within_the_last_modify_time, file_volume_limit, sftp_log_level)
|
|
97
|
+
only_upload_within_the_last_modify_time, file_volume_limit, sftp_log_level, pkey_file_path)
|
|
103
98
|
uploader.upload()
|
|
104
99
|
logger.info(f'上传 本地文件夹代码 {python_proj_dir} 上传到远程 {host} 的 {remote_dir} 文件夹耗时 {round(time.perf_counter() - t_start, 3)} 秒')
|
|
105
100
|
# conn.run(f'''export PYTHONPATH={remote_dir}:$PYTHONPATH''')
|
|
@@ -81,6 +81,14 @@ class BaseJsonAbleModel(BaseModel):
|
|
|
81
81
|
# allow_mutation = False
|
|
82
82
|
extra = "forbid"
|
|
83
83
|
|
|
84
|
+
@staticmethod
|
|
85
|
+
def init_by_another_model(model_type: typing.Type[BaseModel], modelx: BaseModel):
|
|
86
|
+
init_dict = {}
|
|
87
|
+
for k, v in modelx.dict().items():
|
|
88
|
+
if k in model_type.__fields__.keys():
|
|
89
|
+
init_dict[k] = v
|
|
90
|
+
return model_type(**init_dict)
|
|
91
|
+
|
|
84
92
|
|
|
85
93
|
class FunctionResultStatusPersistanceConfig(BaseJsonAbleModel):
|
|
86
94
|
is_save_status: bool # 是否保存函数的运行状态信息
|
|
@@ -175,6 +183,8 @@ class BoosterParams(BaseJsonAbleModel):
|
|
|
175
183
|
broker_exclusive_config: dict = {} # 加上一个不同种类中间件非通用的配置,不同中间件自身独有的配置,不是所有中间件都兼容的配置,因为框架支持30种消息队列,消息队列不仅仅是一般的先进先出queue这么简单的概念,
|
|
176
184
|
# 例如kafka支持消费者组,rabbitmq也支持各种独特概念例如各种ack机制 复杂路由机制,有的中间件原生能支持消息优先级有的中间件不支持,每一种消息队列都有独特的配置参数意义,可以通过这里传递。每种中间件能传递的键值对可以看consumer类的 BROKER_EXCLUSIVE_CONFIG_DEFAULT
|
|
177
185
|
|
|
186
|
+
should_check_publish_func_params: bool = True # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字, 如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
|
|
187
|
+
|
|
178
188
|
auto_generate_info: dict = {} # 自动生成的信息,不需要用户主动传参.
|
|
179
189
|
|
|
180
190
|
@root_validator(skip_on_failure=True)
|
|
@@ -194,7 +204,7 @@ class BoosterParams(BaseJsonAbleModel):
|
|
|
194
204
|
# if not set(values.keys()).issubset(set(BoosterParams.__fields__.keys())):
|
|
195
205
|
# raise ValueError(f'{cls.__name__} 的字段包含了父类 BoosterParams 不存在的字段')
|
|
196
206
|
for k in values.keys():
|
|
197
|
-
if k not in BoosterParams.
|
|
207
|
+
if k not in BoosterParams.__fields__.keys():
|
|
198
208
|
raise ValueError(f'{cls.__name__} 的字段新增了父类 BoosterParams 不存在的字段 "{k}"') # 使 BoosterParams的子类,不能增加字段,只能覆盖字段.
|
|
199
209
|
return values
|
|
200
210
|
|
|
@@ -255,6 +265,7 @@ class PublisherParams(BaseJsonAbleModel):
|
|
|
255
265
|
consuming_function: typing.Callable = None # consuming_function 作用是 inspect 模块获取函数的入参信息
|
|
256
266
|
broker_kind: str = None
|
|
257
267
|
broker_exclusive_config: dict = {}
|
|
268
|
+
should_check_publish_func_params: bool = True # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字, 如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
|
|
258
269
|
|
|
259
270
|
|
|
260
271
|
if __name__ == '__main__':
|
funboost/core/helper_funs.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import copy
|
|
2
2
|
import time
|
|
3
|
+
import uuid
|
|
3
4
|
|
|
4
5
|
|
|
5
6
|
def get_publish_time(paramsx: dict):
|
|
@@ -42,4 +43,7 @@ def _try_get_user_funboost_common_config(funboost_common_conf_field:str):
|
|
|
42
43
|
return getattr(funboost_config.FunboostCommonConfig,funboost_common_conf_field)
|
|
43
44
|
except Exception as e:
|
|
44
45
|
print(e)
|
|
45
|
-
return None
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
def generate_task_id(queue_name:str):
|
|
49
|
+
return f'{queue_name}_result:{uuid.uuid4()}'
|
funboost/core/lazy_impoter.py
CHANGED
|
@@ -79,6 +79,20 @@ class EventletImporter:
|
|
|
79
79
|
self.patcher = patcher
|
|
80
80
|
self.Timeout = Timeout
|
|
81
81
|
|
|
82
|
+
@singleton
|
|
83
|
+
class PeeweeImporter:
|
|
84
|
+
def __init__(self):
|
|
85
|
+
'''pip install peewee == 3.17'''
|
|
86
|
+
from peewee import ModelSelect, Model, BigAutoField, CharField, DateTimeField, MySQLDatabase
|
|
87
|
+
from playhouse.shortcuts import model_to_dict, dict_to_model
|
|
88
|
+
self.ModelSelect = ModelSelect
|
|
89
|
+
self.Model = Model
|
|
90
|
+
self.BigAutoField = BigAutoField
|
|
91
|
+
self.CharField = CharField
|
|
92
|
+
self.DateTimeField =DateTimeField
|
|
93
|
+
self.MySQLDatabase = MySQLDatabase
|
|
94
|
+
self.model_to_dict = model_to_dict
|
|
95
|
+
self.dict_to_model =dict_to_model
|
|
82
96
|
|
|
83
97
|
if __name__ == '__main__':
|
|
84
98
|
for i in range(10000):
|
|
@@ -19,6 +19,7 @@ import amqpstorm
|
|
|
19
19
|
|
|
20
20
|
import nb_log
|
|
21
21
|
from funboost.core.func_params_model import PublisherParams, PriorityConsumingControlConfig
|
|
22
|
+
from funboost.core.helper_funs import generate_task_id
|
|
22
23
|
from funboost.core.loggers import develop_logger
|
|
23
24
|
|
|
24
25
|
from pikav1.exceptions import AMQPError as PikaAMQPError
|
|
@@ -188,9 +189,9 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
188
189
|
if 'extra' in msg:
|
|
189
190
|
msg_function_kw.pop('extra')
|
|
190
191
|
raw_extra = msg['extra']
|
|
191
|
-
if self.publish_params_checker:
|
|
192
|
+
if self.publish_params_checker and self.publisher_params.should_check_publish_func_params:
|
|
192
193
|
self.publish_params_checker.check_params(msg_function_kw)
|
|
193
|
-
task_id = task_id or
|
|
194
|
+
task_id = task_id or generate_task_id(self._queue_name)
|
|
194
195
|
extra_params = {'task_id': task_id, 'publish_time': round(time.time(), 4),
|
|
195
196
|
'publish_time_format': time.strftime('%Y-%m-%d %H:%M:%S')}
|
|
196
197
|
if priority_control_config:
|
funboost/queues/peewee_queue.py
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import datetime
|
|
2
2
|
import time
|
|
3
|
-
|
|
4
|
-
from peewee import ModelSelect
|
|
5
|
-
from playhouse.shortcuts import model_to_dict, dict_to_model
|
|
3
|
+
#
|
|
4
|
+
# from peewee import ModelSelect, Model, BigAutoField, CharField, DateTimeField, MySQLDatabase
|
|
5
|
+
# from playhouse.shortcuts import model_to_dict, dict_to_model
|
|
6
6
|
|
|
7
7
|
# from nb_log import LoggerMixin, LoggerLevelSetterMixin
|
|
8
8
|
from funboost.core.loggers import LoggerLevelSetterMixin,FunboostFileLoggerMixin
|
|
9
9
|
from funboost.funboost_config_deafult import BrokerConnConfig
|
|
10
|
-
from peewee import *
|
|
11
|
-
|
|
10
|
+
# from peewee import *
|
|
11
|
+
from funboost.core.lazy_impoter import PeeweeImporter
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
class TaskStatus:
|
|
@@ -30,13 +30,13 @@ class PeeweeQueue(FunboostFileLoggerMixin, LoggerLevelSetterMixin):
|
|
|
30
30
|
self._create_table()
|
|
31
31
|
|
|
32
32
|
def _create_table(self):
|
|
33
|
-
class FunboostMessage(Model):
|
|
33
|
+
class FunboostMessage(PeeweeImporter().Model):
|
|
34
34
|
"""数据库的一行模拟一条消息"""
|
|
35
|
-
job_id = BigAutoField(primary_key=True, )
|
|
36
|
-
body = CharField(max_length=10240, null=False)
|
|
37
|
-
publish_timestamp = DateTimeField(default=datetime.datetime.now)
|
|
38
|
-
status = CharField(max_length=40, null=False)
|
|
39
|
-
consume_start_timestamp = DateTimeField(default=None, null=True)
|
|
35
|
+
job_id = PeeweeImporter().BigAutoField(primary_key=True, )
|
|
36
|
+
body = PeeweeImporter().CharField(max_length=10240, null=False)
|
|
37
|
+
publish_timestamp = PeeweeImporter().DateTimeField(default=datetime.datetime.now)
|
|
38
|
+
status = PeeweeImporter().CharField(max_length=40, null=False)
|
|
39
|
+
consume_start_timestamp = PeeweeImporter().DateTimeField(default=None, null=True)
|
|
40
40
|
|
|
41
41
|
class Meta:
|
|
42
42
|
db_table = self.queue_name
|
|
@@ -47,7 +47,7 @@ class PeeweeQueue(FunboostFileLoggerMixin, LoggerLevelSetterMixin):
|
|
|
47
47
|
passwd=BrokerConnConfig.MYSQL_PASSWORD,
|
|
48
48
|
database=BrokerConnConfig.MYSQL_DATABASE,
|
|
49
49
|
)
|
|
50
|
-
database = MySQLDatabase(**conn_params)
|
|
50
|
+
database = PeeweeImporter().MySQLDatabase(**conn_params)
|
|
51
51
|
|
|
52
52
|
FunboostMessage.create_table()
|
|
53
53
|
self.FunboostMessage = FunboostMessage
|
|
@@ -68,7 +68,7 @@ class PeeweeQueue(FunboostFileLoggerMixin, LoggerLevelSetterMixin):
|
|
|
68
68
|
# print(ret)
|
|
69
69
|
if len(ret) == 1:
|
|
70
70
|
row_obj = ret[0]
|
|
71
|
-
row = model_to_dict(row_obj)
|
|
71
|
+
row = PeeweeImporter().model_to_dict(row_obj)
|
|
72
72
|
self.FunboostMessage.update(status=TaskStatus.PENGDING, consume_start_timestamp=datetime.datetime.now()
|
|
73
73
|
).where(self.FunboostMessage.job_id == row['job_id']).execute()
|
|
74
74
|
return row
|
funboost/utils/paramiko_util.py
CHANGED
|
@@ -16,7 +16,7 @@ class ParamikoFolderUploader(LoggerMixin, LoggerLevelSetterMixin):
|
|
|
16
16
|
path_pattern_exluded_tuple=('/.git/', '/.idea/', '/dist/', '/build/'),
|
|
17
17
|
file_suffix_tuple_exluded=('.pyc', '.log', '.gz'),
|
|
18
18
|
only_upload_within_the_last_modify_time=3650 * 24 * 60 * 60,
|
|
19
|
-
file_volume_limit=1000 * 1000, sftp_log_level=20):
|
|
19
|
+
file_volume_limit=1000 * 1000, sftp_log_level=20, pkey_file_path=None):
|
|
20
20
|
"""
|
|
21
21
|
|
|
22
22
|
:param host:
|
|
@@ -30,6 +30,7 @@ class ParamikoFolderUploader(LoggerMixin, LoggerLevelSetterMixin):
|
|
|
30
30
|
:param only_upload_within_the_last_modify_time: 仅仅上传最近多少天修改的文件
|
|
31
31
|
:param file_volume_limit: 大于这个体积的不上传,单位b。
|
|
32
32
|
:param sftp_log_level:日志级别
|
|
33
|
+
:param pkey_file_path: 私钥文件路径,如果设置了这个,那么使用私钥登录。
|
|
33
34
|
"""
|
|
34
35
|
self._host = host
|
|
35
36
|
self._port = port
|
|
@@ -48,7 +49,11 @@ class ParamikoFolderUploader(LoggerMixin, LoggerLevelSetterMixin):
|
|
|
48
49
|
self._file_volume_limit = file_volume_limit
|
|
49
50
|
|
|
50
51
|
t = paramiko.Transport((host, port))
|
|
51
|
-
|
|
52
|
+
if pkey_file_path is not None and os.path.exists(pkey_file_path):
|
|
53
|
+
pkey = paramiko.RSAKey.from_private_key_file(open(pkey_file_path))
|
|
54
|
+
t.connect(username=user, pkey=pkey)
|
|
55
|
+
else:
|
|
56
|
+
t.connect(username=user, password=password)
|
|
52
57
|
self.sftp = paramiko.SFTPClient.from_transport(t)
|
|
53
58
|
|
|
54
59
|
ssh = paramiko.SSHClient()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: funboost
|
|
3
|
-
Version: 43.
|
|
3
|
+
Version: 43.9
|
|
4
4
|
Summary: pip install funboost,python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件,支持如 celery dramatiq等框架整体作为funboost中间件,python函数加速器,框架包罗万象,用户能想到的控制功能全都有。一统编程思维,兼容50% python业务场景,适用范围广。只需要一行代码即可分布式执行python一切函数,99%用过funboost的pythoner 感受是 简易 方便 强劲 强大,相见恨晚
|
|
5
5
|
Home-page: https://github.com/ydf0509/funboost
|
|
6
6
|
Author: bfzs
|
|
@@ -44,7 +44,6 @@ Requires-Dist: kafka-python ==2.0.2
|
|
|
44
44
|
Requires-Dist: requests
|
|
45
45
|
Requires-Dist: gnsq ==1.0.1
|
|
46
46
|
Requires-Dist: psutil
|
|
47
|
-
Requires-Dist: peewee ==3.15.1
|
|
48
47
|
Requires-Dist: apscheduler ==3.10.1
|
|
49
48
|
Requires-Dist: pikav0
|
|
50
49
|
Requires-Dist: pikav1
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
funboost/__init__.py,sha256=
|
|
1
|
+
funboost/__init__.py,sha256=H-RYbw9IE6_YoCnuABJoytt_IAGKkpt-IFzEY8GtSHo,3832
|
|
2
2
|
funboost/__init__old.py,sha256=07A1MLsxLtuRQQOIfDyphddOwNBrGe34enoHWAnjV14,20379
|
|
3
3
|
funboost/__main__.py,sha256=-6Nogi666Y0LN8fVm3JmHGTOk8xEGWvotW_GDbSaZME,1065
|
|
4
4
|
funboost/constant.py,sha256=Yxt3WJt9D8ybcrgiojOy0qjnq5mffwTnTJplerGL0Oo,7188
|
|
@@ -33,7 +33,7 @@ funboost/concurrent_pool/backup/async_pool_executor0223.py,sha256=RVUZiylUvpTm6U
|
|
|
33
33
|
funboost/concurrent_pool/backup/async_pool_executor_back.py,sha256=KL6zEQaa1KkZOlAO85mCC1gwLm-YC5Ghn21IUom0UKM,9598
|
|
34
34
|
funboost/concurrent_pool/backup/async_pool_executor_janus.py,sha256=OHMWJ9l3EYTpPpcrPrGGKd4K0tmQ2PN8HiX0Dta0EOo,5728
|
|
35
35
|
funboost/consumers/__init__.py,sha256=ZXY_6Kut1VYNQiF5aWEgIWobsW1ht9YUP0TdRZRWFqI,126
|
|
36
|
-
funboost/consumers/base_consumer.py,sha256=
|
|
36
|
+
funboost/consumers/base_consumer.py,sha256=rUxBGuvadSBxNRw2uoZpC9plMUxwQBWgjQqAf4Wo1mU,75738
|
|
37
37
|
funboost/consumers/celery_consumer.py,sha256=9gtz7nlZkmv3ErmaseT0_Q__ltSPx-fOcwi-TMPoaLA,9220
|
|
38
38
|
funboost/consumers/confirm_mixin.py,sha256=eY6fNwx51Hn4bQSYRjyTRwOqfCGsikVnd2Ga_Ep31N4,6062
|
|
39
39
|
funboost/consumers/dramatiq_consumer.py,sha256=ozmeAfeF0U-YNYHK4suQB0N264h5AZdfMH0O45Mh-8A,2229
|
|
@@ -81,16 +81,16 @@ funboost/contrib/redis_consume_latest_msg_broker.py,sha256=ESortBZ2qu_4PBCa3e3Fe
|
|
|
81
81
|
funboost/contrib/save_result_status_to_sqldb.py,sha256=AxvD7nHs4sjr9U0kwEZzyPKrsGdU_JzEgzzhh_V1_4w,4071
|
|
82
82
|
funboost/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
83
83
|
funboost/core/active_cousumer_info_getter.py,sha256=09fEc-BTEIRfDDfHmOvKnMjLjtOyp4edLsUlAXUR_Qs,4966
|
|
84
|
-
funboost/core/booster.py,sha256=
|
|
84
|
+
funboost/core/booster.py,sha256=BMSrQbpvIfhigzHjloFwHb7mlw69bvk48RJN43dDiTQ,15914
|
|
85
85
|
funboost/core/current_task.py,sha256=rAIQVLqYqtlHVRkjl17yki-mqvuMb640ssGmto4RSdA,4864
|
|
86
86
|
funboost/core/exceptions.py,sha256=pLF7BkRJAfDiWp2_xGZqencmwdPiSQI1NENbImExknY,1311
|
|
87
|
-
funboost/core/fabric_deploy_helper.py,sha256=
|
|
88
|
-
funboost/core/func_params_model.py,sha256=
|
|
87
|
+
funboost/core/fabric_deploy_helper.py,sha256=AbiQqCCDCnCf9vU4rhaAGLRiS4XBe5_WMnoHmAysYto,9958
|
|
88
|
+
funboost/core/func_params_model.py,sha256=9LbfO0HnRmTph0qy8BDvBtoYurGXPfnX6hAiUzPM4ZI,19115
|
|
89
89
|
funboost/core/function_result_status_config.py,sha256=PyjqAQOiwsLt28sRtH-eYRjiI3edPFO4Nde0ILFRReE,1764
|
|
90
90
|
funboost/core/function_result_status_saver.py,sha256=UdokGSwU630t70AZnT9Ecj7GpYXORBDivlc9kadoI2E,9172
|
|
91
|
-
funboost/core/helper_funs.py,sha256=
|
|
91
|
+
funboost/core/helper_funs.py,sha256=k5HttGoz-E9doLBFz0P0b8SCiCWXuk3ZYD_A-tAfn0Q,1682
|
|
92
92
|
funboost/core/kill_remote_task.py,sha256=MZ5vWLGt6SxyN76h5Lf_id9tyVUzjR-qXNyJwXaGlZY,8129
|
|
93
|
-
funboost/core/lazy_impoter.py,sha256=
|
|
93
|
+
funboost/core/lazy_impoter.py,sha256=k0qZs39VL5qb7FspMt0x1noxqh3RIzt_czlFBwcyvKA,2722
|
|
94
94
|
funboost/core/loggers.py,sha256=173aXdqE8nAe8t6OcVMNAFsCUClVrWQovdQhTAg9IyM,2407
|
|
95
95
|
funboost/core/msg_result_getter.py,sha256=oZDuLDR5XQNzzvgDTsA7wroICToPwrkU9-OAaXXUQSk,8031
|
|
96
96
|
funboost/core/muliti_process_enhance.py,sha256=64rkVa5Eel-0EY2B7lc1dQTRwX4ehARVvcxQVDa6jr0,3568
|
|
@@ -122,7 +122,7 @@ funboost/function_result_web/static/js/jquery-1.11.0.min.js,sha256=ryQZ3RXgnqkTz
|
|
|
122
122
|
funboost/function_result_web/templates/index.html,sha256=YM0582Q4t2da-xBf3Ga0McIfcsT9H98rjZck-irMkGo,20387
|
|
123
123
|
funboost/function_result_web/templates/login.html,sha256=q37dj7O0LeyiV38Zd5P1Qn_qmhjdFomuYTRY1Yk48Bo,2007
|
|
124
124
|
funboost/publishers/__init__.py,sha256=xqBHlvsJQVPfbdvP84G0LHmVB7-pFBS7vDnX1Uo9pVY,131
|
|
125
|
-
funboost/publishers/base_publisher.py,sha256=
|
|
125
|
+
funboost/publishers/base_publisher.py,sha256=81Lj8atdD9_Hyh4cGJCa84zhxKQzqV13x2bax58fQwc,15176
|
|
126
126
|
funboost/publishers/celery_publisher.py,sha256=uc9N1uLW74skUCw8dsnvxORM2O3cy4SiI7tUZRmvkHA,2336
|
|
127
127
|
funboost/publishers/celery_publisher000.py,sha256=2XLOyU2__vlIUTi5L15uf0BJqAIjxbc3kCLIRDSOY9w,3966
|
|
128
128
|
funboost/publishers/confluent_kafka_publisher.py,sha256=gC4SUk6I_7zjSngcbTI7oTJ7sza3oE3PE19KQkwCpn4,4802
|
|
@@ -160,7 +160,7 @@ funboost/publishers/txt_file_publisher.py,sha256=dqrfBt1ejjwJbwFS3UqPo4VFDCa6NWb
|
|
|
160
160
|
funboost/publishers/udp_publisher.py,sha256=TOiKrhmCMjx4teqIgdUwRic0lxyK2cupinafsz--wzY,1194
|
|
161
161
|
funboost/publishers/zeromq_publisher.py,sha256=tZStCARZVznYKN5vUEt65hy5WRsVkNsMC5rdyy9JdeM,956
|
|
162
162
|
funboost/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
163
|
-
funboost/queues/peewee_queue.py,sha256=
|
|
163
|
+
funboost/queues/peewee_queue.py,sha256=FrwegrilkHZG6Y1cGdl5Bt_UtA25f7m5TJQJMZ9YnJI,5280
|
|
164
164
|
funboost/queues/sqla_queue.py,sha256=b-2QPvvuMxklm41ibZhGKehaAV9trXBQFCOHzgThx_s,11080
|
|
165
165
|
funboost/timing_job/__init__.py,sha256=uMKIiZOqr8aXGPxZhDcVVeeNTsDi-S7kOkLc1nHaLQw,9099
|
|
166
166
|
funboost/timing_job/apscheduler_use_mysql_store.py,sha256=VoDtjdICIL99s3LdE1aaFvGFNZ27vtDWD3jj4X_SEio,372
|
|
@@ -179,7 +179,7 @@ funboost/utils/mongo_util.py,sha256=g2AdsanKm2v9X-OaTCS6hx_0JvRw5WukXIttN3TD9sI,
|
|
|
179
179
|
funboost/utils/monkey_color_log.py,sha256=QChhQMTB6phZ2eBaPq-9tFZF1n7pWeJgmJPIB_ugkvs,7367
|
|
180
180
|
funboost/utils/monkey_patches.py,sha256=vGmtPuTwNLbS8T3gpufSC_cD8_Vnp85roZrCpJZUSE4,2890
|
|
181
181
|
funboost/utils/mqtt_util.py,sha256=BfCmyYwI-B8VL9499_IuYlJDCbv6ZhwyWThMf8dANOU,3199
|
|
182
|
-
funboost/utils/paramiko_util.py,sha256=
|
|
182
|
+
funboost/utils/paramiko_util.py,sha256=Abtid-dnM2OxK6tByaA3nhpgRxpkWRYWvdziV8rE7cQ,5250
|
|
183
183
|
funboost/utils/rabbitmq_factory.py,sha256=ifDCn2RxSGL4MccmktJc5FYQhAcboCgHBlEo3WGpq3g,2910
|
|
184
184
|
funboost/utils/redis_manager.py,sha256=iG3e9k3oedtKcGwDnjKA0hUFsk_MSlvVM2C3m3K97Y4,3657
|
|
185
185
|
funboost/utils/redis_manager_old.py,sha256=c3RBXN6dM3kjVBqkCdotpZuYmFs4d9emfp5iJgC61Us,5516
|
|
@@ -264,9 +264,9 @@ funboost/utils/pysnooper_ydf/utils.py,sha256=evSmGi_Oul7vSP47AJ0DLjFwoCYCfunJZ1m
|
|
|
264
264
|
funboost/utils/pysnooper_ydf/variables.py,sha256=QejRDESBA06KG9OH4sBT4J1M55eaU29EIHg8K_igaXo,3693
|
|
265
265
|
funboost/utils/times/__init__.py,sha256=Y4bQD3SIA_E7W2YvHq2Qdi0dGM4H2DxyFNdDOuFOq1w,2417
|
|
266
266
|
funboost/utils/times/version.py,sha256=11XfnZVVzOgIhXXdeN_mYfdXThfrsbQHpA0wCjz-hpg,17
|
|
267
|
-
funboost-43.
|
|
268
|
-
funboost-43.
|
|
269
|
-
funboost-43.
|
|
270
|
-
funboost-43.
|
|
271
|
-
funboost-43.
|
|
272
|
-
funboost-43.
|
|
267
|
+
funboost-43.9.dist-info/LICENSE,sha256=9EPP2ktG_lAPB8PjmWV-c9BiaJHc_FP6pPLcUrUwx0E,11562
|
|
268
|
+
funboost-43.9.dist-info/METADATA,sha256=7fO2oWHqPhj3WOk1X00OvefH5GGGx8Rsx2n7wYPEzWI,30408
|
|
269
|
+
funboost-43.9.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
|
270
|
+
funboost-43.9.dist-info/entry_points.txt,sha256=yMSSAGRzRAAhGyNNQHw24MooKlDZsaJ499_D6fPl58A,96
|
|
271
|
+
funboost-43.9.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
|
|
272
|
+
funboost-43.9.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|