funboost 43.7__py3-none-any.whl → 43.8__py3-none-any.whl

funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "43.7"
+__version__ = "43.8"
 
 from funboost.set_frame_config import show_frame_config
 

funboost/core/booster.py

@@ -155,7 +155,9 @@ class Booster:
                       file_volume_limit=1000 * 1000, sftp_log_level=20, extra_shell_str='',
                       invoke_runner_kwargs={'hide': None, 'pty': True, 'warn': False},
                       python_interpreter='python3',
-                      process_num=1):
+                      process_num=1,
+                      pkey_file_path=None,
+                      ):
         """
         入参见 fabric_deploy 函数。这里重复入参是为了代码在pycharm补全提示。
         """

funboost/core/fabric_deploy_helper.py

@@ -1,17 +1,12 @@
 # noinspection PyDefaultArgument
-import re
 import sys
 import threading
 import time
-import os
-import typing
 from pathlib import Path
 from fabric2 import Connection
 from nb_libs.path_helper import PathHelper
-# from funboost.core.booster import Booster
 from funboost.utils.paramiko_util import ParamikoFolderUploader
 
-# import nb_log
 from funboost.core.loggers import get_funboost_file_logger
 from funboost.core.booster import Booster
 
@@ -19,9 +14,6 @@ logger = get_funboost_file_logger(__name__)
 
 
 # noinspection PyDefaultArgument
-
-
-
 def fabric_deploy(booster: Booster, host, port, user, password,
                   path_pattern_exluded_tuple=('/.git/', '/.idea/', '/dist/', '/build/'),
                   file_suffix_tuple_exluded=('.pyc', '.log', '.gz'),
@@ -29,7 +21,9 @@ def fabric_deploy(booster: Booster, host, port, user, password,
                   file_volume_limit=1000 * 1000, sftp_log_level=20, extra_shell_str='',
                   invoke_runner_kwargs={'hide': None, 'pty': True, 'warn': False},
                   python_interpreter='python3',
-                  process_num=1):
+                  process_num=1,
+                  pkey_file_path=None,
+                  ):
     """
     不依赖阿里云codepipeline 和任何运维发布管理工具，只需要在python代码层面就能实现多机器远程部署。
     这实现了函数级别的精确部署，而非是部署一个 .py的代码，远程部署一个函数实现难度比远程部署一个脚本更高一点，部署更灵活。
@@ -65,6 +59,7 @@ def fabric_deploy(booster: Booster, host, port, user, password,
                                  pty 的意思是，远程机器的部署的代码进程是否随着当前脚本的结束而结束。如果为True，本机代码结束远程进程就会结束。如果为False，即使本机代码被关闭结束，远程机器还在运行代码。
                                  warn 的意思是如果远程机器控制台返回了异常码本机代码是否立即退出。warn为True这只是警告一下，warn为False,远程机器返回异常code码则本机代码直接终止退出。
     :param process_num:启动几个进程，要达到最大cpu性能就开启cpu核数个进程就可以了。每个进程内部都有任务函数本身指定的并发方式和并发数量，所以是多进程+线程/协程。
+    :param pkey_file_path: 私钥文件路径，如果设置了这个参数，则使用ssh私钥登录远程机器，如果没设置，则使用密码登录。
     :return:
 
 
@@ -80,7 +75,7 @@ def fabric_deploy(booster: Booster, host, port, user, password,
     func_name = booster.consuming_function.__name__
 
     """以下这种是为了兼容 函数没有@boost,而是使用 boosterxx = BoostersManager.build_booster() 来创建的booster. 下面的 python_exec_str 中需要用到 func_name 
-    也可以远程时候使用 BoostersManager.get_booster(queue_name),然后启动消费.  因为import模块后,就能启动了.
+    也可以远程时候使用 BoostersManager.get_booster(queue_name),然后启动消费.  因为import模块后,就注册booster信息到BoostersManager,所以能启动了.
     """
     module_obj = PathHelper(sys._getframe(2).f_code.co_filename).import_as_module()  # noqa
     for var_name,var_value in module_obj.__dict__.items():
@@ -99,7 +94,7 @@ def fabric_deploy(booster: Booster, host, port, user, password,
         t_start = time.perf_counter()
         uploader = ParamikoFolderUploader(host, port, user, password, python_proj_dir, remote_dir,
                                           path_pattern_exluded_tuple, file_suffix_tuple_exluded,
-                                          only_upload_within_the_last_modify_time, file_volume_limit, sftp_log_level)
+                                          only_upload_within_the_last_modify_time, file_volume_limit, sftp_log_level, pkey_file_path)
         uploader.upload()
         logger.info(f'上传 本地文件夹代码 {python_proj_dir}  上传到远程 {host} 的 {remote_dir} 文件夹耗时 {round(time.perf_counter() - t_start, 3)} 秒')
         # conn.run(f'''export PYTHONPATH={remote_dir}:$PYTHONPATH''')

funboost/core/lazy_impoter.py

@@ -79,6 +79,20 @@ class EventletImporter:
         self.patcher = patcher
         self.Timeout = Timeout
 
+@singleton
+class PeeweeImporter:
+    def __init__(self):
+        '''pip install peewee == 3.17'''
+        from peewee import ModelSelect, Model, BigAutoField, CharField, DateTimeField, MySQLDatabase
+        from playhouse.shortcuts import model_to_dict, dict_to_model
+        self.ModelSelect = ModelSelect
+        self.Model = Model
+        self.BigAutoField = BigAutoField
+        self.CharField = CharField
+        self.DateTimeField =DateTimeField
+        self.MySQLDatabase = MySQLDatabase
+        self.model_to_dict = model_to_dict
+        self.dict_to_model =dict_to_model
 
 if __name__ == '__main__':
     for i in range(10000):

funboost/queues/peewee_queue.py

@@ -1,14 +1,14 @@
 import datetime
 import time
-
-from peewee import ModelSelect
-from playhouse.shortcuts import model_to_dict, dict_to_model
+#
+# from peewee import ModelSelect, Model, BigAutoField, CharField, DateTimeField, MySQLDatabase
+# from playhouse.shortcuts import model_to_dict, dict_to_model
 
 # from nb_log import LoggerMixin, LoggerLevelSetterMixin
 from funboost.core.loggers import LoggerLevelSetterMixin,FunboostFileLoggerMixin
 from funboost.funboost_config_deafult import BrokerConnConfig
-from peewee import *
-
+# from peewee import *
+from funboost.core.lazy_impoter import PeeweeImporter
 
 
 class TaskStatus:
@@ -30,13 +30,13 @@ class PeeweeQueue(FunboostFileLoggerMixin, LoggerLevelSetterMixin):
         self._create_table()
 
     def _create_table(self):
-        class FunboostMessage(Model):
+        class FunboostMessage(PeeweeImporter().Model):
             """数据库的一行模拟一条消息"""
-            job_id = BigAutoField(primary_key=True, )
-            body = CharField(max_length=10240, null=False)
-            publish_timestamp = DateTimeField(default=datetime.datetime.now)
-            status = CharField(max_length=40, null=False)
-            consume_start_timestamp = DateTimeField(default=None, null=True)
+            job_id = PeeweeImporter().BigAutoField(primary_key=True, )
+            body = PeeweeImporter().CharField(max_length=10240, null=False)
+            publish_timestamp = PeeweeImporter().DateTimeField(default=datetime.datetime.now)
+            status = PeeweeImporter().CharField(max_length=40, null=False)
+            consume_start_timestamp = PeeweeImporter().DateTimeField(default=None, null=True)
 
             class Meta:
                 db_table = self.queue_name
@@ -47,7 +47,7 @@ class PeeweeQueue(FunboostFileLoggerMixin, LoggerLevelSetterMixin):
                     passwd=BrokerConnConfig.MYSQL_PASSWORD,
                     database=BrokerConnConfig.MYSQL_DATABASE,
                 )
-                database = MySQLDatabase(**conn_params)
+                database = PeeweeImporter().MySQLDatabase(**conn_params)
 
         FunboostMessage.create_table()
         self.FunboostMessage = FunboostMessage
@@ -68,7 +68,7 @@ class PeeweeQueue(FunboostFileLoggerMixin, LoggerLevelSetterMixin):
             # print(ret)
             if len(ret) == 1:
                 row_obj = ret[0]
-                row = model_to_dict(row_obj)
+                row = PeeweeImporter().model_to_dict(row_obj)
                 self.FunboostMessage.update(status=TaskStatus.PENGDING, consume_start_timestamp=datetime.datetime.now()
                                             ).where(self.FunboostMessage.job_id == row['job_id']).execute()
                 return row

funboost/utils/paramiko_util.py

@@ -16,7 +16,7 @@ class ParamikoFolderUploader(LoggerMixin, LoggerLevelSetterMixin):
                  path_pattern_exluded_tuple=('/.git/', '/.idea/', '/dist/', '/build/'),
                  file_suffix_tuple_exluded=('.pyc', '.log', '.gz'),
                  only_upload_within_the_last_modify_time=3650 * 24 * 60 * 60,
-                 file_volume_limit=1000 * 1000, sftp_log_level=20):
+                 file_volume_limit=1000 * 1000, sftp_log_level=20, pkey_file_path=None):
         """
 
         :param host:
@@ -30,6 +30,7 @@ class ParamikoFolderUploader(LoggerMixin, LoggerLevelSetterMixin):
         :param only_upload_within_the_last_modify_time: 仅仅上传最近多少天修改的文件
         :param file_volume_limit: 大于这个体积的不上传，单位b。
         :param sftp_log_level:日志级别
+        :param pkey_file_path: 私钥文件路径，如果设置了这个，那么使用私钥登录。
         """
         self._host = host
         self._port = port
@@ -48,7 +49,11 @@ class ParamikoFolderUploader(LoggerMixin, LoggerLevelSetterMixin):
         self._file_volume_limit = file_volume_limit
 
         t = paramiko.Transport((host, port))
-        t.connect(username=user, password=password)
+        if pkey_file_path is not None and os.path.exists(pkey_file_path):
+            pkey = paramiko.RSAKey.from_private_key_file(open(pkey_file_path))
+            t.connect(username=user, pkey=pkey)
+        else:
+            t.connect(username=user, password=password)
         self.sftp = paramiko.SFTPClient.from_transport(t)
 
         ssh = paramiko.SSHClient()

{funboost-43.7.dist-info → funboost-43.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 43.7
+Version: 43.8
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs
@@ -44,7 +44,6 @@ Requires-Dist: kafka-python ==2.0.2
 Requires-Dist: requests
 Requires-Dist: gnsq ==1.0.1
 Requires-Dist: psutil
-Requires-Dist: peewee ==3.15.1
 Requires-Dist: apscheduler ==3.10.1
 Requires-Dist: pikav0
 Requires-Dist: pikav1

{funboost-43.7.dist-info → funboost-43.8.dist-info}/RECORD

@@ -1,4 +1,4 @@
-funboost/__init__.py,sha256=PLzAqSyCMps0EATLKAAgKgwQnNDonPSH37xnvU7gv7A,3832
+funboost/__init__.py,sha256=wESWldi73wbFUQRKyLjd4WcKNod-qHTxLtJNXoOnIII,3832
 funboost/__init__old.py,sha256=07A1MLsxLtuRQQOIfDyphddOwNBrGe34enoHWAnjV14,20379
 funboost/__main__.py,sha256=-6Nogi666Y0LN8fVm3JmHGTOk8xEGWvotW_GDbSaZME,1065
 funboost/constant.py,sha256=Yxt3WJt9D8ybcrgiojOy0qjnq5mffwTnTJplerGL0Oo,7188
@@ -81,16 +81,16 @@ funboost/contrib/redis_consume_latest_msg_broker.py,sha256=ESortBZ2qu_4PBCa3e3Fe
 funboost/contrib/save_result_status_to_sqldb.py,sha256=AxvD7nHs4sjr9U0kwEZzyPKrsGdU_JzEgzzhh_V1_4w,4071
 funboost/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 funboost/core/active_cousumer_info_getter.py,sha256=09fEc-BTEIRfDDfHmOvKnMjLjtOyp4edLsUlAXUR_Qs,4966
-funboost/core/booster.py,sha256=UjkMjhEYchUB-FyaFhrVM1XeMTpDdONBKjZXa4ADlgQ,15845
+funboost/core/booster.py,sha256=BMSrQbpvIfhigzHjloFwHb7mlw69bvk48RJN43dDiTQ,15914
 funboost/core/current_task.py,sha256=rAIQVLqYqtlHVRkjl17yki-mqvuMb640ssGmto4RSdA,4864
 funboost/core/exceptions.py,sha256=pLF7BkRJAfDiWp2_xGZqencmwdPiSQI1NENbImExknY,1311
-funboost/core/fabric_deploy_helper.py,sha256=D72f7rP6Qj6t9MRspiWriBns6cRyBvl48S0aBoavYEA,9781
+funboost/core/fabric_deploy_helper.py,sha256=AbiQqCCDCnCf9vU4rhaAGLRiS4XBe5_WMnoHmAysYto,9958
 funboost/core/func_params_model.py,sha256=ChxvSq_iVuYnGR_CdN9h9qNtvBq8sZ4BocdzCjgsQHc,18167
 funboost/core/function_result_status_config.py,sha256=PyjqAQOiwsLt28sRtH-eYRjiI3edPFO4Nde0ILFRReE,1764
 funboost/core/function_result_status_saver.py,sha256=UdokGSwU630t70AZnT9Ecj7GpYXORBDivlc9kadoI2E,9172
 funboost/core/helper_funs.py,sha256=M9Ad9EzgHdP581X-vuFgCavJRoezLGXlXSFg7zyMWD0,1578
 funboost/core/kill_remote_task.py,sha256=MZ5vWLGt6SxyN76h5Lf_id9tyVUzjR-qXNyJwXaGlZY,8129
-funboost/core/lazy_impoter.py,sha256=BnnCcTanjn-ghBXxQQE3BfhajVc60-2_cAe4XtmHDmI,2128
+funboost/core/lazy_impoter.py,sha256=k0qZs39VL5qb7FspMt0x1noxqh3RIzt_czlFBwcyvKA,2722
 funboost/core/loggers.py,sha256=173aXdqE8nAe8t6OcVMNAFsCUClVrWQovdQhTAg9IyM,2407
 funboost/core/msg_result_getter.py,sha256=oZDuLDR5XQNzzvgDTsA7wroICToPwrkU9-OAaXXUQSk,8031
 funboost/core/muliti_process_enhance.py,sha256=64rkVa5Eel-0EY2B7lc1dQTRwX4ehARVvcxQVDa6jr0,3568
@@ -160,7 +160,7 @@ funboost/publishers/txt_file_publisher.py,sha256=dqrfBt1ejjwJbwFS3UqPo4VFDCa6NWb
 funboost/publishers/udp_publisher.py,sha256=TOiKrhmCMjx4teqIgdUwRic0lxyK2cupinafsz--wzY,1194
 funboost/publishers/zeromq_publisher.py,sha256=tZStCARZVznYKN5vUEt65hy5WRsVkNsMC5rdyy9JdeM,956
 funboost/queues/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-funboost/queues/peewee_queue.py,sha256=CFHRxTt1jTYPBfzJMxTklHaNP9Wc0e8s725blu1zYw0,5022
+funboost/queues/peewee_queue.py,sha256=FrwegrilkHZG6Y1cGdl5Bt_UtA25f7m5TJQJMZ9YnJI,5280
 funboost/queues/sqla_queue.py,sha256=b-2QPvvuMxklm41ibZhGKehaAV9trXBQFCOHzgThx_s,11080
 funboost/timing_job/__init__.py,sha256=uMKIiZOqr8aXGPxZhDcVVeeNTsDi-S7kOkLc1nHaLQw,9099
 funboost/timing_job/apscheduler_use_mysql_store.py,sha256=VoDtjdICIL99s3LdE1aaFvGFNZ27vtDWD3jj4X_SEio,372
@@ -179,7 +179,7 @@ funboost/utils/mongo_util.py,sha256=g2AdsanKm2v9X-OaTCS6hx_0JvRw5WukXIttN3TD9sI,
 funboost/utils/monkey_color_log.py,sha256=QChhQMTB6phZ2eBaPq-9tFZF1n7pWeJgmJPIB_ugkvs,7367
 funboost/utils/monkey_patches.py,sha256=vGmtPuTwNLbS8T3gpufSC_cD8_Vnp85roZrCpJZUSE4,2890
 funboost/utils/mqtt_util.py,sha256=BfCmyYwI-B8VL9499_IuYlJDCbv6ZhwyWThMf8dANOU,3199
-funboost/utils/paramiko_util.py,sha256=pu67zkgptqNSV9m2Lznezb3zF1AFYvkWJp_6DVKFSPU,4901
+funboost/utils/paramiko_util.py,sha256=Abtid-dnM2OxK6tByaA3nhpgRxpkWRYWvdziV8rE7cQ,5250
 funboost/utils/rabbitmq_factory.py,sha256=ifDCn2RxSGL4MccmktJc5FYQhAcboCgHBlEo3WGpq3g,2910
 funboost/utils/redis_manager.py,sha256=iG3e9k3oedtKcGwDnjKA0hUFsk_MSlvVM2C3m3K97Y4,3657
 funboost/utils/redis_manager_old.py,sha256=c3RBXN6dM3kjVBqkCdotpZuYmFs4d9emfp5iJgC61Us,5516
@@ -264,9 +264,9 @@ funboost/utils/pysnooper_ydf/utils.py,sha256=evSmGi_Oul7vSP47AJ0DLjFwoCYCfunJZ1m
 funboost/utils/pysnooper_ydf/variables.py,sha256=QejRDESBA06KG9OH4sBT4J1M55eaU29EIHg8K_igaXo,3693
 funboost/utils/times/__init__.py,sha256=Y4bQD3SIA_E7W2YvHq2Qdi0dGM4H2DxyFNdDOuFOq1w,2417
 funboost/utils/times/version.py,sha256=11XfnZVVzOgIhXXdeN_mYfdXThfrsbQHpA0wCjz-hpg,17
-funboost-43.7.dist-info/LICENSE,sha256=9EPP2ktG_lAPB8PjmWV-c9BiaJHc_FP6pPLcUrUwx0E,11562
-funboost-43.7.dist-info/METADATA,sha256=HTH5jburyOsmSvV6PPagicB17XHYJu6C0rohPnzBqdM,30440
-funboost-43.7.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-funboost-43.7.dist-info/entry_points.txt,sha256=yMSSAGRzRAAhGyNNQHw24MooKlDZsaJ499_D6fPl58A,96
-funboost-43.7.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
-funboost-43.7.dist-info/RECORD,,
+funboost-43.8.dist-info/LICENSE,sha256=9EPP2ktG_lAPB8PjmWV-c9BiaJHc_FP6pPLcUrUwx0E,11562
+funboost-43.8.dist-info/METADATA,sha256=a6DT6uVuO6B454V1el0hNEW69IZU4kSh52Qow-3qrWM,30408
+funboost-43.8.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+funboost-43.8.dist-info/entry_points.txt,sha256=yMSSAGRzRAAhGyNNQHw24MooKlDZsaJ499_D6fPl58A,96
+funboost-43.8.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
+funboost-43.8.dist-info/RECORD,,