funboost 43.6__tar.gz → 43.8__tar.gz

{funboost-43.6/funboost.egg-info → funboost-43.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 43.6
+Version: 43.8
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-43.6 → funboost-43.8}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "43.6"
+__version__ = "43.8"
 
 from funboost.set_frame_config import show_frame_config
 

{funboost-43.6 → funboost-43.8}/funboost/core/booster.py

@@ -155,7 +155,9 @@ class Booster:
                       file_volume_limit=1000 * 1000, sftp_log_level=20, extra_shell_str='',
                       invoke_runner_kwargs={'hide': None, 'pty': True, 'warn': False},
                       python_interpreter='python3',
-                      process_num=1):
+                      process_num=1,
+                      pkey_file_path=None,
+                      ):
         """
         入参见 fabric_deploy 函数。这里重复入参是为了代码在pycharm补全提示。
         """

{funboost-43.6 → funboost-43.8}/funboost/core/fabric_deploy_helper.py

@@ -1,17 +1,12 @@
 # noinspection PyDefaultArgument
-import re
 import sys
 import threading
 import time
-import os
-import typing
 from pathlib import Path
 from fabric2 import Connection
 from nb_libs.path_helper import PathHelper
-# from funboost.core.booster import Booster
 from funboost.utils.paramiko_util import ParamikoFolderUploader
 
-# import nb_log
 from funboost.core.loggers import get_funboost_file_logger
 from funboost.core.booster import Booster
 
@@ -19,9 +14,6 @@ logger = get_funboost_file_logger(__name__)
 
 
 # noinspection PyDefaultArgument
-
-
-
 def fabric_deploy(booster: Booster, host, port, user, password,
                   path_pattern_exluded_tuple=('/.git/', '/.idea/', '/dist/', '/build/'),
                   file_suffix_tuple_exluded=('.pyc', '.log', '.gz'),
@@ -29,7 +21,9 @@ def fabric_deploy(booster: Booster, host, port, user, password,
                   file_volume_limit=1000 * 1000, sftp_log_level=20, extra_shell_str='',
                   invoke_runner_kwargs={'hide': None, 'pty': True, 'warn': False},
                   python_interpreter='python3',
-                  process_num=1):
+                  process_num=1,
+                  pkey_file_path=None,
+                  ):
     """
     不依赖阿里云codepipeline 和任何运维发布管理工具，只需要在python代码层面就能实现多机器远程部署。
     这实现了函数级别的精确部署，而非是部署一个 .py的代码，远程部署一个函数实现难度比远程部署一个脚本更高一点，部署更灵活。
@@ -65,27 +59,30 @@ def fabric_deploy(booster: Booster, host, port, user, password,
                                  pty 的意思是，远程机器的部署的代码进程是否随着当前脚本的结束而结束。如果为True，本机代码结束远程进程就会结束。如果为False，即使本机代码被关闭结束，远程机器还在运行代码。
                                  warn 的意思是如果远程机器控制台返回了异常码本机代码是否立即退出。warn为True这只是警告一下，warn为False,远程机器返回异常code码则本机代码直接终止退出。
     :param process_num:启动几个进程，要达到最大cpu性能就开启cpu核数个进程就可以了。每个进程内部都有任务函数本身指定的并发方式和并发数量，所以是多进程+线程/协程。
+    :param pkey_file_path: 私钥文件路径，如果设置了这个参数，则使用ssh私钥登录远程机器，如果没设置，则使用密码登录。
     :return:
 
 
     task_fun.fabric_deploy('192.168.6.133', 22, 'ydf', '123456', process_num=2) 只需要这样就可以自动部署在远程机器运行，无需任何额外操作。
     """
     # print(locals())
-    python_proj_dir = sys.path[1].replace('\\', '/') + '/'
+    python_proj_dir = Path(sys.path[1]).resolve().as_posix() + '/'
     python_proj_dir_short = python_proj_dir.split('/')[-2]
     # 获取被调用函数所在模块文件名
-    file_name = sys._getframe(2).f_code.co_filename.replace('\\', '/')  # noqa
+    file_name = Path(sys._getframe(2).f_code.co_filename).resolve().as_posix() # noqa
     relative_file_name = Path(file_name).relative_to(Path(python_proj_dir)).as_posix()
     relative_module = relative_file_name.replace('/', '.')[:-3]  # -3是去掉.py
     func_name = booster.consuming_function.__name__
-    module_obj = PathHelper(sys._getframe(2).f_code.co_filename).import_as_module()  # noqa
 
-    """以下这种是为了兼容 函数没有@boost.而是使用 boosterxx = BoostersManager.build_booster() 来创建的booster. 下面的 python_exec_str 中需要用到 func_name """
+    """以下这种是为了兼容 函数没有@boost,而是使用 boosterxx = BoostersManager.build_booster() 来创建的booster. 下面的 python_exec_str 中需要用到 func_name 
+    也可以远程时候使用 BoostersManager.get_booster(queue_name),然后启动消费.  因为import模块后,就注册booster信息到BoostersManager,所以能启动了.
+    """
+    module_obj = PathHelper(sys._getframe(2).f_code.co_filename).import_as_module()  # noqa
     for var_name,var_value in module_obj.__dict__.items():
         if isinstance(var_value,Booster) and var_value.queue_name == booster.queue_name:
             func_name = var_name
 
-    # print(file_name, python_proj_dir, relative_module, func_name)
+    logger.debug([file_name, python_proj_dir, python_proj_dir_short,relative_module, func_name])
     # print(relative_module)
     if user == 'root':  # 文件夹会被自动创建，无需用户创建。
         remote_dir = f'/codes/{python_proj_dir_short}'
@@ -97,7 +94,7 @@ def fabric_deploy(booster: Booster, host, port, user, password,
         t_start = time.perf_counter()
         uploader = ParamikoFolderUploader(host, port, user, password, python_proj_dir, remote_dir,
                                           path_pattern_exluded_tuple, file_suffix_tuple_exluded,
-                                          only_upload_within_the_last_modify_time, file_volume_limit, sftp_log_level)
+                                          only_upload_within_the_last_modify_time, file_volume_limit, sftp_log_level, pkey_file_path)
         uploader.upload()
         logger.info(f'上传 本地文件夹代码 {python_proj_dir}  上传到远程 {host} 的 {remote_dir} 文件夹耗时 {round(time.perf_counter() - t_start, 3)} 秒')
         # conn.run(f'''export PYTHONPATH={remote_dir}:$PYTHONPATH''')

{funboost-43.6 → funboost-43.8}/funboost/core/lazy_impoter.py

@@ -79,6 +79,20 @@ class EventletImporter:
         self.patcher = patcher
         self.Timeout = Timeout
 
+@singleton
+class PeeweeImporter:
+    def __init__(self):
+        '''pip install peewee == 3.17'''
+        from peewee import ModelSelect, Model, BigAutoField, CharField, DateTimeField, MySQLDatabase
+        from playhouse.shortcuts import model_to_dict, dict_to_model
+        self.ModelSelect = ModelSelect
+        self.Model = Model
+        self.BigAutoField = BigAutoField
+        self.CharField = CharField
+        self.DateTimeField =DateTimeField
+        self.MySQLDatabase = MySQLDatabase
+        self.model_to_dict = model_to_dict
+        self.dict_to_model =dict_to_model
 
 if __name__ == '__main__':
     for i in range(10000):

{funboost-43.6 → funboost-43.8}/funboost/queues/peewee_queue.py

@@ -1,14 +1,14 @@
 import datetime
 import time
-
-from peewee import ModelSelect
-from playhouse.shortcuts import model_to_dict, dict_to_model
+#
+# from peewee import ModelSelect, Model, BigAutoField, CharField, DateTimeField, MySQLDatabase
+# from playhouse.shortcuts import model_to_dict, dict_to_model
 
 # from nb_log import LoggerMixin, LoggerLevelSetterMixin
 from funboost.core.loggers import LoggerLevelSetterMixin,FunboostFileLoggerMixin
 from funboost.funboost_config_deafult import BrokerConnConfig
-from peewee import *
-
+# from peewee import *
+from funboost.core.lazy_impoter import PeeweeImporter
 
 
 class TaskStatus:
@@ -30,13 +30,13 @@ class PeeweeQueue(FunboostFileLoggerMixin, LoggerLevelSetterMixin):
         self._create_table()
 
     def _create_table(self):
-        class FunboostMessage(Model):
+        class FunboostMessage(PeeweeImporter().Model):
             """数据库的一行模拟一条消息"""
-            job_id = BigAutoField(primary_key=True, )
-            body = CharField(max_length=10240, null=False)
-            publish_timestamp = DateTimeField(default=datetime.datetime.now)
-            status = CharField(max_length=40, null=False)
-            consume_start_timestamp = DateTimeField(default=None, null=True)
+            job_id = PeeweeImporter().BigAutoField(primary_key=True, )
+            body = PeeweeImporter().CharField(max_length=10240, null=False)
+            publish_timestamp = PeeweeImporter().DateTimeField(default=datetime.datetime.now)
+            status = PeeweeImporter().CharField(max_length=40, null=False)
+            consume_start_timestamp = PeeweeImporter().DateTimeField(default=None, null=True)
 
             class Meta:
                 db_table = self.queue_name
@@ -47,7 +47,7 @@ class PeeweeQueue(FunboostFileLoggerMixin, LoggerLevelSetterMixin):
                     passwd=BrokerConnConfig.MYSQL_PASSWORD,
                     database=BrokerConnConfig.MYSQL_DATABASE,
                 )
-                database = MySQLDatabase(**conn_params)
+                database = PeeweeImporter().MySQLDatabase(**conn_params)
 
         FunboostMessage.create_table()
         self.FunboostMessage = FunboostMessage
@@ -68,7 +68,7 @@ class PeeweeQueue(FunboostFileLoggerMixin, LoggerLevelSetterMixin):
             # print(ret)
             if len(ret) == 1:
                 row_obj = ret[0]
-                row = model_to_dict(row_obj)
+                row = PeeweeImporter().model_to_dict(row_obj)
                 self.FunboostMessage.update(status=TaskStatus.PENGDING, consume_start_timestamp=datetime.datetime.now()
                                             ).where(self.FunboostMessage.job_id == row['job_id']).execute()
                 return row

{funboost-43.6 → funboost-43.8}/funboost/utils/paramiko_util.py

@@ -16,7 +16,7 @@ class ParamikoFolderUploader(LoggerMixin, LoggerLevelSetterMixin):
                  path_pattern_exluded_tuple=('/.git/', '/.idea/', '/dist/', '/build/'),
                  file_suffix_tuple_exluded=('.pyc', '.log', '.gz'),
                  only_upload_within_the_last_modify_time=3650 * 24 * 60 * 60,
-                 file_volume_limit=1000 * 1000, sftp_log_level=20):
+                 file_volume_limit=1000 * 1000, sftp_log_level=20, pkey_file_path=None):
         """
 
         :param host:
@@ -30,6 +30,7 @@ class ParamikoFolderUploader(LoggerMixin, LoggerLevelSetterMixin):
         :param only_upload_within_the_last_modify_time: 仅仅上传最近多少天修改的文件
         :param file_volume_limit: 大于这个体积的不上传，单位b。
         :param sftp_log_level:日志级别
+        :param pkey_file_path: 私钥文件路径，如果设置了这个，那么使用私钥登录。
         """
         self._host = host
         self._port = port
@@ -48,7 +49,11 @@ class ParamikoFolderUploader(LoggerMixin, LoggerLevelSetterMixin):
         self._file_volume_limit = file_volume_limit
 
         t = paramiko.Transport((host, port))
-        t.connect(username=user, password=password)
+        if pkey_file_path is not None and os.path.exists(pkey_file_path):
+            pkey = paramiko.RSAKey.from_private_key_file(open(pkey_file_path))
+            t.connect(username=user, pkey=pkey)
+        else:
+            t.connect(username=user, password=password)
         self.sftp = paramiko.SFTPClient.from_transport(t)
 
         ssh = paramiko.SSHClient()

{funboost-43.6 → funboost-43.8/funboost.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 43.6
+Version: 43.8
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-43.6 → funboost-43.8}/funboost.egg-info/requires.txt

@@ -16,7 +16,6 @@ kafka-python==2.0.2
 requests
 gnsq==1.0.1
 psutil
-peewee==3.15.1
 apscheduler==3.10.1
 pikav0
 pikav1

{funboost-43.6 → funboost-43.8}/setup.py

@@ -85,7 +85,7 @@ setup(
         'psutil',
         # 'sqlalchemy==1.3.10',
         # 'sqlalchemy_utils==0.36.1',    # 用户使用数据库作为消息队列时候，自行安装，不自动安装这个包。也可以使用kombu中间件的sqlalchemy模式来操作数据库或者peewee操作。
-        'peewee==3.15.1',
+        # 'peewee==3.17.3',    # 'peewee==3.15.1',  # 惰性安装
         'apscheduler==3.10.1',
         'pikav0',
         'pikav1',